<commit_before># Copyright 2015, Oliver Nagy <olitheolix@gmail.com>
#
# This file is part of Azrael (https://github.com/olitheolix/azrael)
#
# Azrael is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Azrael is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Azrael. If not, see <http://www.gnu.org/licenses/>.

"""
Converts the JSON data from Clients to the native Python types Clerk expects,
and vice versa.

The codecs in this module specify the JSON format between Clerk and the
clients.

``ToClerk_*_Encode``: Converts the Client's JSON data to native (Python) types.
``FromClerk_*_Encode``: Converts Clerk's response to plain JSON.

The JSON only protocoly should make it possible to write clients in other
languages.
"""
import base64
import azrael.util
import azrael.aztypes as aztypes

from azrael.aztypes import typecheck
from IPython import embed as ipshell

# ---------------------------------------------------------------------------
# Ping
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_Ping_Decode(payload):
    return payload

@typecheck
def FromClerk_Ping_Encode(payload: str):
    return payload

# ---------------------------------------------------------------------------
# AddTemplates
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_AddTemplates_Decode(payload: dict):
    """
    Undo the Base64 encoding from all files. Otherwise the templates remain
    unchanged.
    """

    # Convenience.
    T = aztypes.Template
    dec = base64.b64decode
    with azrael.util.Timeit('clerk.decode'):
        decoded = []
        # Iterate over all templates.
        for template in payload['templates']:
            # Iterate over all fragments in current template.
            for fragname in template['fragments']:
                # Undo the Base64 encoding for each fragment.
                files = template['fragments'][fragname]['files']
                files = {k: dec(v.encode('utf8')) for k, v in files.items()}

                # Overwrite the original 'file' dictionary.
                template['fragments'][fragname]['files'] = files

            # Turn the input data into a Template type.
            decoded.append(T(**template))

    # Overwrite the original template with the updated version.
    payload['templates'] = decoded
    return payload

@typecheck
def FromClerk_AddTemplates_Encode(dummyarg):
    return None

# ---------------------------------------------------------------------------
# GetTemplates
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_GetTemplates_Decode(payload: dict):
    return payload

@typecheck
def FromClerk_GetTemplates_Encode(templates):
    out = {}
    for aid, data in templates.items():
        out[aid] = {'url_frag': data['url_frag'],
                    'template': data['template']._asdict()}
    return out

# ---------------------------------------------------------------------------
# Spawn
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_Spawn_Decode(payload: dict):
    return payload

@typecheck
def FromClerk_Spawn_Encode(objIDs: (list, tuple)):
    return objIDs

# ---------------------------------------------------------------------------
# GetTemplateID
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_GetTemplateID_Decode(payload: dict):
    return payload

@typecheck
def FromClerk_GetTemplateID_Encode(templateID: str):
    return templateID

# ---------------------------------------------------------------------------
# GetAllObjectIDs
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_GetAllObjectIDs_Decode(payload):
    return payload

@typecheck
def FromClerk_GetAllObjectIDs_Encode(objIDs: (list, tuple)):
    return objIDs

# ---------------------------------------------------------------------------
# RemoveObject
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_RemoveObject_Decode(payload: dict):
    return payload

@typecheck
def FromClerk_RemoveObject_Encode(dummyarg):
    return None

# ---------------------------------------------------------------------------
# ControlParts
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_ControlParts_Decode(payload: dict):
    # Convenience.
    CmdF, CmdB = aztypes.CmdFactory, aztypes.CmdBooster

    # Compile- and sanity check the commands.
    cmds_b = {k: CmdB(**v) for (k, v) in payload['cmd_boosters'].items()}
    cmds_f = {k: CmdF(**v) for (k, v) in payload['cmd_factories'].items()}

    # Overwrite the data in the payload.
    payload['cmd_boosters'] = cmds_b
    payload['cmd_factories'] = cmds_f
    return payload

@typecheck
def FromClerk_ControlParts_Encode(objIDs: (list, tuple)):
    return objIDs

# ---------------------------------------------------------------------------
# GetFragments
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_GetFragments_Decode(payload: dict):
    # Convert the IDs to integers.
    payload['objIDs'] = [int(_) for _ in payload['objIDs']]
    return payload

@typecheck
def FromClerk_GetFragments_Encode(geo):
    return geo

# ---------------------------------------------------------------------------
# SetFragments
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_SetFragments_Decode(payload: dict):
    # Convert the IDs to integers.
    payload['fragments'] = {int(k): v for (k, v) in payload['fragments'].items()}

    def dec(filedata):
        return base64.b64decode(filedata.encode('utf8'))

    # Undo the Base64 encoding.
    for objID in payload['fragments']:
        for fragname, fragdata in payload['fragments'][objID].items():
            if 'del' in fragdata:
                fragdata['del'] = [dec(v) for v in fragdata['del']]
            if 'put' in fragdata:
                fragdata['put'] = {k: dec(v) for k, v in fragdata['put'].items()}
    return payload

@typecheck
def FromClerk_SetFragments_Encode(response):
    return response

# ---------------------------------------------------------------------------
# SetRigidBodies
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_SetRigidBodies_Decode(payload: dict):
    # Convert the IDs to integers.
    payload['bodies'] = {int(k): v for (k, v) in payload['bodies'].items()}
    return payload

@typecheck
def FromClerk_SetRigidBodies_Encode(dummyarg):
    return None

# ---------------------------------------------------------------------------
# GetRigidBodies
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_GetRigidBodies_Decode(payload: dict):
    return payload

@typecheck
def FromClerk_GetRigidBodies_Encode(payload: dict):
    out = {}
    for objID, data in payload.items():
        if data is None:
            # Clerk did not find this particular object.
            out[objID] = None
            continue

        # Replace the original 'rbs' and 'frag' entries with the new ones.
        out[objID] = {'rbs': data['rbs']._asdict()}
    return out

# ---------------------------------------------------------------------------
# GetObjectStates
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_GetObjectStates_Decode(payload: dict):
    return payload

@typecheck
def FromClerk_GetObjectStates_Encode(payload: dict):
    return payload

# ---------------------------------------------------------------------------
# SetForce
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_SetForce_Decode(payload: dict):
    return payload

@typecheck
def FromClerk_SetForce_Encode(dummyarg):
    return None

# ---------------------------------------------------------------------------
# AddConstraints
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_AddConstraints_Decode(payload: dict):
    # Compile- and sanity check the constraints.
    C = aztypes.ConstraintMeta
    payload['constraints'] = [C(**_) for _ in payload['constraints']]
    return payload

@typecheck
def FromClerk_AddConstraints_Encode(num_added):
    return num_added

# ---------------------------------------------------------------------------
# DeleteConstraints
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_DeleteConstraints_Decode(payload: dict):
    return ToClerk_AddConstraints_Decode(payload)

@typecheck
def FromClerk_DeleteConstraints_Encode(num_added):
    return FromClerk_AddConstraints_Encode(num_added)

# ---------------------------------------------------------------------------
# getConstraints
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_GetConstraints_Decode(payload: dict):
    return payload

@typecheck
def FromClerk_GetConstraints_Encode(constraints):
    return [_._asdict() for _ in constraints]

# ---------------------------------------------------------------------------
# setCustomData
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_SetCustomData_Decode(payload: dict):
    # Convert the IDs to integers.
    payload['data'] = {int(k): v for (k, v) in payload['data'].items()}
    return payload

@typecheck
def FromClerk_SetCustomData_Encode(payload):
    return payload

# ---------------------------------------------------------------------------
# getCustomData
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_GetCustomData_Decode(payload: dict):
    return payload

@typecheck
def FromClerk_GetCustomData_Encode(payload):
    return payload
<commit_msg>Protocol module for 'addTemplate' now validates input against JSON schema.<commit_after># Copyright 2015, Oliver Nagy <olitheolix@gmail.com>
#
# This file is part of Azrael (https://github.com/olitheolix/azrael)
#
# Azrael is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Azrael is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Azrael. If not, see <http://www.gnu.org/licenses/>.

"""
Converts the JSON data from Clients to the native Python types Clerk expects,
and vice versa.

The codecs in this module specify the JSON format between Clerk and the
clients.

``ToClerk_*_Encode``: Converts the Client's JSON data to native (Python) types.
``FromClerk_*_Encode``: Converts Clerk's response to plain JSON.

The JSON only protocoly should make it possible to write clients in other
languages.
"""
import base64
import jsonschema
import azrael.util
import azrael.azschemas
import azrael.aztypes as aztypes

from azrael.aztypes import typecheck
from IPython import embed as ipshell

# ---------------------------------------------------------------------------
# Ping
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_Ping_Decode(payload):
    return payload

@typecheck
def FromClerk_Ping_Encode(payload: str):
    return payload

# ---------------------------------------------------------------------------
# AddTemplates
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_AddTemplates_Decode(payload: dict):
    """
    Undo the Base64 encoding from all files. Otherwise the templates remain
    unchanged.
    """

    # Convenience.
    T = aztypes.Template
    dec = base64.b64decode
    with azrael.util.Timeit('clerk.decode'):
        decoded = []
        # Iterate over all templates.
        for template in payload['templates']:
            # Validate the template against the schema.
            jsonschema.validate(template, azrael.azschemas.Template)

            # Iterate over all fragments in current template.
            for fragname in template['fragments']:
                # Undo the Base64 encoding for each fragment geometry file.
                files = template['fragments'][fragname]['files']
                files = {k: dec(v.encode('utf8')) for k, v in files.items()}

                # Overwrite the original 'file' dictionary.
                template['fragments'][fragname]['files'] = files

            # Turn the input data into a Template type.
            decoded.append(T(**template))

    # Overwrite the original template with the updated version.
    payload['templates'] = decoded
    return payload

@typecheck
def FromClerk_AddTemplates_Encode(dummyarg):
    return None

# ---------------------------------------------------------------------------
# GetTemplates
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_GetTemplates_Decode(payload: dict):
    return payload

@typecheck
def FromClerk_GetTemplates_Encode(templates):
    out = {}
    for aid, data in templates.items():
        out[aid] = {'url_frag': data['url_frag'],
                    'template': data['template']._asdict()}
    return out

# ---------------------------------------------------------------------------
# Spawn
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_Spawn_Decode(payload: dict):
    return payload

@typecheck
def FromClerk_Spawn_Encode(objIDs: (list, tuple)):
    return objIDs

# ---------------------------------------------------------------------------
# GetTemplateID
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_GetTemplateID_Decode(payload: dict):
    return payload

@typecheck
def FromClerk_GetTemplateID_Encode(templateID: str):
    return templateID

# ---------------------------------------------------------------------------
# GetAllObjectIDs
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_GetAllObjectIDs_Decode(payload):
    return payload

@typecheck
def FromClerk_GetAllObjectIDs_Encode(objIDs: (list, tuple)):
    return objIDs

# ---------------------------------------------------------------------------
# RemoveObject
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_RemoveObject_Decode(payload: dict):
    return payload

@typecheck
def FromClerk_RemoveObject_Encode(dummyarg):
    return None

# ---------------------------------------------------------------------------
# ControlParts
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_ControlParts_Decode(payload: dict):
    # Convenience.
    CmdF, CmdB = aztypes.CmdFactory, aztypes.CmdBooster

    # Compile- and sanity check the commands.
    cmds_b = {k: CmdB(**v) for (k, v) in payload['cmd_boosters'].items()}
    cmds_f = {k: CmdF(**v) for (k, v) in payload['cmd_factories'].items()}

    # Overwrite the data in the payload.
    payload['cmd_boosters'] = cmds_b
    payload['cmd_factories'] = cmds_f
    return payload

@typecheck
def FromClerk_ControlParts_Encode(objIDs: (list, tuple)):
    return objIDs

# ---------------------------------------------------------------------------
# GetFragments
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_GetFragments_Decode(payload: dict):
    # Convert the IDs to integers.
    payload['objIDs'] = [int(_) for _ in payload['objIDs']]
    return payload

@typecheck
def FromClerk_GetFragments_Encode(geo):
    return geo

# ---------------------------------------------------------------------------
# SetFragments
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_SetFragments_Decode(payload: dict):
    # Convert the IDs to integers.
    payload['fragments'] = {int(k): v for (k, v) in payload['fragments'].items()}

    def dec(filedata):
        return base64.b64decode(filedata.encode('utf8'))

    # Undo the Base64 encoding.
    for objID in payload['fragments']:
        for fragname, fragdata in payload['fragments'][objID].items():
            if 'del' in fragdata:
                fragdata['del'] = [dec(v) for v in fragdata['del']]
            if 'put' in fragdata:
                fragdata['put'] = {k: dec(v) for k, v in fragdata['put'].items()}
    return payload

@typecheck
def FromClerk_SetFragments_Encode(response):
    return response

# ---------------------------------------------------------------------------
# SetRigidBodies
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_SetRigidBodies_Decode(payload: dict):
    # Convert the IDs to integers.
    payload['bodies'] = {int(k): v for (k, v) in payload['bodies'].items()}
    return payload

@typecheck
def FromClerk_SetRigidBodies_Encode(dummyarg):
    return None

# ---------------------------------------------------------------------------
# GetRigidBodies
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_GetRigidBodies_Decode(payload: dict):
    return payload

@typecheck
def FromClerk_GetRigidBodies_Encode(payload: dict):
    out = {}
    for objID, data in payload.items():
        if data is None:
            # Clerk did not find this particular object.
            out[objID] = None
            continue

        # Replace the original 'rbs' and 'frag' entries with the new ones.
        out[objID] = {'rbs': data['rbs']._asdict()}
    return out

# ---------------------------------------------------------------------------
# GetObjectStates
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_GetObjectStates_Decode(payload: dict):
    return payload

@typecheck
def FromClerk_GetObjectStates_Encode(payload: dict):
    return payload

# ---------------------------------------------------------------------------
# SetForce
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_SetForce_Decode(payload: dict):
    return payload

@typecheck
def FromClerk_SetForce_Encode(dummyarg):
    return None

# ---------------------------------------------------------------------------
# AddConstraints
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_AddConstraints_Decode(payload: dict):
    # Compile- and sanity check the constraints.
    C = aztypes.ConstraintMeta
    payload['constraints'] = [C(**_) for _ in payload['constraints']]
    return payload

@typecheck
def FromClerk_AddConstraints_Encode(num_added):
    return num_added

# ---------------------------------------------------------------------------
# DeleteConstraints
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_DeleteConstraints_Decode(payload: dict):
    return ToClerk_AddConstraints_Decode(payload)

@typecheck
def FromClerk_DeleteConstraints_Encode(num_added):
    return FromClerk_AddConstraints_Encode(num_added)

# ---------------------------------------------------------------------------
# getConstraints
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_GetConstraints_Decode(payload: dict):
    return payload

@typecheck
def FromClerk_GetConstraints_Encode(constraints):
    return [_._asdict() for _ in constraints]

# ---------------------------------------------------------------------------
# setCustomData
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_SetCustomData_Decode(payload: dict):
    # Convert the IDs to integers.
    payload['data'] = {int(k): v for (k, v) in payload['data'].items()}
    return payload

@typecheck
def FromClerk_SetCustomData_Encode(payload):
    return payload

# ---------------------------------------------------------------------------
# getCustomData
# ---------------------------------------------------------------------------

@typecheck
def ToClerk_GetCustomData_Decode(payload: dict):
    return payload

@typecheck
def FromClerk_GetCustomData_Encode(payload):
    return payload
<|endoftext|>


<commit_before>from test_support import TESTFN, TestFailed

import os
import random
import sys

# Brief digression to test that import is case-sensitive:  if we got this
# far, we know for sure that "random" exists.
try:
    import RAnDoM
except ImportError:
    pass
else:
    raise TestFailed("import of RAnDoM should have failed (case mismatch)")

# Another brief digression to test the accuracy of manifest float constants.
import double_const  # don't blink -- that *was* the test

sys.path.insert(0, os.curdir)

source = TESTFN + ".py"
pyc = TESTFN + ".pyc"
pyo = TESTFN + ".pyo"

f = open(source, "w")
print >> f, "# This will test Python's ability to import a .py file"
a = random.randrange(1000)
b = random.randrange(1000)
print >> f, "a =", a
print >> f, "b =", b
f.close()

try:
    try:
        mod = __import__(TESTFN)
    except ImportError, err:
        raise ValueError, "import from .py failed: %s" % err

    if mod.a != a or mod.b != b:
        print a, "!=", mod.a
        print b, "!=", mod.b
        raise ValueError, "module loaded (%s) but contents invalid" % mod
finally:
    os.unlink(source)

try:
    try:
        reload(mod)
    except ImportError, err:
        raise ValueError, "import from .pyc/.pyo failed: %s" % err
finally:
    try:
        os.unlink(pyc)
    except os.error:
        pass
    try:
        os.unlink(pyo)
    except os.error:
        pass

del sys.path[0]
<commit_msg>Rewritten version of Finn Bock's SF patch #446907 (Allow jython to complete test_import).<commit_after>from test_support import TESTFN, TestFailed

import os
import random
import sys

# Brief digression to test that import is case-sensitive:  if we got this
# far, we know for sure that "random" exists.
try:
    import RAnDoM
except ImportError:
    pass
else:
    raise TestFailed("import of RAnDoM should have failed (case mismatch)")

# Another brief digression to test the accuracy of manifest float constants.
import double_const  # don't blink -- that *was* the test

sys.path.insert(0, os.curdir)

source = TESTFN + ".py"
pyo = TESTFN + ".pyo"
if sys.platform.endswith('java'):
    pyc = TESTFN + "$py.class"
else:
    pyc = TESTFN + ".pyc"

f = open(source, "w")
print >> f, "# This will test Python's ability to import a .py file"
a = random.randrange(1000)
b = random.randrange(1000)
print >> f, "a =", a
print >> f, "b =", b
f.close()

try:
    try:
        mod = __import__(TESTFN)
    except ImportError, err:
        raise ValueError, "import from .py failed: %s" % err

    if mod.a != a or mod.b != b:
        print a, "!=", mod.a
        print b, "!=", mod.b
        raise ValueError, "module loaded (%s) but contents invalid" % mod
finally:
    os.unlink(source)

try:
    try:
        reload(mod)
    except ImportError, err:
        raise ValueError, "import from .pyc/.pyo failed: %s" % err
finally:
    try:
        os.unlink(pyc)
    except os.error:
        pass
    try:
        os.unlink(pyo)
    except os.error:
        pass

del sys.path[0]
<|endoftext|>


<commit_before>import collections
import email.utils
from http import HTTPStatus
import io
import typing

import httpolice
from mitmproxy import ctx
import mitmproxy.flow
import mitmproxy.http
import mitmproxy.net.http
import mitmproxy.types

__version__ = '0.9.0'

class MitmproxyHTTPolice:

    def __init__(self):
        self.last_report = None

    def load(self, loader):
        loader.add_option(
            name='httpolice_silence',
            # ``typing.Sequence[int]`` would be better, but is not supported.
            typespec=typing.Sequence[str],
            default=[],
            help='Silence these HTTPolice notice IDs.',
        )
        loader.add_option(
            name='httpolice_mark',
            # Could make this a ``typing.Optional[str]``, but
            # that doesn't work well with the interactive editors,
            # so make "disable" an explicit choice.
            typespec=str,
            choices=[''] + [sev.name for sev in httpolice.Severity],
            default='',
            help=
                'Mark flows where HTTPolice found notices of this severity '
                'or higher (empty to disable).'
        )

    def request(self, flow):
        if flow.request.path == '/+httpolice/':
            flow.response = self.serve_report()

    def response(self, flow):
        exch = flow_to_exchange(flow)
        attach_report(exch, flow)
        mark_exchange(exch, flow)
        log_exchange(exch, flow)

    @mitmproxy.command.command('httpolice.report.html')
    def html_report(self,
                    flows: typing.Sequence[mitmproxy.flow.Flow],
                    path: mitmproxy.types.Path) -> None:
        """Produce an HTTPolice report (HTML) on flows."""
        self.report(flows, httpolice.html_report, path)

    @mitmproxy.command.command('httpolice.report.text')
    def text_report(self,
                    flows: typing.Sequence[mitmproxy.flow.Flow],
                    path: mitmproxy.types.Path) -> None:
        """Produce an HTTPolice report (text) on flows."""
        self.report(flows, httpolice.text_report, path)

    def report(self, flows, report_func, path):
        exchanges = (flow_to_exchange(flow) for flow in flows)
        if path == '-':
            buf = io.BytesIO()
            report_func(exchanges, buf)
            self.last_report = buf.getvalue()
            ctx.log.alert(
                f'HTTPolice: saved report on {len(flows)} flows in memory')
        else:
            with open(path, 'wb') as f:
                report_func(exchanges, f)
            ctx.log.alert(
                f'HTTPolice: wrote report on {len(flows)} flows to {path}')

    def serve_report(self):
        if self.last_report is None:
            status_code = HTTPStatus.NOT_FOUND.value
            content = (
                f'<!DOCTYPE html>
No report has been produced yet.
'
            ).encode('utf-8')
        else:
            status_code = HTTPStatus.OK.value
            content = self.last_report
        headers = {
            'Date': email.utils.formatdate(usegmt=True),
            'Content-Type': (
                'text/html; charset=utf-8' if content.startswith(b'<!')
                else 'text/plain; charset=utf-8'
            ),
            'Cache-Control': 'no-store',
        }
        return mitmproxy.http.HTTPResponse.wrap(
            mitmproxy.net.http.Response.make(status_code, content, headers),
        )

def flow_to_exchange(flow):
    req = construct_request(flow)
    resp = construct_response(flow)
    exch = httpolice.Exchange(req, [resp] if resp else [])
    exch.silence([int(id_) for id_ in ctx.options.httpolice_silence])
    httpolice.check_exchange(exch)
    return exch

def construct_request(flow):
    version, headers, body = extract_message_basics(flow.request)
    scheme = decode(flow.request.scheme)
    method = decode(flow.request.method)

    # Authority-form and absolute-form requests in the tunnel
    # are simply rejected as errors by mitmproxy, closing the connection.
    target = decode(flow.request.path)

    if version == 'HTTP/2':
        pseudo_headers = httpolice.helpers.pop_pseudo_headers(headers)
        authority = pseudo_headers.get(':authority')
        has_host = any(k.lower() == 'host' for (k, v) in headers)
        if authority and not has_host and target.startswith('/'):
            # Reconstruct HTTP/2's equivalent of
            # the "absolute form" of request target (RFC 7540 Section 8.1.2.3).
            target = scheme + '://' + decode(authority) + target

    return httpolice.Request(scheme, method, target, version, headers, body)

def construct_response(flow):
    if flow.response is None:
        return None
    version, headers, body = extract_message_basics(flow.response)
    status = flow.response.status_code
    reason = decode(flow.response.reason)
    if version == 'HTTP/2':
        httpolice.helpers.pop_pseudo_headers(headers)
    return httpolice.Response(version, status, reason, headers, body)

def extract_message_basics(msg):
    version = decode(msg.http_version)
    if version == 'HTTP/2.0':
        version = 'HTTP/2'
    headers = [(decode(k), v) for (k, v) in msg.headers.fields]
    body = msg.raw_content
    return version, headers, body

def attach_report(exch, flow):
    buf = io.BytesIO()
    httpolice.text_report([exch], buf)
    report = buf.getvalue().decode('utf-8')
    for_request, for_response = parse_report(report)
    for title, lines in [('HTTPolice: request', for_request),
                         ('HTTPolice: response', for_response)]:
        if lines:
            text = u'\n'.join(lines) + u'\n'
            try:
                # If this script is being run on a flow previously loaded
                # from file, `flow.metadata` might already contain our keys
                # in the wrong order. Reinsert them instead of just updating.
                flow.metadata.pop(title, None)
                flow.metadata[title] = ReprString(text)
            except Exception:
                # `flow.metadata` is not public API,
                # so could theoretically fail.
                pass

def parse_report(report):
    # `report` is a text report as produced by HTTPolice. From it, we want to
    # extract the notices (titles) for the request and for the response.
    # This may sound stupid: why not just make HTTPolice return them
    # in a structured form? But that would have to be a public API
    # (mitmproxy-HTTPolice doesn't use any private APIs from HTTPolice), and I
    # don't want to add public APIs to HTTPolice without a clear picture of
    # how and by whom they will be used. I want some sort of "JSON report"
    # in HTTPolice eventually, but I don't know the details yet. So for now,
    # let's just parse semi-structured text -- a great Unix tradition.
    for_request, for_response = [], []
    target = for_request
    for line in report.splitlines():
        if line.startswith('------------ request:'):
            target = for_request
        elif line.startswith('------------ response:'):
            target = for_response
        else:
            target.append(line)
    return for_request, for_response

def mark_exchange(exch, flow):
    if ctx.options.httpolice_mark:
        mark_severity = httpolice.Severity[ctx.options.httpolice_mark]
        if any(notice.severity >= mark_severity
               for msg in [exch.request] + exch.responses
               for notice in msg.notices):
            flow.marked = True

def log_exchange(exch, flow):
    # Produce lines like "1 errors, 2 comments" without hardcoding severities.
    severities = collections.Counter(notice.severity
                                     for msg in [exch.request] + exch.responses
                                     for notice in msg.notices)
    pieces = [f'{n} {severity.name}s'
              for (severity, n) in sorted(severities.items(), reverse=True)
              if severity > httpolice.Severity.debug]
    if pieces:
        log_func = (ctx.log.warn
                    if max(severities) >= httpolice.Severity.error
                    else ctx.log.info)
        log_func('HTTPolice: {0} in: {1} {2} ← {3}'.format(
            ', '.join(pieces),
            flow.request.method, ellipsize(flow.request.path),
            flow.response.status_code,
        ))

def decode(s):
    if isinstance(s, bytes):
        return s.decode('iso-8859-1')
    return s

def ellipsize(s, max_length=40):
    if len(s) <= max_length:
        return s
    ellipsis = '...'
    return s[:(max_length - len(ellipsis))] + ellipsis

class ReprString(str):

    # Currently mitmproxy displays ``repr()`` in details view, not ``str()``.
    # See also https://discourse.mitmproxy.org/t/extending-the-ui/359/5

    __slots__ = []

    def __repr__(self):
        return str(self)

addons = [MitmproxyHTTPolice()]

if __name__ == '__main__':
    # Print the path to this script,
    # for substitution into the mitmproxy command.
    print(__file__)
<commit_msg>Bump version to 0.10.0.dev1<commit_after>import collections
import email.utils
from http import HTTPStatus
import io
import typing

import httpolice
from mitmproxy import ctx
import mitmproxy.flow
import mitmproxy.http
import mitmproxy.net.http
import mitmproxy.types

__version__ = '0.10.0.dev1'

class MitmproxyHTTPolice:

    def __init__(self):
        self.last_report = None

    def load(self, loader):
        loader.add_option(
            name='httpolice_silence',
            # ``typing.Sequence[int]`` would be better, but is not supported.
            typespec=typing.Sequence[str],
            default=[],
            help='Silence these HTTPolice notice IDs.',
        )
        loader.add_option(
            name='httpolice_mark',
            # Could make this a ``typing.Optional[str]``, but
            # that doesn't work well with the interactive editors,
            # so make "disable" an explicit choice.
            typespec=str,
            choices=[''] + [sev.name for sev in httpolice.Severity],
            default='',
            help=
                'Mark flows where HTTPolice found notices of this severity '
                'or higher (empty to disable).'
        )

    def request(self, flow):
        if flow.request.path == '/+httpolice/':
            flow.response = self.serve_report()

    def response(self, flow):
        exch = flow_to_exchange(flow)
        attach_report(exch, flow)
        mark_exchange(exch, flow)
        log_exchange(exch, flow)

    @mitmproxy.command.command('httpolice.report.html')
    def html_report(self,
                    flows: typing.Sequence[mitmproxy.flow.Flow],
                    path: mitmproxy.types.Path) -> None:
        """Produce an HTTPolice report (HTML) on flows."""
        self.report(flows, httpolice.html_report, path)

    @mitmproxy.command.command('httpolice.report.text')
    def text_report(self,
                    flows: typing.Sequence[mitmproxy.flow.Flow],
                    path: mitmproxy.types.Path) -> None:
        """Produce an HTTPolice report (text) on flows."""
        self.report(flows, httpolice.text_report, path)

    def report(self, flows, report_func, path):
        exchanges = (flow_to_exchange(flow) for flow in flows)
        if path == '-':
            buf = io.BytesIO()
            report_func(exchanges, buf)
            self.last_report = buf.getvalue()
            ctx.log.alert(
                f'HTTPolice: saved report on {len(flows)} flows in memory')
        else:
            with open(path, 'wb') as f:
                report_func(exchanges, f)
            ctx.log.alert(
                f'HTTPolice: wrote report on {len(flows)} flows to {path}')

    def serve_report(self):
        if self.last_report is None:
            status_code = HTTPStatus.NOT_FOUND.value
            content = (
                f'<!DOCTYPE html>
No report has been produced yet.
'
            ).encode('utf-8')
        else:
            status_code = HTTPStatus.OK.value
            content = self.last_report
        headers = {
            'Date': email.utils.formatdate(usegmt=True),
            'Content-Type': (
                'text/html; charset=utf-8' if content.startswith(b'<!')
                else 'text/plain; charset=utf-8'
            ),
            'Cache-Control': 'no-store',
        }
        return mitmproxy.http.HTTPResponse.wrap(
            mitmproxy.net.http.Response.make(status_code, content, headers),
        )

def flow_to_exchange(flow):
    req = construct_request(flow)
    resp = construct_response(flow)
    exch = httpolice.Exchange(req, [resp] if resp else [])
    exch.silence([int(id_) for id_ in ctx.options.httpolice_silence])
    httpolice.check_exchange(exch)
    return exch

def construct_request(flow):
    version, headers, body = extract_message_basics(flow.request)
    scheme = decode(flow.request.scheme)
    method = decode(flow.request.method)

    # Authority-form and absolute-form requests in the tunnel
    # are simply rejected as errors by mitmproxy, closing the connection.
    target = decode(flow.request.path)

    if version == 'HTTP/2':
        pseudo_headers = httpolice.helpers.pop_pseudo_headers(headers)
        authority = pseudo_headers.get(':authority')
        has_host = any(k.lower() == 'host' for (k, v) in headers)
        if authority and not has_host and target.startswith('/'):
            # Reconstruct HTTP/2's equivalent of
            # the "absolute form" of request target (RFC 7540 Section 8.1.2.3).
            target = scheme + '://' + decode(authority) + target

    return httpolice.Request(scheme, method, target, version, headers, body)

def construct_response(flow):
    if flow.response is None:
        return None
    version, headers, body = extract_message_basics(flow.response)
    status = flow.response.status_code
    reason = decode(flow.response.reason)
    if version == 'HTTP/2':
        httpolice.helpers.pop_pseudo_headers(headers)
    return httpolice.Response(version, status, reason, headers, body)

def extract_message_basics(msg):
    version = decode(msg.http_version)
    if version == 'HTTP/2.0':
        version = 'HTTP/2'
    headers = [(decode(k), v) for (k, v) in msg.headers.fields]
    body = msg.raw_content
    return version, headers, body

def attach_report(exch, flow):
    buf = io.BytesIO()
    httpolice.text_report([exch], buf)
    report = buf.getvalue().decode('utf-8')
    for_request, for_response = parse_report(report)
    for title, lines in [('HTTPolice: request', for_request),
                         ('HTTPolice: response', for_response)]:
        if lines:
            text = u'\n'.join(lines) + u'\n'
            try:
                # If this script is being run on a flow previously loaded
                # from file, `flow.metadata` might already contain our keys
                # in the wrong order. Reinsert them instead of just updating.
                flow.metadata.pop(title, None)
                flow.metadata[title] = ReprString(text)
            except Exception:
                # `flow.metadata` is not public API,
                # so could theoretically fail.
                pass

def parse_report(report):
    # `report` is a text report as produced by HTTPolice. From it, we want to
    # extract the notices (titles) for the request and for the response.
    # This may sound stupid: why not just make HTTPolice return them
    # in a structured form? But that would have to be a public API
    # (mitmproxy-HTTPolice doesn't use any private APIs from HTTPolice), and I
    # don't want to add public APIs to HTTPolice without a clear picture of
    # how and by whom they will be used. I want some sort of "JSON report"
    # in HTTPolice eventually, but I don't know the details yet. So for now,
    # let's just parse semi-structured text -- a great Unix tradition.
    for_request, for_response = [], []
    target = for_request
    for line in report.splitlines():
        if line.startswith('------------ request:'):
            target = for_request
        elif line.startswith('------------ response:'):
            target = for_response
        else:
            target.append(line)
    return for_request, for_response

def mark_exchange(exch, flow):
    if ctx.options.httpolice_mark:
        mark_severity = httpolice.Severity[ctx.options.httpolice_mark]
        if any(notice.severity >= mark_severity
               for msg in [exch.request] + exch.responses
               for notice in msg.notices):
            flow.marked = True

def log_exchange(exch, flow):
    # Produce lines like "1 errors, 2 comments" without hardcoding severities.
    severities = collections.Counter(notice.severity
                                     for msg in [exch.request] + exch.responses
                                     for notice in msg.notices)
    pieces = [f'{n} {severity.name}s'
              for (severity, n) in sorted(severities.items(), reverse=True)
              if severity > httpolice.Severity.debug]
    if pieces:
        log_func = (ctx.log.warn
                    if max(severities) >= httpolice.Severity.error
                    else ctx.log.info)
        log_func('HTTPolice: {0} in: {1} {2} ← {3}'.format(
            ', '.join(pieces),
            flow.request.method, ellipsize(flow.request.path),
            flow.response.status_code,
        ))

def decode(s):
    if isinstance(s, bytes):
        return s.decode('iso-8859-1')
    return s

def ellipsize(s, max_length=40):
    if len(s) <= max_length:
        return s
    ellipsis = '...'
    return s[:(max_length - len(ellipsis))] + ellipsis

class ReprString(str):

    # Currently mitmproxy displays ``repr()`` in details view, not ``str()``.
    # See also https://discourse.mitmproxy.org/t/extending-the-ui/359/5

    __slots__ = []

    def __repr__(self):
        return str(self)

addons = [MitmproxyHTTPolice()]

if __name__ == '__main__':
    # Print the path to this script,
    # for substitution into the mitmproxy command.
    print(__file__)
<|endoftext|>


<commit_before>'''
   Test cases for pyclbr.py
   Nick Mathewson
'''
from test.test_support import run_unittest
import unittest, sys
from types import ClassType, FunctionType, MethodType
import pyclbr
from unittest import TestCase

# This next line triggers an error on old versions of pyclbr.

from commands import getstatus

# Here we test the python class browser code.
#
# The main function in this suite, 'testModule', compares the output
# of pyclbr with the introspected members of a module.  Because pyclbr
# is imperfect (as designed), testModule is called with a set of
# members to ignore.

class PyclbrTest(TestCase):

    def assertListEq(self, l1, l2, ignore):
        ''' succeed iff {l1} - {ignore} == {l2} - {ignore} '''
        try:
            for p1, p2 in (l1, l2), (l2, l1):
                for item in p1:
                    ok = (item in p2) or (item in ignore)
                    if not ok:
                        self.fail("%r missing" % item)
        except:
            print >>sys.stderr, "l1=%r, l2=%r, ignore=%r" % (l1, l2, ignore)
            raise

    def assertHasattr(self, obj, attr, ignore):
        ''' succeed iff hasattr(obj,attr) or attr in ignore. '''
        if attr in ignore: return
        if not hasattr(obj, attr): print "???", attr
        self.failUnless(hasattr(obj, attr),
                        'expected hasattr(%r, %r)' % (obj, attr))

    def assertHaskey(self, obj, key, ignore):
        ''' succeed iff obj.has_key(key) or key in ignore. '''
        if key in ignore: return
        if not obj.has_key(key):
            print >>sys.stderr, "***",key
        self.failUnless(obj.has_key(key))

    def assertEquals(self, a, b, ignore=None):
        ''' succeed iff a == b or a in ignore or b in ignore '''
        if (ignore == None) or (a in ignore) or (b in ignore): return

        unittest.TestCase.assertEquals(self, a, b)

    def checkModule(self, moduleName, module=None, ignore=()):
        ''' succeed iff pyclbr.readmodule_ex(modulename) corresponds
            to the actual module object, module.  Any identifiers in
            ignore are ignored.   If no module is provided, the appropriate
            module is loaded with __import__.'''

        if module == None:
            # Import it.
            # ('<silly>' is to work around an API silliness in __import__)
            module = __import__(moduleName, globals(), {}, ['<silly>'])

        dict = pyclbr.readmodule_ex(moduleName)

        # Make sure the toplevel functions and classes are the same.
        for name, value in dict.items():
            if name in ignore:
                continue
            self.assertHasattr(module, name, ignore)
            py_item = getattr(module, name)
            if isinstance(value, pyclbr.Function):
                self.assertEquals(type(py_item), FunctionType)
            else:
                self.assertEquals(type(py_item), ClassType)
                real_bases = [base.__name__ for base in py_item.__bases__]
                pyclbr_bases = [ getattr(base, 'name', base)
                                 for base in value.super ]

                try:
                    self.assertListEq(real_bases, pyclbr_bases, ignore)
                except:
                    print >>sys.stderr, "class=%s" % py_item
                    raise

                actualMethods = []
                for m in py_item.__dict__.keys():
                    if type(getattr(py_item, m)) == MethodType:
                        actualMethods.append(m)
                foundMethods = []
                for m in value.methods.keys():
                    if m[:2] == '__' and m[-2:] != '__':
                        foundMethods.append('_'+name+m)
                    else:
                        foundMethods.append(m)

                self.assertListEq(foundMethods, actualMethods, ignore)
                self.assertEquals(py_item.__module__, value.module)

                self.assertEquals(py_item.__name__, value.name, ignore)
                # can't check file or lineno

        # Now check for missing stuff.
        def defined_in(item, module):
            if isinstance(item, ClassType):
                return item.__module__ == module.__name__
            if isinstance(item, FunctionType):
                return item.func_globals is module.__dict__
            return False
        for name in dir(module):
            item = getattr(module, name)
            if isinstance(item,  (ClassType, FunctionType)):
                if defined_in(item, module):
                    self.assertHaskey(dict, name, ignore)

    def test_easy(self):
        self.checkModule('pyclbr')
        self.checkModule('doctest',
                         ignore=['_isclass',
                                 '_isfunction',
                                 '_ismodule',
                                 '_classify_class_attrs'])
        self.checkModule('rfc822', ignore=["get"])
        self.checkModule('difflib')

    def test_others(self):
        cm = self.checkModule

        # these are about the 20 longest modules.

        cm('random', ignore=('_verify',)) # deleted

        cm('cgi', ignore=('f', 'g',       # nested declarations
                          'log'))         # set with =, not def

        cm('mhlib', ignore=('do',          # nested declaration
                            'bisect'))     # imported method, set with =

        cm('urllib', ignore=('getproxies_environment', # set with =
                             'getproxies_registry',    # set with =
                             'open_https'))            # not on all platforms

        cm('pickle', ignore=('g',))       # deleted declaration

        cm('aifc', ignore=('openfp',))    # set with =

        cm('Cookie', ignore=('__str__', 'Cookie')) # set with =

        cm('sre_parse', ignore=('literal', # nested def
                                'makedict', 'dump' # from sre_constants
                                ))

        # Tests for modules inside packages
        cm('email.Parser')

        cm('test.test_pyclbr', ignore=('defined_in',))

        # pydoc doesn't work because of string issues
        # cm('pydoc', pydoc)

        # pdb plays too many dynamic games
        # cm('pdb', pdb)

def test_main():
    run_unittest(PyclbrTest)

if __name__ == "__main__":
    test_main()
<commit_msg>Add more sophistication to the comparison between pyclbr output and real module, by filtering out aliased methods.  This, combined with the recent fixes to pyclbr, make it possible to enable more tests with fewer exceptions.<commit_after>'''
   Test cases for pyclbr.py
   Nick Mathewson
'''
from test.test_support import run_unittest
import unittest, sys
from types import ClassType, FunctionType, MethodType
import pyclbr
from unittest import TestCase

# This next line triggers an error on old versions of pyclbr.

from commands import getstatus

# Here we test the python class browser code.
#
# The main function in this suite, 'testModule', compares the output
# of pyclbr with the introspected members of a module.  Because pyclbr
# is imperfect (as designed), testModule is called with a set of
# members to ignore.

class PyclbrTest(TestCase):

    def assertListEq(self, l1, l2, ignore):
        ''' succeed iff {l1} - {ignore} == {l2} - {ignore} '''
        l1.sort()
        l2.sort()
        try:
            for p1, p2 in (l1, l2), (l2, l1):
                for item in p1:
                    ok = (item in p2) or (item in ignore)
                    if not ok:
                        self.fail("%r missing" % item)
        except:
            print >>sys.stderr, "l1=%r\nl2=%r\nignore=%r" % (l1, l2, ignore)
            raise

    def assertHasattr(self, obj, attr, ignore):
        ''' succeed iff hasattr(obj,attr) or attr in ignore. '''
        if attr in ignore: return
        if not hasattr(obj, attr): print "???", attr
        self.failUnless(hasattr(obj, attr),
                        'expected hasattr(%r, %r)' % (obj, attr))

    def assertHaskey(self, obj, key, ignore):
        ''' succeed iff obj.has_key(key) or key in ignore. '''
        if key in ignore: return
        if not obj.has_key(key):
            print >>sys.stderr, "***",key
        self.failUnless(obj.has_key(key))

    def assertEquals(self, a, b, ignore=None):
        ''' succeed iff a == b or a in ignore or b in ignore '''
        if (ignore == None) or (a in ignore) or (b in ignore): return

        unittest.TestCase.assertEquals(self, a, b)

    def checkModule(self, moduleName, module=None, ignore=()):
        ''' succeed iff pyclbr.readmodule_ex(modulename) corresponds
            to the actual module object, module.  Any identifiers in
            ignore are ignored.   If no module is provided, the appropriate
            module is loaded with __import__.'''

        if module == None:
            # Import it.
            # ('<silly>' is to work around an API silliness in __import__)
            module = __import__(moduleName, globals(), {}, ['<silly>'])

        dict = pyclbr.readmodule_ex(moduleName)

        def ismethod(obj, name):
            if not  isinstance(obj, MethodType):
                return False
            if obj.im_self is not None:
                return False
            objname = obj.__name__
            if objname.startswith("__") and not objname.endswith("__"):
                objname = "_%s%s" % (obj.im_class.__name__, objname)
            return objname == name

        # Make sure the toplevel functions and classes are the same.
        for name, value in dict.items():
            if name in ignore:
                continue
            self.assertHasattr(module, name, ignore)
            py_item = getattr(module, name)
            if isinstance(value, pyclbr.Function):
                self.assertEquals(type(py_item), FunctionType)
            else:
                self.assertEquals(type(py_item), ClassType)
                real_bases = [base.__name__ for base in py_item.__bases__]
                pyclbr_bases = [ getattr(base, 'name', base)
                                 for base in value.super ]

                try:
                    self.assertListEq(real_bases, pyclbr_bases, ignore)
                except:
                    print >>sys.stderr, "class=%s" % py_item
                    raise

                actualMethods = []
                for m in py_item.__dict__.keys():
                    if ismethod(getattr(py_item, m), m):
                        actualMethods.append(m)
                foundMethods = []
                for m in value.methods.keys():
                    if m[:2] == '__' and m[-2:] != '__':
                        foundMethods.append('_'+name+m)
                    else:
                        foundMethods.append(m)

                try:
                    self.assertListEq(foundMethods, actualMethods, ignore)
                    self.assertEquals(py_item.__module__, value.module)

                    self.assertEquals(py_item.__name__, value.name, ignore)
                    # can't check file or lineno
                except:
                    print >>sys.stderr, "class=%s" % py_item
                    raise

        # Now check for missing stuff.
        def defined_in(item, module):
            if isinstance(item, ClassType):
                return item.__module__ == module.__name__
            if isinstance(item, FunctionType):
                return item.func_globals is module.__dict__
            return False
        for name in dir(module):
            item = getattr(module, name)
            if isinstance(item,  (ClassType, FunctionType)):
                if defined_in(item, module):
                    self.assertHaskey(dict, name, ignore)

    def test_easy(self):
        self.checkModule('pyclbr')
        self.checkModule('doctest')
        self.checkModule('rfc822')
        self.checkModule('difflib')

    def test_others(self):
        cm = self.checkModule

        # These were once about the 10 longest modules
        cm('random', ignore=('_verify',)) # deleted
        cm('cgi', ignore=('log',))      # set with = in module
        cm('mhlib')
        cm('urllib', ignore=('getproxies_registry',
                             'open_https')) # not on all platforms
        cm('pickle', ignore=('g',))     # from types import *
        cm('aifc', ignore=('openfp',))  # set with = in module
        cm('Cookie')
        cm('sre_parse', ignore=('dump',)) # from sre_constants import *
        cm('pdb')
        cm('pydoc')

        # Tests for modules inside packages
        cm('email.Parser')
        cm('test.test_pyclbr')

def test_main():
    run_unittest(PyclbrTest)

if __name__ == "__main__":
    test_main()
<|endoftext|>


<commit_before>import asyncio
import logging
import threading
from asyncio import QueueEmpty as AsyncQueueEmpty
from asyncio import QueueFull as AsyncQueueFull
from collections import deque
from heapq import heappop, heappush
from queue import Empty as SyncQueueEmpty
from queue import Full as SyncQueueFull
from time import monotonic

__version__ = '0.0.1'

log = logging.getLogger(__package__)

class Queue:
    def __init__(self, maxsize=0, *, loop=None):
        if loop is None:
            loop = asyncio.get_event_loop()

        self._loop = loop
        self._maxsize = maxsize

        self._init(maxsize)

        self._unfinished_tasks = 0

        self._sync_mutex = threading.Lock()
        self._sync_not_empty = threading.Condition(self._sync_mutex)
        self._sync_not_full = threading.Condition(self._sync_mutex)
        self._all_tasks_done = threading.Condition(self._sync_mutex)

        self._async_mutex = asyncio.Lock(loop=loop)
        self._async_not_empty = asyncio.Condition(self._async_mutex, loop=loop)
        self._async_not_full = asyncio.Condition(self._async_mutex, loop=loop)
        self._finished = asyncio.Event(loop=self._loop)
        self._finished.set()

        self._closing = False
        self._pending = set()

        self._sync_queue = SyncQueue(self)
        self._async_queue = AsyncQueue(self)

    def close(self):
        with self._sync_mutex:
            # TODO: forbid queue modification if closing is set
            self._closing = True
            for fut in self._pending:
                fut.cancel()

    @asyncio.coroutine
    def wait_closed(self):
        # should be called from loop after close().
        # Nobody should put/get at this point,
        # so lock acquiring is not required
        if not self._closing:
            raise RuntimeError("Waiting for non-closed queue")
        if not self._pending:
            return
        yield from asyncio.wait(self._pending, loop=self._loop)

    @property
    def maxsize(self):
        return self._maxsize

    @property
    def sync_queue(self):
        return self._sync_queue

    @property
    def async_queue(self):
        return self._async_queue

    # Override these methods to implement other queue organizations
    # (e.g. stack or priority queue).
    # These will only be called with appropriate locks held

    def _init(self, maxsize):
        self._queue = deque()

    def _qsize(self):
        return len(self._queue)

    # Put a new item in the queue
    def _put(self, item):
        self._queue.append(item)

    # Get an item from the queue
    def _get(self):
        return self._queue.popleft()

    def _put_internal(self, item):
        self._put(item)
        self._unfinished_tasks += 1
        self._finished.clear()

    def _notify_sync_not_empty(self):
        def f():
            with self._sync_mutex:
                self._sync_not_empty.notify()

        self._loop.run_in_executor(None, f)

    def _notify_sync_not_full(self):
        def f():
            with self._sync_mutex:
                self._sync_not_full.notify()

        fut = self._loop.run_in_executor(None, f)
        fut.add_done_callback(self._pending.discard)
        self._pending.add(fut)

    def _notify_async_not_empty(self, *, threadsafe):
        @asyncio.coroutine
        def f():
            with (yield from self._async_mutex):
                self._async_not_empty.notify()

        def task_maker():
            task = asyncio.async(f(), loop=self._loop)
            task.add_done_callback(self._pending.discard)
            self._pending.add(task)

        if threadsafe:
            self._loop.call_soon_threadsafe(task_maker)
        else:
            self._loop.call_soon(task_maker)

    def _notify_async_not_full(self, *, threadsafe):
        @asyncio.coroutine
        def f():
            with (yield from self._async_mutex):
                self._async_not_full.notify()

        def task_maker():
            task = asyncio.async(f(), loop=self._loop)
            task.add_done_callback(self._pending.discard)
            self._pending.add(task)

        if threadsafe:
            self._loop.call_soon_threadsafe(task_maker)
        else:
            self._loop.call_soon(task_maker)

class SyncQueue:
    '''Create a queue object with a given maximum size.

    If maxsize is <= 0, the queue size is infinite.
    '''

    def __init__(self, parent):
        self._parent = parent

    @property
    def maxsize(self):
        return self._parent._maxsize

    def task_done(self):
        '''Indicate that a formerly enqueued task is complete.

        Used by Queue consumer threads.  For each get() used to fetch a task,
        a subsequent call to task_done() tells the queue that the processing
        on the task is complete.

        If a join() is currently blocking, it will resume when all items
        have been processed (meaning that a task_done() call was received
        for every item that had been put() into the queue).

        Raises a ValueError if called more times than there were items
        placed in the queue.
        '''
        with self._parent._all_tasks_done:
            unfinished = self._parent._unfinished_tasks - 1
            if unfinished <= 0:
                if unfinished < 0:
                    raise ValueError('task_done() called too many times')
                self._parent._all_tasks_done.notify_all()
                self._parent._loop.call_soon_threadsafe(
                    self._parent._finished.set)
            self._parent._unfinished_tasks = unfinished

    def join(self):
        '''Blocks until all items in the Queue have been gotten and processed.

        The count of unfinished tasks goes up whenever an item is added to the
        queue. The count goes down whenever a consumer thread calls task_done()
        to indicate the item was retrieved and all work on it is complete.

        When the count of unfinished tasks drops to zero, join() unblocks.
        '''
        with self._parent._all_tasks_done:
            while self._parent._unfinished_tasks:
                self._parent._all_tasks_done.wait()

    def qsize(self):
        '''Return the approximate size of the queue (not reliable!).'''
        return self._parent._qsize()

    def empty(self):
        '''Return True if the queue is empty, False otherwise (not reliable!).

        This method is likely to be removed at some point.  Use qsize() == 0
        as a direct substitute, but be aware that either approach risks a race
        condition where a queue can grow before the result of empty() or
        qsize() can be used.

        To create code that needs to wait for all queued tasks to be
        completed, the preferred technique is to use the join() method.
        '''
        return not self._parent._qsize()

    def full(self):
        '''Return True if the queue is full, False otherwise (not reliable!).

        This method is likely to be removed at some point.  Use qsize() >= n
        as a direct substitute, but be aware that either approach risks a race
        condition where a queue can shrink before the result of full() or
        qsize() can be used.
        '''
        return 0 < self._parent._maxsize <= self._parent._qsize()

    def put(self, item, block=True, timeout=None):
        '''Put an item into the queue.

        If optional args 'block' is true and 'timeout' is None (the default),
        block if necessary until a free slot is available. If 'timeout' is
        a non-negative number, it blocks at most 'timeout' seconds and raises
        the Full exception if no free slot was available within that time.
        Otherwise ('block' is false), put an item on the queue if a free slot
        is immediately available, else raise the Full exception ('timeout'
        is ignored in that case).
        '''
        with self._parent._sync_not_full:
            if self._parent._maxsize > 0:
                if not block:
                    if self._parent._qsize() >= self._parent._maxsize:
                        raise SyncQueueFull
                elif timeout is None:
                    while self._parent._qsize() >= self._parent._maxsize:
                        self._parent._sync_not_full.wait()
                elif timeout < 0:
                    raise ValueError("'timeout' must be a non-negative number")
                else:
                    endtime = monotonic() + timeout
                    while self._parent._qsize() >= self._parent._maxsize:
                        remaining = endtime - monotonic()
                        if remaining <= 0.0:
                            raise SyncQueueFull
                        self._parent._sync_not_full.wait(remaining)
            self._parent._put(item)
            self._parent._unfinished_tasks += 1
            self._parent._sync_not_empty.notify()

    def get(self, block=True, timeout=None):
        '''Remove and return an item from the queue.

        If optional args 'block' is true and 'timeout' is None (the default),
        block if necessary until an item is available. If 'timeout' is
        a non-negative number, it blocks at most 'timeout' seconds and raises
        the Empty exception if no item was available within that time.
        Otherwise ('block' is false), return an item if one is immediately
        available, else raise the Empty exception ('timeout' is ignored
        in that case).
        '''
        with self._parent._sync_not_empty:
            if not block:
                if not self._parent._qsize():
                    raise SyncQueueEmpty
            elif timeout is None:
                while not self._parent._qsize():
                    self._parent._sync_not_empty.wait()
            elif timeout < 0:
                raise ValueError("'timeout' must be a non-negative number")
            else:
                endtime = monotonic() + timeout
                while not self._parent._qsize():
                    remaining = endtime - monotonic()
                    if remaining <= 0.0:
                        raise SyncQueueEmpty
                    self._parent._sync_not_empty.wait(remaining)
            item = self._parent._get()
            self._parent._sync_not_full.notify()
            return item

    def put_nowait(self, item):
        '''Put an item into the queue without blocking.

        Only enqueue the item if a free slot is immediately available.
        Otherwise raise the Full exception.
        '''
        return self.put(item, block=False)

    def get_nowait(self):
        '''Remove and return an item from the queue without blocking.

        Only get an item if one is immediately available. Otherwise
        raise the Empty exception.
        '''
        return self.get(block=False)

class AsyncQueue:
    '''Create a queue object with a given maximum size.

    If maxsize is <= 0, the queue size is infinite.
    '''

    def __init__(self, parent):
        self._parent = parent

    def qsize(self):
        """Number of items in the queue."""
        return self._parent._qsize()

    @property
    def maxsize(self):
        """Number of items allowed in the queue."""
        return self._parent._maxsize

    def empty(self):
        """Return True if the queue is empty, False otherwise."""
        return self.qsize() == 0

    def full(self):
        """Return True if there are maxsize items in the queue.

        Note: if the Queue was initialized with maxsize=0 (the default),
        then full() is never True.
        """
        if self._parent._maxsize <= 0:
            return False
        else:
            return self.qsize() >= self._parent._maxsize

    @asyncio.coroutine
    def put(self, item):
        """Put an item into the queue.

        Put an item into the queue. If the queue is full, wait until a free
        slot is available before adding item.

        This method is a coroutine.
        """
        with (yield from self._parent._async_not_full):
            self._parent._sync_mutex.acquire()
            locked = True
            try:
                if self._parent._maxsize > 0:
                    do_wait = True
                    while do_wait:
                        do_wait = (
                            self._parent._qsize() >= self._parent._maxsize
                        )
                        if do_wait:
                            locked = False
                            self._parent._sync_mutex.release()
                            yield from self._parent._async_not_full.wait()
                            self._parent._sync_mutex.acquire()
                            locked = True

                self._parent._put_internal(item)
                self._parent._async_not_empty.notify()
                self._parent._notify_sync_not_empty()
            finally:
                if locked:
                    self._parent._sync_mutex.release()

    def put_nowait(self, item):
        """Put an item into the queue without blocking.

        If no free slot is immediately available, raise QueueFull.
        """
        with self._parent._sync_mutex:
            if self._parent._maxsize > 0:
                if self._parent._qsize() >= self._parent._maxsize:
                    raise AsyncQueueFull

            self._parent._put_internal(item)
            self._parent._notify_async_not_empty(threadsafe=False)
            self._parent._notify_sync_not_empty()

    @asyncio.coroutine
    def get(self):
        """Remove and return an item from the queue.

        If queue is empty, wait until an item is available.

        This method is a coroutine.
        """
        with (yield from self._parent._async_not_empty):
            self._parent._sync_mutex.acquire()
            locked = True
            try:
                do_wait = True
                while do_wait:
                    do_wait = self._parent._qsize() == 0

                    if do_wait:
                        locked = False
                        self._parent._sync_mutex.release()
                        yield from self._parent._async_not_empty.wait()
                        self._parent._sync_mutex.acquire()
                        locked = True

                item = self._parent._get()
                self._parent._async_not_full.notify()
                self._parent._notify_sync_not_full()
                return item
            finally:
                if locked:
                    self._parent._sync_mutex.release()

    def get_nowait(self):
        """Remove and return an item from the queue.

        Return an item if one is immediately available, else raise QueueEmpty.
        """
        with self._parent._sync_mutex:
            if self._parent._qsize() == 0:
                raise AsyncQueueEmpty

            item = self._parent._get()
            self._parent._notify_async_not_full(threadsafe=False)
            self._parent._notify_sync_not_full()
            return item

    def task_done(self):
        """Indicate that a formerly enqueued task is complete.

        Used by queue consumers. For each get() used to fetch a task,
        a subsequent call to task_done() tells the queue that the processing
        on the task is complete.

        If a join() is currently blocking, it will resume when all items have
        been processed (meaning that a task_done() call was received for every
        item that had been put() into the queue).

        Raises ValueError if called more times than there were items placed in
        the queue.
        """
        with self._parent._all_tasks_done:
            if self._parent._unfinished_tasks <= 0:
                raise ValueError('task_done() called too many times')
            self._parent._unfinished_tasks -= 1
            if self._parent._unfinished_tasks == 0:
                self._parent._finished.set()
                self._parent._all_tasks_done.notify_all()

    @asyncio.coroutine
    def join(self):
        """Block until all items in the queue have been gotten and processed.

        The count of unfinished tasks goes up whenever an item is added to the
        queue. The count goes down whenever a consumer calls task_done() to
        indicate that the item was retrieved and all work on it is complete.
        When the count of unfinished tasks drops to zero, join() unblocks.
        """
        while True:
            with self._parent._sync_mutex:
                if self._parent._unfinished_tasks == 0:
                    break
            yield from self._parent._finished.wait()

class PriorityQueue(Queue):
    '''Variant of Queue that retrieves open entries in priority order
    (lowest first).

    Entries are typically tuples of the form:  (priority number, data).

    '''

    def _init(self, maxsize):
        self._queue = []

    def _qsize(self):
        return len(self._queue)

    def _put(self, item):
        heappush(self._queue, item)

    def _get(self):
        return heappop(self._queue)

class LifoQueue(Queue):
    '''Variant of Queue that retrieves most recently added entries first.'''

    def _init(self, maxsize):
        self._queue = deque()

    def _qsize(self):
        return len(self._queue)

    def _put(self, item):
        self._queue.append(item)

    def _get(self):
        return self._queue.pop()
<commit_msg>Send async not_full/not_empty notifications from sync api<commit_after>import asyncio
import logging
import threading
from asyncio import QueueEmpty as AsyncQueueEmpty
from asyncio import QueueFull as AsyncQueueFull
from collections import deque
from heapq import heappop, heappush
from queue import Empty as SyncQueueEmpty
from queue import Full as SyncQueueFull
from time import monotonic

__version__ = '0.0.1'

log = logging.getLogger(__package__)

class Queue:
    def __init__(self, maxsize=0, *, loop=None):
        if loop is None:
            loop = asyncio.get_event_loop()

        self._loop = loop
        self._maxsize = maxsize

        self._init(maxsize)

        self._unfinished_tasks = 0

        self._sync_mutex = threading.Lock()
        self._sync_not_empty = threading.Condition(self._sync_mutex)
        self._sync_not_full = threading.Condition(self._sync_mutex)
        self._all_tasks_done = threading.Condition(self._sync_mutex)

        self._async_mutex = asyncio.Lock(loop=loop)
        self._async_not_empty = asyncio.Condition(self._async_mutex, loop=loop)
        self._async_not_full = asyncio.Condition(self._async_mutex, loop=loop)
        self._finished = asyncio.Event(loop=self._loop)
        self._finished.set()

        self._closing = False
        self._pending = set()

        self._sync_queue = SyncQueue(self)
        self._async_queue = AsyncQueue(self)

    def close(self):
        with self._sync_mutex:
            # TODO: forbid queue modification if closing is set
            self._closing = True
            for fut in self._pending:
                fut.cancel()

    @asyncio.coroutine
    def wait_closed(self):
        # should be called from loop after close().
        # Nobody should put/get at this point,
        # so lock acquiring is not required
        if not self._closing:
            raise RuntimeError("Waiting for non-closed queue")
        if not self._pending:
            return
        yield from asyncio.wait(self._pending, loop=self._loop)

    @property
    def maxsize(self):
        return self._maxsize

    @property
    def sync_queue(self):
        return self._sync_queue

    @property
    def async_queue(self):
        return self._async_queue

    # Override these methods to implement other queue organizations
    # (e.g. stack or priority queue).
    # These will only be called with appropriate locks held

    def _init(self, maxsize):
        self._queue = deque()

    def _qsize(self):
        return len(self._queue)

    # Put a new item in the queue
    def _put(self, item):
        self._queue.append(item)

    # Get an item from the queue
    def _get(self):
        return self._queue.popleft()

    def _put_internal(self, item):
        self._put(item)
        self._unfinished_tasks += 1
        self._finished.clear()

    def _notify_sync_not_empty(self):
        def f():
            with self._sync_mutex:
                self._sync_not_empty.notify()

        self._loop.run_in_executor(None, f)

    def _notify_sync_not_full(self):
        def f():
            with self._sync_mutex:
                self._sync_not_full.notify()

        fut = self._loop.run_in_executor(None, f)
        fut.add_done_callback(self._pending.discard)
        self._pending.add(fut)

    def _notify_async_not_empty(self, *, threadsafe):
        @asyncio.coroutine
        def f():
            with (yield from self._async_mutex):
                self._async_not_empty.notify()

        def task_maker():
            task = asyncio.async(f(), loop=self._loop)
            task.add_done_callback(self._pending.discard)
            self._pending.add(task)

        if threadsafe:
            self._loop.call_soon_threadsafe(task_maker)
        else:
            self._loop.call_soon(task_maker)

    def _notify_async_not_full(self, *, threadsafe):
        @asyncio.coroutine
        def f():
            with (yield from self._async_mutex):
                self._async_not_full.notify()

        def task_maker():
            task = asyncio.async(f(), loop=self._loop)
            task.add_done_callback(self._pending.discard)
            self._pending.add(task)

        if threadsafe:
            self._loop.call_soon_threadsafe(task_maker)
        else:
            self._loop.call_soon(task_maker)

class SyncQueue:
    '''Create a queue object with a given maximum size.

    If maxsize is <= 0, the queue size is infinite.
    '''

    def __init__(self, parent):
        self._parent = parent

    @property
    def maxsize(self):
        return self._parent._maxsize

    def task_done(self):
        '''Indicate that a formerly enqueued task is complete.

        Used by Queue consumer threads.  For each get() used to fetch a task,
        a subsequent call to task_done() tells the queue that the processing
        on the task is complete.

        If a join() is currently blocking, it will resume when all items
        have been processed (meaning that a task_done() call was received
        for every item that had been put() into the queue).

        Raises a ValueError if called more times than there were items
        placed in the queue.
        '''
        with self._parent._all_tasks_done:
            unfinished = self._parent._unfinished_tasks - 1
            if unfinished <= 0:
                if unfinished < 0:
                    raise ValueError('task_done() called too many times')
                self._parent._all_tasks_done.notify_all()
                self._parent._loop.call_soon_threadsafe(
                    self._parent._finished.set)
            self._parent._unfinished_tasks = unfinished

    def join(self):
        '''Blocks until all items in the Queue have been gotten and processed.

        The count of unfinished tasks goes up whenever an item is added to the
        queue. The count goes down whenever a consumer thread calls task_done()
        to indicate the item was retrieved and all work on it is complete.

        When the count of unfinished tasks drops to zero, join() unblocks.
        '''
        with self._parent._all_tasks_done:
            while self._parent._unfinished_tasks:
                self._parent._all_tasks_done.wait()

    def qsize(self):
        '''Return the approximate size of the queue (not reliable!).'''
        return self._parent._qsize()

    def empty(self):
        '''Return True if the queue is empty, False otherwise (not reliable!).

        This method is likely to be removed at some point.  Use qsize() == 0
        as a direct substitute, but be aware that either approach risks a race
        condition where a queue can grow before the result of empty() or
        qsize() can be used.

        To create code that needs to wait for all queued tasks to be
        completed, the preferred technique is to use the join() method.
        '''
        return not self._parent._qsize()

    def full(self):
        '''Return True if the queue is full, False otherwise (not reliable!).

        This method is likely to be removed at some point.  Use qsize() >= n
        as a direct substitute, but be aware that either approach risks a race
        condition where a queue can shrink before the result of full() or
        qsize() can be used.
        '''
        return 0 < self._parent._maxsize <= self._parent._qsize()

    def put(self, item, block=True, timeout=None):
        '''Put an item into the queue.

        If optional args 'block' is true and 'timeout' is None (the default),
        block if necessary until a free slot is available. If 'timeout' is
        a non-negative number, it blocks at most 'timeout' seconds and raises
        the Full exception if no free slot was available within that time.
        Otherwise ('block' is false), put an item on the queue if a free slot
        is immediately available, else raise the Full exception ('timeout'
        is ignored in that case).
        '''
        with self._parent._sync_not_full:
            if self._parent._maxsize > 0:
                if not block:
                    if self._parent._qsize() >= self._parent._maxsize:
                        raise SyncQueueFull
                elif timeout is None:
                    while self._parent._qsize() >= self._parent._maxsize:
                        self._parent._sync_not_full.wait()
                elif timeout < 0:
                    raise ValueError("'timeout' must be a non-negative number")
                else:
                    endtime = monotonic() + timeout
                    while self._parent._qsize() >= self._parent._maxsize:
                        remaining = endtime - monotonic()
                        if remaining <= 0.0:
                            raise SyncQueueFull
                        self._parent._sync_not_full.wait(remaining)
            self._parent._put(item)
            self._parent._unfinished_tasks += 1
            self._parent._sync_not_empty.notify()
            self._parent._notify_async_not_empty(threadsafe=True)

    def get(self, block=True, timeout=None):
        '''Remove and return an item from the queue.

        If optional args 'block' is true and 'timeout' is None (the default),
        block if necessary until an item is available. If 'timeout' is
        a non-negative number, it blocks at most 'timeout' seconds and raises
        the Empty exception if no item was available within that time.
        Otherwise ('block' is false), return an item if one is immediately
        available, else raise the Empty exception ('timeout' is ignored
        in that case).
        '''
        with self._parent._sync_not_empty:
            if not block:
                if not self._parent._qsize():
                    raise SyncQueueEmpty
            elif timeout is None:
                while not self._parent._qsize():
                    self._parent._sync_not_empty.wait()
            elif timeout < 0:
                raise ValueError("'timeout' must be a non-negative number")
            else:
                endtime = monotonic() + timeout
                while not self._parent._qsize():
                    remaining = endtime - monotonic()
                    if remaining <= 0.0:
                        raise SyncQueueEmpty
                    self._parent._sync_not_empty.wait(remaining)
            item = self._parent._get()
            self._parent._sync_not_full.notify()
            self._parent._notify_async_not_full(threadsafe=True)
            return item

    def put_nowait(self, item):
        '''Put an item into the queue without blocking.

        Only enqueue the item if a free slot is immediately available.
        Otherwise raise the Full exception.
        '''
        return self.put(item, block=False)

    def get_nowait(self):
        '''Remove and return an item from the queue without blocking.

        Only get an item if one is immediately available. Otherwise
        raise the Empty exception.
        '''
        return self.get(block=False)

class AsyncQueue:
    '''Create a queue object with a given maximum size.

    If maxsize is <= 0, the queue size is infinite.
    '''

    def __init__(self, parent):
        self._parent = parent

    def qsize(self):
        """Number of items in the queue."""
        return self._parent._qsize()

    @property
    def maxsize(self):
        """Number of items allowed in the queue."""
        return self._parent._maxsize

    def empty(self):
        """Return True if the queue is empty, False otherwise."""
        return self.qsize() == 0

    def full(self):
        """Return True if there are maxsize items in the queue.

        Note: if the Queue was initialized with maxsize=0 (the default),
        then full() is never True.
        """
        if self._parent._maxsize <= 0:
            return False
        else:
            return self.qsize() >= self._parent._maxsize

    @asyncio.coroutine
    def put(self, item):
        """Put an item into the queue.

        Put an item into the queue. If the queue is full, wait until a free
        slot is available before adding item.

        This method is a coroutine.
        """
        with (yield from self._parent._async_not_full):
            self._parent._sync_mutex.acquire()
            locked = True
            try:
                if self._parent._maxsize > 0:
                    do_wait = True
                    while do_wait:
                        do_wait = (
                            self._parent._qsize() >= self._parent._maxsize
                        )
                        if do_wait:
                            locked = False
                            self._parent._sync_mutex.release()
                            yield from self._parent._async_not_full.wait()
                            self._parent._sync_mutex.acquire()
                            locked = True

                self._parent._put_internal(item)
                self._parent._async_not_empty.notify()
                self._parent._notify_sync_not_empty()
            finally:
                if locked:
                    self._parent._sync_mutex.release()

    def put_nowait(self, item):
        """Put an item into the queue without blocking.

        If no free slot is immediately available, raise QueueFull.
        """
        with self._parent._sync_mutex:
            if self._parent._maxsize > 0:
                if self._parent._qsize() >= self._parent._maxsize:
                    raise AsyncQueueFull

            self._parent._put_internal(item)
            self._parent._notify_async_not_empty(threadsafe=False)
            self._parent._notify_sync_not_empty()

    @asyncio.coroutine
    def get(self):
        """Remove and return an item from the queue.

        If queue is empty, wait until an item is available.

        This method is a coroutine.
        """
        with (yield from self._parent._async_not_empty):
            self._parent._sync_mutex.acquire()
            locked = True
            try:
                do_wait = True
                while do_wait:
                    do_wait = self._parent._qsize() == 0

                    if do_wait:
                        locked = False
                        self._parent._sync_mutex.release()
                        yield from self._parent._async_not_empty.wait()
                        self._parent._sync_mutex.acquire()
                        locked = True

                item = self._parent._get()
                self._parent._async_not_full.notify()
                self._parent._notify_sync_not_full()
                return item
            finally:
                if locked:
                    self._parent._sync_mutex.release()

    def get_nowait(self):
        """Remove and return an item from the queue.

        Return an item if one is immediately available, else raise QueueEmpty.
        """
        with self._parent._sync_mutex:
            if self._parent._qsize() == 0:
                raise AsyncQueueEmpty

            item = self._parent._get()
            self._parent._notify_async_not_full(threadsafe=False)
            self._parent._notify_sync_not_full()
            return item

    def task_done(self):
        """Indicate that a formerly enqueued task is complete.

        Used by queue consumers. For each get() used to fetch a task,
        a subsequent call to task_done() tells the queue that the processing
        on the task is complete.

        If a join() is currently blocking, it will resume when all items have
        been processed (meaning that a task_done() call was received for every
        item that had been put() into the queue).

        Raises ValueError if called more times than there were items placed in
        the queue.
        """
        with self._parent._all_tasks_done:
            if self._parent._unfinished_tasks <= 0:
                raise ValueError('task_done() called too many times')
            self._parent._unfinished_tasks -= 1
            if self._parent._unfinished_tasks == 0:
                self._parent._finished.set()
                self._parent._all_tasks_done.notify_all()

    @asyncio.coroutine
    def join(self):
        """Block until all items in the queue have been gotten and processed.

        The count of unfinished tasks goes up whenever an item is added to the
        queue. The count goes down whenever a consumer calls task_done() to
        indicate that the item was retrieved and all work on it is complete.
        When the count of unfinished tasks drops to zero, join() unblocks.
        """
        while True:
            with self._parent._sync_mutex:
                if self._parent._unfinished_tasks == 0:
                    break
            yield from self._parent._finished.wait()

class PriorityQueue(Queue):
    '''Variant of Queue that retrieves open entries in priority order
    (lowest first).

    Entries are typically tuples of the form:  (priority number, data).

    '''

    def _init(self, maxsize):
        self._queue = []

    def _qsize(self):
        return len(self._queue)

    def _put(self, item):
        heappush(self._queue, item)

    def _get(self):
        return heappop(self._queue)

class LifoQueue(Queue):
    '''Variant of Queue that retrieves most recently added entries first.'''

    def _init(self, maxsize):
        self._queue = deque()

    def _qsize(self):
        return len(self._queue)

    def _put(self, item):
        self._queue.append(item)

    def _get(self):
        return self._queue.pop()
<|endoftext|>