<commit_before># -*- encoding: utf-8 -*-
from aiohttp.web import Response as aioResponse
from aiohttp.helpers import sentinel
from plone.server.interfaces import IRendered
from plone.server.interfaces import IRenderFormats
from plone.server.interfaces import IFrameFormats
from plone.server.interfaces import IRequest
from plone.server.interfaces import IView
from plone.server.browser import Response
from zope.component import adapter
from zope.interface import implementer
from zope.component import queryAdapter
import json

# JSON Decoder
from zope.securitypolicy.settings import PermissionSetting

class PServerJSONEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, complex):
            return [obj.real, obj.imag]
        try:
            iterable = iter(obj)
        except TypeError:
            pass
        else:
            return list(iterable)

        if isinstance(obj, PermissionSetting):
            return obj.getName()
        # Let the base class default method raise the TypeError
        return json.JSONEncoder.default(self, obj)

def json_response(data=sentinel, *, text=None, body=None, status=200,
                  reason=None, headers=None, content_type='application/json',
                  dumps=json.dumps):
    if data is not sentinel:
        if text or body:
            raise ValueError(
                "only one of data, text, or body should be specified"
            )
        else:
            text = dumps(data, cls=PServerJSONEncoder)
    return aioResponse(text=text, body=body, status=status, reason=reason,
                    headers=headers, content_type=content_type)

# Marker objects/interfaces to look for

class IRendererFormatHtml(IRenderFormats):
    pass

class IRendererFormatJson(IRenderFormats):
    pass

class IRendererFormatRaw(IRenderFormats):
    pass

class IFrameFormatsJson(IFrameFormats):
    pass

@adapter(IRequest)
@implementer(IRenderFormats)
class RendererFormats(object):
    def __init__(self, request):
        self.request = request

@adapter(IRequest)
@implementer(IRendererFormatJson)
class RendererFormatJson(object):
    def __init__(self, request):
        self.request = request

@adapter(IRequest)
@implementer(IRendererFormatHtml)
class RendererFormatHtml(object):
    def __init__(self, request):
        self.request = request

@adapter(IRequest)
@implementer(IRendererFormatRaw)
class RendererFormatRaw(object):
    def __init__(self, request):
        self.request = request

# Real objects

@adapter(IRenderFormats, IView, IRequest)
@implementer(IRendered)
class Renderer(object):

    def __init__(self, renderformat, view, request):
        self.view = view
        self.request = request
        self.renderformat = renderformat

@adapter(IRendererFormatJson, IView, IRequest)
@implementer(IRendered)
class RendererJson(Renderer):
    async def __call__(self, value):
        headers = {}
        if hasattr(value, '__class__') and issubclass(value.__class__, Response):
            json_value = value.response
            headers = value.headers
            status = value.status
        else:
            # Not a Response object, don't convert
            return value
        # Framing of options
        frame = self.request.get('frame')
        frame = self.request.GET['frame'] if 'frame' in self.request.GET else ''
        if frame:
            framer = queryAdapter(self.request, IFrameFormatsJson, frame)
            json_value = framer(json_value)
        resp = json_response(json_value)
        resp.headers.update(headers)
        resp.headers.update(
            {'Content-Type': 'application/json'})
        resp.set_status(status)
        # Actions / workflow / roles

        return resp

@adapter(IRendererFormatHtml, IView, IRequest)
@implementer(IRendered)
class RendererHtml(Renderer):
    async def __call__(self, value):
        # Safe html transformation
        return value

@adapter(IRendererFormatRaw, IView, IRequest)
@implementer(IRendered)
class RendererRaw(Renderer):
    async def __call__(self, value):
        if hasattr(value, '__class__') and issubclass(value.__class__, Response):
            resp = value.response
            if isinstance(resp, dict):
                resp = aioResponse(body=bytes(json.dumps(resp), 'utf-8'))
            resp.headers.update(value.headers)
            resp.set_status(value.status)
        else:
            resp = value
        return resp

<commit_msg>add json content type if automatic conversation to json string<commit_after># -*- encoding: utf-8 -*-
from aiohttp.web import Response as aioResponse
from aiohttp.helpers import sentinel
from plone.server.interfaces import IRendered
from plone.server.interfaces import IRenderFormats
from plone.server.interfaces import IFrameFormats
from plone.server.interfaces import IRequest
from plone.server.interfaces import IView
from plone.server.browser import Response
from zope.component import adapter
from zope.interface import implementer
from zope.component import queryAdapter
import json

# JSON Decoder
from zope.securitypolicy.settings import PermissionSetting

class PServerJSONEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, complex):
            return [obj.real, obj.imag]
        try:
            iterable = iter(obj)
        except TypeError:
            pass
        else:
            return list(iterable)

        if isinstance(obj, PermissionSetting):
            return obj.getName()
        # Let the base class default method raise the TypeError
        return json.JSONEncoder.default(self, obj)

def json_response(data=sentinel, *, text=None, body=None, status=200,
                  reason=None, headers=None, content_type='application/json',
                  dumps=json.dumps):
    if data is not sentinel:
        if text or body:
            raise ValueError(
                "only one of data, text, or body should be specified"
            )
        else:
            text = dumps(data, cls=PServerJSONEncoder)
    return aioResponse(text=text, body=body, status=status, reason=reason,
                    headers=headers, content_type=content_type)

# Marker objects/interfaces to look for

class IRendererFormatHtml(IRenderFormats):
    pass

class IRendererFormatJson(IRenderFormats):
    pass

class IRendererFormatRaw(IRenderFormats):
    pass

class IFrameFormatsJson(IFrameFormats):
    pass

@adapter(IRequest)
@implementer(IRenderFormats)
class RendererFormats(object):
    def __init__(self, request):
        self.request = request

@adapter(IRequest)
@implementer(IRendererFormatJson)
class RendererFormatJson(object):
    def __init__(self, request):
        self.request = request

@adapter(IRequest)
@implementer(IRendererFormatHtml)
class RendererFormatHtml(object):
    def __init__(self, request):
        self.request = request

@adapter(IRequest)
@implementer(IRendererFormatRaw)
class RendererFormatRaw(object):
    def __init__(self, request):
        self.request = request

# Real objects

@adapter(IRenderFormats, IView, IRequest)
@implementer(IRendered)
class Renderer(object):

    def __init__(self, renderformat, view, request):
        self.view = view
        self.request = request
        self.renderformat = renderformat

@adapter(IRendererFormatJson, IView, IRequest)
@implementer(IRendered)
class RendererJson(Renderer):
    async def __call__(self, value):
        headers = {}
        if hasattr(value, '__class__') and issubclass(value.__class__, Response):
            json_value = value.response
            headers = value.headers
            status = value.status
        else:
            # Not a Response object, don't convert
            return value
        # Framing of options
        frame = self.request.get('frame')
        frame = self.request.GET['frame'] if 'frame' in self.request.GET else ''
        if frame:
            framer = queryAdapter(self.request, IFrameFormatsJson, frame)
            json_value = framer(json_value)
        resp = json_response(json_value)
        resp.headers.update(headers)
        resp.headers.update(
            {'Content-Type': 'application/json'})
        resp.set_status(status)
        # Actions / workflow / roles

        return resp

@adapter(IRendererFormatHtml, IView, IRequest)
@implementer(IRendered)
class RendererHtml(Renderer):
    async def __call__(self, value):
        # Safe html transformation
        return value

@adapter(IRendererFormatRaw, IView, IRequest)
@implementer(IRendered)
class RendererRaw(Renderer):
    async def __call__(self, value):
        if hasattr(value, '__class__') and issubclass(value.__class__, Response):
            resp = value.response
            if isinstance(resp, dict):
                resp = aioResponse(body=bytes(json.dumps(resp), 'utf-8'))
                resp.headers['Content-Type'] = 'application/json'
            resp.headers.update(value.headers)
            resp.set_status(value.status)
        else:
            resp = value
        return resp
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import asyncio
import inspect
import multiprocessing

from cli_common.log import get_logger

logger = get_logger(__name__)

class MessageBus(object):
    '''
    Communication bus between processes
    '''
    def __init__(self, max_messages=None):
        self.queues = {}
        self.max_messages = max_messages
        self.nb_messages = 0

    def add_queue(self, name, mp=False):
        '''
        Create a new queue on the message bus
        * asyncio by default
        * multiprocessing when mp=True
        '''
        assert name not in self.queues, 'Queue {} already setup'.format(name)
        if mp:
            self.queues[name] = multiprocessing.Queue()
        else:
            self.queues[name] = asyncio.Queue()

    def is_full(self):
        '''
        Helper for unit tests runtimes
        '''
        if self.max_messages is None:
            return False
        return self.nb_messages > self.max_messages

    async def send(self, name, payload):
        '''
        Send a message on a specific queue
        '''
        assert name in self.queues, 'Missing queue {}'.format(name)
        queue = self.queues[name]
        if isinstance(queue, asyncio.Queue):
            await queue.put(payload)
        else:
            queue.put(payload)
        self.nb_messages += 1

    async def receive(self, name):
        '''
        Wait for a message on a specific queue
        This is a blocking operation
        '''
        assert name in self.queues, 'Missing queue {}'.format(name)
        queue = self.queues[name]
        if isinstance(queue, asyncio.Queue):
            return await queue.get()
        else:
            return queue.get()

    async def run(self, input_name, output_name, method):
        '''
        Pass messages from input to output
        Optionally applies some conversions methods
        This is also the "ideal" usage between 2 queues
        '''
        assert input_name in self.queues, 'Missing queue {}'.format(input_name)
        assert output_name in self.queues, 'Missing queue {}'.format(output_name)

        while not self.is_full():

            message = await self.receive(input_name)

            # Run async or sync methods
            if inspect.iscoroutinefunction(method):
                new_message = await method(message)
            else:
                new_message = method(message)

            if not new_message:
                logger.info('Skipping new message creation: no result', message=message)
                continue

            await self.send(output_name, new_message)
<commit_msg>pulselistener: Use run_in_executor for sync calls in bus (#2228)<commit_after># -*- coding: utf-8 -*-
import asyncio
import inspect
import multiprocessing

from cli_common.log import get_logger

logger = get_logger(__name__)

class MessageBus(object):
    '''
    Communication bus between processes
    '''
    def __init__(self, max_messages=None):
        self.queues = {}
        self.max_messages = max_messages
        self.nb_messages = 0

    def add_queue(self, name, mp=False):
        '''
        Create a new queue on the message bus
        * asyncio by default
        * multiprocessing when mp=True
        '''
        assert name not in self.queues, 'Queue {} already setup'.format(name)
        if mp:
            self.queues[name] = multiprocessing.Queue()
        else:
            self.queues[name] = asyncio.Queue()

    def is_full(self):
        '''
        Helper for unit tests runtimes
        '''
        if self.max_messages is None:
            return False
        return self.nb_messages > self.max_messages

    async def send(self, name, payload):
        '''
        Send a message on a specific queue
        '''
        assert name in self.queues, 'Missing queue {}'.format(name)
        queue = self.queues[name]
        if isinstance(queue, asyncio.Queue):
            await queue.put(payload)
        else:
            # Run the synchronous mp queue.put in the asynchronous loop
            await asyncio.get_running_loop().run_in_executor(None, lambda: queue.put(payload))
        self.nb_messages += 1

    async def receive(self, name):
        '''
        Wait for a message on a specific queue
        This is a blocking operation
        '''
        assert name in self.queues, 'Missing queue {}'.format(name)
        queue = self.queues[name]
        if isinstance(queue, asyncio.Queue):
            return await queue.get()
        else:
            # Run the synchronous mp queue.get in the asynchronous loop
            return await asyncio.get_running_loop().run_in_executor(None, queue.get)

    async def run(self, input_name, output_name, method):
        '''
        Pass messages from input to output
        Optionally applies some conversions methods
        This is also the "ideal" usage between 2 queues
        '''
        assert input_name in self.queues, 'Missing queue {}'.format(input_name)
        assert output_name in self.queues, 'Missing queue {}'.format(output_name)

        while not self.is_full():

            message = await self.receive(input_name)

            # Run async or sync methods
            if inspect.iscoroutinefunction(method):
                new_message = await method(message)
            else:
                new_message = method(message)

            if not new_message:
                logger.info('Skipping new message creation: no result', message=message)
                continue

            await self.send(output_name, new_message)
<|endoftext|>


<commit_before>"""
@file
@brief Encryption fucntionalities

Inspired from `AES encryption of files in Python with PyCrypto <http://eli.thegreenplace.net/2010/06/25/aes-encryption-of-files-in-python-with-pycrypto>`_

.. versionadded:: 1.3
"""
import random
import sys
import os
import struct
import base64

if sys.version_info[0] == 2:
    from StringIO import StringIO as StreamIO
    FileNotFoundError = Exception
else:
    from io import BytesIO as StreamIO

class EncryptionError(Exception):
    """
    raised if an issue happen during encryption
    """
    pass

def open_input_output(filename, out_filename=None):
    """
    convert filename and out_filename as streams

    @param      filename        bytes or filename or BytesIO
    @param      out_filename    BytesIO or filename or None
    @return                     in_size, in_close, in_stream, out_close, out_return, out_stream
    """
    # input
    typstr = str  # unicode #
    if isinstance(filename, typstr):
        if not os.path.exists(filename):
            raise FileNotFoundError(filename)
        st = open(filename, "rb")
        close = True
        filesize = os.path.getsize(filename)
    elif isinstance(filename, StreamIO):
        st = filename
        close = False
        filesize = len(st.getvalue())
    else:
        st = StreamIO(filename)
        close = False
        filesize = len(filename)

    # output
    if out_filename is None:
        sto = StreamIO()
        ret = True
        out_close = False
    elif isinstance(out_filename, StreamIO):
        sto = out_filename
        ret = False
        out_close = False
    else:
        sto = open(out_filename, "wb")
        ret = False
        out_close = True

    return filesize, close, st, out_close, ret, sto

def close_input_output(in_size, in_close, in_stream, out_close, out_return, out_stream):
    """
    takes the output of @see fn open_input_output and closes streams
    and return expected values

    @param      in_size         size of input
    @param      in_close        should we close the input stream
    @param      in_stream       input stream
    @param      out_close       should we close the output stream
    @param      out_return      should we return something
    @param      out_stream      output stream
    @return                     None or content of output stream
    """
    if in_close:
        in_stream.close()

    if out_close:
        if out_return:
            raise EncryptionError("incompability")
        out_stream.close()

    if out_return:
        return out_stream.getvalue()

def get_encryptor(key, algo="AES", chunksize=2 ** 24, **params):
    """
    return a encryptor with method encrypt and decrypt

    @param      algo        AES or fernet
    @param      key         key
    @param      iv          extra parameter for AES
    @param      chunksize   Fernet does not allow streaming
    @return                 encryptor, origsize
    """
    if algo == "fernet":
        from cryptography.fernet import Fernet
        bkey = base64.b64encode(key.encode())
        encryptor = Fernet(bkey)
        origsize = None
        chunksize = None
    elif algo == "AES":
        from Cryptodome.Cipher import AES
        ksize = {16, 32, 64, 128, 256}
        chunksize = chunksize
        if len(key) not in ksize:
            raise EncryptionError(
                "len(key)=={0} should be of length {1}".format(len(key), str(ksize)))
        if "out_stream" in params:
            iv = bytes([random.randint(0, 0xFF) for i in range(16)])
            params["out_stream"].write(struct.pack('<Q', params["in_size"]))
            params["out_stream"].write(iv)
            encryptor = AES.new(key, AES.MODE_CBC, iv)
            origsize = params["in_size"]
        else:
            origsize = struct.unpack(
                '<Q', params["in_stream"].read(struct.calcsize('Q')))[0]
            iv = params["in_stream"].read(16)
            encryptor = AES.new(key, AES.MODE_CBC, iv)  # decryptor
    else:
        raise ValueError("unknown algorithm: {0}, should be in {1}".format(
            algo, ["fernet", "AES"]))
    return encryptor, origsize, chunksize

def encrypt_stream(key, filename, out_filename=None, chunksize=2 ** 18, algo="AES"):
    """
    Encrypts a file using AES (CBC mode) with the given key.

    @param      key             The encryption key - a string that must be
                                either 16, 24 or 32 bytes long. Longer keys
                                are more secure. If the data to encrypt is in bytes,
                                the key must be given in bytes too.

    @param      filename        bytes or Name of the input file
    @param      out_filename    if None, the returns bytes

    @param      chunksize       Sets the size of the chunk which the function
                                uses to read and encrypt the file. Larger chunk
                                sizes can be faster for some files and machines.
                                chunksize must be divisible by 16.

    @param      algo            AES (PyCryptodomex) of or fernet (cryptography)

    @return                     filename or bytes

    The function relies on module `pycrypto <https://pypi.python.org/pypi/pycrypto>`_,
    `cryptography <http://cryptography.readthedocs.org/>`_,
    algoritm `AES <https://fr.wikipedia.org/wiki/Advanced_Encryption_Standard>`_,
    `Fernet <http://cryptography.readthedocs.org/en/latest/fernet/>`_.

    .. versionadded:: 1.3
    """

    in_size, in_close, in_stream, out_close, out_return, out_stream = open_input_output(
        filename, out_filename)

    encryptor, origsize, chunksize = get_encryptor(
        key, algo, out_stream=out_stream, in_size=in_size, chunksize=chunksize)

    while True:
        chunk = in_stream.read(chunksize)
        if len(chunk) == 0:
            break
        elif len(chunk) % 16 != 0 and origsize is not None:
            chunk += b' ' * (16 - len(chunk) % 16)

        out_stream.write(encryptor.encrypt(chunk))

    return close_input_output(in_size, in_close, in_stream, out_close, out_return, out_stream)

def decrypt_stream(key, filename, out_filename=None, chunksize=3 * 2 ** 13, algo="AES"):
    """
    Decrypts a file using AES (CBC mode) with the given key.

    @param      key             The encryption key - a string that must be
                                either 16, 24 or 32 bytes long. Longer keys
                                are more secure. If the data to encrypt is in bytes,
                                the key must be given in bytes too.

    @param      filename        bytes or Name of the input file
    @param      out_filename    if None, the returns bytes

    @param      chunksize       Sets the size of the chunk which the function
                                uses to read and encrypt the file. Larger chunk
                                sizes can be faster for some files and machines.
                                chunksize must be divisible by 16.

    @param      algo            AES (PyCryptodomex) of or fernet (cryptography)

    @return                     filename or bytes

    The function relies on module `pycrypto <https://pypi.python.org/pypi/pycrypto>`_,
    `cryptography <http://cryptography.readthedocs.org/>`_,
    algoritm `AES <https://fr.wikipedia.org/wiki/Advanced_Encryption_Standard>`_,
    `Fernet <http://cryptography.readthedocs.org/en/latest/fernet/>`_.

    .. versionadded:: 1.3
    """
    in_size, in_close, in_stream, out_close, out_return, out_stream = open_input_output(
        filename, out_filename)

    decryptor, origsize, chunksize = get_encryptor(
        key, algo, in_stream=in_stream, chunksize=chunksize)

    while True:
        chunk = in_stream.read(chunksize)
        if len(chunk) == 0:
            break
        out_stream.write(decryptor.decrypt(chunk))
        out_stream.truncate(origsize)

    return close_input_output(in_size, in_close, in_stream, out_close, out_return, out_stream)
<commit_msg>fix type issue for linux<commit_after>"""
@file
@brief Encryption fucntionalities

Inspired from `AES encryption of files in Python with PyCrypto <http://eli.thegreenplace.net/2010/06/25/aes-encryption-of-files-in-python-with-pycrypto>`_

.. versionadded:: 1.3
"""
import random
import sys
import os
import struct
import base64

if sys.version_info[0] == 2:
    from StringIO import StringIO as StreamIO
    FileNotFoundError = Exception
else:
    from io import BytesIO as StreamIO

class EncryptionError(Exception):
    """
    raised if an issue happen during encryption
    """
    pass

def open_input_output(filename, out_filename=None):
    """
    convert filename and out_filename as streams

    @param      filename        bytes or filename or BytesIO
    @param      out_filename    BytesIO or filename or None
    @return                     in_size, in_close, in_stream, out_close, out_return, out_stream
    """
    # input
    typstr = str  # unicode #
    if isinstance(filename, typstr):
        if not os.path.exists(filename):
            raise FileNotFoundError(filename)
        st = open(filename, "rb")
        close = True
        filesize = os.path.getsize(filename)
    elif isinstance(filename, StreamIO):
        st = filename
        close = False
        filesize = len(st.getvalue())
    else:
        st = StreamIO(filename)
        close = False
        filesize = len(filename)

    # output
    if out_filename is None:
        sto = StreamIO()
        ret = True
        out_close = False
    elif isinstance(out_filename, StreamIO):
        sto = out_filename
        ret = False
        out_close = False
    else:
        sto = open(out_filename, "wb")
        ret = False
        out_close = True

    return filesize, close, st, out_close, ret, sto

def close_input_output(in_size, in_close, in_stream, out_close, out_return, out_stream):
    """
    takes the output of @see fn open_input_output and closes streams
    and return expected values

    @param      in_size         size of input
    @param      in_close        should we close the input stream
    @param      in_stream       input stream
    @param      out_close       should we close the output stream
    @param      out_return      should we return something
    @param      out_stream      output stream
    @return                     None or content of output stream
    """
    if in_close:
        in_stream.close()

    if out_close:
        if out_return:
            raise EncryptionError("incompability")
        out_stream.close()

    if out_return:
        return out_stream.getvalue()

def get_encryptor(key, algo="AES", chunksize=2 ** 24, **params):
    """
    return a encryptor with method encrypt and decrypt

    @param      algo        AES or fernet
    @param      key         key
    @param      iv          extra parameter for AES
    @param      chunksize   Fernet does not allow streaming
    @return                 encryptor, origsize
    """
    if algo == "fernet":
        from cryptography.fernet import Fernet
        if hasattr(key, "encode"):
            # it a string
            bkey = key.encode()
        else:
            bkey = key
        bkey = base64.b64encode(bkey)
        encryptor = Fernet(bkey)
        origsize = None
        chunksize = None
    elif algo == "AES":
        from Cryptodome.Cipher import AES
        ksize = {16, 32, 64, 128, 256}
        chunksize = chunksize
        if len(key) not in ksize:
            raise EncryptionError(
                "len(key)=={0} should be of length {1}".format(len(key), str(ksize)))
        if "out_stream" in params:
            iv = bytes([random.randint(0, 0xFF) for i in range(16)])
            params["out_stream"].write(struct.pack('<Q', params["in_size"]))
            params["out_stream"].write(iv)
            encryptor = AES.new(key, AES.MODE_CBC, iv)
            origsize = params["in_size"]
        else:
            origsize = struct.unpack(
                '<Q', params["in_stream"].read(struct.calcsize('Q')))[0]
            iv = params["in_stream"].read(16)
            encryptor = AES.new(key, AES.MODE_CBC, iv)  # decryptor
    else:
        raise ValueError("unknown algorithm: {0}, should be in {1}".format(
            algo, ["fernet", "AES"]))
    return encryptor, origsize, chunksize

def encrypt_stream(key, filename, out_filename=None, chunksize=2 ** 18, algo="AES"):
    """
    Encrypts a file using AES (CBC mode) with the given key.

    @param      key             The encryption key - a string that must be
                                either 16, 24 or 32 bytes long. Longer keys
                                are more secure. If the data to encrypt is in bytes,
                                the key must be given in bytes too.

    @param      filename        bytes or Name of the input file
    @param      out_filename    if None, the returns bytes

    @param      chunksize       Sets the size of the chunk which the function
                                uses to read and encrypt the file. Larger chunk
                                sizes can be faster for some files and machines.
                                chunksize must be divisible by 16.

    @param      algo            AES (PyCryptodomex) of or fernet (cryptography)

    @return                     filename or bytes

    The function relies on module `pycrypto <https://pypi.python.org/pypi/pycrypto>`_,
    `cryptography <http://cryptography.readthedocs.org/>`_,
    algoritm `AES <https://fr.wikipedia.org/wiki/Advanced_Encryption_Standard>`_,
    `Fernet <http://cryptography.readthedocs.org/en/latest/fernet/>`_.

    .. versionadded:: 1.3
    """

    in_size, in_close, in_stream, out_close, out_return, out_stream = open_input_output(
        filename, out_filename)

    encryptor, origsize, chunksize = get_encryptor(
        key, algo, out_stream=out_stream, in_size=in_size, chunksize=chunksize)

    while True:
        chunk = in_stream.read(chunksize)
        if len(chunk) == 0:
            break
        elif len(chunk) % 16 != 0 and origsize is not None:
            chunk += b' ' * (16 - len(chunk) % 16)

        out_stream.write(encryptor.encrypt(chunk))

    return close_input_output(in_size, in_close, in_stream, out_close, out_return, out_stream)

def decrypt_stream(key, filename, out_filename=None, chunksize=3 * 2 ** 13, algo="AES"):
    """
    Decrypts a file using AES (CBC mode) with the given key.

    @param      key             The encryption key - a string that must be
                                either 16, 24 or 32 bytes long. Longer keys
                                are more secure. If the data to encrypt is in bytes,
                                the key must be given in bytes too.

    @param      filename        bytes or Name of the input file
    @param      out_filename    if None, the returns bytes

    @param      chunksize       Sets the size of the chunk which the function
                                uses to read and encrypt the file. Larger chunk
                                sizes can be faster for some files and machines.
                                chunksize must be divisible by 16.

    @param      algo            AES (PyCryptodomex) of or fernet (cryptography)

    @return                     filename or bytes

    The function relies on module `pycrypto <https://pypi.python.org/pypi/pycrypto>`_,
    `cryptography <http://cryptography.readthedocs.org/>`_,
    algoritm `AES <https://fr.wikipedia.org/wiki/Advanced_Encryption_Standard>`_,
    `Fernet <http://cryptography.readthedocs.org/en/latest/fernet/>`_.

    .. versionadded:: 1.3
    """
    in_size, in_close, in_stream, out_close, out_return, out_stream = open_input_output(
        filename, out_filename)

    decryptor, origsize, chunksize = get_encryptor(
        key, algo, in_stream=in_stream, chunksize=chunksize)

    while True:
        chunk = in_stream.read(chunksize)
        if len(chunk) == 0:
            break
        out_stream.write(decryptor.decrypt(chunk))
        out_stream.truncate(origsize)

    return close_input_output(in_size, in_close, in_stream, out_close, out_return, out_stream)
<|endoftext|>


<commit_before># ==================================================================================================
# Copyright 2011 Twitter, Inc.
# --------------------------------------------------------------------------------------------------
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this work except in compliance with the License.
# You may obtain a copy of the License in the LICENSE file, or at:
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==================================================================================================

import itertools
import os
import sys

from contextlib import contextmanager
from multiprocessing.pool import ThreadPool

from twitter.common.collections.orderedset import OrderedSet

from twitter.pants.base.artifact_cache import create_artifact_cache
from twitter.pants.base.hash_utils import hash_file
from twitter.pants.base.build_invalidator import CacheKeyGenerator
from twitter.pants.reporting.reporting_utils import items_to_report_element
from twitter.pants.tasks.cache_manager import CacheManager, InvalidationCheck

class TaskError(Exception):
  """Raised to indicate a task has failed."""

class Task(object):
  @classmethod
  def setup_parser(cls, option_group, args, mkflag):
    """Set up the cmd-line parser.

    Subclasses can add flags to the pants command line using the given option group.
    Flag names should be created with mkflag([name]) to ensure flags are properly namespaced
    amongst other tasks.
    """

  def __init__(self, context):
    self.context = context
    self.dry_run = self.can_dry_run() and self.context.options.dry_run
    self._cache_key_generator = CacheKeyGenerator()
    self._artifact_cache = None
    self._build_invalidator_dir = os.path.join(context.config.get('tasks', 'build_invalidator'),
                                               self.product_type())

  def setup_artifact_cache(self, spec):
    """Subclasses can call this in their __init__() to set up artifact caching for that task type.

    spec should be a list of urls/file path prefixes, which are used in that order.
    By default, no artifact caching is used.
    """
    if len(spec) > 0:
      pants_workdir = self.context.config.getdefault('pants_workdir')
      self._artifact_cache = create_artifact_cache(self.context, pants_workdir, spec)

  def product_type(self):
    """Set the product type for this task.

    By default, each task is considered as creating a unique product type.
    Subclasses can override this to specify a shared product type, e.g., 'classes'.

    Tasks with the same product type can invalidate each other's targets, e.g., if a ScalaLibrary
    depends on a JavaLibrary, a change to the JavaLibrary will invalidate the ScalaLibrary because
    they both have the same product type.
    """
    return self.__class__.__name__

  def can_dry_run(self):
    """Subclasses can override this to indicate that they respect the --dry-run flag.

    It's the subclass task's responsibility to do the right thing if this flag is set.

    Note that tasks such as codegen and ivy resolution cannot dry-run, because subsequent
    cache key computation will fail on missing sources/external deps.
    """
    return False

  def execute(self, targets):
    """Executes this task against targets, which may be a subset of the current context targets."""
    raise TaskError('execute() not implemented')

  def invalidate_for(self):
    """Provides extra objects that participate in invalidation.

    Subclasses can override and return an object that should be checked for changes when
    managing target invalidation.  If the pickled form of returned object changes
    between runs all targets will be invalidated.
    """
    return None

  def invalidate_for_files(self):
    """Provides extra files that participate in invalidation.

    Subclasses can override and return a list of full paths to extra, non-source files that should
    be checked for changes when managing target invalidation. This is useful for tracking
    changes to pre-built build tools, e.g., the thrift compiler.
    """
    return []

  @contextmanager
  def invalidated(self, targets, only_buildfiles=False, invalidate_dependents=False,
                  partition_size_hint=sys.maxint):
    """Checks targets for invalidation. Subclasses call this to figure out what to work on.

    targets:               The targets to check for changes.
    only_buildfiles:       If True, then only the target's BUILD files are checked for changes, not
                           its sources.
    invalidate_dependents: If True then any targets depending on changed targets are invalidated.
    partition_size_hint:   Each VersionedTargetSet in the yielded list will represent targets
                           containing roughly this number of source files, if possible. Set to
                           sys.maxint for a single VersionedTargetSet. Set to 0 for one
                           VersionedTargetSet per target. It is up to the caller to do the right
                           thing with whatever partitioning it asks for.

    Yields an InvalidationCheck object reflecting the (partitioned) targets.

    If no exceptions are thrown by work in the block, the build cache is updated for the targets.
    Note: the artifact cache is not updated, that must be done manually.
    """
    with self.invalidated_with_artifact_cache_check(targets,
                                                    only_buildfiles,
                                                    invalidate_dependents,
                                                    partition_size_hint) as check:
      yield check

  @contextmanager
  def invalidated_with_artifact_cache_check(self,
                                            targets,
                                            only_buildfiles=False,
                                            invalidate_dependents=False,
                                            partition_size_hint=sys.maxint):
    """Checks targets for invalidation, first checking the artifact cache.
    Subclasses call this to figure out what to work on.

    targets:               The targets to check for changes.
    only_buildfiles:       If True, then only the target's BUILD files are checked for changes, not
                           its sources.
    invalidate_dependents: If True then any targets depending on changed targets are invalidated.
    partition_size_hint:   Each VersionedTargetSet in the yielded list will represent targets
                           containing roughly this number of source files, if possible. Set to
                           sys.maxint for a single VersionedTargetSet. Set to 0 for one
                           VersionedTargetSet per target. It is up to the caller to do the right
                           thing with whatever partitioning it asks for.

    Yields a pair of (invalidation_check, cached_vts) where invalidation_check is an
    InvalidationCheck object reflecting the (partitioned) targets, and cached_vts is a list of
    VersionedTargets that were satisfied from the artifact cache.

    If no exceptions are thrown by work in the block, the build cache is updated for the targets.
    Note: the artifact cache is not updated, that must be done manually.
    """
    extra_data = []
    extra_data.append(self.invalidate_for())

    for f in self.invalidate_for_files():
      extra_data.append(hash_file(f))

    cache_manager = CacheManager(self._cache_key_generator,
                                 self._build_invalidator_dir,
                                 invalidate_dependents,
                                 extra_data,
                                 only_externaldeps=only_buildfiles)

    invalidation_check = cache_manager.check(targets, partition_size_hint)

    # See if we have entire partitions cached.
    if self._artifact_cache and self.context.options.read_from_artifact_cache:
      with self.context.new_workunit('cache'):
        all_cached_targets = []
        partitions_to_check = \
          [vt for vt in invalidation_check.all_vts_partitioned if not vt.valid]
        cached_partitions, uncached_partitions = self.check_artifact_cache(partitions_to_check)
        for vt in cached_partitions:
          for t in vt.targets:
            all_cached_targets.append(t)

        # See if we have any individual targets from the uncached partitions.
        vts_to_check = [vt for vt in itertools.chain.from_iterable(
          [x.versioned_targets for x in uncached_partitions]) if not vt.valid]
        cached_targets, uncached_targets = self.check_artifact_cache(vts_to_check)
        for vt in cached_targets:
          all_cached_targets.append(vt.target)

      if all_cached_targets:
        # Do some reporting.
        for t in all_cached_targets:
          self.context.run_tracker.artifact_cache_stats.add_hit('default', t)
        self._report_targets('Using cached artifacts for ', all_cached_targets, '.')

      # Now that we've checked the cache, re-partition whatever is still invalid.
      if uncached_targets:
        for vts in uncached_targets:
          self.context.run_tracker.artifact_cache_stats.add_miss('default', vts.target)
        self._report_targets('No cached artifacts for ',
                             [vt.target for vt in uncached_targets], '.')
      invalidation_check = \
        InvalidationCheck(invalidation_check.all_vts, uncached_targets, partition_size_hint)

    # Do some reporting.
    targets = []
    sources = []
    num_invalid_partitions = len(invalidation_check.invalid_vts_partitioned)
    for vt in invalidation_check.invalid_vts_partitioned:
      targets.extend(vt.targets)
      sources.extend(vt.cache_key.sources)
    if len(targets):
      msg_elements = ['Invalidated ',
                      items_to_report_element([t.address.reference() for t in targets], 'target')]
      if len(sources) > 0:
        msg_elements.append(' containing ')
        msg_elements.append(items_to_report_element(sources, 'source file'))
      if num_invalid_partitions > 1:
        msg_elements.append(' in %d target partitions' % num_invalid_partitions)
      msg_elements.append('.')
      self.context.log.info(*msg_elements)

    # Yield the result, and then mark the targets as up to date.
    yield invalidation_check
    if not self.dry_run:
      for vt in invalidation_check.invalid_vts:
        vt.update()  # In case the caller doesn't update.

  def check_artifact_cache(self, vts):
    """Checks the artifact cache for the specified VersionedTargetSets.

    Returns a list of the ones that were satisfied from the cache. These don't require building.
    """
    if not vts:
      return [], []

    cached_vts = []
    uncached_vts = OrderedSet(vts)

    with self.context.new_workunit('check'):
      pool = ThreadPool(processes=6)
      res = pool.map(lambda vt: self._artifact_cache.use_cached_files(vt.cache_key),
                     vts, chunksize=1)
      pool.close()
      pool.join()
      for vt, was_in_cache in zip(vts, res):
        if was_in_cache:
          cached_vts.append(vt)
          uncached_vts.discard(vt)
          vt.update()
    return cached_vts, list(uncached_vts)

  def update_artifact_cache(self, vts_artifactfiles_pairs):
    """Write to the artifact cache, if we're configured to.

    vts_artifactfiles_pairs - a list of pairs (vts, artifactfiles) where
      - vts is single VersionedTargetSet.
      - artifactfiles is a list of paths to artifacts for the VersionedTargetSet.
    """
    if self._artifact_cache and self.context.options.write_to_artifact_cache:
      with self.context.new_workunit('cache'):
        with self.context.new_workunit('update'):
          targets = set()
          for vts, artifactfiles in vts_artifactfiles_pairs:
            targets.update([x.address.reference() for x in vts.targets])
          for vts, artifactfiles in vts_artifactfiles_pairs:
            if self.context.options.verify_artifact_cache:
              pass  # TODO: Verify that the artifact we just built is identical to the cached one.
            self._report_targets('Caching artifacts for ', list(targets), '.')
            self._artifact_cache.insert(vts.cache_key, artifactfiles)

  def _report_targets(self, prefix, targets, suffix):
    self.context.log.info(
      prefix,
      items_to_report_element([t.address.reference() for t in targets], 'target'),
      suffix)

__all__ = (
  'TaskError',
  'Task'
)
<commit_msg>Fix logging bug.<commit_after># ==================================================================================================
# Copyright 2011 Twitter, Inc.
# --------------------------------------------------------------------------------------------------
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this work except in compliance with the License.
# You may obtain a copy of the License in the LICENSE file, or at:
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==================================================================================================

import itertools
import os
import sys

from contextlib import contextmanager
from multiprocessing.pool import ThreadPool

from twitter.common.collections.orderedset import OrderedSet

from twitter.pants.base.artifact_cache import create_artifact_cache
from twitter.pants.base.hash_utils import hash_file
from twitter.pants.base.build_invalidator import CacheKeyGenerator
from twitter.pants.reporting.reporting_utils import items_to_report_element
from twitter.pants.tasks.cache_manager import CacheManager, InvalidationCheck

class TaskError(Exception):
  """Raised to indicate a task has failed."""

class Task(object):
  @classmethod
  def setup_parser(cls, option_group, args, mkflag):
    """Set up the cmd-line parser.

    Subclasses can add flags to the pants command line using the given option group.
    Flag names should be created with mkflag([name]) to ensure flags are properly namespaced
    amongst other tasks.
    """

  def __init__(self, context):
    self.context = context
    self.dry_run = self.can_dry_run() and self.context.options.dry_run
    self._cache_key_generator = CacheKeyGenerator()
    self._artifact_cache = None
    self._build_invalidator_dir = os.path.join(context.config.get('tasks', 'build_invalidator'),
                                               self.product_type())

  def setup_artifact_cache(self, spec):
    """Subclasses can call this in their __init__() to set up artifact caching for that task type.

    spec should be a list of urls/file path prefixes, which are used in that order.
    By default, no artifact caching is used.
    """
    if len(spec) > 0:
      pants_workdir = self.context.config.getdefault('pants_workdir')
      self._artifact_cache = create_artifact_cache(self.context, pants_workdir, spec)

  def product_type(self):
    """Set the product type for this task.

    By default, each task is considered as creating a unique product type.
    Subclasses can override this to specify a shared product type, e.g., 'classes'.

    Tasks with the same product type can invalidate each other's targets, e.g., if a ScalaLibrary
    depends on a JavaLibrary, a change to the JavaLibrary will invalidate the ScalaLibrary because
    they both have the same product type.
    """
    return self.__class__.__name__

  def can_dry_run(self):
    """Subclasses can override this to indicate that they respect the --dry-run flag.

    It's the subclass task's responsibility to do the right thing if this flag is set.

    Note that tasks such as codegen and ivy resolution cannot dry-run, because subsequent
    cache key computation will fail on missing sources/external deps.
    """
    return False

  def execute(self, targets):
    """Executes this task against targets, which may be a subset of the current context targets."""
    raise TaskError('execute() not implemented')

  def invalidate_for(self):
    """Provides extra objects that participate in invalidation.

    Subclasses can override and return an object that should be checked for changes when
    managing target invalidation.  If the pickled form of returned object changes
    between runs all targets will be invalidated.
    """
    return None

  def invalidate_for_files(self):
    """Provides extra files that participate in invalidation.

    Subclasses can override and return a list of full paths to extra, non-source files that should
    be checked for changes when managing target invalidation. This is useful for tracking
    changes to pre-built build tools, e.g., the thrift compiler.
    """
    return []

  @contextmanager
  def invalidated(self, targets, only_buildfiles=False, invalidate_dependents=False,
                  partition_size_hint=sys.maxint):
    """Checks targets for invalidation. Subclasses call this to figure out what to work on.

    targets:               The targets to check for changes.
    only_buildfiles:       If True, then only the target's BUILD files are checked for changes, not
                           its sources.
    invalidate_dependents: If True then any targets depending on changed targets are invalidated.
    partition_size_hint:   Each VersionedTargetSet in the yielded list will represent targets
                           containing roughly this number of source files, if possible. Set to
                           sys.maxint for a single VersionedTargetSet. Set to 0 for one
                           VersionedTargetSet per target. It is up to the caller to do the right
                           thing with whatever partitioning it asks for.

    Yields an InvalidationCheck object reflecting the (partitioned) targets.

    If no exceptions are thrown by work in the block, the build cache is updated for the targets.
    Note: the artifact cache is not updated, that must be done manually.
    """
    with self.invalidated_with_artifact_cache_check(targets,
                                                    only_buildfiles,
                                                    invalidate_dependents,
                                                    partition_size_hint) as check:
      yield check

  @contextmanager
  def invalidated_with_artifact_cache_check(self,
                                            targets,
                                            only_buildfiles=False,
                                            invalidate_dependents=False,
                                            partition_size_hint=sys.maxint):
    """Checks targets for invalidation, first checking the artifact cache.
    Subclasses call this to figure out what to work on.

    targets:               The targets to check for changes.
    only_buildfiles:       If True, then only the target's BUILD files are checked for changes, not
                           its sources.
    invalidate_dependents: If True then any targets depending on changed targets are invalidated.
    partition_size_hint:   Each VersionedTargetSet in the yielded list will represent targets
                           containing roughly this number of source files, if possible. Set to
                           sys.maxint for a single VersionedTargetSet. Set to 0 for one
                           VersionedTargetSet per target. It is up to the caller to do the right
                           thing with whatever partitioning it asks for.

    Yields a pair of (invalidation_check, cached_vts) where invalidation_check is an
    InvalidationCheck object reflecting the (partitioned) targets, and cached_vts is a list of
    VersionedTargets that were satisfied from the artifact cache.

    If no exceptions are thrown by work in the block, the build cache is updated for the targets.
    Note: the artifact cache is not updated, that must be done manually.
    """
    extra_data = []
    extra_data.append(self.invalidate_for())

    for f in self.invalidate_for_files():
      extra_data.append(hash_file(f))

    cache_manager = CacheManager(self._cache_key_generator,
                                 self._build_invalidator_dir,
                                 invalidate_dependents,
                                 extra_data,
                                 only_externaldeps=only_buildfiles)

    invalidation_check = cache_manager.check(targets, partition_size_hint)

    # See if we have entire partitions cached.
    if self._artifact_cache and self.context.options.read_from_artifact_cache:
      with self.context.new_workunit('cache'):
        all_cached_targets = []
        partitions_to_check = \
          [vt for vt in invalidation_check.all_vts_partitioned if not vt.valid]
        cached_partitions, uncached_partitions = self.check_artifact_cache(partitions_to_check)
        for vt in cached_partitions:
          for t in vt.targets:
            all_cached_targets.append(t)

        # See if we have any individual targets from the uncached partitions.
        vts_to_check = [vt for vt in itertools.chain.from_iterable(
          [x.versioned_targets for x in uncached_partitions]) if not vt.valid]
        cached_targets, uncached_targets = self.check_artifact_cache(vts_to_check)
        for vt in cached_targets:
          all_cached_targets.append(vt.target)

      if all_cached_targets:
        # Do some reporting.
        for t in all_cached_targets:
          self.context.run_tracker.artifact_cache_stats.add_hit('default', t)
        self._report_targets('Using cached artifacts for ', all_cached_targets, '.')

      # Now that we've checked the cache, re-partition whatever is still invalid.
      if uncached_targets:
        for vts in uncached_targets:
          self.context.run_tracker.artifact_cache_stats.add_miss('default', vts.target)
        self._report_targets('No cached artifacts for ',
                             [vt.target for vt in uncached_targets], '.')
      invalidation_check = \
        InvalidationCheck(invalidation_check.all_vts, uncached_targets, partition_size_hint)

    # Do some reporting.
    targets = []
    sources = []
    num_invalid_partitions = len(invalidation_check.invalid_vts_partitioned)
    for vt in invalidation_check.invalid_vts_partitioned:
      targets.extend(vt.targets)
      sources.extend(vt.cache_key.sources)
    if len(targets):
      msg_elements = ['Invalidated ',
                      items_to_report_element([t.address.reference() for t in targets], 'target')]
      if len(sources) > 0:
        msg_elements.append(' containing ')
        msg_elements.append(items_to_report_element(sources, 'source file'))
      if num_invalid_partitions > 1:
        msg_elements.append(' in %d target partitions' % num_invalid_partitions)
      msg_elements.append('.')
      self.context.log.info(*msg_elements)

    # Yield the result, and then mark the targets as up to date.
    yield invalidation_check
    if not self.dry_run:
      for vt in invalidation_check.invalid_vts:
        vt.update()  # In case the caller doesn't update.

  def check_artifact_cache(self, vts):
    """Checks the artifact cache for the specified VersionedTargetSets.

    Returns a list of the ones that were satisfied from the cache. These don't require building.
    """
    if not vts:
      return [], []

    cached_vts = []
    uncached_vts = OrderedSet(vts)

    with self.context.new_workunit('check'):
      pool = ThreadPool(processes=6)
      res = pool.map(lambda vt: self._artifact_cache.use_cached_files(vt.cache_key),
                     vts, chunksize=1)
      pool.close()
      pool.join()
      for vt, was_in_cache in zip(vts, res):
        if was_in_cache:
          cached_vts.append(vt)
          uncached_vts.discard(vt)
          vt.update()
    return cached_vts, list(uncached_vts)

  def update_artifact_cache(self, vts_artifactfiles_pairs):
    """Write to the artifact cache, if we're configured to.

    vts_artifactfiles_pairs - a list of pairs (vts, artifactfiles) where
      - vts is single VersionedTargetSet.
      - artifactfiles is a list of paths to artifacts for the VersionedTargetSet.
    """
    if self._artifact_cache and self.context.options.write_to_artifact_cache:
      with self.context.new_workunit('cache'):
        with self.context.new_workunit('update'):
          # Do some reporting.
          targets = set()
          for vts, artifactfiles in vts_artifactfiles_pairs:
            targets.update(vts.targets)
          self._report_targets('Caching artifacts for ', list(targets), '.')

          # Cache the artifacts.
          for vts, artifactfiles in vts_artifactfiles_pairs:
            if self.context.options.verify_artifact_cache:
              pass  # TODO: Verify that the artifact we just built is identical to the cached one.
            self._artifact_cache.insert(vts.cache_key, artifactfiles)

  def _report_targets(self, prefix, targets, suffix):
    self.context.log.info(
      prefix,
      items_to_report_element([t.address.reference() for t in targets], 'target'),
      suffix)

__all__ = (
  'TaskError',
  'Task'
)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
Created on Jan 25, 2018

.. codeauthor: Max Goebel <goebel@weblyzard.com>
'''
from __future__ import unicode_literals
from builtins import object
import json

from datetime import datetime
from decimal import Decimal
from itertools import chain

from weblyzard_api.model.parsers.xml_2013 import XML2013
from weblyzard_api.model import Sentence, SpanFactory, TokenCharSpan
from weblyzard_api.model.parsers.xml_2005 import XML2005
from weblyzard_api.model.parsers.xml_deprecated import XMLDeprecated
from weblyzard_api.model.exceptions import (MissingFieldException,
                                            UnexpectedFieldException)

class Document(object):
    # supported partition keys
    FRAGMENT_KEY = 'FRAGMENT'
    SENTENCE_KEY = u'SENTENCE'
    TITLE_KEY = u'TITLE'
    BODY_KEY = u'BODY'
    TOKEN_KEY = u'TOKEN'
    SENTIMENT_KEY = u'SENTIMENT_SCOPE'
    # NEGATION_KEY = u'NEGATION_SCOPE'
    MULTIPLIER_KEY = 'MULTIPLIER_SCOPE'

    # mapping from document attributes to serialized JSON fields
    MAPPING = {"content_id": "id",
               "md5sum": "id",
               "span_type": "@type",
               "header": "header",
               "content_type": "format",
               "sem_orient": "semOrient"}

    # list of required attributes
    REQUIRED_FIELDS = ['id', 'format', 'lang',
                       'content']

    # list of optional attributes
    OPTIONAL_FIELDS = ['partitions', 'annotations', 'encoding',
                       'features', 'relations', 'header', 'nilsimsa']

    SUPPORTED_XML_VERSIONS = {XML2005.VERSION: XML2005,
                              XML2013.VERSION: XML2013,
                              XMLDeprecated.VERSION: XMLDeprecated}

    def __init__(self, content_id, content, content_type, lang, nilsimsa=None,
                 partitions=None, header=None, annotations=None):
        ''' '''
        self.content_id = content_id
        self.content = content
        self.content_type = content_type
        self.lang = lang.lower() if lang else lang
        self.nilsimsa = nilsimsa

        # populate default dicts:
        self.partitions = {label: [SpanFactory.new_span(span) for span in spans]
                                for label, spans in partitions.items()} \
                                    if len(partitions) else {}
        self.header = header if header else {}
        self.annotations = annotations if annotations else []

    def get_body(self):
        if self.content is None or len(self.content) == 0:
            return ''
        if self.BODY_KEY in self.partitions:
            body_spans = self.partitions[self.BODY_KEY]
            spans = [self.content[span.start:span.end] for span in body_spans]
            return ' '.join(spans)
        return ''

    def get_title(self):
        if self.content is None or len(self.content) == 0:
            return ''
        if self.TITLE_KEY in self.partitions:
            title_spans = self.partitions[self.TITLE_KEY]
            titles = [self.content[span.start:span.end] for span in title_spans]
            return ' '.join(titles)
        return ''

    def set_title(self, title):
        """ """
        assert title in self.content
        start_index = self.content.index(title)
        end_index = start_index + len(title)
        self.partitions[self.TITLE_KEY] = [{
            "@type": "CharSpan",
            "start": start_index,
            "end": end_index
        }]

    title = property(get_title, set_title)

    def __repr__(self):
        return 'Document: {}'.format(self.__dict__)

    @classmethod
    def _dict_transform(cls, data, mapping=None):
        '''
        Recursively transform a document object to a JSON serializable dict,
        with MAPPING applied as well as empty results removed.
        :param data, the data to be transformed to a dict
        :return a dictionary of a document, ready for JSON serialization
        '''
        if mapping is None:
            mapping = cls.MAPPING
        if data is None:
            return None
        if isinstance(data, str):
            return data
        if isinstance(data, int):
            return data
        if isinstance(data, float):
            return data
        if isinstance(data, int):
            return data
        if isinstance(data, bool):
            return data
        if isinstance(data, Decimal):
            return str(data)  # needed for e.g. GEO coordinates
        if isinstance(data, tuple):
            if len(data) == 1:
                return data[0]
            return data
        if isinstance(data, memoryview):
            data = bytes(data)
        if isinstance(data, bytes):
            return data.decode("utf-8")
        if isinstance(data, datetime):
            return data.strftime("%Y-%m-%d %H:%M:%S")
        if isinstance(data, list):
            result = []
            for item in data:
                result.append(cls._dict_transform(item, mapping=mapping))
            return result
        if isinstance(data, dict):
            result = {}
            for key, value in data.items():
                key = mapping.get(key, key)
                if value is not None:
                    value = cls._dict_transform(value, mapping=mapping)
                    if value is not None:
                        result[key] = value
            #             if not len(result):
            #                 return None
            return result
        if isinstance(data, object):
            result = {}
            for key, value in data.__dict__.items():
                if key in mapping:
                    key = mapping[key]
                elif key.startswith('_'):
                    continue

                if value is not None:
                    if key == 'lang':
                        value = value.upper()
                    value = cls._dict_transform(value, mapping=mapping)
                    if value is not None:
                        result[key] = cls._dict_transform(
                            value, mapping=mapping)
            return result
        return None

    @classmethod
    def from_json(cls, json_payload):
        '''
        Convert a JSON object into a content model.
        :param json_payload, the string representation of the JSON content model
        '''
        parsed_content = json.loads(json_payload, strict=False)
        return cls.from_dict(dict_=parsed_content)

    @classmethod
    def from_dict(cls, dict_):
        '''
        Convert a `dict` object corresponding to the JSON serialisation
        into a Document object.
        :param dict_, the `dict` representing the Document.
        '''
        # validation
        for required_field in cls.REQUIRED_FIELDS:
            if not required_field in dict_:
                raise MissingFieldException(required_field)

        for key in dict_.keys():
            if not key in cls.REQUIRED_FIELDS + cls.OPTIONAL_FIELDS:
                raise UnexpectedFieldException(key)

        # This is tricky ... the mapping cannot be easily inversed
        # making the md5sum to content_id conversion at the top level necessary
        inverse_mapping = {v: k for k, v in cls.MAPPING.items() if
                                                    k != 'content_id'}
        parsed_content = Document._dict_transform(dict_,
                                                  mapping=inverse_mapping)
        parsed_content['content_id'] = parsed_content.pop('md5sum')

        # populate default dicts:
        partitions = {label: [SpanFactory.new_span(span) for span in spans]
                      for label, spans in parsed_content['partitions'].items()} \
            if 'partitions' in parsed_content else {}

        header = parsed_content['header'] \
            if 'header' in parsed_content and parsed_content[
            'header'] is not None else {}

        if not len(header):
            header = parsed_content['header'] \
                if 'header' in parsed_content else {}

        annotations = parsed_content['annotations'] \
            if 'annotations' in parsed_content else {}

        return Document(content_id=int(parsed_content['content_id']),
                        content=parsed_content.get('content'),
                        nilsimsa=parsed_content.get('nilsimsa'),
                        lang=parsed_content.get('lang'),
                        content_type=parsed_content.get('content_type'),
                        partitions=partitions,
                        header=header,
                        annotations=annotations)

    def to_json(self):
        '''
        Serialize a document to JSON '''
        return json.dumps(self.to_dict())

    def to_dict(self):
        '''
        Create a dict representing the Document analogous to the JSON structure.
        '''
        result = self._dict_transform(self)
        return result

    def to_xml(self, ignore_title=False, include_fragments=False,
               xml_version=XML2013.VERSION):
        '''
        Serialize a document to XML.
        :param ignore_titles: if set, titles will not be serialized.
        :param include_fragments: non-sentence fragments will be included.
        :param xml_version: the version of XML to be used (defaults to 2013)
        :return: the serialized XML
        '''
        if not hasattr(self, 'features'):
            self.features = {}

        if not hasattr(self, 'relations'):
            self.relations = {}

        if not hasattr(self, 'titles'):
            self.titles = []
        titles = self.titles
        if ignore_title:
            titles = []

        return self.SUPPORTED_XML_VERSIONS[xml_version].dump_xml(
            titles=titles,
            attributes=self.header,
            sentences=self.get_sentences(include_fragments=include_fragments))

    def get_text_by_span(self, span):
        '''
        Return the textual content of a span.
        :param span, the span to extract content for.
        '''
        return self.content[span.start:span.end]

    @classmethod
    def overlapping(cls, spanA, spanB):
        ''' Return whether two spans overlap. '''
        return (spanB.start <= spanA.start and spanB.end > spanA.start) or \
                (spanA.start <= spanB.start and spanA.end > spanB.start)

    def get_partition_overlaps(self, search_span, target_partition_key):
        ''' Return all spans from a given target_partition_key that overlap
        the search span.
        :param search_span, the span to search for overlaps by.
        :param target_partition_key, the target partition'''
        result = []

        if not target_partition_key in self.partitions:
            return result

        for other_span in self.partitions[target_partition_key]:
            if self.overlapping(other_span, search_span):
                result.append(other_span)

        return result

    def get_pos_for_annotation(self, annotation):
        """
        Get the part-of-speech for a given annotation.
        :param annotation
        :return: the POS of the annotation
        """
        for token_span in self.partitions[self.TOKEN_KEY]:
            if token_span.start >= annotation['start']:
                return token_span.pos
        return None

    def get_sentences(self, zero_based=False, include_title=True,
                      include_fragments=False):
        """
        Legacy method to extract webLyzard sentences from content model.
        :param zero_based: if True, enforce token indices starting at 0
        :param include_title: if True, include title sentences
        :param include_fragments: if True, include fragments (non-sentence text)
        """
        result = []
        offset = 0
        requested_keys = [self.SENTENCE_KEY]

        if include_fragments:
            requested_keys.append(self.FRAGMENT_KEY)
        if not any([key in self.partitions for key in requested_keys]):
            return result
        sentence_spans = chain(
            *(self.partitions.get(key, []) for key in requested_keys)
        )
        sentence_spans = sorted(sentence_spans, key=lambda span: span.start)

        for sentence_span in sentence_spans:
            if zero_based:
                offset = sentence_span.start
            if not sentence_span.span_type == 'SentenceCharSpan':
                raise Exception('Bad sentence span')

            # get tokens
            token_spans = self.get_partition_overlaps(search_span=sentence_span,
                                                      target_partition_key=self.TOKEN_KEY)
            is_title = len(
                self.get_partition_overlaps(search_span=sentence_span,
                                            target_partition_key=self.TITLE_KEY)) > 0
            if not include_title and is_title:
                continue

            # serialize POS, tokens, and dependecy to string
            pos_sequence = ' '.join([ts.pos for ts in token_spans])
            tok_sequence = ' '.join(
                ['{},{}'.format(ts.start - offset, ts.end - offset) for ts in
                 token_spans])
            try:
                dep_sequence = ' '.join(
                    ['{}:{}'.format(*ts.dependency.values()) for ts in
                     token_spans])
            except AttributeError:
                dep_sequence = None

            # prefer semOrient over sem_orient, if both are annotated and non-zero
            sem_orient = None
            if getattr(sentence_span, 'semOrient', None):
                sem_orient = sentence_span.semOrient
            else:
                sem_orient = sentence_span.sem_orient

            # finally, extract the sentence text.
            value = self.get_text_by_span(sentence_span)

            result.append(Sentence(md5sum=sentence_span.md5sum,
                                   sem_orient=sem_orient,
                                   significance=sentence_span.significance,
                                   pos=pos_sequence, token=tok_sequence,
                                   value=value, is_title=is_title,
                                   dependency=dep_sequence))

        return result
<commit_msg>fix: deal with None partitions<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
Created on Jan 25, 2018

.. codeauthor: Max Goebel <goebel@weblyzard.com>
'''
from __future__ import unicode_literals
from builtins import object
import json

from datetime import datetime
from decimal import Decimal
from itertools import chain

from weblyzard_api.model.parsers.xml_2013 import XML2013
from weblyzard_api.model import Sentence, SpanFactory, TokenCharSpan
from weblyzard_api.model.parsers.xml_2005 import XML2005
from weblyzard_api.model.parsers.xml_deprecated import XMLDeprecated
from weblyzard_api.model.exceptions import (MissingFieldException,
                                            UnexpectedFieldException)

class Document(object):
    # supported partition keys
    FRAGMENT_KEY = 'FRAGMENT'
    SENTENCE_KEY = u'SENTENCE'
    TITLE_KEY = u'TITLE'
    BODY_KEY = u'BODY'
    TOKEN_KEY = u'TOKEN'
    SENTIMENT_KEY = u'SENTIMENT_SCOPE'
    # NEGATION_KEY = u'NEGATION_SCOPE'
    MULTIPLIER_KEY = 'MULTIPLIER_SCOPE'

    # mapping from document attributes to serialized JSON fields
    MAPPING = {"content_id": "id",
               "md5sum": "id",
               "span_type": "@type",
               "header": "header",
               "content_type": "format",
               "sem_orient": "semOrient"}

    # list of required attributes
    REQUIRED_FIELDS = ['id', 'format', 'lang',
                       'content']

    # list of optional attributes
    OPTIONAL_FIELDS = ['partitions', 'annotations', 'encoding',
                       'features', 'relations', 'header', 'nilsimsa']

    SUPPORTED_XML_VERSIONS = {XML2005.VERSION: XML2005,
                              XML2013.VERSION: XML2013,
                              XMLDeprecated.VERSION: XMLDeprecated}

    def __init__(self, content_id, content, content_type, lang, nilsimsa=None,
                 partitions=None, header=None, annotations=None):
        ''' '''
        self.content_id = content_id
        self.content = content
        self.content_type = content_type
        self.lang = lang.lower() if lang else lang
        self.nilsimsa = nilsimsa

        # populate default dicts:
        if partitions is None:
            self.partitions = {}
        else:
            self.partitions = {label: [SpanFactory.new_span(span) for span in spans]
                                for label, spans in partitions.items()}
        self.header = header if header else {}
        self.annotations = annotations if annotations else []

    def get_body(self):
        if self.content is None or len(self.content) == 0:
            return ''
        if self.BODY_KEY in self.partitions:
            body_spans = self.partitions[self.BODY_KEY]
            spans = [self.content[span.start:span.end] for span in body_spans]
            return ' '.join(spans)
        return ''

    def get_title(self):
        if self.content is None or len(self.content) == 0:
            return ''
        if self.TITLE_KEY in self.partitions:
            title_spans = self.partitions[self.TITLE_KEY]
            titles = [self.content[span.start:span.end] for span in title_spans]
            return ' '.join(titles)
        return ''

    def set_title(self, title):
        """ """
        assert title in self.content
        start_index = self.content.index(title)
        end_index = start_index + len(title)
        self.partitions[self.TITLE_KEY] = [{
            "@type": "CharSpan",
            "start": start_index,
            "end": end_index
        }]

    title = property(get_title, set_title)

    def __repr__(self):
        return 'Document: {}'.format(self.__dict__)

    @classmethod
    def _dict_transform(cls, data, mapping=None):
        '''
        Recursively transform a document object to a JSON serializable dict,
        with MAPPING applied as well as empty results removed.
        :param data, the data to be transformed to a dict
        :return a dictionary of a document, ready for JSON serialization
        '''
        if mapping is None:
            mapping = cls.MAPPING
        if data is None:
            return None
        if isinstance(data, str):
            return data
        if isinstance(data, int):
            return data
        if isinstance(data, float):
            return data
        if isinstance(data, int):
            return data
        if isinstance(data, bool):
            return data
        if isinstance(data, Decimal):
            return str(data)  # needed for e.g. GEO coordinates
        if isinstance(data, tuple):
            if len(data) == 1:
                return data[0]
            return data
        if isinstance(data, memoryview):
            data = bytes(data)
        if isinstance(data, bytes):
            return data.decode("utf-8")
        if isinstance(data, datetime):
            return data.strftime("%Y-%m-%d %H:%M:%S")
        if isinstance(data, list):
            result = []
            for item in data:
                result.append(cls._dict_transform(item, mapping=mapping))
            return result
        if isinstance(data, dict):
            result = {}
            for key, value in data.items():
                key = mapping.get(key, key)
                if value is not None:
                    value = cls._dict_transform(value, mapping=mapping)
                    if value is not None:
                        result[key] = value
            #             if not len(result):
            #                 return None
            return result
        if isinstance(data, object):
            result = {}
            for key, value in data.__dict__.items():
                if key in mapping:
                    key = mapping[key]
                elif key.startswith('_'):
                    continue

                if value is not None:
                    if key == 'lang':
                        value = value.upper()
                    value = cls._dict_transform(value, mapping=mapping)
                    if value is not None:
                        result[key] = cls._dict_transform(
                            value, mapping=mapping)
            return result
        return None

    @classmethod
    def from_json(cls, json_payload):
        '''
        Convert a JSON object into a content model.
        :param json_payload, the string representation of the JSON content model
        '''
        parsed_content = json.loads(json_payload, strict=False)
        return cls.from_dict(dict_=parsed_content)

    @classmethod
    def from_dict(cls, dict_):
        '''
        Convert a `dict` object corresponding to the JSON serialisation
        into a Document object.
        :param dict_, the `dict` representing the Document.
        '''
        # validation
        for required_field in cls.REQUIRED_FIELDS:
            if not required_field in dict_:
                raise MissingFieldException(required_field)

        for key in dict_.keys():
            if not key in cls.REQUIRED_FIELDS + cls.OPTIONAL_FIELDS:
                raise UnexpectedFieldException(key)

        # This is tricky ... the mapping cannot be easily inversed
        # making the md5sum to content_id conversion at the top level necessary
        inverse_mapping = {v: k for k, v in cls.MAPPING.items() if
                                                    k != 'content_id'}
        parsed_content = Document._dict_transform(dict_,
                                                  mapping=inverse_mapping)
        parsed_content['content_id'] = parsed_content.pop('md5sum')

        # populate default dicts:
        partitions = {label: [SpanFactory.new_span(span) for span in spans]
                      for label, spans in parsed_content['partitions'].items()} \
            if 'partitions' in parsed_content else {}

        header = parsed_content['header'] \
            if 'header' in parsed_content and parsed_content[
            'header'] is not None else {}

        if not len(header):
            header = parsed_content['header'] \
                if 'header' in parsed_content else {}

        annotations = parsed_content['annotations'] \
            if 'annotations' in parsed_content else {}

        return Document(content_id=int(parsed_content['content_id']),
                        content=parsed_content.get('content'),
                        nilsimsa=parsed_content.get('nilsimsa'),
                        lang=parsed_content.get('lang'),
                        content_type=parsed_content.get('content_type'),
                        partitions=partitions,
                        header=header,
                        annotations=annotations)

    def to_json(self):
        '''
        Serialize a document to JSON '''
        return json.dumps(self.to_dict())

    def to_dict(self):
        '''
        Create a dict representing the Document analogous to the JSON structure.
        '''
        result = self._dict_transform(self)
        return result

    def to_xml(self, ignore_title=False, include_fragments=False,
               xml_version=XML2013.VERSION):
        '''
        Serialize a document to XML.
        :param ignore_titles: if set, titles will not be serialized.
        :param include_fragments: non-sentence fragments will be included.
        :param xml_version: the version of XML to be used (defaults to 2013)
        :return: the serialized XML
        '''
        if not hasattr(self, 'features'):
            self.features = {}

        if not hasattr(self, 'relations'):
            self.relations = {}

        if not hasattr(self, 'titles'):
            self.titles = []
        titles = self.titles
        if ignore_title:
            titles = []

        return self.SUPPORTED_XML_VERSIONS[xml_version].dump_xml(
            titles=titles,
            attributes=self.header,
            sentences=self.get_sentences(include_fragments=include_fragments))

    def get_text_by_span(self, span):
        '''
        Return the textual content of a span.
        :param span, the span to extract content for.
        '''
        return self.content[span.start:span.end]

    @classmethod
    def overlapping(cls, spanA, spanB):
        ''' Return whether two spans overlap. '''
        return (spanB.start <= spanA.start and spanB.end > spanA.start) or \
                (spanA.start <= spanB.start and spanA.end > spanB.start)

    def get_partition_overlaps(self, search_span, target_partition_key):
        ''' Return all spans from a given target_partition_key that overlap
        the search span.
        :param search_span, the span to search for overlaps by.
        :param target_partition_key, the target partition'''
        result = []

        if not target_partition_key in self.partitions:
            return result

        for other_span in self.partitions[target_partition_key]:
            if self.overlapping(other_span, search_span):
                result.append(other_span)

        return result

    def get_pos_for_annotation(self, annotation):
        """
        Get the part-of-speech for a given annotation.
        :param annotation
        :return: the POS of the annotation
        """
        for token_span in self.partitions[self.TOKEN_KEY]:
            if token_span.start >= annotation['start']:
                return token_span.pos
        return None

    def get_sentences(self, zero_based=False, include_title=True,
                      include_fragments=False):
        """
        Legacy method to extract webLyzard sentences from content model.
        :param zero_based: if True, enforce token indices starting at 0
        :param include_title: if True, include title sentences
        :param include_fragments: if True, include fragments (non-sentence text)
        """
        result = []
        offset = 0
        requested_keys = [self.SENTENCE_KEY]

        if include_fragments:
            requested_keys.append(self.FRAGMENT_KEY)
        if not any([key in self.partitions for key in requested_keys]):
            return result
        sentence_spans = chain(
            *(self.partitions.get(key, []) for key in requested_keys)
        )
        sentence_spans = sorted(sentence_spans, key=lambda span: span.start)

        for sentence_span in sentence_spans:
            if zero_based:
                offset = sentence_span.start
            if not sentence_span.span_type == 'SentenceCharSpan':
                raise Exception('Bad sentence span')

            # get tokens
            token_spans = self.get_partition_overlaps(search_span=sentence_span,
                                                      target_partition_key=self.TOKEN_KEY)
            is_title = len(
                self.get_partition_overlaps(search_span=sentence_span,
                                            target_partition_key=self.TITLE_KEY)) > 0
            if not include_title and is_title:
                continue

            # serialize POS, tokens, and dependecy to string
            pos_sequence = ' '.join([ts.pos for ts in token_spans])
            tok_sequence = ' '.join(
                ['{},{}'.format(ts.start - offset, ts.end - offset) for ts in
                 token_spans])
            try:
                dep_sequence = ' '.join(
                    ['{}:{}'.format(*ts.dependency.values()) for ts in
                     token_spans])
            except AttributeError:
                dep_sequence = None

            # prefer semOrient over sem_orient, if both are annotated and non-zero
            sem_orient = None
            if getattr(sentence_span, 'semOrient', None):
                sem_orient = sentence_span.semOrient
            else:
                sem_orient = sentence_span.sem_orient

            # finally, extract the sentence text.
            value = self.get_text_by_span(sentence_span)

            result.append(Sentence(md5sum=sentence_span.md5sum,
                                   sem_orient=sem_orient,
                                   significance=sentence_span.significance,
                                   pos=pos_sequence, token=tok_sequence,
                                   value=value, is_title=is_title,
                                   dependency=dep_sequence))

        return result
<|endoftext|>


<commit_before>'''
Provide a series Element Expected Condition Functors

@date 2015-08-03
@author Hong-She Liang <starofrainnight@gmail.com>
'''

# Import all expected conditions from selenium
from selenium.webdriver.support.expected_conditions import *

class eecf_stale_of(object):
    def __init__(self, check_status):
        self.__check_status = check_status

    def __call__(self, element):
        try:
            # Calling any method forces a staleness check
            element.is_enabled()
            return self.__check_status
        except StaleElementReferenceException:
            return not self.__check_status

class eecf_visible_of(object):
    def __init__(self, check_status):
        self.__check_status = check_status

    def __call__(self, element):
        try:
            return element.is_displayed() == self.__check_status
        except (NoSuchElementException, StaleElementReferenceException):
            # In the case of NoSuchElement, returns true because the element is
            # not present in DOM. The try block checks if the element is present
            # but is invisible.
            # In the case of StaleElementReference, returns true because stale
            # element reference implies that element is no longer visible.
            return not self.__check_status

class eecf_select_of(object):
    def __init__(self, check_status):
        self.__check_status = check_status

    def __call__(self, element):
        return element.is_selected() == self.__check_status

class eecf_enable_of(object):
    def __init__(self, check_status):
        self.__check_status = check_status

    def __call__(self, element):
        """ Wait until an element is enabled
        returns False otherwise.
        """
        # Calling any method forces a staleness check
        return element.is_enabled() == self.__check_status

class conditions_matched(object):
    """
    Use for wait a series conditions and store them to a named dict.
    """

    def __init__(self, condition_dict={}, matched_at_least=1):
        self.__condition_dict = condition_dict
        self.__matched_at_least = matched_at_least

    def __call__(self, driver):
        result = {}
        for k, v in self.__condition_dict.iteritems():
            value = v["condition"](driver)
            if False != value:
                result[k] = value
            elif ("optional" not in v) or (not v["optional"]):
                # If optional flag is True and value is False, we return False
                return False

        if len(result) < self.__matched_at_least:
            return False

        return result

class xpath_find(object):
    def __init__(self, xpath_expr, conditions=[]):
        self.__xpath_expr = xpath_expr
        self.__conditions = conditions

    def __call__(self, driver):
        return driver.xpath_find(self.__xpath_expr,
            conditions=self.__conditions)

class xpath_find_all(object):
    """
    Easily to wait until find all elements recursively.

    Example :

    @code
    elements = WebDriverWait(driver, 10).until(EC.xpath_find(
        "//input[@id='fm-login-id']",
        conditions=[eecf_visible_of(True)]))
    @endcode
    """
    def __init__(self, xpath_expr, conditions=[]):
        self.__xpath_expr = xpath_expr
        self.__conditions = conditions

    def __call__(self, driver):
        elements = driver.xpath_find_all(self.__xpath_expr,
            conditions=self.__conditions)
        if len(elements) > 0:
            return elements
        else:
            return False
<commit_msg>Added sample comments<commit_after>'''
Provide a series Element Expected Condition Functors

@date 2015-08-03
@author Hong-She Liang <starofrainnight@gmail.com>
'''

# Import all expected conditions from selenium
from selenium.webdriver.support.expected_conditions import *

class eecf_stale_of(object):
    def __init__(self, check_status):
        self.__check_status = check_status

    def __call__(self, element):
        try:
            # Calling any method forces a staleness check
            element.is_enabled()
            return self.__check_status
        except StaleElementReferenceException:
            return not self.__check_status

class eecf_visible_of(object):
    def __init__(self, check_status):
        self.__check_status = check_status

    def __call__(self, element):
        try:
            return element.is_displayed() == self.__check_status
        except (NoSuchElementException, StaleElementReferenceException):
            # In the case of NoSuchElement, returns true because the element is
            # not present in DOM. The try block checks if the element is present
            # but is invisible.
            # In the case of StaleElementReference, returns true because stale
            # element reference implies that element is no longer visible.
            return not self.__check_status

class eecf_select_of(object):
    def __init__(self, check_status):
        self.__check_status = check_status

    def __call__(self, element):
        return element.is_selected() == self.__check_status

class eecf_enable_of(object):
    def __init__(self, check_status):
        self.__check_status = check_status

    def __call__(self, element):
        """ Wait until an element is enabled
        returns False otherwise.
        """
        # Calling any method forces a staleness check
        return element.is_enabled() == self.__check_status

class conditions_matched(object):
    """
    Use for wait a series conditions and store them to a named dict.

    @code
    matched_elements = self.waiter.until(EC.conditions_matched({
        "checkcode":{"condition":EC.xpath_find("//input[@id='fm-login-checkcode']")},
    }))
    @endcode
    """

    def __init__(self, condition_dict={}, matched_at_least=1):
        self.__condition_dict = condition_dict
        self.__matched_at_least = matched_at_least

    def __call__(self, driver):
        result = {}
        for k, v in self.__condition_dict.iteritems():
            value = v["condition"](driver)
            if False != value:
                result[k] = value
            elif ("optional" not in v) or (not v["optional"]):
                # If optional flag is True and value is False, we return False
                return False

        if len(result) < self.__matched_at_least:
            return False

        return result

class xpath_find(object):
    def __init__(self, xpath_expr, conditions=[]):
        self.__xpath_expr = xpath_expr
        self.__conditions = conditions

    def __call__(self, driver):
        return driver.xpath_find(self.__xpath_expr,
            conditions=self.__conditions)

class xpath_find_all(object):
    """
    Easily to wait until find all elements recursively.

    Example :

    @code
    elements = WebDriverWait(driver, 10).until(EC.xpath_find(
        "//input[@id='fm-login-id']",
        conditions=[eecf_visible_of(True)]))
    @endcode
    """
    def __init__(self, xpath_expr, conditions=[]):
        self.__xpath_expr = xpath_expr
        self.__conditions = conditions

    def __call__(self, driver):
        elements = driver.xpath_find_all(self.__xpath_expr,
            conditions=self.__conditions)
        if len(elements) > 0:
            return elements
        else:
            return False
<|endoftext|>