<commit_before>'''
Jinja loading utils to enable a more powerful backend for jinja templates
'''
# Import python libs
from os import path
import logging

# Import third-party libs
from jinja2 import (Template, BaseLoader, Environment, StrictUndefined,
                    FileSystemLoader)
from jinja2.exceptions import TemplateNotFound

# Import Salt libs
import salt
import salt.fileclient

log = logging.getLogger(__name__)

def get_template(filename, opts, env):
    loader = SaltCacheLoader(opts, env)
    if filename.startswith(loader.searchpath):
        if opts.get('allow_undefined', False):
            jinja = Environment(loader=loader)
        else:
            jinja = Environment(loader=loader, undefined=StrictUndefined)
        relpath = path.relpath(filename, loader.searchpath)
        # the template was already fetched
        loader.cached.append(relpath)
        return jinja.get_template(relpath)
    else:
        # fallback for templates outside the state tree
        loader = FileSystemLoader(path.dirname(filename))
        if opts.get('allow_undefined', False):
            jinja = Environment(loader=loader)
        else:
            jinja = Environment(loader=loader, undefined=StrictUndefined)
        relpath = path.relpath(filename, path.dirname(filename))
        return jinja.get_template(relpath)

class SaltCacheLoader(BaseLoader):
    '''
    A special jinja Template Loader for salt.
    Requested templates are always fetched from the server
    to guarantee that the file is up to date.
    Templates are cached like regular salt states
    and only loaded once per loader instance.
    '''
    def __init__(self, opts, env='base', encoding='utf-8'):
        self.opts = opts
        self.env = env
        self.encoding = encoding
        self.searchpath = path.join(opts['cachedir'], 'files', env)
        log.debug('Jinja search path: \'{0}\''.format(self.searchpath))
        self._file_client = None
        self.cached = []

    def file_client(self):
        '''
        Return a file client. Instantiates on first call.
        '''
        if not self._file_client:
            self._file_client = salt.fileclient.get_file_client(self.opts)
        return self._file_client

    def cache_file(self, template):
        '''
        Cache a file from the salt master
        '''
        saltpath = path.join('salt://', template)
        self.file_client().get_file(saltpath, '', True, self.env)

    def check_cache(self, template):
        '''
        Cache a file only once
        '''
        if template not in self.cached:
            self.cache_file(template)
            self.cached.append(template)

    def get_source(self, environment, template):
        # checks for relative '..' paths
        if '..' in template:
            log.warning(
                'Discarded template path \'{0}\', relative paths are '
                'prohibited'.format(template)
            )
            raise TemplateNotFound(template)
        self.check_cache(template)
        filepath = path.join(self.searchpath, template)
        with open(filepath, 'rb') as f:
            try:
                contents = f.read().decode(self.encoding)
            except IOError:
                raise TemplateNotFound(template)
        mtime = path.getmtime(filepath)

        def uptodate():
            try:
                return path.getmtime(filepath) == mtime
            except OSError:
                return False
        return contents, filepath, uptodate
<commit_msg>pylint fix<commit_after>'''
Jinja loading utils to enable a more powerful backend for jinja templates
'''
# Import python libs
from os import path
import logging

# Import third-party libs
from jinja2 import (BaseLoader, Environment, StrictUndefined,
                    FileSystemLoader)
from jinja2.exceptions import TemplateNotFound

# Import Salt libs
import salt
import salt.fileclient

log = logging.getLogger(__name__)

def get_template(filename, opts, env):
    loader = SaltCacheLoader(opts, env)
    if filename.startswith(loader.searchpath):
        if opts.get('allow_undefined', False):
            jinja = Environment(loader=loader)
        else:
            jinja = Environment(loader=loader, undefined=StrictUndefined)
        relpath = path.relpath(filename, loader.searchpath)
        # the template was already fetched
        loader.cached.append(relpath)
        return jinja.get_template(relpath)
    else:
        # fallback for templates outside the state tree
        loader = FileSystemLoader(path.dirname(filename))
        if opts.get('allow_undefined', False):
            jinja = Environment(loader=loader)
        else:
            jinja = Environment(loader=loader, undefined=StrictUndefined)
        relpath = path.relpath(filename, path.dirname(filename))
        return jinja.get_template(relpath)

class SaltCacheLoader(BaseLoader):
    '''
    A special jinja Template Loader for salt.
    Requested templates are always fetched from the server
    to guarantee that the file is up to date.
    Templates are cached like regular salt states
    and only loaded once per loader instance.
    '''
    def __init__(self, opts, env='base', encoding='utf-8'):
        self.opts = opts
        self.env = env
        self.encoding = encoding
        self.searchpath = path.join(opts['cachedir'], 'files', env)
        log.debug('Jinja search path: \'{0}\''.format(self.searchpath))
        self._file_client = None
        self.cached = []

    def file_client(self):
        '''
        Return a file client. Instantiates on first call.
        '''
        if not self._file_client:
            self._file_client = salt.fileclient.get_file_client(self.opts)
        return self._file_client

    def cache_file(self, template):
        '''
        Cache a file from the salt master
        '''
        saltpath = path.join('salt://', template)
        self.file_client().get_file(saltpath, '', True, self.env)

    def check_cache(self, template):
        '''
        Cache a file only once
        '''
        if template not in self.cached:
            self.cache_file(template)
            self.cached.append(template)

    def get_source(self, environment, template):
        # checks for relative '..' paths
        if '..' in template:
            log.warning(
                'Discarded template path \'{0}\', relative paths are '
                'prohibited'.format(template)
            )
            raise TemplateNotFound(template)
        self.check_cache(template)
        filepath = path.join(self.searchpath, template)
        with open(filepath, 'rb') as f:
            try:
                contents = f.read().decode(self.encoding)
            except IOError:
                raise TemplateNotFound(template)
        mtime = path.getmtime(filepath)

        def uptodate():
            try:
                return path.getmtime(filepath) == mtime
            except OSError:
                return False
        return contents, filepath, uptodate
<|endoftext|>


<commit_before># coding=utf-8
from bluebottle.bluebottle_dashboard.dashboard import CustomAppIndexDashboard
from bluebottle.projects.dashboard import AppIndexDashboard as ProjectAppIndexDashboard
from django.test.client import RequestFactory
from django.test.utils import override_settings
from jet.dashboard.dashboard import DefaultAppIndexDashboard
from tenant_schemas.urlresolvers import reverse

from bluebottle.test.utils import BluebottleAdminTestCase

class MainDashboardTest(BluebottleAdminTestCase):
    """
    Test main admin dashboard
    """

    def setUp(self):
        super(MainDashboardTest, self).setUp()
        self.init_projects()
        self.client.force_login(self.superuser)
        self.admin_url = reverse('admin:index')

    def test_main_dashboard(self):
        response = self.client.get(self.admin_url)
        self.assertContains(response, 'Recently submitted initiatives')
        self.assertContains(response, 'Recently submitted events')
        self.assertContains(response, "Initiatives I'm reviewing")
        self.assertContains(response, 'Recently joined users')
        self.assertContains(response, 'Export metrics')
        # Stand settings don't show export options
        self.assertNotContains(response, 'Download report')
        self.assertNotContains(response, 'Request complete participation metrics')

    @override_settings(REPORTING_BACKOFFICE_ENABLED=True, PARTICIPATION_BACKOFFICE_ENABLED=True)
    def test_main_dashboard_export_options(self):
        # Override settings to show export options
        response = self.client.get(self.admin_url)
        self.assertContains(response, 'Download report')
        self.assertContains(response, 'Request complete participation metrics')

class CustomAppDashboardTest(BluebottleAdminTestCase):
    """
    Test custom app dashboard loading
    """

    def setUp(self):
        super(CustomAppDashboardTest, self).setUp()
        self.client.force_login(self.superuser)
        self.admin_url = reverse('admin:index')
        self.request = RequestFactory().get(self.admin_url)
        self.request.user = self.superuser

    def test_existing_app_dashboard(self):
        dash = CustomAppIndexDashboard({'request': self.request}, app_label='projects')
        self.assertTrue(isinstance(dash, ProjectAppIndexDashboard))

    def test_non_existing_app_dashboard(self):
        dash = CustomAppIndexDashboard({'request': self.request}, app_label='orders')
        self.assertTrue(isinstance(dash, DefaultAppIndexDashboard))
<commit_msg>Fix test<commit_after># coding=utf-8
from bluebottle.bluebottle_dashboard.dashboard import CustomAppIndexDashboard
from bluebottle.projects.dashboard import AppIndexDashboard as ProjectAppIndexDashboard
from django.test.client import RequestFactory
from django.test.utils import override_settings
from jet.dashboard.dashboard import DefaultAppIndexDashboard
from tenant_schemas.urlresolvers import reverse

from bluebottle.test.utils import BluebottleAdminTestCase

class MainDashboardTest(BluebottleAdminTestCase):
    """
    Test main admin dashboard
    """

    def setUp(self):
        super(MainDashboardTest, self).setUp()
        self.init_projects()
        self.client.force_login(self.superuser)
        self.admin_url = reverse('admin:index')

    def test_main_dashboard(self):
        response = self.client.get(self.admin_url)
        self.assertContains(response, 'Recently submitted initiatives')
        self.assertContains(response, 'Recently submitted events')
        self.assertContains(response, 'Recently joined users')
        self.assertContains(response, 'Export metrics')
        # Stand settings don't show export options
        self.assertNotContains(response, 'Download report')
        self.assertNotContains(response, 'Request complete participation metrics')

    @override_settings(REPORTING_BACKOFFICE_ENABLED=True, PARTICIPATION_BACKOFFICE_ENABLED=True)
    def test_main_dashboard_export_options(self):
        # Override settings to show export options
        response = self.client.get(self.admin_url)
        self.assertContains(response, 'Download report')
        self.assertContains(response, 'Request complete participation metrics')

class CustomAppDashboardTest(BluebottleAdminTestCase):
    """
    Test custom app dashboard loading
    """

    def setUp(self):
        super(CustomAppDashboardTest, self).setUp()
        self.client.force_login(self.superuser)
        self.admin_url = reverse('admin:index')
        self.request = RequestFactory().get(self.admin_url)
        self.request.user = self.superuser

    def test_existing_app_dashboard(self):
        dash = CustomAppIndexDashboard({'request': self.request}, app_label='projects')
        self.assertTrue(isinstance(dash, ProjectAppIndexDashboard))

    def test_non_existing_app_dashboard(self):
        dash = CustomAppIndexDashboard({'request': self.request}, app_label='orders')
        self.assertTrue(isinstance(dash, DefaultAppIndexDashboard))
<|endoftext|>


<commit_before># Copyright 2017 Creu Blanca
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).

from odoo import models

class PartnerXlsx(models.AbstractModel):
    _name = "report.report_xlsx.partner_xlsx"
    _inherit = "report.report_xlsx.abstract"
    _description = "Partner XLSX Report"

    def generate_xlsx_report(self, workbook, data, partners):
        sheet = workbook.add_worksheet("Report")
        i = 0
        for obj in partners:
            bold = workbook.add_format({"bold": True})
            sheet.write(i, 0, obj.name, bold)
            i += 1
<commit_msg>[12.0][FIX] Report xlsx res_partner<commit_after># Copyright 2017 Creu Blanca
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).

from odoo import models

class PartnerXlsx(models.AbstractModel):
    _name = "report.report_xlsx.partner_xlsx"
    _inherit = "report.report_xlsx.abstract"
    _description = "Partner XLSX Report"

    def generate_xlsx_report(self, workbook, data, partners):
        sheet = workbook.add_worksheet("Report")
        for i, obj in enumerate(partners):
            bold = workbook.add_format({"bold": True})
            sheet.write(i, 0, obj.name, bold)
<|endoftext|>


<commit_before>import letters
import letters2x

class PixetSet(object):
    def get_pixel_set(self):
        pass

class LettersPixelSet(PixetSet):
    def __init__(self, string, size=1):
        self.__letters = self.__load_letters(size)
        self.__pixel_set = self.__form_pixel_set(str(string).lower())
        self.__pixel_set = self.__normalize_pixel_set(self.__pixel_set)

    def get_pixel_set(self):
        return self.__pixel_set

    def __load_letters(self, size=1):
        if size == 2:
            return letters2x
        else:
            return letters

    def __form_pixel_set(self, string):
        pixel_set = ()
        space_pixel_set = self.__get_space_pixel_set()
        for letter in string:
            pixel_set = pixel_set + (space_pixel_set, self.__get_letter_position(letter), )

        return pixel_set

    def __normalize_pixel_set(self, pixel_set):
        rows = [[] for _ in range(self.__letters.HEIGHT)]
        for letter in pixel_set:
            for i in range(self.__letters.HEIGHT):
                rows[i] = rows[i] + letter[i]

        return rows

    def __get_space_pixel_set(self):
        try:
            return getattr(self.__letters, 'space')
        except AttributeError:
            raise SystemExit('Space letter is not exists')

    def __get_letter_position(self, letter):
        try:
            return getattr(self.__letters, letter)
        except AttributeError:
            raise SystemExit('Letter ' + letter + ' is not exists')
<commit_msg>dictionary as module support<commit_after>import importlib

class PixetSet(object):
    def get_pixel_set(self):
        pass

class LettersPixelSet(PixetSet):
    def __init__(self, string, dictionary):
        self.__letters = self.__load_letters(dictionary)
        self.__pixel_set = self.__form_pixel_set(str(string).lower())
        self.__pixel_set = self.__normalize_pixel_set(self.__pixel_set)

    def get_pixel_set(self):
        return self.__pixel_set

    def __load_letters(self, dictionary):
        return importlib.import_module('github_art.dictionaries.' + dictionary)

    def __form_pixel_set(self, string):
        pixel_set = ()
        space_pixel_set = self.__get_space_pixel_set()

        for index, letter in enumerate(string):
            pixel_set = pixel_set + (space_pixel_set, self.__get_letter_position(letter), ) if index > 0 \
                else (self.__get_letter_position(letter), )

        return pixel_set

    def __normalize_pixel_set(self, pixel_set):
        rows = [[] for _ in range(self.__letters.HEIGHT)]
        for letter in pixel_set:
            for i in range(self.__letters.HEIGHT):
                rows[i] = rows[i] + letter[i]

        return rows

    def __get_space_pixel_set(self):
        try:
            return getattr(self.__letters, 'space')
        except AttributeError:
            raise SystemExit('Space letter is not exists')

    def __get_letter_position(self, letter):
        try:
            return getattr(self.__letters, letter)
        except AttributeError:
            raise SystemExit('Letter ' + letter + ' is not exists')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2015, 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from collections import namedtuple

import six

from twisted.internet import defer
from twisted.internet.defer import DeferredList

from synapse.api.constants import MAX_DEPTH, EventTypes, Membership, RoomVersions
from synapse.api.errors import Codes, SynapseError
from synapse.crypto.event_signing import check_event_content_hash
from synapse.events import event_type_from_format_version
from synapse.events.utils import prune_event
from synapse.http.servlet import assert_params_in_dict
from synapse.types import get_domain_from_id
from synapse.util import logcontext, unwrapFirstError

logger = logging.getLogger(__name__)

class FederationBase(object):
    def __init__(self, hs):
        self.hs = hs

        self.server_name = hs.hostname
        self.keyring = hs.get_keyring()
        self.spam_checker = hs.get_spam_checker()
        self.store = hs.get_datastore()
        self._clock = hs.get_clock()

    @defer.inlineCallbacks
    def _check_sigs_and_hash_and_fetch(self, origin, pdus, room_version,
                                       outlier=False, include_none=False):
        """Takes a list of PDUs and checks the signatures and hashs of each
        one. If a PDU fails its signature check then we check if we have it in
        the database and if not then request if from the originating server of
        that PDU.

        If a PDU fails its content hash check then it is redacted.

        The given list of PDUs are not modified, instead the function returns
        a new list.

        Args:
            origin (str)
            pdu (list)
            room_version (str)
            outlier (bool): Whether the events are outliers or not
            include_none (str): Whether to include None in the returned list
                for events that have failed their checks

        Returns:
            Deferred : A list of PDUs that have valid signatures and hashes.
        """
        deferreds = self._check_sigs_and_hashes(room_version, pdus)

        @defer.inlineCallbacks
        def handle_check_result(pdu, deferred):
            try:
                res = yield logcontext.make_deferred_yieldable(deferred)
            except SynapseError:
                res = None

            if not res:
                # Check local db.
                res = yield self.store.get_event(
                    pdu.event_id,
                    allow_rejected=True,
                    allow_none=True,
                )

            if not res and pdu.origin != origin:
                try:
                    res = yield self.get_pdu(
                        destinations=[pdu.origin],
                        event_id=pdu.event_id,
                        room_version=room_version,
                        outlier=outlier,
                        timeout=10000,
                    )
                except SynapseError:
                    pass

            if not res:
                logger.warn(
                    "Failed to find copy of %s with valid signature",
                    pdu.event_id,
                )

            defer.returnValue(res)

        handle = logcontext.preserve_fn(handle_check_result)
        deferreds2 = [
            handle(pdu, deferred)
            for pdu, deferred in zip(pdus, deferreds)
        ]

        valid_pdus = yield logcontext.make_deferred_yieldable(
            defer.gatherResults(
                deferreds2,
                consumeErrors=True,
            )
        ).addErrback(unwrapFirstError)

        if include_none:
            defer.returnValue(valid_pdus)
        else:
            defer.returnValue([p for p in valid_pdus if p])

    def _check_sigs_and_hash(self, room_version, pdu):
        return logcontext.make_deferred_yieldable(
            self._check_sigs_and_hashes(room_version, [pdu])[0],
        )

    def _check_sigs_and_hashes(self, room_version, pdus):
        """Checks that each of the received events is correctly signed by the
        sending server.

        Args:
            room_version (str): The room version of the PDUs
            pdus (list[FrozenEvent]): the events to be checked

        Returns:
            list[Deferred]: for each input event, a deferred which:
              * returns the original event if the checks pass
              * returns a redacted version of the event (if the signature
                matched but the hash did not)
              * throws a SynapseError if the signature check failed.
            The deferreds run their callbacks in the sentinel logcontext.
        """
        deferreds = _check_sigs_on_pdus(self.keyring, room_version, pdus)

        ctx = logcontext.LoggingContext.current_context()

        def callback(_, pdu):
            with logcontext.PreserveLoggingContext(ctx):
                if not check_event_content_hash(pdu):
                    # let's try to distinguish between failures because the event was
                    # redacted (which are somewhat expected) vs actual ball-tampering
                    # incidents.
                    #
                    # This is just a heuristic, so we just assume that if the keys are
                    # about the same between the redacted and received events, then the
                    # received event was probably a redacted copy (but we then use our
                    # *actual* redacted copy to be on the safe side.)
                    redacted_event = prune_event(pdu)
                    if (
                        set(redacted_event.keys()) == set(pdu.keys()) and
                        set(six.iterkeys(redacted_event.content))
                            == set(six.iterkeys(pdu.content))
                    ):
                        logger.info(
                            "Event %s seems to have been redacted; using our redacted "
                            "copy",
                            pdu.event_id,
                        )
                    else:
                        logger.warning(
                            "Event %s content has been tampered, redacting",
                            pdu.event_id, pdu.get_pdu_json(),
                        )
                    return redacted_event

                if self.spam_checker.check_event_for_spam(pdu):
                    logger.warn(
                        "Event contains spam, redacting %s: %s",
                        pdu.event_id, pdu.get_pdu_json()
                    )
                    return prune_event(pdu)

                return pdu

        def errback(failure, pdu):
            failure.trap(SynapseError)
            with logcontext.PreserveLoggingContext(ctx):
                logger.warn(
                    "Signature check failed for %s: %s",
                    pdu.event_id, failure.getErrorMessage(),
                )
            return failure

        for deferred, pdu in zip(deferreds, pdus):
            deferred.addCallbacks(
                callback, errback,
                callbackArgs=[pdu],
                errbackArgs=[pdu],
            )

        return deferreds

class PduToCheckSig(namedtuple("PduToCheckSig", [
    "pdu", "redacted_pdu_json", "sender_domain", "deferreds",
])):
    pass

def _check_sigs_on_pdus(keyring, room_version, pdus):
    """Check that the given events are correctly signed

    Args:
        keyring (synapse.crypto.Keyring): keyring object to do the checks
        room_version (str): the room version of the PDUs
        pdus (Collection[EventBase]): the events to be checked

    Returns:
        List[Deferred]: a Deferred for each event in pdus, which will either succeed if
           the signatures are valid, or fail (with a SynapseError) if not.
    """

    # (currently this is written assuming the v1 room structure; we'll probably want a
    # separate function for checking v2 rooms)

    # we want to check that the event is signed by:
    #
    # (a) the sender's server
    #
    #     - except in the case of invites created from a 3pid invite, which are exempt
    #     from this check, because the sender has to match that of the original 3pid
    #     invite, but the event may come from a different HS, for reasons that I don't
    #     entirely grok (why do the senders have to match? and if they do, why doesn't the
    #     joining server ask the inviting server to do the switcheroo with
    #     exchange_third_party_invite?).
    #
    #     That's pretty awful, since redacting such an invite will render it invalid
    #     (because it will then look like a regular invite without a valid signature),
    #     and signatures are *supposed* to be valid whether or not an event has been
    #     redacted. But this isn't the worst of the ways that 3pid invites are broken.
    #
    # (b) for V1 and V2 rooms, the server which created the event_id
    #
    # let's start by getting the domain for each pdu, and flattening the event back
    # to JSON.

    pdus_to_check = [
        PduToCheckSig(
            pdu=p,
            redacted_pdu_json=prune_event(p).get_pdu_json(),
            sender_domain=get_domain_from_id(p.sender),
            deferreds=[],
        )
        for p in pdus
    ]

    # First we check that the sender event is signed by the sender's domain
    # (except if its a 3pid invite, in which case it may be sent by any server)
    pdus_to_check_sender = [
        p for p in pdus_to_check
        if not _is_invite_via_3pid(p.pdu)
    ]

    more_deferreds = keyring.verify_json_objects_for_server([
        (p.sender_domain, p.redacted_pdu_json)
        for p in pdus_to_check_sender
    ])

    for p, d in zip(pdus_to_check_sender, more_deferreds):
        p.deferreds.append(d)

    # now let's look for events where the sender's domain is different to the
    # event id's domain (normally only the case for joins/leaves), and add additional
    # checks. Only do this if the room version has a concept of event ID domain
    if room_version in (
        RoomVersions.V1, RoomVersions.V2, RoomVersions.VDH_TEST,
        RoomVersions.STATE_V2_TEST,
    ):
        pdus_to_check_event_id = [
            p for p in pdus_to_check
            if p.sender_domain != get_domain_from_id(p.pdu.event_id)
        ]

        more_deferreds = keyring.verify_json_objects_for_server([
            (get_domain_from_id(p.pdu.event_id), p.redacted_pdu_json)
            for p in pdus_to_check_event_id
        ])

        for p, d in zip(pdus_to_check_event_id, more_deferreds):
            p.deferreds.append(d)
    elif room_version in (RoomVersions.V3,):
        pass  # No further checks needed, as event IDs are hashes here
    else:
        raise RuntimeError("Unrecognized room version %s" % (room_version,))

    # replace lists of deferreds with single Deferreds
    return [_flatten_deferred_list(p.deferreds) for p in pdus_to_check]

def _flatten_deferred_list(deferreds):
    """Given a list of deferreds, either return the single deferred,
    combine into a DeferredList, or return an already resolved deferred.
    """
    if len(deferreds) > 1:
        return DeferredList(deferreds, fireOnOneErrback=True, consumeErrors=True)
    elif len(deferreds) == 1:
        return deferreds[0]
    else:
        return defer.succeed(None)

def _is_invite_via_3pid(event):
    return (
        event.type == EventTypes.Member
        and event.membership == Membership.INVITE
        and "third_party_invite" in event.content
    )

def event_from_pdu_json(pdu_json, event_format_version, outlier=False):
    """Construct a FrozenEvent from an event json received over federation

    Args:
        pdu_json (object): pdu as received over federation
        event_format_version (int): The event format version
        outlier (bool): True to mark this event as an outlier

    Returns:
        FrozenEvent

    Raises:
        SynapseError: if the pdu is missing required fields or is otherwise
            not a valid matrix event
    """
    # we could probably enforce a bunch of other fields here (room_id, sender,
    # origin, etc etc)
    assert_params_in_dict(pdu_json, ('type', 'depth'))

    depth = pdu_json['depth']
    if not isinstance(depth, six.integer_types):
        raise SynapseError(400, "Depth %r not an intger" % (depth, ),
                           Codes.BAD_JSON)

    if depth < 0:
        raise SynapseError(400, "Depth too small", Codes.BAD_JSON)
    elif depth > MAX_DEPTH:
        raise SynapseError(400, "Depth too large", Codes.BAD_JSON)

    event = event_type_from_format_version(event_format_version)(
        pdu_json,
    )

    event.internal_metadata.outlier = outlier

    return event
<commit_msg>No vdh tests!<commit_after># -*- coding: utf-8 -*-
# Copyright 2015, 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from collections import namedtuple

import six

from twisted.internet import defer
from twisted.internet.defer import DeferredList

from synapse.api.constants import MAX_DEPTH, EventTypes, Membership, RoomVersions
from synapse.api.errors import Codes, SynapseError
from synapse.crypto.event_signing import check_event_content_hash
from synapse.events import event_type_from_format_version
from synapse.events.utils import prune_event
from synapse.http.servlet import assert_params_in_dict
from synapse.types import get_domain_from_id
from synapse.util import logcontext, unwrapFirstError

logger = logging.getLogger(__name__)

class FederationBase(object):
    def __init__(self, hs):
        self.hs = hs

        self.server_name = hs.hostname
        self.keyring = hs.get_keyring()
        self.spam_checker = hs.get_spam_checker()
        self.store = hs.get_datastore()
        self._clock = hs.get_clock()

    @defer.inlineCallbacks
    def _check_sigs_and_hash_and_fetch(self, origin, pdus, room_version,
                                       outlier=False, include_none=False):
        """Takes a list of PDUs and checks the signatures and hashs of each
        one. If a PDU fails its signature check then we check if we have it in
        the database and if not then request if from the originating server of
        that PDU.

        If a PDU fails its content hash check then it is redacted.

        The given list of PDUs are not modified, instead the function returns
        a new list.

        Args:
            origin (str)
            pdu (list)
            room_version (str)
            outlier (bool): Whether the events are outliers or not
            include_none (str): Whether to include None in the returned list
                for events that have failed their checks

        Returns:
            Deferred : A list of PDUs that have valid signatures and hashes.
        """
        deferreds = self._check_sigs_and_hashes(room_version, pdus)

        @defer.inlineCallbacks
        def handle_check_result(pdu, deferred):
            try:
                res = yield logcontext.make_deferred_yieldable(deferred)
            except SynapseError:
                res = None

            if not res:
                # Check local db.
                res = yield self.store.get_event(
                    pdu.event_id,
                    allow_rejected=True,
                    allow_none=True,
                )

            if not res and pdu.origin != origin:
                try:
                    res = yield self.get_pdu(
                        destinations=[pdu.origin],
                        event_id=pdu.event_id,
                        room_version=room_version,
                        outlier=outlier,
                        timeout=10000,
                    )
                except SynapseError:
                    pass

            if not res:
                logger.warn(
                    "Failed to find copy of %s with valid signature",
                    pdu.event_id,
                )

            defer.returnValue(res)

        handle = logcontext.preserve_fn(handle_check_result)
        deferreds2 = [
            handle(pdu, deferred)
            for pdu, deferred in zip(pdus, deferreds)
        ]

        valid_pdus = yield logcontext.make_deferred_yieldable(
            defer.gatherResults(
                deferreds2,
                consumeErrors=True,
            )
        ).addErrback(unwrapFirstError)

        if include_none:
            defer.returnValue(valid_pdus)
        else:
            defer.returnValue([p for p in valid_pdus if p])

    def _check_sigs_and_hash(self, room_version, pdu):
        return logcontext.make_deferred_yieldable(
            self._check_sigs_and_hashes(room_version, [pdu])[0],
        )

    def _check_sigs_and_hashes(self, room_version, pdus):
        """Checks that each of the received events is correctly signed by the
        sending server.

        Args:
            room_version (str): The room version of the PDUs
            pdus (list[FrozenEvent]): the events to be checked

        Returns:
            list[Deferred]: for each input event, a deferred which:
              * returns the original event if the checks pass
              * returns a redacted version of the event (if the signature
                matched but the hash did not)
              * throws a SynapseError if the signature check failed.
            The deferreds run their callbacks in the sentinel logcontext.
        """
        deferreds = _check_sigs_on_pdus(self.keyring, room_version, pdus)

        ctx = logcontext.LoggingContext.current_context()

        def callback(_, pdu):
            with logcontext.PreserveLoggingContext(ctx):
                if not check_event_content_hash(pdu):
                    # let's try to distinguish between failures because the event was
                    # redacted (which are somewhat expected) vs actual ball-tampering
                    # incidents.
                    #
                    # This is just a heuristic, so we just assume that if the keys are
                    # about the same between the redacted and received events, then the
                    # received event was probably a redacted copy (but we then use our
                    # *actual* redacted copy to be on the safe side.)
                    redacted_event = prune_event(pdu)
                    if (
                        set(redacted_event.keys()) == set(pdu.keys()) and
                        set(six.iterkeys(redacted_event.content))
                            == set(six.iterkeys(pdu.content))
                    ):
                        logger.info(
                            "Event %s seems to have been redacted; using our redacted "
                            "copy",
                            pdu.event_id,
                        )
                    else:
                        logger.warning(
                            "Event %s content has been tampered, redacting",
                            pdu.event_id, pdu.get_pdu_json(),
                        )
                    return redacted_event

                if self.spam_checker.check_event_for_spam(pdu):
                    logger.warn(
                        "Event contains spam, redacting %s: %s",
                        pdu.event_id, pdu.get_pdu_json()
                    )
                    return prune_event(pdu)

                return pdu

        def errback(failure, pdu):
            failure.trap(SynapseError)
            with logcontext.PreserveLoggingContext(ctx):
                logger.warn(
                    "Signature check failed for %s: %s",
                    pdu.event_id, failure.getErrorMessage(),
                )
            return failure

        for deferred, pdu in zip(deferreds, pdus):
            deferred.addCallbacks(
                callback, errback,
                callbackArgs=[pdu],
                errbackArgs=[pdu],
            )

        return deferreds

class PduToCheckSig(namedtuple("PduToCheckSig", [
    "pdu", "redacted_pdu_json", "sender_domain", "deferreds",
])):
    pass

def _check_sigs_on_pdus(keyring, room_version, pdus):
    """Check that the given events are correctly signed

    Args:
        keyring (synapse.crypto.Keyring): keyring object to do the checks
        room_version (str): the room version of the PDUs
        pdus (Collection[EventBase]): the events to be checked

    Returns:
        List[Deferred]: a Deferred for each event in pdus, which will either succeed if
           the signatures are valid, or fail (with a SynapseError) if not.
    """

    # (currently this is written assuming the v1 room structure; we'll probably want a
    # separate function for checking v2 rooms)

    # we want to check that the event is signed by:
    #
    # (a) the sender's server
    #
    #     - except in the case of invites created from a 3pid invite, which are exempt
    #     from this check, because the sender has to match that of the original 3pid
    #     invite, but the event may come from a different HS, for reasons that I don't
    #     entirely grok (why do the senders have to match? and if they do, why doesn't the
    #     joining server ask the inviting server to do the switcheroo with
    #     exchange_third_party_invite?).
    #
    #     That's pretty awful, since redacting such an invite will render it invalid
    #     (because it will then look like a regular invite without a valid signature),
    #     and signatures are *supposed* to be valid whether or not an event has been
    #     redacted. But this isn't the worst of the ways that 3pid invites are broken.
    #
    # (b) for V1 and V2 rooms, the server which created the event_id
    #
    # let's start by getting the domain for each pdu, and flattening the event back
    # to JSON.

    pdus_to_check = [
        PduToCheckSig(
            pdu=p,
            redacted_pdu_json=prune_event(p).get_pdu_json(),
            sender_domain=get_domain_from_id(p.sender),
            deferreds=[],
        )
        for p in pdus
    ]

    # First we check that the sender event is signed by the sender's domain
    # (except if its a 3pid invite, in which case it may be sent by any server)
    pdus_to_check_sender = [
        p for p in pdus_to_check
        if not _is_invite_via_3pid(p.pdu)
    ]

    more_deferreds = keyring.verify_json_objects_for_server([
        (p.sender_domain, p.redacted_pdu_json)
        for p in pdus_to_check_sender
    ])

    for p, d in zip(pdus_to_check_sender, more_deferreds):
        p.deferreds.append(d)

    # now let's look for events where the sender's domain is different to the
    # event id's domain (normally only the case for joins/leaves), and add additional
    # checks. Only do this if the room version has a concept of event ID domain
    if room_version in (
        RoomVersions.V1, RoomVersions.V2, RoomVersions.STATE_V2_TEST,
    ):
        pdus_to_check_event_id = [
            p for p in pdus_to_check
            if p.sender_domain != get_domain_from_id(p.pdu.event_id)
        ]

        more_deferreds = keyring.verify_json_objects_for_server([
            (get_domain_from_id(p.pdu.event_id), p.redacted_pdu_json)
            for p in pdus_to_check_event_id
        ])

        for p, d in zip(pdus_to_check_event_id, more_deferreds):
            p.deferreds.append(d)
    elif room_version in (RoomVersions.V3,):
        pass  # No further checks needed, as event IDs are hashes here
    else:
        raise RuntimeError("Unrecognized room version %s" % (room_version,))

    # replace lists of deferreds with single Deferreds
    return [_flatten_deferred_list(p.deferreds) for p in pdus_to_check]

def _flatten_deferred_list(deferreds):
    """Given a list of deferreds, either return the single deferred,
    combine into a DeferredList, or return an already resolved deferred.
    """
    if len(deferreds) > 1:
        return DeferredList(deferreds, fireOnOneErrback=True, consumeErrors=True)
    elif len(deferreds) == 1:
        return deferreds[0]
    else:
        return defer.succeed(None)

def _is_invite_via_3pid(event):
    return (
        event.type == EventTypes.Member
        and event.membership == Membership.INVITE
        and "third_party_invite" in event.content
    )

def event_from_pdu_json(pdu_json, event_format_version, outlier=False):
    """Construct a FrozenEvent from an event json received over federation

    Args:
        pdu_json (object): pdu as received over federation
        event_format_version (int): The event format version
        outlier (bool): True to mark this event as an outlier

    Returns:
        FrozenEvent

    Raises:
        SynapseError: if the pdu is missing required fields or is otherwise
            not a valid matrix event
    """
    # we could probably enforce a bunch of other fields here (room_id, sender,
    # origin, etc etc)
    assert_params_in_dict(pdu_json, ('type', 'depth'))

    depth = pdu_json['depth']
    if not isinstance(depth, six.integer_types):
        raise SynapseError(400, "Depth %r not an intger" % (depth, ),
                           Codes.BAD_JSON)

    if depth < 0:
        raise SynapseError(400, "Depth too small", Codes.BAD_JSON)
    elif depth > MAX_DEPTH:
        raise SynapseError(400, "Depth too large", Codes.BAD_JSON)

    event = event_type_from_format_version(event_format_version)(
        pdu_json,
    )

    event.internal_metadata.outlier = outlier

    return event
<|endoftext|>


<commit_before>#!/usr/bin/env python
'''
The setup script for salt
'''

import os
import sys
import urllib2
from distutils import log
from distutils.core import setup
from distutils.cmd import Command
from distutils.command.sdist import sdist as original_sdist

setup_kwargs = {}
USE_SETUPTOOLS = False
SALTCLOUD_SOURCE_DIR = os.path.abspath(os.path.dirname(__file__))

BOOTSTRAP_SCRIPT_DISTRIBUTED_VERSION = os.environ.get(
    # The user can provide a different bootstrap-script version.
    # ATTENTION: A tag for that version MUST exist
    'BOOTSTRAP_SCRIPT_VERSION',
    # If no bootstrap-script version was provided from the environment, let's
    # provide the one we define.
    'v1.5.8'
)

if 'USE_SETUPTOOLS' in os.environ:
    try:
        from setuptools import setup
        from setuptools.command.sdist import sdist as original_sdist
        USE_SETUPTOOLS = True
        saltcloud_reqs = os.path.join(SALTCLOUD_SOURCE_DIR, 'requirements.txt')
        requirements = ''
        with open(saltcloud_reqs) as f:
            requirements = f.read()

        setup_kwargs['install_requires'] = requirements
        setup_kwargs['dependency_links'] = [
            'https://github.com/saltstack/salt-testing/tarball/develop#egg=SaltTesting'
        ]
        setup_kwargs['tests_require'] = ['SaltTesting']
    except:
        USE_SETUPTOOLS = False

if USE_SETUPTOOLS is False:
    from distutils.core import setup

exec(
    compile(
        open('saltcloud/version.py').read(), 'saltcloud/version.py', 'exec'
    )
)

class sdist(original_sdist):
    user_options = original_sdist.user_options + [
        ('skip-bootstrap-download', None,
         'Skip downloading the bootstrap-salt.sh script. This can also be '
         'triggered by having `SKIP_BOOTSTRAP_DOWNLOAD=1` as an environment '
         'variable.')
    ]
    boolean_options = original_sdist.boolean_options + [
        'skip-bootstrap-download'
    ]

    def initialize_options(self):
        original_sdist.initialize_options(self)
        self.skip_bootstrap_download = False

    def finalize_options(self):
        original_sdist.finalize_options(self)
        if 'SKIP_BOOTSTRAP_DOWNLOAD' in os.environ:
            skip_bootstrap_download = os.environ.get(
                'SKIP_BOOTSTRAP_DOWNLOAD', '0'
            )
            self.skip_bootstrap_download = skip_bootstrap_download == '1'

    def run(self):
        if self.skip_bootstrap_download is False:
            # Let's update the bootstrap-script to the version defined to be
            # distributed. See BOOTSTRAP_SCRIPT_DISTRIBUTED_VERSION above.
            url = (
                'https://github.com/saltstack/salt-bootstrap/raw/{0}'
                '/bootstrap-salt.sh'.format(
                    BOOTSTRAP_SCRIPT_DISTRIBUTED_VERSION
                )
            )
            req = urllib2.urlopen(url)
            deploy_path = os.path.join(
                SALTCLOUD_SOURCE_DIR,
                'saltcloud',
                'deploy',
                'bootstrap-salt.sh'
            )
            if req.getcode() == 200:
                try:
                    log.info(
                        'Updating bootstrap-salt.sh.'
                        '\n\tSource:      {0}'
                        '\n\tDestination: {1}'.format(
                            url,
                            deploy_path
                        )
                    )
                    with open(deploy_path, 'w') as fp_:
                        fp_.write(req.read())
                except (OSError, IOError), err:
                    log.error(
                        'Failed to write the updated script: {0}'.format(err)
                    )
            else:
                log.error(
                    'Failed to update the bootstrap-salt.sh script. HTTP '
                    'Error code: {0}'.format(
                        req.getcode()
                    )
                )

        # Let's the rest of the build command
        original_sdist.run(self)

class TestCommand(Command):
    description = 'Run tests'
    user_options = [
        ('runtests-opts=', 'R', 'Command line options to pass to runtests.py')
    ]

    def initialize_options(self):
        self.runtests_opts = None

    def finalize_options(self):
        pass

    def run(self):
        from subprocess import Popen
        self.run_command('build')
        build_cmd = self.get_finalized_command('build_ext')
        runner = os.path.abspath('tests/runtests.py')
        test_cmd = sys.executable + ' {0}'.format(runner)
        if self.runtests_opts:
            test_cmd += ' {0}'.format(self.runtests_opts)

        print('running test')
        test_process = Popen(
            test_cmd, shell=True,
            stdout=sys.stdout, stderr=sys.stderr,
            cwd=build_cmd.build_lib
        )
        test_process.communicate()
        sys.exit(test_process.returncode)

NAME = 'salt-cloud'
VER = __version__
DESC = ('Generic cloud provisioning system with build in functions ')

setup(name=NAME,
      version=VER,
      description=DESC,
      author='Thomas S Hatch',
      author_email='thatch@saltstack.com',
      url='http://saltstack.org',
      classifiers=[
          'Programming Language :: Python',
          'Programming Language :: Python :: 2.6',
          'Programming Language :: Python :: 2.7',
          'Development Status :: 3 - Alpha',
          'Environment :: Console',
          'Intended Audience :: Developers',
          'Intended Audience :: Information Technology',
          'Intended Audience :: System Administrators',
          'License :: OSI Approved :: Apache Software License',
          'Operating System :: POSIX :: Linux',
          'Topic :: System :: Distributed Computing',
      ],
      packages=['saltcloud',
                'saltcloud/utils',
                'saltcloud/clouds',
                ],
      package_data={
          'saltcloud': ['deploy/*.sh'],
      },
      data_files=[('share/man/man1', ['doc/man/salt-cloud.1']),
                  ('share/man/man7', ['doc/man/salt-cloud.7'])
                  ],
      scripts=['scripts/salt-cloud'],
      cmdclass={
          'sdist': sdist,
          'test': TestCommand
      },
      **setup_kwargs
      )
<commit_msg>One less file in salt_cloud<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    :codeauthor: Pedro Algarvio (pedro@algarvio.me)

    tests.support.mock
    ~~~~~~~~~~~~~~~~~~

    Helper module that wraps `mock` and provides some fake objects in order to
    properly set the function/class decorators and yet skip the test case's
    execution.

    Note: mock >= 2.0.0 required since unittest.mock does not have
    MagicMock.assert_called in Python < 3.6.
'''
# pylint: disable=unused-import,function-redefined,blacklisted-module,blacklisted-external-module

from __future__ import absolute_import
import errno
import fnmatch
import sys

# Import salt libs
from salt.ext import six
import salt.utils.stringutils

try:
    from mock import (
        Mock,
        MagicMock,
        patch,
        sentinel,
        DEFAULT,
        # ANY and call will be imported further down
        create_autospec,
        FILTER_DIR,
        NonCallableMock,
        NonCallableMagicMock,
        PropertyMock,
        __version__
    )
    NO_MOCK = False
    NO_MOCK_REASON = ''
    mock_version = []
    for __part in __version__.split('.'):
        try:
            mock_version.append(int(__part))
        except ValueError:
            # Non-integer value (ex. '1a')
            mock_version.append(__part)
    mock_version = tuple(mock_version)
except ImportError as exc:
    NO_MOCK = True
    NO_MOCK_REASON = 'mock python module is unavailable'
    mock_version = (0, 0, 0)

    # Let's not fail on imports by providing fake objects and classes

    class MagicMock(object):

        # __name__ can't be assigned a unicode
        __name__ = str('{0}.fakemock').format(__name__)  # future lint: disable=blacklisted-function

        def __init__(self, *args, **kwargs):
            pass

        def dict(self, *args, **kwargs):
            return self

        def multiple(self, *args, **kwargs):
            return self

        def __call__(self, *args, **kwargs):
            return self

    Mock = MagicMock
    patch = MagicMock()
    sentinel = object()
    DEFAULT = object()
    create_autospec = MagicMock()
    FILTER_DIR = True
    NonCallableMock = MagicMock()
    NonCallableMagicMock = MagicMock()
    mock_open = object()
    PropertyMock = object()
    call = tuple
    ANY = object()

if NO_MOCK is False:
    try:
        from mock import call, ANY
    except ImportError:
        NO_MOCK = True
        NO_MOCK_REASON = 'you need to upgrade your mock version to >= 0.8.0'

class MockFH(object):
    def __init__(self, filename, read_data):
        self.filename = filename
        self.empty_string = b'' if isinstance(read_data, six.binary_type) else ''
        self.read_data = self._iterate_read_data(read_data)
        self.write = Mock()
        self.writelines = Mock()

    def _iterate_read_data(self, read_data):
        '''
        Helper for mock_open:
        Retrieve lines from read_data via a generator so that separate calls to
        readline, read, and readlines are properly interleaved
        '''
        # Newline will always be a bytestring on PY2 because mock_open will have
        # normalized it to one.
        newline = b'\n' if isinstance(read_data, six.binary_type) else '\n'

        read_data = [line + newline for line in read_data.split(newline)]

        if read_data[-1] == newline:
            # If the last line ended in a newline, the list comprehension will have an
            # extra entry that's just a newline. Remove this.
            read_data = read_data[:-1]
        else:
            # If there wasn't an extra newline by itself, then the file being
            # emulated doesn't have a newline to end the last line, so remove the
            # newline that we added in the list comprehension.
            read_data[-1] = read_data[-1][:-1]

        for line in read_data:
            yield line

    def read(self, size=0):
        if not isinstance(size, six.integer_types) or size < 0:
            raise TypeError('a positive integer is required')

        joined = self.empty_string.join(self.read_data)
        if not size:
            # read() called with no args, return everything
            return joined
        else:
            # read() called with an explicit size. Return a slice matching the
            # requested size, but before doing so, reset read_data to reflect
            # what we read.
            self.read_data = self._iterate_read_data(joined[size:])
            return joined[:size]

    def readlines(self, size=None):  # pylint: disable=unused-argument
        # TODO: Implement "size" argument
        return list(self.read_data)

    def readline(self, size=None):  # pylint: disable=unused-argument
        # TODO: Implement "size" argument
        try:
            return next(self.read_data)
        except StopIteration:
            return self.empty_string

    def __iter__(self):
        while True:
            try:
                yield next(self.read_data)
            except StopIteration:
                break

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):  # pylint: disable=unused-argument
        pass

# reimplement mock_open to support multiple filehandles
def mock_open(read_data=''):
    '''
    A helper function to create a mock to replace the use of `open`. It works
    for "open" called directly or used as a context manager.

    The "mock" argument is the mock object to configure. If "None" (the
    default) then a "MagicMock" will be created for you, with the API limited
    to methods or attributes available on standard file handles.

    "read_data" is a string representing the contents of the file to be read.
    By default, this is an empty string.

    Optionally, "read_data" can be a dictionary mapping fnmatch.fnmatch()
    patterns to strings. This allows the mocked filehandle to serve content for
    more than one file path.

    .. code-block:: python

        data = {
            '/etc/foo.conf': textwrap.dedent("""\
                Foo
                Bar
                Baz
                """),
            '/etc/bar.conf': textwrap.dedent("""\
                A
                B
                C
                """),
        }
        with patch('salt.utils.files.fopen', mock_open(read_data=data):
            do stuff

    If the file path being opened does not match any of the glob expressions,
    an IOError will be raised to simulate the file not existing.

    Glob expressions will be attempted in iteration order, so if a file path
    matches more than one glob expression it will match whichever is iterated
    first. If a specifc iteration order is desired (and you are not running
    Python >= 3.6), consider passing "read_data" as an OrderedDict.

    Passing "read_data" as a string is equivalent to passing it with a glob
    expression of "*".
    '''
    # Normalize read_data, Python 2 filehandles should never produce unicode
    # types on read.
    if not isinstance(read_data, dict):
        read_data = {'*': read_data}

    if six.PY2:
        # .__class__() used here to preserve the dict class in the event that
        # an OrderedDict was used.
        read_data = read_data.__class__(
            [(x, salt.utils.stringutils.to_str(y))
             for x, y in six.iteritems(read_data)]
        )

    def _fopen_side_effect(name, *args, **kwargs):
        for pat in read_data:
            if pat == '*':
                continue
            if fnmatch.fnmatch(name, pat):
                matched_pattern = pat
                break
        else:
            # No non-glob match in read_data, fall back to '*'
            matched_pattern = '*'
        try:
            return MockFH(name, read_data[matched_pattern])
        except KeyError:
            # No matching glob in read_data, treat this as a file that does
            # not exist and raise the appropriate exception.
            raise IOError(errno.ENOENT, 'No such file or directory', name)

    return MagicMock(name='open', spec=open, side_effect=_fopen_side_effect)
<commit_msg>Add mocked close() function to MockFH<commit_after># -*- coding: utf-8 -*-
'''
    :codeauthor: Pedro Algarvio (pedro@algarvio.me)

    tests.support.mock
    ~~~~~~~~~~~~~~~~~~

    Helper module that wraps `mock` and provides some fake objects in order to
    properly set the function/class decorators and yet skip the test case's
    execution.

    Note: mock >= 2.0.0 required since unittest.mock does not have
    MagicMock.assert_called in Python < 3.6.
'''
# pylint: disable=unused-import,function-redefined,blacklisted-module,blacklisted-external-module

from __future__ import absolute_import
import errno
import fnmatch
import sys

# Import salt libs
from salt.ext import six
import salt.utils.stringutils

try:
    from mock import (
        Mock,
        MagicMock,
        patch,
        sentinel,
        DEFAULT,
        # ANY and call will be imported further down
        create_autospec,
        FILTER_DIR,
        NonCallableMock,
        NonCallableMagicMock,
        PropertyMock,
        __version__
    )
    NO_MOCK = False
    NO_MOCK_REASON = ''
    mock_version = []
    for __part in __version__.split('.'):
        try:
            mock_version.append(int(__part))
        except ValueError:
            # Non-integer value (ex. '1a')
            mock_version.append(__part)
    mock_version = tuple(mock_version)
except ImportError as exc:
    NO_MOCK = True
    NO_MOCK_REASON = 'mock python module is unavailable'
    mock_version = (0, 0, 0)

    # Let's not fail on imports by providing fake objects and classes

    class MagicMock(object):

        # __name__ can't be assigned a unicode
        __name__ = str('{0}.fakemock').format(__name__)  # future lint: disable=blacklisted-function

        def __init__(self, *args, **kwargs):
            pass

        def dict(self, *args, **kwargs):
            return self

        def multiple(self, *args, **kwargs):
            return self

        def __call__(self, *args, **kwargs):
            return self

    Mock = MagicMock
    patch = MagicMock()
    sentinel = object()
    DEFAULT = object()
    create_autospec = MagicMock()
    FILTER_DIR = True
    NonCallableMock = MagicMock()
    NonCallableMagicMock = MagicMock()
    mock_open = object()
    PropertyMock = object()
    call = tuple
    ANY = object()

if NO_MOCK is False:
    try:
        from mock import call, ANY
    except ImportError:
        NO_MOCK = True
        NO_MOCK_REASON = 'you need to upgrade your mock version to >= 0.8.0'

class MockFH(object):
    def __init__(self, filename, read_data):
        self.filename = filename
        self.empty_string = b'' if isinstance(read_data, six.binary_type) else ''
        self.read_data = self._iterate_read_data(read_data)
        self.close = Mock()
        self.write = Mock()
        self.writelines = Mock()

    def _iterate_read_data(self, read_data):
        '''
        Helper for mock_open:
        Retrieve lines from read_data via a generator so that separate calls to
        readline, read, and readlines are properly interleaved
        '''
        # Newline will always be a bytestring on PY2 because mock_open will have
        # normalized it to one.
        newline = b'\n' if isinstance(read_data, six.binary_type) else '\n'

        read_data = [line + newline for line in read_data.split(newline)]

        if read_data[-1] == newline:
            # If the last line ended in a newline, the list comprehension will have an
            # extra entry that's just a newline. Remove this.
            read_data = read_data[:-1]
        else:
            # If there wasn't an extra newline by itself, then the file being
            # emulated doesn't have a newline to end the last line, so remove the
            # newline that we added in the list comprehension.
            read_data[-1] = read_data[-1][:-1]

        for line in read_data:
            yield line

    def read(self, size=0):
        if not isinstance(size, six.integer_types) or size < 0:
            raise TypeError('a positive integer is required')

        joined = self.empty_string.join(self.read_data)
        if not size:
            # read() called with no args, return everything
            return joined
        else:
            # read() called with an explicit size. Return a slice matching the
            # requested size, but before doing so, reset read_data to reflect
            # what we read.
            self.read_data = self._iterate_read_data(joined[size:])
            return joined[:size]

    def readlines(self, size=None):  # pylint: disable=unused-argument
        # TODO: Implement "size" argument
        return list(self.read_data)

    def readline(self, size=None):  # pylint: disable=unused-argument
        # TODO: Implement "size" argument
        try:
            return next(self.read_data)
        except StopIteration:
            return self.empty_string

    def __iter__(self):
        while True:
            try:
                yield next(self.read_data)
            except StopIteration:
                break

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):  # pylint: disable=unused-argument
        pass

# reimplement mock_open to support multiple filehandles
def mock_open(read_data=''):
    '''
    A helper function to create a mock to replace the use of `open`. It works
    for "open" called directly or used as a context manager.

    The "mock" argument is the mock object to configure. If "None" (the
    default) then a "MagicMock" will be created for you, with the API limited
    to methods or attributes available on standard file handles.

    "read_data" is a string representing the contents of the file to be read.
    By default, this is an empty string.

    Optionally, "read_data" can be a dictionary mapping fnmatch.fnmatch()
    patterns to strings. This allows the mocked filehandle to serve content for
    more than one file path.

    .. code-block:: python

        data = {
            '/etc/foo.conf': textwrap.dedent("""\
                Foo
                Bar
                Baz
                """),
            '/etc/bar.conf': textwrap.dedent("""\
                A
                B
                C
                """),
        }
        with patch('salt.utils.files.fopen', mock_open(read_data=data):
            do stuff

    If the file path being opened does not match any of the glob expressions,
    an IOError will be raised to simulate the file not existing.

    Glob expressions will be attempted in iteration order, so if a file path
    matches more than one glob expression it will match whichever is iterated
    first. If a specifc iteration order is desired (and you are not running
    Python >= 3.6), consider passing "read_data" as an OrderedDict.

    Passing "read_data" as a string is equivalent to passing it with a glob
    expression of "*".
    '''
    # Normalize read_data, Python 2 filehandles should never produce unicode
    # types on read.
    if not isinstance(read_data, dict):
        read_data = {'*': read_data}

    if six.PY2:
        # .__class__() used here to preserve the dict class in the event that
        # an OrderedDict was used.
        read_data = read_data.__class__(
            [(x, salt.utils.stringutils.to_str(y))
             for x, y in six.iteritems(read_data)]
        )

    def _fopen_side_effect(name, *args, **kwargs):
        for pat in read_data:
            if pat == '*':
                continue
            if fnmatch.fnmatch(name, pat):
                matched_pattern = pat
                break
        else:
            # No non-glob match in read_data, fall back to '*'
            matched_pattern = '*'
        try:
            return MockFH(name, read_data[matched_pattern])
        except KeyError:
            # No matching glob in read_data, treat this as a file that does
            # not exist and raise the appropriate exception.
            raise IOError(errno.ENOENT, 'No such file or directory', name)

    return MagicMock(name='open', spec=open, side_effect=_fopen_side_effect)
<|endoftext|>


<commit_before>from gi.repository import Gtk

class CameraParamWindow(Gtk.Window):
    def __init__(self):
        Gtk.Window.__init__(self, title="Camera Parameters")
        self.timesettings = Gtk.Table()
        self.timesettings.set_row_spacings(5)
        self.timesettings.set_col_spacings(5)
        self.add(self.timesettings)

        self.shutterentry = Gtk.Entry()
        shutterlabel = Gtk.Label()
        shutterlabel.set_text("Shutter Speed")
        self.timesettings.attach(self.shutterentry, 1, 2, 0, 1)
        self.timesettings.attach(shutterlabel, 0, 1, 0, 1)

        self.frame_entry = Gtk.Entry()
        frame_label = Gtk.Label()
        frame_label.set_text("Length of a Frame")
        self.timesettings.attach(self.frame_entry, 1, 2, 1, 2)
        self.timesettings.attach(frame_label, 0, 1, 1, 2)

        self.total_shots_entry = Gtk.Entry()
        total_shots_label = Gtk.Label()
        total_shots_label.set_text("Total Number of Shots")
        self.timesettings.attach(self.total_shots_entry, 1, 2, 2, 3)
        self.timesettings.attach(total_shots_label, 0, 1, 2, 3)

        self.play_time_entry = Gtk.Entry()
        play_time_label = Gtk.Label()
        play_time_label.set_text("Playback Time")
        self.timesettings.attach(self.play_time_entry, 3, 4, 0, 1)
        self.timesettings.attach(play_time_label, 2, 3, 0, 1)

        self.shooting_time_entry = Gtk.Entry()
        shooting_time_label = Gtk.Label()
        shooting_time_label.set_text("Time to shoot sequence")
        self.timesettings.attach(self.shooting_time_entry, 3, 4, 1, 2)
        self.timesettings.attach(shooting_time_label, 2, 3, 1, 2)

if __name__=="__main__":
    win = CameraParamWindow()
    win.connect("delete-event", Gtk.main_quit)
    win.show_all()
    Gtk.main()
<commit_msg>Update fields when values are changed<commit_after>from gi.repository import Gtk

def _mkspin(maxval):
        adj = Gtk.Adjustment(0, 0, maxval, 1, 10, 0)
        spin = Gtk.SpinButton()
        spin.set_adjustment(adj)
        return spin

class TimeEntry(Gtk.Box):
    def __init__(self):
        Gtk.Box.__init__(self, spacing=2)
        self.hoursf = _mkspin(48)
        self.minutesf = _mkspin(59)
        self.secondsf = _mkspin(59)
        self.pack_start(self.hoursf, False, True, 0)
        self.pack_start(self.minutesf, False, True, 0)
        self.pack_start(self.secondsf, False, True, 0)

    def connect(self, event, func):
        self.hoursf.connect(event, func)
        self.minutesf.connect(event, func)
        self.secondsf.connect(event, func)

    @property
    def totalseconds(self):
        return ((self.hoursf.get_value()*60 + self.minutesf.get_value())*60 + self.secondsf.get_value())

    @totalseconds.setter
    def totalseconds(self, value):
        self.hoursf.set_value(value//60//60)
        self.minutesf.set_value((value//60)%60)
        self.secondsf.set_value((value%60)%60)

class CameraParamWindow(Gtk.Window):
    def __init__(self):
        Gtk.Window.__init__(self, title="Camera Parameters")
        self.set_border_width(10)
        self.timesettings = Gtk.ListBox()
        self.add(self.timesettings)

        self.shutterentry = Gtk.Entry() # TODO show the list from gphoto
        self._make_row(self.timesettings, "Shutter Time", self.shutterentry)

        self.frame_entry = _mkspin(90)
        self.frame_entry.connect("value-changed", self.update_times)
        self._make_row(self.timesettings, "Length of Frame", self.frame_entry)

        self.total_shots_entry = _mkspin(6000) # TODO adjust based on mem card space
        self.total_shots_entry.connect("value-changed", self.update_times)
        self._make_row(self.timesettings, "Total Shots", self.total_shots_entry)

        self.play_time_entry = TimeEntry()
        self.play_time_entry.connect("value-changed", self.update_numshots)
        self._make_row(self.timesettings, "Playback Time", self.play_time_entry)

        self.shooting_time_entry = TimeEntry()
        self._make_row(self.timesettings, "Shooting Time", self.shooting_time_entry)

    def _make_row(self, lbox, labeltxt, entry):
        label = Gtk.Label(labeltxt, xalign=0)
        hbox = Gtk.Box(spacing=6)
        row = Gtk.ListBoxRow()
        hbox.pack_start(label, True, True, 0)
        hbox.pack_start(entry, False, True, 0)
        row.add(hbox)
        self.timesettings.add(row)

    def update_times(self, sender):
        framelen = self.frame_entry.get_value()
        total_shots = self.total_shots_entry.get_value()
        self.shooting_time_entry.totalseconds = framelen*total_shots
        self.play_time_entry.totalseconds = total_shots/24 # TODO specify FPS of video

    def update_numshots(self, sender):
        playtime = self.play_time_entry.totalseconds
        self.total_shots_entry.set_value(playtime*24)
        self.update_times(self)

if __name__=="__main__":
    win = CameraParamWindow()
    win.connect("delete-event", Gtk.main_quit)
    win.show_all()
    Gtk.main()
<|endoftext|>


<commit_before>from itertools import izip
from parse import parser
from types import TranslateType, q_str
from UserDict import DictMixin

#NOTE: Dealing with the impedance mismatch between Q symbols, chars and char lists and Python strings is a major pain-point
CHAR_CODE=10

class q_list(list):
  def __init__(self,l=[],code=None):
    super(q_list,self).__init__(l)
    if 0 == len(self):
      self.type=parser.code_dict[0]
    else:
      if code:
        self.type=parser.code_dict[code]
      elif isinstance(l,q_str):
        self.type=parser.code_dict[CHAR_CODE]
      else:
        self.type=self._determine_iter_type(l)
      if self.type.code == 0:
        self = _recurse_collections(self)

  @staticmethod
  def convert_sequence(val):
    if (not (isinstance(val,q_list) or isinstance(val,q_dict) or isinstance(val,table))) and ('__iter__' in dir(val) or '__getitem__' in dir(val) or (isinstance(val,q_str) and not val.is_char)):
      return q_list(val)
    else:
      return val

  @staticmethod
  def _determine_iter_type(l,reeval=False):
    if (not reeval) and 'type' in dir(l):
      return l.type
    tc=map(lambda x:parser.type(x).list_code,l)
    for code in tc[1:]:
      if tc[0] != code:
        return parser.types['list']
    return parser.code_dict[tc[0]]

  def _determine_type(self):
      self.type=self._determine_iter_type(self,True)

  def __getslice__(self,i,j):
    return q_list(super(q_list,self).__getslice__(i,j))

  @staticmethod
  def _add_list_helper(type,y):
    if type.list_code > 0 and q_list._determine_iter_type(y).code != type.code:
      return parser.types['list']
    else:
      return type

  def __setitem__(self,i,y):
    super(q_list,self).__setitem__(i,_recurse_collections(y))
    self.type = self._add_list_helper(self.type,[y])

  def __setslice__(self,i,j,y):
    super(q_list,self).__setslice__(i,j,_recurse_collections(y))
    self.type = self._add_list_helper(self.type,y)

  def __delitem__(self,y):
    super(q_list,self).__delitem__(y)
    self._determine_type()

  def __delslice__(self,i,y):
    super(q_list,self).__delslice__(i,y)
    self._determine_type()

  def __mul__(self,n):
    return q_list(super(q_list,self).__rmul__(n),code=self.type.code)

  def __rmul__(self,n): return self.__mul__(n)

  def __add__(self,y):
    if self.type.code == self._determine_iter_type(y):
      return q_list(super(q_list,self).__add__(_recurse_collections(y)),code=self.type.code)
    else:
      return q_list(super(q_list,self).__add__(_recurse_collections(y)))

  def __radd__(self,y): return self.__add__(y)
  def __iadd__(self,y): return self.__add__(y)

  def append(self,x):
    self[len(self):len(self)] = [_recurse_collections(x)]

  def extend(self,x):
    self[len(self):len(self)] = _recurse_collections(x)

  def insert(self,i,x):
    self[i:i] = [_recurse_collections(x)]

  def pop(self,i=None):
    if i is None:
      i = len(self) - 1
    x = self[i]
    del self[i]
    return x

  def remove(self,x):
    del self[self.index(x)]

  @staticmethod
  def _write(val,message):
    return val._write_self(message)

  def _write_self(self,message):
    message.fromstring(parser.write_byte(0)+parser.write_integer(len(self)))
    for el in self:
      if self.type.code != 0:
        message.fromstring(self.type.write_data(el))
      else:
        message.fromstring(parser.write(el))
    return message

  @staticmethod
  def _read(reader,tcode,endianness,offset,bytes):
    (n, offset) = parser.read_integer(endianness,offset+1,bytes)
    val = q_list([])
    for i in range(0, n):
        (item,offset) = reader(endianness,offset,bytes)
        val.append(item)
    if tcode is CHAR_CODE:
      return (q_str(''.join(val)),offset)
    else:
      return (val,offset)

class q_dict(object,DictMixin):
  def __init__(self,arg=None,**kwargs):
    self._keys=q_list()
    self._values=q_list()
    if getattr(arg,"keys",False) and getattr(arg,"values",False):
      self._keys.extend(arg.keys())
      self._values.extend(arg.values())
    elif getattr(arg,"iteritems",False):
      for (k,v) in arg.iteritems():
        self._keys.append(k)
        self._values.append(v)
    elif getattr(arg,"items",False):
      for (k,v) in arg.items():
        self._keys.append(k)
        self._values.append(v)
    elif arg:
      for (k,v) in arg:
        self._keys.append(k)
        self._values.append(v)
    else:
      self._keys.extend(kwargs.keys())
      self._values.extend(kwargs.values())

  def keys(self):
    return list(self._keys)

  def values(self):
    return list(self._values)

  def __getitem__(self,i):
    try:
      return self._values[self._keys.index(i)]
    except ValueError:
      raise KeyError(i)

  def __setitem__(self,i,y):
    try:
      self._values[self._keys.index(i)]=y
    except ValueError:
      self._keys.append(i)
      self._values.append(y)

  def __delitem__(self,i):
    try:
      p = self._keys.index(i)
      del self.keys[p]
      del self.keys[p]
    except ValueError:
      raise KeyError(i)

  def __contains__(self,i):
    return i in self._keys

  def __iter__(self):
    return iter(self._keys)

  def iteritems(self):
    return izip(iter(self._keys),iter(self._values))

  def __repr__(self):
    return "{" + ", ".join([repr(self._keys[x])+": " + repr(self._values[x]) for x in xrange(0,len(self._keys))]) + "}"

  @staticmethod
  def _smallest_diff_key(a,b):
    return min([k for k in a if a.get(k) != b.get(k)])

  def __cmp__(self,other):
    if len(self) != len(other):
      return cmp(len(self),len(other))
    self_diff = self._smallest_diff_key(self,other)
    other_diff = self._smallest_diff_key(other,self)
    if self_diff != other_diff:
      return cmp(self_diff,other_diff)
    return cmp(self[self_diff],other[other_diff])

  @staticmethod
  def _read(endianness,offset,bytes):
    (keys,offset) = parser.read(endianness,offset,bytes)
    (values,offset) = parser.read(endianness,offset,bytes)
    dictionary=q_dict()
    dictionary._keys=keys
    dictionary._values=values
    if isinstance(keys,table) and isinstance(values,table):
      return (table(dictionary),offset)
    else:
      return (dictionary,offset)

  @staticmethod
  def _write(val,message):
    if isinstance(val,table):
      return parser.write(val._data._values,parser.write(val._data._keys,message))
    return parser.write(val.values(),parser.write(val.keys(),message))

class flip(q_dict):

  @staticmethod
  def _read(endianness,offset,bytes):
    (dictionary,offset) = parser.read(endianness,offset+1,bytes)
    return (table(flip(dictionary)),offset)

  @staticmethod
  def _write(val,message):
    if isinstance(val,table):
      val=val._data
    message.fromstring(parser.write_byte(0)+parser.write_byte(99))
    return q_dict._write(val,message)

class table(object):
  def __init__(self,val,keys=[]):
    if isinstance(val,flip):
      self._data=val
      self._type=flip
    elif (isinstance(val,q_dict) and isinstance(val._keys,flip) and isinstance(val._values,flip)):
      self._data=val
      self._type=q_dict
    elif 0 < len(keys):
      self._data = q_dict()
      self._data._keys = table(izip(keys,[val[x] for x in keys]))
      non_keys = list(set(val.keys()).difference(keys))
      self._data._values = table(izip(non_keys,[val[x] for x in non_keys]))
      self._type=q_dict
    else:
      self._data=flip(val)
      self._type=flip

  def __eq__(self,other):
    if sorted(self.cols()) != sorted(other.cols()):
      return False
    if len(self) != len(other):
      return False
    for col in self.cols:
      if self[col] != other[col]:
        return False
    return True

  def __ne__(self,other):
    return not self == other

  def __len__(self):
    return len(self[self.cols()[0]])

  def cols(self):
    if self._type==flip:
      return self._data.keys()
    else:
      return self._data._keys.cols() + self._data._values.cols()

  def keys(self):
    if self._type==flip:
      raise ValueError
    else:
      return self._data._keys

  @staticmethod
  def _validate_row(val,row):
    kfunc = getattr(row,"cols",getattr(row,"keys",False))
    if kfunc:
      return sorted(val.cols()) == sorted(kfunc())
    if len(val.cols()) > 1:
      if len(val.cols()) != len(row):
        return False
    else:
      if getattr(row,"__len__",False):
        return False
    return True

  def __getitem__(self,key):
    if self._type==flip:
      if isinstance(key,int):
        return q_dict(izip(self.cols(),[self._data[c][key] for c in self.cols()]))
      elif isinstance(key,slice):
        return table(flip(q_dict(izip(self.cols(),[self._data[c][key] for c in self.cols()]))))
      else:
        return self._data[key]
    else:
      if not self._validate_row(self.keys(),key):
        raise KeyError
      return self._data._values[self._data._keys.index(key)]

  def __setitem__(self,key,value):
    if self._type==flip:
      if isinstance(key,int) or isinstance(key,slice):
        if not self._validate_row(self,value):
          raise ValueError
        kfunc = getattr(value,"cols",getattr(value,"keys",False))
        if kfunc:
          for vk in kfunc():
            self._data[vk][key] = value[vk]
        else:
          for i, vk in enumerate(self.cols()):
            self._data[vk][key] = value[i]
      else:
        if len(value) != len(self._data[key]):
          raise ValueError
        self._data[key] = value
    else:
      if not self._validate_row(self.keys(),key):
        raise KeyError
      self._data._values[self._data._keys.index(key)] = value

  def __delitem__(self,key):
    if self._type==flip:
      if isinstance(key,int) or isinstance(key,slice):
        for c in self.cols():
          del self._data[c][key]
      else:
        del self._data[key]
    else:
      if not self._validate_row(self.keys(),key):
        raise KeyError
      del self._data._values[self._data._keys.index(key)]
      del self._data._keys[self._data._keys.index(key)]

  def index(self,val,start=0,stop=None,raise_miss=True):
    if getattr(val,"keys",False):
      cols = val.keys()
    elif not getattr(val,"__len__",False):
      cols = [self.cols()[0]]
      val = dict([(cols[0],val)])
    else:
      cols = self.cols()[0:len(val)]
      val = dict(izip(cols,val))
    for i in xrange(start,stop or len(self)):
      row = self[i]
      match = True
      for col in cols:
        if row[col] != val[col]:
          match = False
      if match:
        return i
    if raise_miss:
      raise ValueError
    else:
      return None

  def __iter__(self):
    if self._type == flip:
      for i in xrange(0,len(self)):
        yield self[i]
    else:
      for i in xrange(0,len(self._data._values)):
        yield self._data._values[i]

  def __contains__(self,item):
    if self._type == flip:
      return None != self.index(item,raise_miss=False)
    else:
      if not self._validate_row(self.keys(),item):
        raise KeyError
      return None != self._data._keys.index(item,raise_miss=False)

def _unknown_collection(x):
  return '__iter__' in dir(x) and not (isinstance(x,q_list) or isinstance(x,q_dict) or isinstance(x,table))

def _recurse_collections(c):
  if not _unknown_collection(c):
    return c
  if isinstance(c,dict):
    return q_dict(c)
  elif isinstance(c,list):
    for (i,e) in enumerate(c):
      if _unknown_collection(e):
        c[i] = _recurse_collections(e)
    return q_list(c)
  else:
    raise ValueError("Unknown collection type: " + str(type(c)))

parser.types['dict'] = TranslateType(q_dict,99,overwrite_write=q_dict._write,overwrite_read=q_dict._read)
parser.types['flip'] = TranslateType(flip,98,overwrite_write=flip._write,overwrite_read=flip._read)
parser.types['list'] = TranslateType(q_list,0,overwrite_write=q_list._write,overwrite_read=q_list._read)
parser.types['table'] = TranslateType(table)
<commit_msg>Changed sequence conversion logic<commit_after>from itertools import izip
from parse import parser
from types import TranslateType, q_str
from UserDict import DictMixin

#NOTE: Dealing with the impedance mismatch between Q symbols, chars and char lists and Python strings is a major pain-point
CHAR_CODE=10

class q_list(list):
  def __init__(self,l=[],code=None):
    super(q_list,self).__init__(l)
    if 0 == len(self):
      self.type=parser.code_dict[0]
    else:
      if code:
        self.type=parser.code_dict[code]
      elif isinstance(l,q_str):
        self.type=parser.code_dict[CHAR_CODE]
      else:
        self.type=self._determine_iter_type(l)
      if self.type.code == 0:
        self = _recurse_collections(self)

  @staticmethod
  def convert_sequence(val):
    if isinstance(val,q_str) and not val.is_char:
      return q_list(val)
    elif (not (isinstance(val,str) or isinstance(val,q_list) or isinstance(val,q_dict) or isinstance(val,table))) and ('__iter__' in dir(val) or '__getitem__' in dir(val)):
      return q_list(val)
    else:
      return val

  @staticmethod
  def _determine_iter_type(l,reeval=False):
    if (not reeval) and 'type' in dir(l):
      return l.type
    tc=map(lambda x:parser.type(x).list_code,l)
    for code in tc[1:]:
      if tc[0] != code:
        return parser.types['list']
    return parser.code_dict[tc[0]]

  def _determine_type(self):
      self.type=self._determine_iter_type(self,True)

  def __getslice__(self,i,j):
    return q_list(super(q_list,self).__getslice__(i,j))

  @staticmethod
  def _add_list_helper(type,y):
    if type.list_code > 0 and q_list._determine_iter_type(y).code != type.code:
      return parser.types['list']
    else:
      return type

  def __setitem__(self,i,y):
    super(q_list,self).__setitem__(i,_recurse_collections(y))
    self.type = self._add_list_helper(self.type,[y])

  def __setslice__(self,i,j,y):
    super(q_list,self).__setslice__(i,j,_recurse_collections(y))
    self.type = self._add_list_helper(self.type,y)

  def __delitem__(self,y):
    super(q_list,self).__delitem__(y)
    self._determine_type()

  def __delslice__(self,i,y):
    super(q_list,self).__delslice__(i,y)
    self._determine_type()

  def __mul__(self,n):
    return q_list(super(q_list,self).__rmul__(n),code=self.type.code)

  def __rmul__(self,n): return self.__mul__(n)

  def __add__(self,y):
    if self.type.code == self._determine_iter_type(y):
      return q_list(super(q_list,self).__add__(_recurse_collections(y)),code=self.type.code)
    else:
      return q_list(super(q_list,self).__add__(_recurse_collections(y)))

  def __radd__(self,y): return self.__add__(y)
  def __iadd__(self,y): return self.__add__(y)

  def append(self,x):
    self[len(self):len(self)] = [_recurse_collections(x)]

  def extend(self,x):
    self[len(self):len(self)] = _recurse_collections(x)

  def insert(self,i,x):
    self[i:i] = [_recurse_collections(x)]

  def pop(self,i=None):
    if i is None:
      i = len(self) - 1
    x = self[i]
    del self[i]
    return x

  def remove(self,x):
    del self[self.index(x)]

  @staticmethod
  def _write(val,message):
    return val._write_self(message)

  def _write_self(self,message):
    message.fromstring(parser.write_byte(0)+parser.write_integer(len(self)))
    for el in self:
      if self.type.code != 0:
        message.fromstring(self.type.write_data(el))
      else:
        message.fromstring(parser.write(el))
    return message

  @staticmethod
  def _read(reader,tcode,endianness,offset,bytes):
    (n, offset) = parser.read_integer(endianness,offset+1,bytes)
    val = q_list([])
    for i in range(0, n):
        (item,offset) = reader(endianness,offset,bytes)
        val.append(item)
    if tcode is CHAR_CODE:
      return (q_str(''.join(val)),offset)
    else:
      return (val,offset)

class q_dict(object,DictMixin):
  def __init__(self,arg=None,**kwargs):
    self._keys=q_list()
    self._values=q_list()
    if getattr(arg,"keys",False) and getattr(arg,"values",False):
      self._keys.extend(arg.keys())
      self._values.extend(arg.values())
    elif getattr(arg,"iteritems",False):
      for (k,v) in arg.iteritems():
        self._keys.append(k)
        self._values.append(v)
    elif getattr(arg,"items",False):
      for (k,v) in arg.items():
        self._keys.append(k)
        self._values.append(v)
    elif arg:
      for (k,v) in arg:
        self._keys.append(k)
        self._values.append(v)
    else:
      self._keys.extend(kwargs.keys())
      self._values.extend(kwargs.values())

  def keys(self):
    return list(self._keys)

  def values(self):
    return list(self._values)

  def __getitem__(self,i):
    try:
      return self._values[self._keys.index(i)]
    except ValueError:
      raise KeyError(i)

  def __setitem__(self,i,y):
    try:
      self._values[self._keys.index(i)]=y
    except ValueError:
      self._keys.append(i)
      self._values.append(y)

  def __delitem__(self,i):
    try:
      p = self._keys.index(i)
      del self.keys[p]
      del self.keys[p]
    except ValueError:
      raise KeyError(i)

  def __contains__(self,i):
    return i in self._keys

  def __iter__(self):
    return iter(self._keys)

  def iteritems(self):
    return izip(iter(self._keys),iter(self._values))

  def __repr__(self):
    return "{" + ", ".join([repr(self._keys[x])+": " + repr(self._values[x]) for x in xrange(0,len(self._keys))]) + "}"

  @staticmethod
  def _smallest_diff_key(a,b):
    return min([k for k in a if a.get(k) != b.get(k)])

  def __cmp__(self,other):
    if len(self) != len(other):
      return cmp(len(self),len(other))
    self_diff = self._smallest_diff_key(self,other)
    other_diff = self._smallest_diff_key(other,self)
    if self_diff != other_diff:
      return cmp(self_diff,other_diff)
    return cmp(self[self_diff],other[other_diff])

  @staticmethod
  def _read(endianness,offset,bytes):
    (keys,offset) = parser.read(endianness,offset,bytes)
    (values,offset) = parser.read(endianness,offset,bytes)
    dictionary=q_dict()
    dictionary._keys=keys
    dictionary._values=values
    if isinstance(keys,table) and isinstance(values,table):
      return (table(dictionary),offset)
    else:
      return (dictionary,offset)

  @staticmethod
  def _write(val,message):
    if isinstance(val,table):
      return parser.write(val._data._values,parser.write(val._data._keys,message))
    return parser.write(val.values(),parser.write(val.keys(),message))

class flip(q_dict):

  @staticmethod
  def _read(endianness,offset,bytes):
    (dictionary,offset) = parser.read(endianness,offset+1,bytes)
    return (table(flip(dictionary)),offset)

  @staticmethod
  def _write(val,message):
    if isinstance(val,table):
      val=val._data
    message.fromstring(parser.write_byte(0)+parser.write_byte(99))
    return q_dict._write(val,message)

class table(object):
  def __init__(self,val,keys=[]):
    if isinstance(val,flip):
      self._data=val
      self._type=flip
    elif (isinstance(val,q_dict) and isinstance(val._keys,flip) and isinstance(val._values,flip)):
      self._data=val
      self._type=q_dict
    elif 0 < len(keys):
      self._data = q_dict()
      self._data._keys = table(izip(keys,[val[x] for x in keys]))
      non_keys = list(set(val.keys()).difference(keys))
      self._data._values = table(izip(non_keys,[val[x] for x in non_keys]))
      self._type=q_dict
    else:
      self._data=flip(val)
      self._type=flip

  def __eq__(self,other):
    if sorted(self.cols()) != sorted(other.cols()):
      return False
    if len(self) != len(other):
      return False
    for col in self.cols:
      if self[col] != other[col]:
        return False
    return True

  def __ne__(self,other):
    return not self == other

  def __len__(self):
    return len(self[self.cols()[0]])

  def cols(self):
    if self._type==flip:
      return self._data.keys()
    else:
      return self._data._keys.cols() + self._data._values.cols()

  def keys(self):
    if self._type==flip:
      raise ValueError
    else:
      return self._data._keys

  @staticmethod
  def _validate_row(val,row):
    kfunc = getattr(row,"cols",getattr(row,"keys",False))
    if kfunc:
      return sorted(val.cols()) == sorted(kfunc())
    if len(val.cols()) > 1:
      if len(val.cols()) != len(row):
        return False
    else:
      if getattr(row,"__len__",False):
        return False
    return True

  def __getitem__(self,key):
    if self._type==flip:
      if isinstance(key,int):
        return q_dict(izip(self.cols(),[self._data[c][key] for c in self.cols()]))
      elif isinstance(key,slice):
        return table(flip(q_dict(izip(self.cols(),[self._data[c][key] for c in self.cols()]))))
      else:
        return self._data[key]
    else:
      if not self._validate_row(self.keys(),key):
        raise KeyError
      return self._data._values[self._data._keys.index(key)]

  def __setitem__(self,key,value):
    if self._type==flip:
      if isinstance(key,int) or isinstance(key,slice):
        if not self._validate_row(self,value):
          raise ValueError
        kfunc = getattr(value,"cols",getattr(value,"keys",False))
        if kfunc:
          for vk in kfunc():
            self._data[vk][key] = value[vk]
        else:
          for i, vk in enumerate(self.cols()):
            self._data[vk][key] = value[i]
      else:
        if len(value) != len(self._data[key]):
          raise ValueError
        self._data[key] = value
    else:
      if not self._validate_row(self.keys(),key):
        raise KeyError
      self._data._values[self._data._keys.index(key)] = value

  def __delitem__(self,key):
    if self._type==flip:
      if isinstance(key,int) or isinstance(key,slice):
        for c in self.cols():
          del self._data[c][key]
      else:
        del self._data[key]
    else:
      if not self._validate_row(self.keys(),key):
        raise KeyError
      del self._data._values[self._data._keys.index(key)]
      del self._data._keys[self._data._keys.index(key)]

  def index(self,val,start=0,stop=None,raise_miss=True):
    if getattr(val,"keys",False):
      cols = val.keys()
    elif not getattr(val,"__len__",False):
      cols = [self.cols()[0]]
      val = dict([(cols[0],val)])
    else:
      cols = self.cols()[0:len(val)]
      val = dict(izip(cols,val))
    for i in xrange(start,stop or len(self)):
      row = self[i]
      match = True
      for col in cols:
        if row[col] != val[col]:
          match = False
      if match:
        return i
    if raise_miss:
      raise ValueError
    else:
      return None

  def __iter__(self):
    if self._type == flip:
      for i in xrange(0,len(self)):
        yield self[i]
    else:
      for i in xrange(0,len(self._data._values)):
        yield self._data._values[i]

  def __contains__(self,item):
    if self._type == flip:
      return None != self.index(item,raise_miss=False)
    else:
      if not self._validate_row(self.keys(),item):
        raise KeyError
      return None != self._data._keys.index(item,raise_miss=False)

def _unknown_collection(x):
  return '__iter__' in dir(x) and not (isinstance(x,q_list) or isinstance(x,q_dict) or isinstance(x,table))

def _recurse_collections(c):
  if not _unknown_collection(c):
    return c
  if isinstance(c,dict):
    return q_dict(c)
  elif isinstance(c,list):
    for (i,e) in enumerate(c):
      if _unknown_collection(e):
        c[i] = _recurse_collections(e)
    return q_list(c)
  else:
    raise ValueError("Unknown collection type: " + str(type(c)))

parser.types['dict'] = TranslateType(q_dict,99,overwrite_write=q_dict._write,overwrite_read=q_dict._read)
parser.types['flip'] = TranslateType(flip,98,overwrite_write=flip._write,overwrite_read=flip._read)
parser.types['list'] = TranslateType(q_list,0,overwrite_write=q_list._write,overwrite_read=q_list._read)
parser.types['table'] = TranslateType(table)
<|endoftext|>