<commit_before>
# -*- coding: utf-8 -*-

import requests
import click
import udatetime
import time
import logging
from datetime import datetime, date, time
import dotmap

"""Main module."""

BASE_URL = 'https://reisapi.ruter.no'
GET_PLACES_URL = BASE_URL + '/Place/GetPlaces/'
GET_TRAVELS_URL = BASE_URL + '/Travel/GetTravels'

TRANSPORTATIONS = { 0: 'Walk',
                    2: 'Bus',
                    5: 'Boat',
                    6: 'Train',
                    7: 'Tram',
                    8: 'Subway'}

def place_id(place):

    request = requests.get(GET_PLACES_URL, params={'id': place})
    request = request.json()
    id = request[0].get('ID')

    return id

def trip_proposals(from_id, to_id, after_time=None):

    if not after_time:
        _after_time = datetime.now()
        # 01102017190000
        after_time = _after_time.strftime('%d%m%Y%H%M%S')

    payload = {'fromPlace': from_id,
               'toPlace': to_id,
               'time': after_time,
               'isafter': 'True',
               'proposals': '5'
              }

    request = requests.get(GET_TRAVELS_URL, params=payload)

    return request.json().get('TravelProposals')

def pretty_time(da_time=''):
    # 2017-10-07T16:41:00+02:00
    time = udatetime.from_string(da_time)
    return datetime.strftime(time, '%H:%M')

def trip(from_place, to_place):

    from_id = place_id(from_place)

    to_id = place_id(to_place)

    proposals = trip_proposals(from_id, to_id)

    for position, proposal in enumerate(proposals, start=1):

        if proposal.get('Remarks'):
            print('Remarks! Check app or ruter.no.')

        print('Forslag #{0} (Travel time: {1})\n'.format(position, proposal.get('TotalTravelTime')))

        for pos, stage in enumerate(proposal.get('Stages'), start=1):

            stage = dotmap.DotMap(stage)

            if stage.Transportation == 0:
                text = '[{0}] Walk ({1})'.format(pos, stage.WalkingTime)
            else:
                step_params = dict(pos=pos,
                                   departure_name=stage.DepartureStop.Name,
                                   departure_time=pretty_time(stage.DepartureTime),
                                   line_name=stage.LineName,
                                   arrival_name=stage.ArrivalStop.Name,
                                   arrival_time=pretty_time(stage.ArrivalTime),
                                   transportation=TRANSPORTATIONS[stage.Transportation],
                                   )

                text = '[{pos}] From {departure_name} at {departure_time} [{line_name}] -> {arrival_name} at {arrival_time} [{transportation}]'.format(**step_params)

            click.echo(text)

        click.echo()
<commit_msg>Add crayons and prettier output<commit_after>
# -*- coding: utf-8 -*-

import requests
import click
import udatetime
import time
import logging
from datetime import datetime, date, time
import dotmap
import crayons

"""Main module."""

BASE_URL = 'https://reisapi.ruter.no'
GET_PLACES_URL = BASE_URL + '/Place/GetPlaces/'
GET_TRAVELS_URL = BASE_URL + '/Travel/GetTravels'

TRANSPORTATIONS = { 0: 'Walk',
                    2: 'Bus',
                    5: 'Boat',
                    6: 'Train',
                    7: 'Tram',
                    8: 'Subway'}

def place_id(place):

    request = requests.get(GET_PLACES_URL, params={'id': place})
    request = request.json()
    id = request[0].get('ID')

    return id

def trip_proposals(from_id, to_id, after_time=None):

    if not after_time:
        _after_time = datetime.now()
        # 01102017190000
        after_time = _after_time.strftime('%d%m%Y%H%M%S')

    payload = {'fromPlace': from_id,
               'toPlace': to_id,
               'time': after_time,
               'isafter': 'True',
               'proposals': '5'
              }

    request = requests.get(GET_TRAVELS_URL, params=payload)

    return request.json().get('TravelProposals')

def pretty_time(da_time=''):
    # 2017-10-07T16:41:00+02:00
    time = udatetime.from_string(da_time)
    return datetime.strftime(time, '%H:%M')

def trip(from_place, to_place):

    from_id = place_id(from_place)

    to_id = place_id(to_place)

    proposals = trip_proposals(from_id, to_id)

    for position, proposal in enumerate(proposals, start=1):

        click.echo(crayons.yellow('------------ Suggestion #{0} ({1})-------------\n'.format(position, proposal.get('TotalTravelTime'), bold=True)))

        if proposal.get('Remarks'):
            print('Remarks! Check app or ruter.no.')

        for pos, stage in enumerate(proposal.get('Stages'), start=1):

            stage = dotmap.DotMap(stage)

            if stage.Transportation == 0:
                text = '[{0}] Walk ({1})'.format(pos, stage.WalkingTime)
            else:
                step_params = dict(pos=crayons.cyan(pos, bold=True),
                                   departure_name=stage.DepartureStop.Name,
                                   departure_time=pretty_time(stage.DepartureTime),
                                   line_name=stage.LineName,
                                   arrival_name=stage.ArrivalStop.Name,
                                   arrival_time=pretty_time(stage.ArrivalTime),
                                   transportation=TRANSPORTATIONS[stage.Transportation],
                                   )

                text = '[{pos}] From {departure_name} at {departure_time} [{line_name}] -> {arrival_name} at {arrival_time} [{transportation}]'.format(**step_params)

            click.echo(text)

        click.echo()
<|endoftext|>


<commit_before>from django.core.urlresolvers import reverse
from django.http import Http404
from calaccess_processed.models import ProcessedDataFile
from calaccess_website.views import CalAccessModelListMixin
from calaccess_website.templatetags.calaccess_website_tags import slugify
from bakery.views import BuildableDetailView, BuildableListView

def get_ccdc_model_list():
    """
    Return a list of models classes for the data files published
    """
    file_list = [
        'Division',
        'Membership',
        'Organization',
        'OrganizationIdentifier',
        'OrganizationName',
        'Person',
        'PersonIdentifier',
        'PersonName',
        'Post',
        'BallotMeasureContest',
        'BallotMeasureContestIdentifier',
        'BallotMeasureContestOption',
        'BallotMeasureContestSource',
        'Candidacy',
        'CandidacySource',
        'CandidateContest',
        'CandidateContestPost',
        'CandidateContestSource',
        'Election',
        'ElectionIdentifier',
        'ElectionSource',
        'RetentionContest',
        'RetentionContestIdentifier',
        'RetentionContestOption',
        'RetentionContestSource',
    ]

    model_list = [
        ProcessedDataFile(file_name=f).model for f in file_list
    ]
    return sorted(model_list, key=lambda m: m().object_name)

class CcdcFileList(BuildableListView, CalAccessModelListMixin):
    template_name = 'calaccess_website/ccdc_file_list.html'
    build_path = "documentation/calaccess-files/index.html"

    def get_queryset(self):
        """
        Returns the CCDC model list with grouped by type.
        """
        return self.regroup_by_klass_group(get_ccdc_model_list())

    def get_context_data(self, **kwargs):
        context = super(CcdcFileList, self).get_context_data(**kwargs)
        context['file_num'] = len(get_ccdc_model_list())
        return context

class BaseFileDetailView(BuildableDetailView):
    """
    Base class for views providing information about a CCDC data file.
    """
    def get_queryset(self):
        """
        Returns a list of the ccdc data files as a key dictionary
        with the URL slug as the keys.
        """
        return dict(
            (slugify(str(m().object_name)), m) for m in get_ccdc_model_list()
        )

    def set_kwargs(self, obj):
        self.kwargs = {
            'slug': obj
        }

    def get_object(self):
        """
        Returns the file model from the CAL-ACCESS processed data app that
        matches the provided slug.

        Raises a 404 error if one is not found
        """
        key = self.kwargs['slug']
        try:
            return self.get_queryset()[key.lower()]
        except KeyError:
            raise Http404

    def get_context_data(self, **kwargs):
        """
        Add some extra bits to the template's context
        """
        file_name = self.kwargs['slug'].replace("-", "")
        context = super(BaseFileDetailView, self).get_context_data(**kwargs)
        # Pull all previous versions of the provided file
        context['version_list'] = ProcessedDataFile.objects.filter(
            file_name__icontains=file_name
        ).order_by(
            '-version__raw_version__release_datetime'
        ).exclude(
            version__raw_version__release_datetime__lte='2016-07-27'
        )
        # note if the most recent version of the file is empty
        try:
            context['empty'] = context['version_list'][0].records_count == 0
        except IndexError:
            context['empty'] = True
        return context

    def build_queryset(self):
        [self.build_object(o) for o in self.get_queryset()]

class CcdcFileDownloadsList(BaseFileDetailView):
    """
    A detail page with links to all downloads for the provided CCDC data file.
    """
    template_name = 'calaccess_website/ccdc_file_downloads_list.html'

    def get_url(self, obj):
        return reverse('ccdc_file_downloads_list', kwargs=dict(slug=obj))

class CcdcFileDetail(BaseFileDetailView):
    """
    A detail page with all documentation for the provided CCDC data file.
    """
    template_name = 'calaccess_website/ccdc_file_detail.html'

    def get_url(self, obj):
        return reverse('ccdc_file_detail', kwargs=dict(slug=obj))

    def get_context_data(self, **kwargs):
        """
        Add some extra bits to the template's context
        """
        context = super(CcdcFileDetail, self).get_context_data(**kwargs)
        # Add list of fields to context
        context['fields'] = self.get_sorted_fields()

        return context

    def get_sorted_fields(self):
        """
        Return a list of fields (dicts) sorted by name.
        """
        field_list = []

        for field in self.object().get_field_list():
            field_data = {
                'column': field.column,
                'description': field.description % field.__dict__,
                'help_text': field.help_text,
            }
            if len(field.choices) > 0:
                field_data['choices'] = [c for c in field.choices]
            else:
                field_data['choices'] = None

            field_list.append(field_data)

        return sorted(field_list, key=lambda k: k['column'])
<commit_msg>flake8<commit_after>from django.core.urlresolvers import reverse
from django.http import Http404
from calaccess_processed.models import ProcessedDataFile
from calaccess_website.views import CalAccessModelListMixin
from calaccess_website.templatetags.calaccess_website_tags import slugify
from bakery.views import BuildableDetailView, BuildableListView

def get_ccdc_model_list():
    """
    Return a list of models classes for the data files published
    """
    file_list = [
        'Division',
        'Membership',
        'Organization',
        'OrganizationIdentifier',
        'OrganizationName',
        'Person',
        'PersonIdentifier',
        'PersonName',
        'Post',
        'BallotMeasureContest',
        'BallotMeasureContestIdentifier',
        'BallotMeasureContestOption',
        'BallotMeasureContestSource',
        'Candidacy',
        'CandidacySource',
        'CandidateContest',
        'CandidateContestPost',
        'CandidateContestSource',
        'Election',
        'ElectionIdentifier',
        'ElectionSource',
        'RetentionContest',
        'RetentionContestIdentifier',
        'RetentionContestOption',
        'RetentionContestSource',
    ]

    model_list = [
        ProcessedDataFile(file_name=f).model for f in file_list
    ]
    return sorted(model_list, key=lambda m: m().object_name)

class CcdcFileList(BuildableListView, CalAccessModelListMixin):
    template_name = 'calaccess_website/ccdc_file_list.html'
    build_path = "documentation/calaccess-files/index.html"

    def get_queryset(self):
        """
        Returns the CCDC model list with grouped by type.
        """
        return self.regroup_by_klass_group(get_ccdc_model_list())

    def get_context_data(self, **kwargs):
        context = super(CcdcFileList, self).get_context_data(**kwargs)
        context['file_num'] = len(get_ccdc_model_list())
        return context

class BaseFileDetailView(BuildableDetailView):
    """
    Base class for views providing information about a CCDC data file.
    """
    def get_queryset(self):
        """
        Returns a list of the ccdc data files as a key dictionary
        with the URL slug as the keys.
        """
        return dict(
            (slugify(str(m().object_name)), m) for m in get_ccdc_model_list()
        )

    def set_kwargs(self, obj):
        self.kwargs = {
            'slug': obj
        }

    def get_object(self):
        """
        Returns the file model from the CAL-ACCESS processed data app that
        matches the provided slug.

        Raises a 404 error if one is not found
        """
        key = self.kwargs['slug']
        try:
            return self.get_queryset()[key.lower()]
        except KeyError:
            raise Http404

    def get_context_data(self, **kwargs):
        """
        Add some extra bits to the template's context
        """
        file_name = self.kwargs['slug'].replace("-", "")
        context = super(BaseFileDetailView, self).get_context_data(**kwargs)
        # Pull all previous versions of the provided file
        context['version_list'] = ProcessedDataFile.objects.filter(
            file_name__icontains=file_name
        ).order_by(
            '-version__raw_version__release_datetime'
        ).exclude(
            version__raw_version__release_datetime__lte='2016-07-27'
        )
        # note if the most recent version of the file is empty
        try:
            context['empty'] = context['version_list'][0].records_count == 0
        except IndexError:
            context['empty'] = True
        return context

    def build_queryset(self):
        [self.build_object(o) for o in self.get_queryset()]

class CcdcFileDownloadsList(BaseFileDetailView):
    """
    A detail page with links to all downloads for the provided CCDC data file.
    """
    template_name = 'calaccess_website/ccdc_file_downloads_list.html'

    def get_url(self, obj):
        return reverse('ccdc_file_downloads_list', kwargs=dict(slug=obj))

class CcdcFileDetail(BaseFileDetailView):
    """
    A detail page with all documentation for the provided CCDC data file.
    """
    template_name = 'calaccess_website/ccdc_file_detail.html'

    def get_url(self, obj):
        return reverse('ccdc_file_detail', kwargs=dict(slug=obj))

    def get_context_data(self, **kwargs):
        """
        Add some extra bits to the template's context
        """
        context = super(CcdcFileDetail, self).get_context_data(**kwargs)
        # Add list of fields to context
        context['fields'] = self.get_sorted_fields()

        return context

    def get_sorted_fields(self):
        """
        Return a list of fields (dicts) sorted by name.
        """
        field_list = []

        for field in self.object().get_field_list():
            field_data = {
                'column': field.column,
                'description': field.description % field.__dict__,
                'help_text': field.help_text,
            }
            if len(field.choices) > 0:
                field_data['choices'] = [c for c in field.choices]
            else:
                field_data['choices'] = None

            field_list.append(field_data)

        return sorted(field_list, key=lambda k: k['column'])
<|endoftext|>


<commit_before>########
# Copyright (c) 2013 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

__author__ = 'idanmo'

import logging
from functools import wraps
from manager import get_node_state
from manager import update_node_state

CLOUDIFY_ID_PROPERTY = '__cloudify_id'
CLOUDIFY_NODE_STATE_PROPERTY = 'node_state'

# overridden in celery.py
def operation(method):
    return method

def _inject_argument(arg_name, arg_value, args, kwargs=None):
    """Inject argument to kwargs.
    This is currently done by simply putting the key and value in kwargs
    since Celery's task decorator maps **kwargs to relevant arguments
    and when tasks are executed from workflow all arguments are passed
    in **kwargs.
    Args:
        arg_name: The argument name to inject.
        arg_value: The argument value to inject.
        args: Invocation arguments.
        kwargs: Invocation kwargs (optional).

    Returns:
        An (*args, **kwargs) tuple to be used for invoking method.
    """
    kwargs[arg_name] = arg_value
    return args, kwargs

def with_node_state(func=None, **arguments):
    """Injects node state for the node in context.
    Args:
        arg: argument name to inject the node state as where 'node_state'
            is the default.
    """
    if func is not None:
        @wraps(func)
        def wrapper(*args, **kwargs):
            node_id = _get_node_id_from_args(func, args, kwargs)
            if node_id is None:
                raise RuntimeError(
                    'Node id property [{0}] not present in '
                    'method invocation arguments'.format(CLOUDIFY_ID_PROPERTY))
            node_state = get_node_state(node_id)
            node_state_arg = arguments['arg'] if 'arg' in arguments\
                else 'node_state'
            args, kwargs = _inject_argument(node_state_arg, node_state,
                                            args, kwargs)
            result = func(*args, **kwargs)
            update_node_state(node_state)
            return result
        return wrapper
    else:
        def partial_wrapper(fn):
            return with_node_state(fn, **arguments)
        return partial_wrapper

def with_logger(func=None, **arguments):
    """Injects a Cloudify operation logger.
    Args:
        arg: argument name to inject the logger as where 'logger' is the
            default.
    """
    if func is not None:
        @wraps(func)
        def wrapper(*args, **kwargs):
            logger_arg = arguments['arg'] if 'arg' in arguments \
                else 'logger'
            logger = logging.getLogger('cloudify')
            args, kwargs = _inject_argument(logger_arg, logger,
                                            args, kwargs)
            result = func(*args, **kwargs)
            return result
        return wrapper
    else:
        def partial_wrapper(fn):
            return with_logger(fn, **arguments)
        return partial_wrapper

def _get_node_id_from_args(method, args, kwargs):
    """Gets node id for method invocation.
    Args:
        method: Invoked method.
        args: Invocation *args.
        kwargs: Invocation **kwargs.
    Returns:
        Extracted node id or None if not found.
    """
    if CLOUDIFY_ID_PROPERTY in kwargs:
        return kwargs[CLOUDIFY_ID_PROPERTY]
    try:
        arg_names = method.func_code.co_varnames
        arg_index = None
        for i in range(len(arg_names)):
            if arg_names[i].endswith(CLOUDIFY_ID_PROPERTY):
                arg_index = i
                break
        return args[arg_index]
    except ValueError:
        pass
    return None
<commit_msg>remove unused argument from inject_argument method.<commit_after>########
# Copyright (c) 2013 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

__author__ = 'idanmo'

import logging
from functools import wraps
from manager import get_node_state
from manager import update_node_state

CLOUDIFY_ID_PROPERTY = '__cloudify_id'
CLOUDIFY_NODE_STATE_PROPERTY = 'node_state'

# overridden in celery.py
def operation(method):
    return method

def _inject_argument(arg_name, arg_value, kwargs=None):
    """Inject argument to kwargs.
    This is currently done by simply putting the key and value in kwargs
    since Celery's task decorator maps **kwargs to relevant arguments
    and when tasks are executed from workflow all arguments are passed
    in **kwargs.
    Args:
        arg_name: The argument name to inject.
        arg_value: The argument value to inject.
        args: Invocation arguments.
        kwargs: Invocation kwargs (optional).

    Returns:
        An (*args, **kwargs) tuple to be used for invoking method.
    """
    kwargs[arg_name] = arg_value
    return kwargs

def with_node_state(func=None, **arguments):
    """Injects node state for the node in context.
    Args:
        arg: argument name to inject the node state as where 'node_state'
            is the default.
    """
    if func is not None:
        @wraps(func)
        def wrapper(*args, **kwargs):
            node_id = _get_node_id_from_args(func, args, kwargs)
            if node_id is None:
                raise RuntimeError(
                    'Node id property [{0}] not present in '
                    'method invocation arguments'.format(CLOUDIFY_ID_PROPERTY))
            node_state = get_node_state(node_id)
            node_state_arg = arguments['arg'] if 'arg' in arguments\
                else 'node_state'
            kwargs = _inject_argument(node_state_arg, node_state, kwargs)
            result = func(*args, **kwargs)
            update_node_state(node_state)
            return result
        return wrapper
    else:
        def partial_wrapper(fn):
            return with_node_state(fn, **arguments)
        return partial_wrapper

def with_logger(func=None, **arguments):
    """Injects a Cloudify operation logger.
    Args:
        arg: argument name to inject the logger as where 'logger' is the
            default.
    """
    if func is not None:
        @wraps(func)
        def wrapper(*args, **kwargs):
            logger_arg = arguments['arg'] if 'arg' in arguments \
                else 'logger'
            logger = logging.getLogger('cloudify')
            kwargs = _inject_argument(logger_arg, logger, kwargs)
            result = func(*args, **kwargs)
            return result
        return wrapper
    else:
        def partial_wrapper(fn):
            return with_logger(fn, **arguments)
        return partial_wrapper

def _get_node_id_from_args(method, args, kwargs):
    """Gets node id for method invocation.
    Args:
        method: Invoked method.
        args: Invocation *args.
        kwargs: Invocation **kwargs.
    Returns:
        Extracted node id or None if not found.
    """
    if CLOUDIFY_ID_PROPERTY in kwargs:
        return kwargs[CLOUDIFY_ID_PROPERTY]
    try:
        arg_names = method.func_code.co_varnames
        arg_index = None
        for i in range(len(arg_names)):
            if arg_names[i].endswith(CLOUDIFY_ID_PROPERTY):
                arg_index = i
                break
        return args[arg_index]
    except ValueError:
        pass
    return None
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models
from django.db.models import Count, Subquery, OuterRef
from django.urls import reverse
from address.models import AddressField
from enumfields import EnumIntegerField, Enum
from taggit.managers import TaggableManager
import inspect

class Person(models.Model):
    name = models.CharField(max_length=200)
    email = models.CharField(max_length=200)
    address = AddressField(blank=True)
    created = models.DateTimeField(auto_now_add=True)

    tags = TaggableManager()

    def __unicode__(self):
        ret = self.name.strip()
        if len(ret) == 0:
            return self.email
        return ret
<commit_msg>crm: models: create a blank Address object if one is not supplied<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models
from django.db.models import Count, Subquery, OuterRef
from django.urls import reverse
from address.models import AddressField, Address
from enumfields import EnumIntegerField, Enum
from taggit.managers import TaggableManager
import inspect

class Person(models.Model):
    name = models.CharField(max_length=200)
    email = models.CharField(max_length=200)
    address = AddressField(blank=True)
    created = models.DateTimeField(auto_now_add=True)

    tags = TaggableManager()

    def save(self, *args, **kwargs):
        if not self.address_id:
            self.address, _ = Address.objects.get_or_create(raw='')
        super(Person, self).save(*args, **kwargs)

    def __unicode__(self):
        ret = self.name.strip()
        if len(ret) == 0:
            return self.email
        return ret
<|endoftext|>


<commit_before>"""Tests for the toolchain sub-system"""
import sys
import os
from string import printable
from copy import deepcopy
from hypothesis import given
from hypothesis.strategies import text, lists, fixed_dictionaries

ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..",
                                    ".."))
sys.path.insert(0, ROOT)

from tools.toolchains import TOOLCHAIN_CLASSES, LEGACY_TOOLCHAIN_NAMES
from tools.targets import TARGET_MAP

def test_instantiation():
    """Test that all exported toolchain may be instantiated"""
    for name, tc_class in  TOOLCHAIN_CLASSES.items():
        cls = tc_class(TARGET_MAP["K64F"])
        assert name == cls.name or\
            name == LEGACY_TOOLCHAIN_NAMES[cls.name]

ALPHABET = [char for char in printable if char not in [u'.', u'/']]

@given(fixed_dictionaries({
    'common': lists(text()),
    'c': lists(text()),
    'cxx': lists(text()),
    'asm': lists(text()),
    'ld': lists(text())}),
       lists(text(min_size=1, alphabet=ALPHABET), min_size=1))
def test_toolchain_profile_c(profile, source_file):
    """Test that the appropriate profile parameters are passed to the
    C compiler"""
    filename = deepcopy(source_file)
    filename[-1] += ".c"
    to_compile = os.path.join(*filename)
    for _, tc_class in TOOLCHAIN_CLASSES.items():
        toolchain = tc_class(TARGET_MAP["K64F"], build_profile=profile)
        toolchain.inc_md5 = ""
        toolchain.build_dir = ""
        compile_command = toolchain.compile_command(to_compile,
                                                    to_compile + ".o", [])
        for parameter in profile['c'] + profile['common']:
            assert any(parameter in cmd for cmd in compile_command), \
                "Toolchain %s did not propigate arg %s" % (toolchain.name,
                                                           parameter)

@given(fixed_dictionaries({
    'common': lists(text()),
    'c': lists(text()),
    'cxx': lists(text()),
    'asm': lists(text()),
    'ld': lists(text())}),
       lists(text(min_size=1, alphabet=ALPHABET), min_size=1))
def test_toolchain_profile_cpp(profile, source_file):
    """Test that the appropriate profile parameters are passed to the
    C++ compiler"""
    filename = deepcopy(source_file)
    filename[-1] += ".cpp"
    to_compile = os.path.join(*filename)
    for _, tc_class in TOOLCHAIN_CLASSES.items():
        toolchain = tc_class(TARGET_MAP["K64F"], build_profile=profile)
        toolchain.inc_md5 = ""
        toolchain.build_dir = ""
        compile_command = toolchain.compile_command(to_compile,
                                                    to_compile + ".o", [])
        for parameter in profile['cxx'] + profile['common']:
            assert any(parameter in cmd for cmd in compile_command), \
                "Toolchain %s did not propigate arg %s" % (toolchain.name,
                                                           parameter)

@given(fixed_dictionaries({
    'common': lists(text()),
    'c': lists(text()),
    'cxx': lists(text()),
    'asm': lists(text()),
    'ld': lists(text())}),
       lists(text(min_size=1, alphabet=ALPHABET), min_size=1))
def test_toolchain_profile_asm(profile, source_file):
    """Test that the appropriate profile parameters are passed to the
    Assembler"""
    filename = deepcopy(source_file)
    filename[-1] += ".s"
    to_compile = os.path.join(*filename)
    for _, tc_class in TOOLCHAIN_CLASSES.items():
        toolchain = tc_class(TARGET_MAP["K64F"], build_profile=profile)
        toolchain.inc_md5 = ""
        toolchain.build_dir = ""
        compile_command = toolchain.compile_command(to_compile,
                                                    to_compile + ".o", [])
        if not compile_command:
            assert compile_command, to_compile
        for parameter in profile['asm']:
            assert any(parameter in cmd for cmd in compile_command), \
                "Toolchain %s did not propigate arg %s" % (toolchain.name,
                                                           parameter)

<commit_msg>Add a test of find duplicates<commit_after>"""Tests for the toolchain sub-system"""
import sys
import os
from string import printable
from copy import deepcopy
from mock import MagicMock
from hypothesis import given
from hypothesis.strategies import text, lists, fixed_dictionaries

ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..",
                                    ".."))
sys.path.insert(0, ROOT)

from tools.toolchains import TOOLCHAIN_CLASSES, LEGACY_TOOLCHAIN_NAMES,\
    Resources
from tools.targets import TARGET_MAP

def test_instantiation():
    """Test that all exported toolchain may be instantiated"""
    for name, tc_class in  TOOLCHAIN_CLASSES.items():
        cls = tc_class(TARGET_MAP["K64F"])
        assert name == cls.name or\
            name == LEGACY_TOOLCHAIN_NAMES[cls.name]

ALPHABET = [char for char in printable if char not in [u'.', u'/']]

@given(fixed_dictionaries({
    'common': lists(text()),
    'c': lists(text()),
    'cxx': lists(text()),
    'asm': lists(text()),
    'ld': lists(text())}),
       lists(text(min_size=1, alphabet=ALPHABET), min_size=1))
def test_toolchain_profile_c(profile, source_file):
    """Test that the appropriate profile parameters are passed to the
    C compiler"""
    filename = deepcopy(source_file)
    filename[-1] += ".c"
    to_compile = os.path.join(*filename)
    for _, tc_class in TOOLCHAIN_CLASSES.items():
        toolchain = tc_class(TARGET_MAP["K64F"], build_profile=profile)
        toolchain.inc_md5 = ""
        toolchain.build_dir = ""
        compile_command = toolchain.compile_command(to_compile,
                                                    to_compile + ".o", [])
        for parameter in profile['c'] + profile['common']:
            assert any(parameter in cmd for cmd in compile_command), \
                "Toolchain %s did not propigate arg %s" % (toolchain.name,
                                                           parameter)

@given(fixed_dictionaries({
    'common': lists(text()),
    'c': lists(text()),
    'cxx': lists(text()),
    'asm': lists(text()),
    'ld': lists(text())}),
       lists(text(min_size=1, alphabet=ALPHABET), min_size=1))
def test_toolchain_profile_cpp(profile, source_file):
    """Test that the appropriate profile parameters are passed to the
    C++ compiler"""
    filename = deepcopy(source_file)
    filename[-1] += ".cpp"
    to_compile = os.path.join(*filename)
    for _, tc_class in TOOLCHAIN_CLASSES.items():
        toolchain = tc_class(TARGET_MAP["K64F"], build_profile=profile)
        toolchain.inc_md5 = ""
        toolchain.build_dir = ""
        compile_command = toolchain.compile_command(to_compile,
                                                    to_compile + ".o", [])
        for parameter in profile['cxx'] + profile['common']:
            assert any(parameter in cmd for cmd in compile_command), \
                "Toolchain %s did not propigate arg %s" % (toolchain.name,
                                                           parameter)

@given(fixed_dictionaries({
    'common': lists(text()),
    'c': lists(text()),
    'cxx': lists(text()),
    'asm': lists(text()),
    'ld': lists(text())}),
       lists(text(min_size=1, alphabet=ALPHABET), min_size=1))
def test_toolchain_profile_asm(profile, source_file):
    """Test that the appropriate profile parameters are passed to the
    Assembler"""
    filename = deepcopy(source_file)
    filename[-1] += ".s"
    to_compile = os.path.join(*filename)
    for _, tc_class in TOOLCHAIN_CLASSES.items():
        toolchain = tc_class(TARGET_MAP["K64F"], build_profile=profile)
        toolchain.inc_md5 = ""
        toolchain.build_dir = ""
        compile_command = toolchain.compile_command(to_compile,
                                                    to_compile + ".o", [])
        if not compile_command:
            assert compile_command, to_compile
        for parameter in profile['asm']:
            assert any(parameter in cmd for cmd in compile_command), \
                "Toolchain %s did not propigate arg %s" % (toolchain.name,
                                                           parameter)

    for name, Class in  TOOLCHAIN_CLASSES.items():
        CLS = Class(TARGET_MAP["K64F"])
        assert name == CLS.name or name ==  LEGACY_TOOLCHAIN_NAMES[CLS.name]

@given(lists(text(alphabet=ALPHABET, min_size=1), min_size=1))
def test_detect_duplicates(filenames):
    c_sources = [os.path.join(name, "dupe.c") for name in filenames]
    s_sources = [os.path.join(name, "dupe.s") for name in filenames]
    cpp_sources = [os.path.join(name, "dupe.cpp") for name in filenames]
    with MagicMock() as notify:
        toolchain = TOOLCHAIN_CLASSES["ARM"](TARGET_MAP["K64F"], notify=notify)
        res = Resources()
        res.c_sources = c_sources
        res.s_sources = s_sources
        res.cpp_sources = cpp_sources
        assert res.detect_duplicates(toolchain) == 1,\
            "Not Enough duplicates found"

        _, (notification, _), _ = notify.mock_calls[1]
        assert "dupe.o" in notification["message"]
        assert "dupe.s" in notification["message"]
        assert "dupe.c" in notification["message"]
        assert "dupe.cpp" in notification["message"]
<|endoftext|>


<commit_before># Authors: CommPy contributors
# License: BSD 3-Clause

"""
============================================
Wifi 802.11 simulation (:mod:`commpy.wifi80211`)
============================================

.. autosummary::
   :toctree: generated/

   Wifi80211    -- Class to simulate the transmissions and receiving parameters of physical layer 802.11
"""

import math
from typing import List

import numpy as np

import commpy.channelcoding.convcode as cc
import commpy.links as lk
import commpy.modulation as mod
from commpy.channels import _FlatChannel

# =============================================================================
# Convolutional Code
# =============================================================================

class Wifi80211:
    """
    This class aims to simulate the transmissions and receiving parameters of physical layer 802.11 (currently till VHT (ac))

    First the chunk is coded according to the generator matrix from the standard, having a rate of 1/2.
    Then, depending on the Modulation Coding Scheme (MCS) used, puncturing is applied to achieve other coding rates.
    For more details of which MCS map to which modulation and each coding the standard is *the* recommended place,
    but for a lighter and faster source to check  https://mcsindex.com is a good place.
    Finally the bits are then mapped to the modulation scheme in conformity to the MCS (BPSK, QPSK, 16-QAM, 64-QAM, 256-QAM).

    On the receiving the inverse operations are perform, with depuncture when MCS needs it.

    """
    # Build memory and generator matrix
    # Number of delay elements in the convolutional encoder
    # "The encoder uses a 6-stage shift register."
    # (https://pdfs.semanticscholar.org/c63b/71e43dc23b17ca57267f3b769224c64d5e33.pdf p.19)
    memory = np.array(6, ndmin=1)
    generator_matrix = np.array((133, 171), ndmin=2)  # from 802.11 standard, page 2295

    def get_modem(self) -> mod.Modem:
        """
        Gets the modem that is going to be used for this particular WiFi simulation according to the MCS
        """
        qpsks = [
            2,
            4,
            4,
            16,
            64,
            64,
            64,
            256,
            256
        ]
        if self.mcs == 0:
            # BPSK
            return mod.PSKModem(2)
        else:
            # Modem : QPSK
            return mod.QAMModem(qpsks[self.mcs])

    @staticmethod
    def _get_puncture_matrix(numerator: int, denominator: int) -> List:
        if numerator == 1 and denominator == 2:
            return None
        # from the standard 802.11 2016
        if numerator == 2 and denominator == 3:
            # page 2297
            return [1, 1, 1, 0]
        if numerator == 3 and denominator == 4:
            # page 2297
            return [1, 1, 1, 0, 0, 1]
        if numerator == 5 and denominator == 6:
            # page 2378
            return [1, 1, 1, 0, 0, 1, 1, 0, 0, 1]
        return None

    def _get_coding(self):
        coding = [
            (1, 2),
            (1, 2),
            (3, 4),
            (1, 2),
            (3, 4),
            (2, 3),
            (3, 4),
            (5, 6),
            (3, 4),
            (5, 6),
        ]
        return coding[self.mcs]

    @staticmethod
    def _get_trellis():
        return cc.Trellis(Wifi80211.memory, Wifi80211.generator_matrix)

    def __init__(self, mcs: int):
        """
        Build WiFi 802.11 simulation class
        Parameters
        ----------
        mcs : int
              The Modulation Coding Scheme (MCS) to simulate.
              A list of MCS and which coding and modulations they correspond to
        """
        self.mcs = mcs
        self.modem = None

    def link_performance(self, channel: _FlatChannel, SNRs, tx_max, err_min, send_chunk=None,
                         frame_aggregation=1, receiver=None, stop_on_surpass_error=True):
        """
        Estimate the BER performance of a link model with Monte Carlo simulation as in commpy.links.link_performance

        Parameters
        ----------
        channel : _FlatChannel
                  The channel to be used for the simulation

        SNRs : 1D arraylike
               Signal to Noise ratio in dB defined as :math:`SNR_{dB} = (E_b/N_0)_{dB} + 10 \log_{10}(R_cM_c)`
               where :math:`Rc` is the code rate and :math:`Mc` the modulation rate.

        tx_max : int
                 Maximum number of transmissions for each SNR.

        err_min : int
                  link_performance send bits until it reach err_min errors (see also send_max).

        send_chunk : int
                     Number of bits to be send at each frame. This is also the frame length of the decoder if available
                     so it should be large enough regarding the code type.
                     *Default*: send_chunck = err_min

        frame_aggregation : int
                            Number of frames aggregated per transmission (each frame with size send_chunk)

        receiver  : function
                    Specify a custom receiver function to be used in the simulation.
                    This is particular useful for MIMO simulations.

        stop_on_surpass_error : bool
                                Controls if during simulation of a SNR it should break and move to the next SNR when
                                the bit error is above the err_min parameter

        Returns
        -------
        BERs : 1d ndarray
               Estimated Bit Error Ratio corresponding to each SNRs
        """

        trellis1 = Wifi80211._get_trellis()
        coding = self._get_coding()
        modem = self.get_modem()

        def modulate(bits):
            res = cc.conv_encode(bits, trellis1, 'cont')
            puncture_matrix = Wifi80211._get_puncture_matrix(coding[0], coding[1])
            res_p = res
            if puncture_matrix:
                res_p = cc.puncturing(res, puncture_matrix)

            return modem.modulate(res_p)

        # Receiver function (no process required as there are no fading)
        def _receiver(y, h, constellation, noise_var):
            return modem.demodulate(y, 'soft', noise_var)

        if not receiver:
            receiver = _receiver

        # Decoder function
        def decoder_soft(msg):
            msg_d = msg
            puncture_matrix = Wifi80211._get_puncture_matrix(coding[0], coding[1])
            if puncture_matrix:
                try:
                    msg_d = cc.depuncturing(msg, puncture_matrix, math.ceil(len(msg) * coding[0] / coding[1] * 2))
                except IndexError as e:
                    print(e)
                    print("Decoded message size %d" % (math.ceil(len(msg) * coding[0] / coding[1] * 2)))
                    print("Encoded message size %d" % len(msg))
                    print("Coding %d/%d" % (coding[0], coding[1]))
            return cc.viterbi_decode(msg_d, trellis1, decoding_type='soft')

        self.model = lk.LinkModel(modulate, channel, receiver,
                                  modem.num_bits_symbol, modem.constellation, modem.Es,
                                  decoder_soft, coding[0] / coding[1])
        return self.model.link_performance_full_metrics(SNRs, tx_max,
                                                        err_min=err_min, send_chunk=send_chunk,
                                                        code_rate=coding[0] / coding[1],
                                                        number_chunks_per_send=frame_aggregation,
                                                        stop_on_surpass_error=stop_on_surpass_error
                                                        )
<commit_msg>Add modulation and corresponding coding list to documentation<commit_after># Authors: CommPy contributors
# License: BSD 3-Clause

"""
============================================
Wifi 802.11 simulation (:mod:`commpy.wifi80211`)
============================================

.. autosummary::
   :toctree: generated/

   Wifi80211    -- Class to simulate the transmissions and receiving parameters of physical layer 802.11
"""

import math
from typing import List

import numpy as np

import commpy.channelcoding.convcode as cc
import commpy.links as lk
import commpy.modulation as mod
from commpy.channels import _FlatChannel

# =============================================================================
# Convolutional Code
# =============================================================================

class Wifi80211:
    """
    This class aims to simulate the transmissions and receiving parameters of physical layer 802.11 (currently till VHT (ac))

    First the chunk is coded according to the generator matrix from the standard, having a rate of 1/2.
    Then, depending on the Modulation Coding Scheme (MCS) used, puncturing is applied to achieve other coding rates.
    For more details of which MCS map to which modulation and each coding the standard is *the* recommended place,
    but for a lighter and faster source to check  https://mcsindex.com is a good place.
    Finally the bits are then mapped to the modulation scheme in conformity to the MCS (BPSK, QPSK, 16-QAM, 64-QAM, 256-QAM).

    On the receiving the inverse operations are perform, with depuncture when MCS needs it.

    """
    # Build memory and generator matrix
    # Number of delay elements in the convolutional encoder
    # "The encoder uses a 6-stage shift register."
    # (https://pdfs.semanticscholar.org/c63b/71e43dc23b17ca57267f3b769224c64d5e33.pdf p.19)
    memory = np.array(6, ndmin=1)
    generator_matrix = np.array((133, 171), ndmin=2)  # from 802.11 standard, page 2295

    def get_modem(self) -> mod.Modem:
        """
        Gets the modem that is going to be used for this particular WiFi simulation according to the MCS
        """
        qpsks = [
            2,
            4,
            4,
            16,
            64,
            64,
            64,
            256,
            256
        ]
        if self.mcs == 0:
            # BPSK
            return mod.PSKModem(2)
        else:
            # Modem : QPSK
            return mod.QAMModem(qpsks[self.mcs])

    @staticmethod
    def _get_puncture_matrix(numerator: int, denominator: int) -> List:
        if numerator == 1 and denominator == 2:
            return None
        # from the standard 802.11 2016
        if numerator == 2 and denominator == 3:
            # page 2297
            return [1, 1, 1, 0]
        if numerator == 3 and denominator == 4:
            # page 2297
            return [1, 1, 1, 0, 0, 1]
        if numerator == 5 and denominator == 6:
            # page 2378
            return [1, 1, 1, 0, 0, 1, 1, 0, 0, 1]
        return None

    def _get_coding(self):
        coding = [
            (1, 2),
            (1, 2),
            (3, 4),
            (1, 2),
            (3, 4),
            (2, 3),
            (3, 4),
            (5, 6),
            (3, 4),
            (5, 6),
        ]
        return coding[self.mcs]

    @staticmethod
    def _get_trellis():
        return cc.Trellis(Wifi80211.memory, Wifi80211.generator_matrix)

    def __init__(self, mcs: int):
        """
        Build WiFi 802.11 simulation class
        Parameters
        ----------
        mcs : int
              The Modulation Coding Scheme (MCS) to simulate.
              A list of MCS and which coding and modulations they correspond to is bellow:
                 - 0 : BPSK 1/2
                 - 1 : QPSK 1/2
                 - 2 : QPSK 3/4
                 - 3 : 16-QAM 1/2
                 - 4 : 16-QAM 3/4
                 - 5 : 64-QAM 2/3
                 - 6 : 64-QAM 3/4
                 - 7 : 64-QAM 5/6
                 - 8 : 256-QAM 3/4
                 - 9 : 256-QAM 5/6
        """
        self.mcs = mcs
        self.modem = None

    def link_performance(self, channel: _FlatChannel, SNRs, tx_max, err_min, send_chunk=None,
                         frame_aggregation=1, receiver=None, stop_on_surpass_error=True):
        """
        Estimate the BER performance of a link model with Monte Carlo simulation as in commpy.links.link_performance

        Parameters
        ----------
        channel : _FlatChannel
                  The channel to be used for the simulation

        SNRs : 1D arraylike
               Signal to Noise ratio in dB defined as :math:`SNR_{dB} = (E_b/N_0)_{dB} + 10 \log_{10}(R_cM_c)`
               where :math:`Rc` is the code rate and :math:`Mc` the modulation rate.

        tx_max : int
                 Maximum number of transmissions for each SNR.

        err_min : int
                  link_performance send bits until it reach err_min errors (see also send_max).

        send_chunk : int
                     Number of bits to be send at each frame. This is also the frame length of the decoder if available
                     so it should be large enough regarding the code type.
                     *Default*: send_chunck = err_min

        frame_aggregation : int
                            Number of frames aggregated per transmission (each frame with size send_chunk)

        receiver  : function
                    Specify a custom receiver function to be used in the simulation.
                    This is particular useful for MIMO simulations.

        stop_on_surpass_error : bool
                                Controls if during simulation of a SNR it should break and move to the next SNR when
                                the bit error is above the err_min parameter

        Returns
        -------
        BERs : 1d ndarray
               Estimated Bit Error Ratio corresponding to each SNRs
        """

        trellis1 = Wifi80211._get_trellis()
        coding = self._get_coding()
        modem = self.get_modem()

        def modulate(bits):
            res = cc.conv_encode(bits, trellis1, 'cont')
            puncture_matrix = Wifi80211._get_puncture_matrix(coding[0], coding[1])
            res_p = res
            if puncture_matrix:
                res_p = cc.puncturing(res, puncture_matrix)

            return modem.modulate(res_p)

        # Receiver function (no process required as there are no fading)
        def _receiver(y, h, constellation, noise_var):
            return modem.demodulate(y, 'soft', noise_var)

        if not receiver:
            receiver = _receiver

        # Decoder function
        def decoder_soft(msg):
            msg_d = msg
            puncture_matrix = Wifi80211._get_puncture_matrix(coding[0], coding[1])
            if puncture_matrix:
                try:
                    msg_d = cc.depuncturing(msg, puncture_matrix, math.ceil(len(msg) * coding[0] / coding[1] * 2))
                except IndexError as e:
                    print(e)
                    print("Decoded message size %d" % (math.ceil(len(msg) * coding[0] / coding[1] * 2)))
                    print("Encoded message size %d" % len(msg))
                    print("Coding %d/%d" % (coding[0], coding[1]))
            return cc.viterbi_decode(msg_d, trellis1, decoding_type='soft')

        self.model = lk.LinkModel(modulate, channel, receiver,
                                  modem.num_bits_symbol, modem.constellation, modem.Es,
                                  decoder_soft, coding[0] / coding[1])
        return self.model.link_performance_full_metrics(SNRs, tx_max,
                                                        err_min=err_min, send_chunk=send_chunk,
                                                        code_rate=coding[0] / coding[1],
                                                        number_chunks_per_send=frame_aggregation,
                                                        stop_on_surpass_error=stop_on_surpass_error
                                                        )
<|endoftext|>


<commit_before>from flask import current_app

import json
import requests
from requests_toolbelt import sessions
from . import CRMIntegration
import phonenumbers

class RogueIntegration(CRMIntegration):

    def __init__(self, domain, api_key):
        super(RogueIntegration, self).__init__()
        if api_key:
            self.rogue_session = sessions.BaseUrlSession(
                base_url='https://'+domain)
            self.rogue_session.headers={'X-DS-Importer-API-Key': api_key}
        else:
            raise Exception('unable to authenticate to Rogue')

    def get_user(self, phone_number):
        """Not required for Rogue, we have a one-shot API with phone number"""
        # this is basically a no-op, but wraps the data in the desired format
        return {
            'id': phone_number,
            'phone': phone_number
        }

    def save_action(self, call, crm_campaign_id, crm_user):
        """Given a call and crm_campaign_id
        Save the call attributes (target, duration and status) to the CRM
        Returns a boolean status"""

        # create the call action
        call_action = {
            'mobile': crm_user['phone'], # this is not strictly a verified mobile number
            'callpower_campaign_id': crm_campaign_id,
            'status': call.status,
            'call_timestamp': call.timestamp.isoformat(),
            'call_duration': call.duration,
            'campaign_target_name': call.target.name,
            'campaign_target_title': call.target.title,
            'campaign_target_district': call.target.district,
            'callpower_campaign_name': call.campaign.name,
            'number_dialed_into': call.session.from_number,
        }
        current_app.logger.info("creating action: %s" % json.dumps(call_action))
        rogue_response = self.rogue_session.post('/api/v1/callpower/call', json=call_action)
        current_app.logger.info("rogue response (%s): %s" % (rogue_response.status_code, rogue_response.text))
        return rogue_response.ok

    def save_campaign_meta(self, crm_campaign_id, meta={}):
        """Given a page name (crm_campaign_id)
        Save meta values to pagefields
        Returns a boolean status"""
        raise NotImplementedError()
<commit_msg>Rogue accept json header<commit_after>from flask import current_app

import json
import requests
from requests_toolbelt import sessions
from . import CRMIntegration
import phonenumbers

class RogueIntegration(CRMIntegration):

    def __init__(self, domain, api_key):
        super(RogueIntegration, self).__init__()
        if api_key:
            self.rogue_session = sessions.BaseUrlSession(
                base_url='https://'+domain)
            self.rogue_session.headers={
                'X-DS-Importer-API-Key': api_key,
                'Accept': 'application/json'
            }
        else:
            raise Exception('unable to authenticate to Rogue')

    def get_user(self, phone_number):
        """Not required for Rogue, we have a one-shot API with phone number"""
        # this is basically a no-op, but wraps the data in the desired format
        return {
            'id': phone_number,
            'phone': phone_number
        }

    def save_action(self, call, crm_campaign_id, crm_user):
        """Given a call and crm_campaign_id
        Save the call attributes (target, duration and status) to the CRM
        Returns a boolean status"""

        # create the call action
        call_action = {
            'mobile': crm_user['phone'], # this is not strictly a verified mobile number
            'callpower_campaign_id': crm_campaign_id,
            'status': call.status,
            'call_timestamp': call.timestamp.isoformat(),
            'call_duration': call.duration,
            'campaign_target_name': call.target.name,
            'campaign_target_title': call.target.title,
            'campaign_target_district': call.target.district,
            'callpower_campaign_name': call.campaign.name,
            'number_dialed_into': call.session.from_number,
        }
        current_app.logger.info("creating action: %s" % json.dumps(call_action))
        rogue_response = self.rogue_session.post('/api/v1/callpower/call', json=call_action)
        current_app.logger.info("rogue response (%s): %s" % (rogue_response.status_code, rogue_response.text))
        return rogue_response.ok

    def save_campaign_meta(self, crm_campaign_id, meta={}):
        """Given a page name (crm_campaign_id)
        Save meta values to pagefields
        Returns a boolean status"""
        raise NotImplementedError()
<|endoftext|>


<commit_before>import sys
import base64
import binascii

from .. import DeenPlugin

__all__ = ['DeenPluginBase64',
           'DeenPluginBase64Url',
           'DeenPluginBase32',
           'DeenPluginBase85']

class DeenPluginBase64(DeenPlugin):
    name = 'base64'
    display_name = 'Base64'
    aliases = ['b64']
    cmd_name = 'base64'
    cmd_help = 'Base64 encode/decode data'

    def __init__(self):
        super(DeenPluginBase64, self).__init__()

    def process(self, data):
        super(DeenPluginBase64, self).process(data)
        try:
            data = base64.b64encode(data)
        except Exception as e:
            self.error = e
        return data

    def unprocess(self, data):
        super(DeenPluginBase64, self).unprocess(data)
        # Remove new lines and carriage returns from
        # Base64 encoded data.
        data = data.replace(b'\n', b'').replace(b'\r', b'')
        # If the padding character is missing, restore it.
        padding = len(data) % 4
        if padding != 0:
            data += b'=' * (4 - padding)
        try:
            data = base64.b64decode(data)
        except binascii.Error as e:
            self.error = e
        return data

class DeenPluginBase64Url(DeenPlugin):
    name = 'base64url'
    display_name = 'Base64 URL'
    aliases = ['b64url']
    cmd_name = 'base64url'
    cmd_help='Base64 URL encode/decode data'

    def __init__(self):
        super(DeenPluginBase64Url, self).__init__()

    def process(self, data):
        super(DeenPluginBase64Url, self).process(data)
        try:
            data = base64.urlsafe_b64encode(data)
        except Exception as e:
            self.error = e
        return data

    def unprocess(self, data):
        super(DeenPluginBase64Url, self).unprocess(data)
        # Remove new lines and carriage returns from
        # Base64 encoded data.
        data = data.replace(b'\n', b'').replace(b'\r', b'')
        try:
            data = base64.urlsafe_b64decode(data)
        except binascii.Error as e:
            self.error = e
        return data

class DeenPluginBase32(DeenPlugin):
    name = 'base32'
    display_name = 'Base32'
    aliases = ['b32']
    cmd_name = 'base32'
    cmd_help='Base32 encode/decode data'

    def __init__(self):
        super(DeenPluginBase32, self).__init__()

    def process(self, data):
        super(DeenPluginBase32, self).process(data)
        try:
            data = base64.b32encode(data)
        except Exception as e:
            self.error = e
        return data

    def unprocess(self, data):
        super(DeenPluginBase32, self).unprocess(data)
        # Remove new lines and carriage returns from
        # Base32 encoded data.
        data = data.replace(b'\n', b'').replace(b'\r', b'')
        try:
            data = base64.b32decode(data)
        except binascii.Error as e:
            self.error = e
        return data

class DeenPluginBase85(DeenPlugin):
    name = 'base85'
    display_name = 'Base85'
    aliases = ['b85']
    cmd_name = 'base85'
    cmd_help='Base85 encode/decode data'

    def __init__(self):
        super(DeenPluginBase85, self).__init__()

    @staticmethod
    def prerequisites():
        if sys.version_info.major < 3 or \
            (sys.version_info.major == 3 and
                sys.version_info.minor < 4):
            return False
        else:
            return True

    def process(self, data):
        super(DeenPluginBase85, self).process(data)
        try:
            data = base64.b85encode(data)
        except Exception as e:
            self.error = e
        return data

    def unprocess(self, data):
        super(DeenPluginBase85, self).unprocess(data)
        # Remove new lines and carriage returns from
        # Base85 encoded data.
        data = data.replace(b'\n', b'').replace(b'\r', b'')
        try:
            data = base64.b85decode(data)
        except (binascii.Error, ValueError) as e:
            self.error = e
        return data<commit_msg>Fixed missing padding in DeenPluginBase64.unprocess()<commit_after>import sys
import base64
import binascii

from .. import DeenPlugin

__all__ = ['DeenPluginBase64',
           'DeenPluginBase64Url',
           'DeenPluginBase32',
           'DeenPluginBase85']

class DeenPluginBase64(DeenPlugin):
    name = 'base64'
    display_name = 'Base64'
    aliases = ['b64']
    cmd_name = 'base64'
    cmd_help = 'Base64 encode/decode data'

    def __init__(self):
        super(DeenPluginBase64, self).__init__()

    def process(self, data):
        super(DeenPluginBase64, self).process(data)
        try:
            data = base64.b64encode(data)
        except Exception as e:
            self.error = e
        return data

    def unprocess(self, data):
        super(DeenPluginBase64, self).unprocess(data)
        # Remove new lines and carriage returns from
        # Base64 encoded data.
        data = data.replace(b'\n', b'').replace(b'\r', b'')
        # If the padding character is missing, restore it.
        padding = len(data) % 3
        if padding != 0:
            data += b'=' * (3 - padding)
        try:
            data = base64.b64decode(data)
        except binascii.Error as e:
            self.error = e
        return data

class DeenPluginBase64Url(DeenPlugin):
    name = 'base64url'
    display_name = 'Base64 URL'
    aliases = ['b64url']
    cmd_name = 'base64url'
    cmd_help='Base64 URL encode/decode data'

    def __init__(self):
        super(DeenPluginBase64Url, self).__init__()

    def process(self, data):
        super(DeenPluginBase64Url, self).process(data)
        try:
            data = base64.urlsafe_b64encode(data)
        except Exception as e:
            self.error = e
        return data

    def unprocess(self, data):
        super(DeenPluginBase64Url, self).unprocess(data)
        # Remove new lines and carriage returns from
        # Base64 encoded data.
        data = data.replace(b'\n', b'').replace(b'\r', b'')
        try:
            data = base64.urlsafe_b64decode(data)
        except binascii.Error as e:
            self.error = e
        return data

class DeenPluginBase32(DeenPlugin):
    name = 'base32'
    display_name = 'Base32'
    aliases = ['b32']
    cmd_name = 'base32'
    cmd_help='Base32 encode/decode data'

    def __init__(self):
        super(DeenPluginBase32, self).__init__()

    def process(self, data):
        super(DeenPluginBase32, self).process(data)
        try:
            data = base64.b32encode(data)
        except Exception as e:
            self.error = e
        return data

    def unprocess(self, data):
        super(DeenPluginBase32, self).unprocess(data)
        # Remove new lines and carriage returns from
        # Base32 encoded data.
        data = data.replace(b'\n', b'').replace(b'\r', b'')
        try:
            data = base64.b32decode(data)
        except binascii.Error as e:
            self.error = e
        return data

class DeenPluginBase85(DeenPlugin):
    name = 'base85'
    display_name = 'Base85'
    aliases = ['b85']
    cmd_name = 'base85'
    cmd_help='Base85 encode/decode data'

    def __init__(self):
        super(DeenPluginBase85, self).__init__()

    @staticmethod
    def prerequisites():
        if sys.version_info.major < 3 or \
            (sys.version_info.major == 3 and
                sys.version_info.minor < 4):
            return False
        else:
            return True

    def process(self, data):
        super(DeenPluginBase85, self).process(data)
        try:
            data = base64.b85encode(data)
        except Exception as e:
            self.error = e
        return data

    def unprocess(self, data):
        super(DeenPluginBase85, self).unprocess(data)
        # Remove new lines and carriage returns from
        # Base85 encoded data.
        data = data.replace(b'\n', b'').replace(b'\r', b'')
        try:
            data = base64.b85decode(data)
        except (binascii.Error, ValueError) as e:
            self.error = e
        return data<|endoftext|>


<commit_before>import os
import errno
import subprocess as sub
import datetime

class DotDict(object):
    """
    Immutable dict-like objects accessible by dot notation

    Used because the amount of configuration access is very high and just using
    dots instead of the dict notation feels good.

    It should be noted that this class could definitely have a more elegant
    writing, possibly inheriting from dict(). However, this is probably good
    enough and will probably not cause any trouble.

    """

    def __init__(self, data):
        # TODO: Make this more elegant
        if isinstance(data, DotDict):
            self.data = data.data
        else:
            self.data = data

    def __repr__(self):  # pragma: nocover
        return '<DotDict {}>'.format(self.data)

    def __getattr__(self, key):
        if key in ('values', 'keys', 'items'):
            # Dict methods, just return and run them.
            return getattr(self.data, key)

        # Default to None
        val = self.data.get(key, None)

        if isinstance(val, dict):
            val = DotDict(val)

        return val

    def __eq__(self, other):
        if isinstance(other, dict):
            return self.data == other
        elif isinstance(other, DotDict):
            return self.data == other.data
        else:
            return False

    def __iter__(self):
        for x in self.data:
            yield x

    # So that we can still access as dicts
    __getitem__ = __getattr__

def dynamic_load(target):
    """
    Dynamically import a class and return it

    This is used by the core parts of the main configuration file since
    one of the main features is to let the user specify which class to use.

    """

    split = target.split('.')
    module_name = '.'.join(split[:-1])
    class_name = split[-1]

    mod = __import__(module_name, fromlist=[class_name])
    return getattr(mod, class_name)

def oneshot(cmd):
    """
    Oneshot execution of a subprocess. Returns output as a single string.

    This is only to cut down on boiler.

    """

    popen = sub.Popen(cmd.split(), stdout=sub.PIPE, stderr=sub.PIPE)
    exit = popen.wait()

    if exit != 0:
        raise Exception(
            "Execution of '{0}' failed with exitcode {1}\n\n{2}".format(
                cmd, exit, popen.stderr.read().decode().strip()
            )
        )

    return str(popen.stdout.read().decode().strip())

def mkdir(path):  # pragma: nocover
    """
    Safe implementation of 'mkdir -p'

    """

    # Python 3.2 has os.makedirs(exist_ok=True)...
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

def now():
    """
    Mockable now() method. datetime.datetime is immutable and cannot be patched
    by mock.

    """

    return datetime.datetime.now()
<commit_msg>Make utils.now() use utctime().<commit_after>import os
import errno
import subprocess as sub
import datetime

class DotDict(object):
    """
    Immutable dict-like objects accessible by dot notation

    Used because the amount of configuration access is very high and just using
    dots instead of the dict notation feels good.

    It should be noted that this class could definitely have a more elegant
    writing, possibly inheriting from dict(). However, this is probably good
    enough and will probably not cause any trouble.

    """

    def __init__(self, data):
        # TODO: Make this more elegant
        if isinstance(data, DotDict):
            self.data = data.data
        else:
            self.data = data

    def __repr__(self):  # pragma: nocover
        return '<DotDict {}>'.format(self.data)

    def __getattr__(self, key):
        if key in ('values', 'keys', 'items'):
            # Dict methods, just return and run them.
            return getattr(self.data, key)

        # Default to None
        val = self.data.get(key, None)

        if isinstance(val, dict):
            val = DotDict(val)

        return val

    def __eq__(self, other):
        if isinstance(other, dict):
            return self.data == other
        elif isinstance(other, DotDict):
            return self.data == other.data
        else:
            return False

    def __iter__(self):
        for x in self.data:
            yield x

    # So that we can still access as dicts
    __getitem__ = __getattr__

def dynamic_load(target):
    """
    Dynamically import a class and return it

    This is used by the core parts of the main configuration file since
    one of the main features is to let the user specify which class to use.

    """

    split = target.split('.')
    module_name = '.'.join(split[:-1])
    class_name = split[-1]

    mod = __import__(module_name, fromlist=[class_name])
    return getattr(mod, class_name)

def oneshot(cmd):
    """
    Oneshot execution of a subprocess. Returns output as a single string.

    This is only to cut down on boiler.

    """

    popen = sub.Popen(cmd.split(), stdout=sub.PIPE, stderr=sub.PIPE)
    exit = popen.wait()

    if exit != 0:
        raise Exception(
            "Execution of '{0}' failed with exitcode {1}\n\n{2}".format(
                cmd, exit, popen.stderr.read().decode().strip()
            )
        )

    return str(popen.stdout.read().decode().strip())

def mkdir(path):  # pragma: nocover
    """
    Safe implementation of 'mkdir -p'

    """

    # Python 3.2 has os.makedirs(exist_ok=True)...
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

def now():
    """
    Mockable now() method. datetime.datetime is immutable and cannot be patched
    by mock.

    """

    return datetime.datetime.utcnow()
<|endoftext|>


<commit_before>from crocs.token import XNode, Token, eof, TokVal
import re
import time

class LexError(Exception):
    pass

class YaccError(Exception):
    pass

class Grammar:
    discard = []

class XSpec:
    pass

class TSeq(list):
    """
    This is meant to be returned by XNode's instances
    that extract strings from a given doc sequentially.
    """

    def __init__(self, *args):
        self.extend(args)
        self.slen = 0
        self.tlen = 0

    def clen(self):
        val = map(len, self)
        return sum(val)

    def __len__(self):
        return self.clen()

class PTree(list):
    """
    """

    def __init__(self, rule, *args):
        super(PTree, self).__init__(args)
        self.rule = rule
        self.result = None

    def eval(self, handles):
        result = self
        for ind in handles:
            result = ind(*result)

        if result is not self:
            self.result = result

    def val(self):
        return self.result

    def tlen(self):
        count = 0
        for ind in self:
            count += ind.tlen()
        return count

class Yacc:
    def __init__(self, grammar):
        self.root    = grammar.root
        self.discard = grammar.discard

    def is_discarded(self, token):
        for indi in self.discard:
            if indi.consume((token, )):
                return True
        return False

    def remove_tokens(self, tokens):
        """
        """
        for indi in tokens:
            if not self.is_discarded(indi):
                yield indi

    def build(self, tokens):
        """
        """

        tokens = self.remove_tokens(tokens)
        tokens = tuple(tokens)

        while True:
            ptree  = self.consume(tokens)
            tokens = tokens[ptree.tlen():]
            if ptree:
                yield ptree
            else:
                break

    def consume(self, tokens):
        """
        """

        ptree = self.root.consume(tokens)
        if not ptree and tokens:
            if tokens[0]:
                self.handle_error(ptree, tokens)
        return ptree

    def handle_error(self, ptree, tokens):
        """
        """

        print('Crocs Yacc error!')
        print('PTree:', ptree)
        print('Tokens:', tokens)
        raise YaccError('Unexpected struct!')

    def add_handle(self, rule, handle):
        """
        """
        rule.hmap.append(handle)

    def del_handle(self, rule, handle):
        """
        """
        rule.hmap.remove(handle)

class Lexer:
    def __init__(self, xspec, no_errors=False):
        """
        """
        self.root   = xspec.root
        self.data   = ''
        self.offset = 0
        self.no_errorss = no_errors

    def run(self):
        """
        """

        while True:
            tseq = self.consume(self.data)
            self.data = self.data[len(tseq):]
            if tseq:
                yield from tseq
            else:
                break

        # The loop stops on eof. It is useful for
        # some rules.
        yield eof

    def consume(self, data):
        """
        """
        tseq = self.root.consume(data)
        if not tseq and data:
            self.handle_error()
        return tseq

    def handle_error(self):
        msg = 'Unexpected token: %s' % repr(self.data[:30])
        raise LexError(msg)

    def feed(self, data):
        """
        """
        self.data = data
        self.offset = 0

class LexMap(XNode):
    def __init__(self):
        """
        """
        self.children = []
        super(LexMap, self).__init__()

    def register(self, xnode):
        self.children.append(xnode)

    def consume(self, data):
        """
        """
        if not data:
            return TSeq(eof)

        for ind in self.children:
            tseq = ind.consume(data)
            if tseq:
                return tseq
        return TSeq()

    def __repr__(self):
        return 'LexMap(%s)' % self.children

class Struct(XNode):
    def __init__(self, recursive=False):
        self.children = []
        super(Struct, self).__init__()
        self.recursive = recursive

    def consume(self, tokens, exclude=[], precedence=[]):
        """
        """

        for ind in self.children:
            if not ind in exclude and not ind in precedence:
                ptree = ind.consume(tokens, exclude, precedence)
                if ptree and self.recursive:
                    return self.reduce(ptree, tokens, precedence)
                elif ptree:
                    return ptree
        return PTree(self)

    def push(self, struct, ptree, tokens, precedence=[]):
        """
        """
        for ind in self.children:
            if not ind in precedence:
               rtree = ind.push(struct, ptree, tokens, precedence)
               if rtree:
                   return rtree
        return PTree(self)

    def reduce(self, ptree, tokens, precedence=[]):
        """
        """

        rtree = None
        while True:
            rtree = self.push(self,
                ptree, tokens, precedence)
            if not rtree:
                return ptree
            ptree = rtree

    def add(self, *args):
        """
        """
        self.children.extend(args)

class Rule(XNode):
    def __init__(self, trigger, *args, up=[]):
        """
        """
        self.trigger = trigger
        self.symbols = args
        self.up   = up
        self.hmap = []

    def push(self, struct, ptree, tokens, precedence=[]):
        """
        """

        if not struct is self.trigger:
            return PTree(self)

        tokens  = tokens[ptree.tlen():]
        exclude = [self]
        rtree = self.validate(ptree, tokens,
            exclude, precedence=self.up)
        return rtree

    def validate(self, ptree, tokens, exclude=[], precedence=[]):
        """
        """

        ntree = PTree(self, ptree)
        for ind in self.symbols:
            rtree = ind.consume(tokens, exclude, precedence)
            if rtree:
                tokens = tokens[rtree.tlen():]
                ntree.append(rtree)
            else:
                return rtree
        ntree.eval(self.hmap)
        return ntree

    def consume(self, tokens, exclude=[], precedence=[]):
        """
        """

        if isinstance(self.trigger, Struct):
            exclude = exclude + [self]

        rtree = self.trigger.consume(tokens, exclude, self.up)
        if not rtree:
            return PTree(self)

        tokens = tokens[rtree.tlen():]
        ntree = self.validate(rtree, tokens, exclude, self.up)
        if not ntree and self.symbols:
            return PTree(self)
        return ntree

class Times(XNode):
    def __init__(self, refer):
        self.refer = refer

    def consume(self, tokens, exclude=[]):
        """
        """
        ptree = PTree(self)
        while True:
            rtree = self.refer.consume(tokens, exclude)
            if rtree:
                ptree.append(rtree)
            else:
                return ptree
            tokens = tokens[rtree.tlen():]

class LexSeq(XNode):
    """
    """
    def __init__(self, lexmap, *args, type=None):
        self.lexmap = lexmap
        self.xnodes = []
        self.type   = type
        self.xnodes.extend(args)
        lexmap.register(self)

    def consume(self, data):
        """
        """
        tseq = TSeq()
        for ind in self.xnodes:
            slice = data[len(tseq):]
            token = ind.consume(slice)
            if token:
                tseq.extend(token)
            else:
                return TSeq()
        pass

        if self.type:
            return TSeq(self.type(tseq))
        return tseq

    def __repr__(self):
        return 'LexSeq(%s)' % self.xnodes

class SeqNode(XNode):
    def __init__(self, regex, type=Token):
        """
        """

        super(XNode, self).__init__()
        self.regex = regex
        self.type  = type

    def consume(self, data):
        """
        """

        regobj = re.match(self.regex, data)
        if regobj:
            return self.mktoken(regobj)

    def mktoken(self, regobj):
        tokval = regobj.group(0)
        token  = self.type(tokval)
        return TSeq(token)

    def __repr__(self):
        return 'SeqNode(%s(%s))' % (
            self.type.__name__, repr(self.regex))

class LexNode(SeqNode):
    def __init__(self, lexmap, regex, type=Token):
        """
        """

        super(LexNode, self).__init__(regex, type)
        self.lexmap = lexmap
        self.lexmap.register(self)

class LexRef(XNode):
    def __init__(self, xnode):
        self.xnode = xnode
        super(LexRef, self).__init__()
        """
        """

    def consume(self, data):
        """
        """

        tseq = TSeq()
        while True:
            slice = data[len(tseq):]
            token = self.xnode.consume(slice)
            if not token:
                break
            else:
                tseq.extend(token)
        return tseq
<commit_msg>Cosmetic changes.<commit_after>from crocs.token import XNode, Token, eof, TokVal
import re
import time

class LexError(Exception):
    pass

class YaccError(Exception):
    pass

class Grammar:
    discard = []

class XSpec:
    pass

class TSeq(list):
    """
    This is meant to be returned by XNode's instances
    that extract strings from a given doc sequentially.
    """

    def __init__(self, *args):
        self.extend(args)
        self.slen = 0
        self.tlen = 0

    def clen(self):
        val = map(len, self)
        return sum(val)

    def __len__(self):
        return self.clen()

class PTree(list):
    """
    """

    def __init__(self, rule, *args):
        super(PTree, self).__init__(args)
        self.rule = rule
        self.result = None

    def eval(self, handles):
        result = self
        for ind in handles:
            result = ind(*result)

        if result is not self:
            self.result = result

    def val(self):
        return self.result

    def tlen(self):
        count = 0
        for ind in self:
            count += ind.tlen()
        return count

class Yacc:
    def __init__(self, grammar):
        self.root    = grammar.root
        self.discard = grammar.discard

    def is_discarded(self, token):
        for indi in self.discard:
            if indi.consume((token, )):
                return True
        return False

    def remove_tokens(self, tokens):
        """
        """
        for indi in tokens:
            if not self.is_discarded(indi):
                yield indi

    def build(self, tokens):
        """
        """

        tokens = self.remove_tokens(tokens)
        tokens = tuple(tokens)

        while True:
            ptree  = self.consume(tokens)
            tokens = tokens[ptree.tlen():]
            if ptree:
                yield ptree
            else:
                break

    def consume(self, tokens):
        """
        """

        ptree = self.root.consume(tokens)
        if not ptree and tokens:
            if tokens[0]:
                self.handle_error(ptree, tokens)
        return ptree

    def handle_error(self, ptree, tokens):
        """
        """

        print('Crocs Yacc error!')
        print('PTree:', ptree)
        print('Tokens:', tokens)
        raise YaccError('Unexpected struct!')

    def add_handle(self, rule, handle):
        """
        """
        rule.hmap.append(handle)

    def del_handle(self, rule, handle):
        """
        """
        rule.hmap.remove(handle)

class Lexer:
    def __init__(self, xspec, no_errors=False):
        """
        """
        self.root   = xspec.root
        self.data   = ''
        self.offset = 0
        self.no_errorss = no_errors

    def run(self):
        """
        """

        while True:
            tseq = self.consume(self.data)
            self.data = self.data[len(tseq):]
            if tseq:
                yield from tseq
            else:
                break

        # The loop stops on eof. It is useful for
        # some rules.
        yield eof

    def consume(self, data):
        """
        """
        tseq = self.root.consume(data)
        if not tseq and data:
            self.handle_error()
        return tseq

    def handle_error(self):
        msg = 'Unexpected token: %s' % repr(self.data[:30])
        raise LexError(msg)

    def feed(self, data):
        """
        """
        self.data = data
        self.offset = 0

class LexMap(XNode):
    def __init__(self):
        """
        """
        self.children = []
        super(LexMap, self).__init__()

    def register(self, xnode):
        self.children.append(xnode)

    def consume(self, data):
        """
        """
        if not data:
            return TSeq(eof)

        for ind in self.children:
            tseq = ind.consume(data)
            if tseq:
                return tseq
        return TSeq()

    def __repr__(self):
        return 'LexMap(%s)' % self.children

class Struct(XNode):
    def __init__(self, recursive=False):
        self.children = []
        super(Struct, self).__init__()
        self.recursive = recursive

    def consume(self, tokens, exclude=[], precedence=[]):
        """
        """

        for ind in self.children:
            if not (ind in exclude or ind in precedence):
                ptree = ind.consume(tokens, exclude, precedence)
                if ptree and self.recursive:
                    return self.reduce(ptree, tokens, precedence)
                elif ptree:
                    return ptree
        return PTree(self)

    def push(self, struct, ptree, tokens, precedence=[]):
        """
        """
        for ind in self.children:
            if not ind in precedence:
               rtree = ind.push(struct, ptree, tokens, precedence)
               if rtree:
                   return rtree
        return PTree(self)

    def reduce(self, ptree, tokens, precedence=[]):
        """
        """

        rtree = None
        while True:
            rtree = self.push(self,
                ptree, tokens, precedence)
            if not rtree:
                return ptree
            ptree = rtree

    def add(self, *args):
        """
        """
        self.children.extend(args)

class Rule(XNode):
    def __init__(self, trigger, *args, up=[]):
        """
        """
        self.trigger = trigger
        self.symbols = args
        self.up   = up
        self.hmap = []

    def push(self, struct, ptree, tokens, precedence=[]):
        """
        """

        if not struct is self.trigger:
            return PTree(self)

        tokens  = tokens[ptree.tlen():]
        exclude = [self]
        rtree = self.validate(ptree, tokens,
            exclude, precedence=self.up)
        return rtree

    def validate(self, ptree, tokens, exclude=[], precedence=[]):
        """
        """

        ntree = PTree(self, ptree)
        for ind in self.symbols:
            rtree = ind.consume(tokens, exclude, precedence)
            if rtree:
                tokens = tokens[rtree.tlen():]
                ntree.append(rtree)
            else:
                return rtree
        ntree.eval(self.hmap)
        return ntree

    def consume(self, tokens, exclude=[], precedence=[]):
        """
        """

        if isinstance(self.trigger, Struct):
            exclude = exclude + [self]

        rtree = self.trigger.consume(tokens, exclude, self.up)
        if not rtree:
            return PTree(self)

        tokens = tokens[rtree.tlen():]
        ntree = self.validate(rtree, tokens, exclude, self.up)
        if not ntree and self.symbols:
            return PTree(self)
        return ntree

class Times(XNode):
    def __init__(self, refer):
        self.refer = refer

    def consume(self, tokens, exclude=[]):
        """
        """
        ptree = PTree(self)
        while True:
            rtree = self.refer.consume(tokens, exclude)
            if rtree:
                ptree.append(rtree)
            else:
                return ptree
            tokens = tokens[rtree.tlen():]

class LexSeq(XNode):
    """
    """
    def __init__(self, lexmap, *args, type=None):
        self.lexmap = lexmap
        self.xnodes = []
        self.type   = type
        self.xnodes.extend(args)
        lexmap.register(self)

    def consume(self, data):
        """
        """
        tseq = TSeq()
        for ind in self.xnodes:
            slice = data[len(tseq):]
            token = ind.consume(slice)
            if token:
                tseq.extend(token)
            else:
                return TSeq()
        pass

        if self.type:
            return TSeq(self.type(tseq))
        return tseq

    def __repr__(self):
        return 'LexSeq(%s)' % self.xnodes

class SeqNode(XNode):
    def __init__(self, regex, type=Token):
        """
        """

        super(XNode, self).__init__()
        self.regex = regex
        self.type  = type

    def consume(self, data):
        """
        """

        regobj = re.match(self.regex, data)
        if regobj:
            return self.mktoken(regobj)

    def mktoken(self, regobj):
        tokval = regobj.group(0)
        token  = self.type(tokval)
        return TSeq(token)

    def __repr__(self):
        return 'SeqNode(%s(%s))' % (
            self.type.__name__, repr(self.regex))

class LexNode(SeqNode):
    def __init__(self, lexmap, regex, type=Token):
        """
        """

        super(LexNode, self).__init__(regex, type)
        self.lexmap = lexmap
        self.lexmap.register(self)

class LexRef(XNode):
    def __init__(self, xnode):
        self.xnode = xnode
        super(LexRef, self).__init__()
        """
        """

    def consume(self, data):
        """
        """

        tseq = TSeq()
        while True:
            slice = data[len(tseq):]
            token = self.xnode.consume(slice)
            if not token:
                break
            else:
                tseq.extend(token)
        return tseq
<|endoftext|>