<commit_before>import sys
import os

from spec import eq_, skip, Spec, raises, ok_, trap

from invoke.runner import Runner, run
from invoke.exceptions import Failure

from _utils import support

class _MockRunner(Runner):
    def run(self, command, warn, hide):
        return "", "", 0, None

class Run(Spec):
    "run()"

    def setup(self):
        os.chdir(support)
        self.both = "echo foo && ./err bar"
        self.out = "echo foo"
        self.err = "./err bar"
        self.sub = "inv -c pty_output hide_%s"

    class return_value:
        def return_code_in_result(self):
            """
            Result has .return_code (and .exited) containing exit code int
            """
            r = run(self.out, hide='both')
            eq_(r.return_code, 0)
            eq_(r.exited, 0)

        def nonzero_return_code_for_failures(self):
            result = run("false", warn=True)
            eq_(result.exited, 1)
            result = run("goobypls", warn=True, hide='both')
            eq_(result.exited, 127)

        def stdout_attribute_contains_stdout(self):
            eq_(run(self.out, hide='both').stdout, 'foo\n')

        def stderr_attribute_contains_stderr(self):
            eq_(run(self.err, hide='both').stderr, 'bar\n')

        def stdout_contains_both_streams_under_pty(self):
            r = run(self.both, hide='both', pty=True)
            eq_(r.stdout, 'foo\r\nbar\r\n')

        def stderr_is_empty_under_pty(self):
            r = run(self.both, hide='both', pty=True)
            eq_(r.stderr, '')

        def ok_attr_indicates_success(self):
            eq_(run("true").ok, True)
            eq_(run("false", warn=True).ok, False)

        def failed_attr_indicates_failure(self):
            eq_(run("true").failed, False)
            eq_(run("false", warn=True).failed, True)

        def has_exception_attr(self):
            eq_(run("meh", runner=_MockRunner).exception, None)

    class failure_handling:
        @raises(Failure)
        def fast_failures(self):
            run("false")

        def run_acts_as_success_boolean(self):
            ok_(not run("false", warn=True))
            ok_(run("true"))

        def non_one_return_codes_still_act_as_False(self):
            ok_(not run("goobypls", warn=True, hide='both'))

        def warn_kwarg_allows_continuing_past_failures(self):
            eq_(run("false", warn=True).exited, 1)

        def Failure_repr_includes_stderr(self):
            try:
                run("./err ohnoz && exit 1", hide='both')
                assert false # Ensure failure to Failure fails
            except Failure as f:
                r = repr(f)
                assert 'ohnoz' in r, "Sentinel 'ohnoz' not found in %r" % r

    class output_controls:
        @trap
        def _hide_both(self, val):
            run(self.both, hide=val)
            eq_(sys.stdall.getvalue(), "")

        def hide_both_hides_everything(self):
            self._hide_both('both')

        def hide_True_hides_everything(self):
            self._hide_both(True)

        @trap
        def hide_out_only_hides_stdout(self):
            run(self.both, hide='out')
            eq_(sys.stdout.getvalue().strip(), "")
            eq_(sys.stderr.getvalue().strip(), "bar")

        @trap
        def hide_err_only_hides_stderr(self):
            run(self.both, hide='err')
            eq_(sys.stdout.getvalue().strip(), "foo")
            eq_(sys.stderr.getvalue().strip(), "")

        @trap
        def hide_accepts_stderr_alias_for_err(self):
            run(self.both, hide='stderr')
            eq_(sys.stdout.getvalue().strip(), "foo")
            eq_(sys.stderr.getvalue().strip(), "")

        @trap
        def hide_accepts_stdout_alias_for_out(self):
            run(self.both, hide='stdout')
            eq_(sys.stdout.getvalue().strip(), "")
            eq_(sys.stderr.getvalue().strip(), "bar")

        def hide_both_hides_both_under_pty(self):
            r = run(self.sub % 'both', hide='both')
            eq_(r.stdout, "")
            eq_(r.stderr, "")

        def hide_out_hides_both_under_pty(self):
            r = run(self.sub % 'out', hide='both')
            eq_(r.stdout, "")
            eq_(r.stderr, "")

        def hide_err_has_no_effect_under_pty(self):
            r = run(self.sub % 'err', hide='both')
            eq_(r.stdout, "foo\r\nbar\r\n")
            eq_(r.stderr, "")

        @trap
        def _no_hiding(self, val):
            r = run(self.both, hide=val)
            eq_(sys.stdout.getvalue().strip(), "foo")
            eq_(sys.stderr.getvalue().strip(), "bar")

        def hide_None_hides_nothing(self):
            self._no_hiding(None)

        def hide_False_hides_nothing(self):
            self._no_hiding(False)

        @raises(ValueError)
        def hide_unknown_vals_raises_ValueError(self):
            run("command", hide="what")

        def hide_unknown_vals_mention_value_given_in_error(self):
            value = "penguinmints"
            try:
                run("command", hide=value)
            except ValueError as e:
                msg = "Error from run(hide=xxx) did not tell user what the bad value was!"
                msg += "\nException msg: %s" % e
                ok_(value in str(e), msg)
            else:
                assert False, "run() did not raise ValueError for bad hide= value"

        def hide_does_not_affect_capturing(self):
            eq_(run(self.out, hide='both').stdout, 'foo\n')

    class pseudo_terminals:
        def return_value_indicates_whether_pty_was_used(self):
            eq_(run("true").pty, False)
            eq_(run("true", pty=True).pty, True)

        def pty_defaults_to_off(self):
            eq_(run("true").pty, False)

    class command_echo:
        @trap
        def does_not_echo_commands_run_by_default(self):
            run("echo hi")
            eq_(sys.stdout.getvalue().strip(), "hi")

        @trap
        def when_echo_True_commands_echoed_in_bold(self):
            run("echo hi", echo=True)
            expected = "\033[1;37mecho hi\033[0m\nhi"
            eq_(sys.stdout.getvalue().strip(), expected)

    #
    # Random edge/corner case junk
    #

    def non_stupid_OSErrors_get_captured(self):
        # Somehow trigger an OSError saying "Input/output error" within
        # pexpect.spawn().interact() & assert it is in result.exception
        skip()

    def KeyboardInterrupt_on_stdin_doesnt_flake(self):
        # E.g. inv test => Ctrl-C halfway => shouldn't get buffer API errors
        skip()

    class funky_characters_in_stdout:
        def basic_nonstandard_characters(self):
            # Crummy "doesn't explode with decode errors" test
            run("cat tree.out", hide='both')

        def nonprinting_bytes(self):
            # Seriously non-printing characters (i.e. non UTF8) also don't asplode
            # load('funky').derp()
            run("echo '\xff'", hide='both')

        def nonprinting_bytes_pty(self):
            # PTY use adds another utf-8 decode spot which can also fail.
            run("echo '\xff'", pty=True, hide='both')
<commit_msg>Trying something new<commit_after>import sys
import os

from spec import eq_, skip, Spec, raises, ok_, trap

from invoke.runner import Runner, run
from invoke.exceptions import Failure

from _utils import support

def _runner(**kwargs):
    """
    Return a Runner subclass whose run() return value reflects ``kwargs``.
    """
    kwargs.setdefault('exited', 0)
    value = map(
        lambda x: kwargs.pop(x, None),
        ('stdout', 'stderr', 'exited', 'exception'),
    )
    class MockRunner(Runner):
        def run(self, command, warn, hide):
            return value
    return MockRunner

class Run(Spec):
    "run()"

    def setup(self):
        os.chdir(support)
        self.both = "echo foo && ./err bar"
        self.out = "echo foo"
        self.err = "./err bar"
        self.sub = "inv -c pty_output hide_%s"

    class return_value:
        def return_code_in_result(self):
            """
            Result has .return_code (and .exited) containing exit code int
            """
            r = run(self.out, hide='both')
            eq_(r.return_code, 0)
            eq_(r.exited, 0)

        def nonzero_return_code_for_failures(self):
            result = run("false", warn=True)
            eq_(result.exited, 1)
            result = run("goobypls", warn=True, hide='both')
            eq_(result.exited, 127)

        def stdout_attribute_contains_stdout(self):
            eq_(run(self.out, hide='both').stdout, 'foo\n')

        def stderr_attribute_contains_stderr(self):
            eq_(run(self.err, hide='both').stderr, 'bar\n')

        def stdout_contains_both_streams_under_pty(self):
            r = run(self.both, hide='both', pty=True)
            eq_(r.stdout, 'foo\r\nbar\r\n')

        def stderr_is_empty_under_pty(self):
            r = run(self.both, hide='both', pty=True)
            eq_(r.stderr, '')

        def ok_attr_indicates_success(self):
            eq_(run("true").ok, True)
            eq_(run("false", warn=True).ok, False)

        def failed_attr_indicates_failure(self):
            eq_(run("true").failed, False)
            eq_(run("false", warn=True).failed, True)

        def has_exception_attr(self):
            eq_(run("meh", runner=_runner()).exception, None)

    class failure_handling:
        @raises(Failure)
        def fast_failures(self):
            run("false")

        def run_acts_as_success_boolean(self):
            ok_(not run("false", warn=True))
            ok_(run("true"))

        def non_one_return_codes_still_act_as_False(self):
            ok_(not run("goobypls", warn=True, hide='both'))

        def warn_kwarg_allows_continuing_past_failures(self):
            eq_(run("false", warn=True).exited, 1)

        def Failure_repr_includes_stderr(self):
            try:
                run("./err ohnoz && exit 1", hide='both')
                assert false # Ensure failure to Failure fails
            except Failure as f:
                r = repr(f)
                assert 'ohnoz' in r, "Sentinel 'ohnoz' not found in %r" % r

    class output_controls:
        @trap
        def _hide_both(self, val):
            run(self.both, hide=val)
            eq_(sys.stdall.getvalue(), "")

        def hide_both_hides_everything(self):
            self._hide_both('both')

        def hide_True_hides_everything(self):
            self._hide_both(True)

        @trap
        def hide_out_only_hides_stdout(self):
            run(self.both, hide='out')
            eq_(sys.stdout.getvalue().strip(), "")
            eq_(sys.stderr.getvalue().strip(), "bar")

        @trap
        def hide_err_only_hides_stderr(self):
            run(self.both, hide='err')
            eq_(sys.stdout.getvalue().strip(), "foo")
            eq_(sys.stderr.getvalue().strip(), "")

        @trap
        def hide_accepts_stderr_alias_for_err(self):
            run(self.both, hide='stderr')
            eq_(sys.stdout.getvalue().strip(), "foo")
            eq_(sys.stderr.getvalue().strip(), "")

        @trap
        def hide_accepts_stdout_alias_for_out(self):
            run(self.both, hide='stdout')
            eq_(sys.stdout.getvalue().strip(), "")
            eq_(sys.stderr.getvalue().strip(), "bar")

        def hide_both_hides_both_under_pty(self):
            r = run(self.sub % 'both', hide='both')
            eq_(r.stdout, "")
            eq_(r.stderr, "")

        def hide_out_hides_both_under_pty(self):
            r = run(self.sub % 'out', hide='both')
            eq_(r.stdout, "")
            eq_(r.stderr, "")

        def hide_err_has_no_effect_under_pty(self):
            r = run(self.sub % 'err', hide='both')
            eq_(r.stdout, "foo\r\nbar\r\n")
            eq_(r.stderr, "")

        @trap
        def _no_hiding(self, val):
            r = run(self.both, hide=val)
            eq_(sys.stdout.getvalue().strip(), "foo")
            eq_(sys.stderr.getvalue().strip(), "bar")

        def hide_None_hides_nothing(self):
            self._no_hiding(None)

        def hide_False_hides_nothing(self):
            self._no_hiding(False)

        @raises(ValueError)
        def hide_unknown_vals_raises_ValueError(self):
            run("command", hide="what")

        def hide_unknown_vals_mention_value_given_in_error(self):
            value = "penguinmints"
            try:
                run("command", hide=value)
            except ValueError as e:
                msg = "Error from run(hide=xxx) did not tell user what the bad value was!"
                msg += "\nException msg: %s" % e
                ok_(value in str(e), msg)
            else:
                assert False, "run() did not raise ValueError for bad hide= value"

        def hide_does_not_affect_capturing(self):
            eq_(run(self.out, hide='both').stdout, 'foo\n')

    class pseudo_terminals:
        def return_value_indicates_whether_pty_was_used(self):
            eq_(run("true").pty, False)
            eq_(run("true", pty=True).pty, True)

        def pty_defaults_to_off(self):
            eq_(run("true").pty, False)

    class command_echo:
        @trap
        def does_not_echo_commands_run_by_default(self):
            run("echo hi")
            eq_(sys.stdout.getvalue().strip(), "hi")

        @trap
        def when_echo_True_commands_echoed_in_bold(self):
            run("echo hi", echo=True)
            expected = "\033[1;37mecho hi\033[0m\nhi"
            eq_(sys.stdout.getvalue().strip(), expected)

    #
    # Random edge/corner case junk
    #

    def non_stupid_OSErrors_get_captured(self):
        # Somehow trigger an OSError saying "Input/output error" within
        # pexpect.spawn().interact() & assert it is in result.exception
        skip()

    def KeyboardInterrupt_on_stdin_doesnt_flake(self):
        # E.g. inv test => Ctrl-C halfway => shouldn't get buffer API errors
        skip()

    class funky_characters_in_stdout:
        def basic_nonstandard_characters(self):
            # Crummy "doesn't explode with decode errors" test
            run("cat tree.out", hide='both')

        def nonprinting_bytes(self):
            # Seriously non-printing characters (i.e. non UTF8) also don't asplode
            # load('funky').derp()
            run("echo '\xff'", hide='both')

        def nonprinting_bytes_pty(self):
            # PTY use adds another utf-8 decode spot which can also fail.
            run("echo '\xff'", pty=True, hide='both')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (c) 2011-2012 Raphaël Barrois

import os
import sys

from django.conf import settings

if not settings.configured:
    settings.configure(
        DATABASES={
            'default': {
                'ENGINE': 'django.db.backends.sqlite3',
            }
        },
        INSTALLED_APPS=[
            'django.contrib.auth',
            'django.contrib.contenttypes',
            'tests.djworkflows',
            'django_xworkflows',
            'django_xworkflows.xworkflow_log',
        ]
    )

from django.test import simple

def runtests(*test_args):
    if not test_args:
        test_args = ('djworkflows',)
    runner = simple.DjangoTestSuiteRunner()
    failures = runner.run_tests(test_args)
    sys.exit(failures)

if __name__ == '__main__':
    runtests(*sys.argv[1:])

<commit_msg>Run entire test suite every time<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (c) 2011-2012 Raphaël Barrois

import os
import sys

from django.conf import settings

if not settings.configured:
    settings.configure(
        DATABASES={
            'default': {
                'ENGINE': 'django.db.backends.sqlite3',
            }
        },
        INSTALLED_APPS=[
            'django.contrib.auth',
            'django.contrib.contenttypes',
            'tests.djworkflows',
            'django_xworkflows',
            'django_xworkflows.xworkflow_log',
        ]
    )

from django.test import simple

def runtests(*test_args):
    if not test_args:
        test_args = ('djworkflows',)
    runner = simple.DjangoTestSuiteRunner(failfast=False)
    failures = runner.run_tests(test_args)
    sys.exit(failures)

if __name__ == '__main__':
    runtests(*sys.argv[1:])

<|endoftext|>


<commit_before>from os import path
from multiprocessing import Process

from bottle import Bottle, run, template

PRESETS_DIR = 'presets'

class TPBApp(Bottle):
    presets = {
            '/search/<query>/<page>/<ordering>/<category>': 'search.html',
            '/recent/<page>': 'recent.html',
            '/top/<category>': 'top.html',
            }

    def __init__(self, host='localhost', port=8000):
        super(TPBApp, self).__init__()
        self.host = host
        self.port = port
        self.process = None
        self.build_routes()

    def build_routes(self):
        for url, preset in self.presets.items():
            with open(path.join(PRESETS_DIR, preset)) as preset:
                content = preset.read()
                self.route(url)(lambda **kwargs: template(content))

    def run(self):
        run(self, host=self.host, port=self.port)

    def start(self):
        self.process = Process(target=self.run)
        self.process.start()

    def stop(self):
        self.process.terminate()
        self.process = None

    @property
    def url(self):
        return 'http://{}:{}'.format(self.host, self.port)

if __name__ == '__main__':
    TPBApp('localhost', 8000).run()
<commit_msg>get the full presets dir path<commit_after>from os import path
from multiprocessing import Process

from bottle import Bottle, run, template

PRESETS_DIR = path.join(path.dirname(__file__), 'presets')

class TPBApp(Bottle):
    presets = {
            '/search/<query>/<page>/<ordering>/<category>': 'search.html',
            '/recent/<page>': 'recent.html',
            '/top/<category>': 'top.html',
            }

    def __init__(self, host='localhost', port=8000):
        super(TPBApp, self).__init__()
        self.host = host
        self.port = port
        self.process = None
        self.build_routes()

    def build_routes(self):
        for url, preset in self.presets.items():
            with open(path.join(PRESETS_DIR, preset)) as preset:
                content = preset.read()
                self.route(url)(lambda **kwargs: template(content))

    def run(self):
        run(self, host=self.host, port=self.port)

    def start(self):
        self.process = Process(target=self.run)
        self.process.start()

    def stop(self):
        self.process.terminate()
        self.process = None

    @property
    def url(self):
        return 'http://{}:{}'.format(self.host, self.port)

if __name__ == '__main__':
    TPBApp('localhost', 8000).run()
<|endoftext|>


<commit_before>from datetime import datetime
import operator

from mongoengine import *
from flask.ext.mongoengine.wtf import model_form

from core.helpers import is_url, is_ip, is_hostname, iterify
from core.database import Node, TagListField
from core.observables import ObservableTag, Tag
from core.entities import Entity
from core.errors import ObservableValidationError

class Observable(Node):

    value = StringField(verbose_name="Value", required=True, unique=True, sparse=True, max_length=1024)
    sources = ListField(StringField(), verbose_name="Sources")
    description = StringField(verbose_name="Description")
    context = ListField(DictField(), verbose_name="Context")
    tags = ListField(EmbeddedDocumentField(ObservableTag), verbose_name="Tags")
    last_analyses = DictField(verbose_name="Last analyses")

    created = DateTimeField(default=datetime.now)

    exclude_fields = ['sources', 'context', 'last_analyses', 'created']

    meta = {
        "allow_inheritance": True,
    }

    @classmethod
    def get_form(klass):
        form = model_form(klass, exclude=klass.exclude_fields)
        form.tags = TagListField()
        return form

    def __unicode__(self):
        return u"{} ({} context)".format(self.value, len(self.context))

    @staticmethod
    def guess_type(string):
        from core.observables import Url, Ip, Hostname
        if string and string.strip() != '':
            if is_url(string):
                return Url
            elif is_ip(string):
                return Ip
            elif is_hostname(string):
                return Hostname
            else:
                raise ObservableValidationError("{} was not recognized as a viable datatype".format(string))

    @classmethod
    def add_text(cls, text):
        return Observable.guess_type(text).get_or_create(value=text)

    @staticmethod
    def change_all_tags(old_tags, new_tag):
        for o in Observable.objects(tags__name__in=iterify(old_tags)):
            for old_tag in old_tags:
                o.change_tag(old_tag, new_tag)

    def add_context(self, context):
        assert 'source' in context
        context = {k: v for k, v in sorted(context.items(), key=operator.itemgetter(0))}
        self.modify(add_to_set__context=context)
        return self.reload()

    def add_source(self, source):
        return self.modify(add_to_set__sources=source)

    def get_tags(self, fresh=True):
        return [t.name for t in self.tags if (t.fresh or not fresh)]

    def find_tags(self):
        # find related tags and count them
        new_tags = {}
        for tag in self.tags:
            tag = Tag.objects.get(name=tag.name)
            for produces in tag.produces:
                new_tags[produces] = new_tags.get(tag, 0) + 1

        # remove already known tags
        localtags = [tag.name for tag in self.tags]
        for tag in new_tags.copy():
            if tag in localtags:
                new_tags.pop(tag)

        return new_tags

    def has_tag(self, tag_to_search):
        for tag in self.tags:
            if tag.name == tag_to_search:
                return True
        else:
            return False

    def change_tag(self, old_tag, new_tag):
        if not self.modify({"tags__name": old_tag, "tags__name__ne": new_tag}, set__tags__S__name=new_tag):
            self.modify({"tags__name": old_tag}, pull__tags__name=old_tag)
            self.modify({"tags__name": new_tag}, set__tags__S__last_seen=datetime.now())
        return self.reload()

    def tag(self, new_tags, strict=False):
        new_tags = iterify(new_tags)

        if strict:
            remove = set([t.name for t in self.tags]) - set(new_tags)
            for tag in remove:
                self.modify(pull__tags__name=tag)

        for new_tag in new_tags:
            if new_tag.strip() != '':
                new_tag = Tag(name=new_tag)
                new_tag.clean()

                try:  # check if tag is a replacement
                    tag = Tag.objects.get(replaces=new_tag.name)
                except DoesNotExist:
                    tag = Tag.get_or_create(name=new_tag.name)

                # search for related entities and link them
                for e in Entity.objects(tags__in=[tag.name]):
                    self.link_to(e, 'Tagged', 'tags')

                if not self.modify({"tags__name": tag.name}, set__tags__S__fresh=True, set__tags__S__last_seen=datetime.now()):
                    self.modify(push__tags=ObservableTag(name=tag.name))
                    tag.modify(inc__count=1)

        return self.reload()

    def check_tags(self):
        for tag in self.tags:
            if tag.expiration and (tag.last_seen + tag.expiration) < datetime.now():
                tag.fresh = False
        return self.save()

    def fresh_tags(self):
        return [tag for tag in self.tags if tag.fresh]

    def analysis_done(self, module_name):
        ts = datetime.now()
        return self.modify(**{"set__last_analyses__{}".format(module_name): ts})

    def info(self):
        i = {k: v for k, v in self._data.items() if k in ["value", "context", "last_analyses", "created", "sources"]}
        i['tags'] = [t.info() for t in self.tags]
        if self.id:
            i['id'] = str(self.id)
        i['type'] = self.__class__.__name__
        return i
<commit_msg>Bugfix<commit_after>from datetime import datetime
import operator

from mongoengine import *
from flask.ext.mongoengine.wtf import model_form

from core.helpers import is_url, is_ip, is_hostname, iterify
from core.database import Node, TagListField
from core.observables import ObservableTag, Tag
from core.entities import Entity
from core.errors import ObservableValidationError

class Observable(Node):

    value = StringField(verbose_name="Value", required=True, unique=True, sparse=True, max_length=1024)
    sources = ListField(StringField(), verbose_name="Sources")
    description = StringField(verbose_name="Description")
    context = ListField(DictField(), verbose_name="Context")
    tags = ListField(EmbeddedDocumentField(ObservableTag), verbose_name="Tags")
    last_analyses = DictField(verbose_name="Last analyses")

    created = DateTimeField(default=datetime.now)

    exclude_fields = ['sources', 'context', 'last_analyses', 'created']

    meta = {
        "allow_inheritance": True,
    }

    @classmethod
    def get_form(klass):
        form = model_form(klass, exclude=klass.exclude_fields)
        form.tags = TagListField()
        return form

    def __unicode__(self):
        return u"{} ({} context)".format(self.value, len(self.context))

    @staticmethod
    def guess_type(string):
        from core.observables import Url, Ip, Hostname
        if string and string.strip() != '':
            if is_url(string):
                return Url
            elif is_ip(string):
                return Ip
            elif is_hostname(string):
                return Hostname
            else:
                raise ObservableValidationError("{} was not recognized as a viable datatype".format(string))

    @classmethod
    def add_text(cls, text):
        return Observable.guess_type(text).get_or_create(value=text)

    @staticmethod
    def change_all_tags(old_tags, new_tag):
        old_tags = iterify(old_tags)
        for o in Observable.objects(tags__name__in=old_tags):
            for old_tag in old_tags:
                o.change_tag(old_tag, new_tag)

    def add_context(self, context):
        assert 'source' in context
        context = {k: v for k, v in sorted(context.items(), key=operator.itemgetter(0))}
        self.modify(add_to_set__context=context)
        return self.reload()

    def add_source(self, source):
        return self.modify(add_to_set__sources=source)

    def get_tags(self, fresh=True):
        return [t.name for t in self.tags if (t.fresh or not fresh)]

    def find_tags(self):
        # find related tags and count them
        new_tags = {}
        for tag in self.tags:
            tag = Tag.objects.get(name=tag.name)
            for produces in tag.produces:
                new_tags[produces] = new_tags.get(tag, 0) + 1

        # remove already known tags
        localtags = [tag.name for tag in self.tags]
        for tag in new_tags.copy():
            if tag in localtags:
                new_tags.pop(tag)

        return new_tags

    def has_tag(self, tag_to_search):
        for tag in self.tags:
            if tag.name == tag_to_search:
                return True
        else:
            return False

    def change_tag(self, old_tag, new_tag):
        if not self.modify({"tags__name": old_tag, "tags__name__ne": new_tag}, set__tags__S__name=new_tag):
            self.modify({"tags__name": old_tag}, pull__tags__name=old_tag)
            self.modify({"tags__name": new_tag}, set__tags__S__last_seen=datetime.now())
        return self.reload()

    def tag(self, new_tags, strict=False):
        new_tags = iterify(new_tags)

        if strict:
            remove = set([t.name for t in self.tags]) - set(new_tags)
            for tag in remove:
                self.modify(pull__tags__name=tag)

        for new_tag in new_tags:
            if new_tag.strip() != '':
                new_tag = Tag(name=new_tag)
                new_tag.clean()

                try:  # check if tag is a replacement
                    tag = Tag.objects.get(replaces=new_tag.name)
                except DoesNotExist:
                    tag = Tag.get_or_create(name=new_tag.name)

                # search for related entities and link them
                for e in Entity.objects(tags__in=[tag.name]):
                    self.link_to(e, 'Tagged', 'tags')

                if not self.modify({"tags__name": tag.name}, set__tags__S__fresh=True, set__tags__S__last_seen=datetime.now()):
                    self.modify(push__tags=ObservableTag(name=tag.name))
                    tag.modify(inc__count=1)

        return self.reload()

    def check_tags(self):
        for tag in self.tags:
            if tag.expiration and (tag.last_seen + tag.expiration) < datetime.now():
                tag.fresh = False
        return self.save()

    def fresh_tags(self):
        return [tag for tag in self.tags if tag.fresh]

    def analysis_done(self, module_name):
        ts = datetime.now()
        return self.modify(**{"set__last_analyses__{}".format(module_name): ts})

    def info(self):
        i = {k: v for k, v in self._data.items() if k in ["value", "context", "last_analyses", "created", "sources"]}
        i['tags'] = [t.info() for t in self.tags]
        if self.id:
            i['id'] = str(self.id)
        i['type'] = self.__class__.__name__
        return i
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import logging
# import json
import boto3
from core import ff_utils
from core.utils import Tibanna, ensure_list

LOG = logging.getLogger(__name__)
s3 = boto3.resource('s3')

def handler(event, context):
    '''
    this is generic function to run awsem workflow
    based on the data passed in

    workflow_uuid : for now, pass this on. Later we can add a code to automatically retrieve this from app_name.
    Note multiple workflow_uuids can be available for an app_name
    (different versions of the same app could have a different uuid)
    '''
    # get incomming data
    input_file_list = event.get('input_files')
    app_name = event.get('app_name')
    print(app_name)
    workflow_uuid = event.get('workflow_uuid')
    output_bucket = event.get('output_bucket')
    tibanna_settings = event.get('_tibanna', {})
    # if they don't pass in env guess it from output_bucket
    env = tibanna_settings.get('env', '-'.join(output_bucket.split('-')[1:-1]))
    # tibanna provides access to keys based on env and stuff like that
    tibanna = Tibanna(env, s3_keys=event.get('s3_keys'), ff_keys=event.get('ff_keys'),
                      settings=tibanna_settings)

    args = dict()

    # get argument format & type info from workflow
    workflow_info = ff_utils.get_metadata(workflow_uuid, key=tibanna.ff_keys)
    LOG.info("workflow info  %s" % workflow_info)
    if 'error' in workflow_info.get('@type', []):
        raise Exception("FATAL, can't lookup workflow info for % fourfront" % workflow_uuid)

    # get cwl info from workflow_info
    for k in ['app_name', 'app_version', 'cwl_directory_url', 'cwl_main_filename', 'cwl_child_filenames']:
        LOG.info(workflow_info.get(k))
        args[k] = workflow_info.get(k)

    # get format-extension map
    try:
        fp_schema = ff_utils.get_metadata("profiles/file_processed.json", key=tibanna.ff_keys)
        fe_map = fp_schema.get('file_format_file_extension')
    except Exception as e:
        LOG.error("Can't get format-extension map from file_processed schema. %s\n" % e)

    # processed file metadata
    output_files = []
    try:
        if 'arguments' in workflow_info:
            pf_meta = []
            for arg in workflow_info.get('arguments'):
                if (arg.get('argument_type') in
                   ['Output processed file', 'Output report file', 'Output QC file']):

                    of = dict()
                    of['workflow_argument_name'] = arg.get('workflow_argument_name')
                    of['type'] = arg.get('argument_type')
                    if 'argument_format' in arg:
                        # These are not processed files but report or QC files.
                        pf = ff_utils.ProcessedFileMetadata(file_format=arg.get('argument_format'))
                        try:
                            resp = pf.post(key=tibanna.ff_keys)  # actually post processed file metadata here
                            resp = resp.get('@graph')[0]
                            of['upload_key'] = resp.get('upload_key')
                            of['value'] = resp.get('uuid')
                        except Exception as e:
                            LOG.error("Failed to post Processed file metadata. %s\n" % e)
                            LOG.error("resp" + str(resp) + "\n")
                            raise e
                        of['format'] = arg.get('argument_format')
                        of['extension'] = fe_map.get(arg.get('argument_format'))
                        pf_meta.append(pf)
                    output_files.append(of)

    except Exception as e:
        LOG.error("output_files = " + str(output_files) + "\n")
        LOG.error("Can't prepare output_files information. %s\n" % e)
        raise e

    # create the ff_meta output info
    input_files = []
    for input_file in input_file_list:
        for idx, uuid in enumerate(ensure_list(input_file['uuid'])):
            input_files.append({'workflow_argument_name': input_file['workflow_argument_name'],
                                'value': uuid, 'ordinal': idx + 1})
    LOG.info("input_files is %s" % input_files)

    ff_meta = ff_utils.create_ffmeta_awsem(workflow_uuid, app_name, input_files,
                                           run_url=tibanna.settings.get('url', ''),
                                           output_files=output_files)

    LOG.info("ff_meta is %s" % ff_meta.__dict__)

    # store metadata so we know the run has started
    ff_meta.post_plain_wrf(key=tibanna.ff_keys)

    # input file args for awsem
    args['input_files'] = dict()
    for input_file in input_file_list:
        if isinstance(input_file['uuid'], unicode):
            input_file['uuid'] = input_file['uuid'].encode('utf-8')
        if isinstance(input_file['object_key'], unicode):
            input_file['object_key'] = input_file['object_key'].encode('utf-8')
        if isinstance(input_file['uuid'], str) and isinstance(input_file['object_key'], str):
            object_key = input_file['uuid'] + '/' + input_file['object_key']
        elif (isinstance(input_file['uuid'], list) and
              isinstance(input_file['object_key'], list) and
              len(input_file['uuid']) == len(input_file['object_key'])):

            object_key = [a + '/' + b for a, b in zip(input_file['uuid'], input_file['object_key'])]
        else:
            raise Exception("input_file uuid and object_key should match in their type and length (if lists) : " +
                            "type{}{} length{}{}".format(type(input_file['uuid']), type(input_file['object_key']),
                            len(input_file['uuid']), len(input_file['object_key'])))
        args['input_files'].update({input_file['workflow_argument_name']: {
                                    'bucket_name': input_file['bucket_name'],
                                    'object_key': object_key}})

    LOG.info("input_file_args is %s" % args['input_files'])
    args['secondary_files'] = dict()   # temporary, later fill in based on the attachment information

    # parameters
    args['input_parameters'] = event.get('parameters')

    # output target
    args['output_target'] = dict()
    for of in ff_meta.output_files:
        arg_name = of.get('workflow_argument_name')
        if of.get('type') == 'Output processed file':
            args['output_target'][arg_name] = of.get('upload_key')
        else:
            args['output_target'][arg_name] = ff_meta.uuid + '/' + arg_name

    # output bucket
    args['output_S3_bucket'] = event.get('output_bucket')

    # let's not pass keys in plain text parameters
    return {"input_file_args": input_file_list,
            "ff_meta": ff_meta.as_dict(),
            'pf_meta': [meta.as_dict() for meta in pf_meta],
            "_tibanna": tibanna.as_dict(),
            "config": event.get("config"),
            "args": args
            }
<commit_msg>fixed trailing whitespace<commit_after># -*- coding: utf-8 -*-
import logging
# import json
import boto3
from core import ff_utils
from core.utils import Tibanna, ensure_list

LOG = logging.getLogger(__name__)
s3 = boto3.resource('s3')

def handler(event, context):
    '''
    this is generic function to run awsem workflow
    based on the data passed in

    workflow_uuid : for now, pass this on. Later we can add a code to automatically retrieve this from app_name.
    Note multiple workflow_uuids can be available for an app_name
    (different versions of the same app could have a different uuid)
    '''
    # get incomming data
    input_file_list = event.get('input_files')
    app_name = event.get('app_name')
    print(app_name)
    workflow_uuid = event.get('workflow_uuid')
    output_bucket = event.get('output_bucket')
    tibanna_settings = event.get('_tibanna', {})
    # if they don't pass in env guess it from output_bucket
    env = tibanna_settings.get('env', '-'.join(output_bucket.split('-')[1:-1]))
    # tibanna provides access to keys based on env and stuff like that
    tibanna = Tibanna(env, s3_keys=event.get('s3_keys'), ff_keys=event.get('ff_keys'),
                      settings=tibanna_settings)

    args = dict()

    # get argument format & type info from workflow
    workflow_info = ff_utils.get_metadata(workflow_uuid, key=tibanna.ff_keys)
    LOG.info("workflow info  %s" % workflow_info)
    if 'error' in workflow_info.get('@type', []):
        raise Exception("FATAL, can't lookup workflow info for % fourfront" % workflow_uuid)

    # get cwl info from workflow_info
    for k in ['app_name', 'app_version', 'cwl_directory_url', 'cwl_main_filename', 'cwl_child_filenames']:
        LOG.info(workflow_info.get(k))
        args[k] = workflow_info.get(k)

    # get format-extension map
    try:
        fp_schema = ff_utils.get_metadata("profiles/file_processed.json", key=tibanna.ff_keys)
        fe_map = fp_schema.get('file_format_file_extension')
    except Exception as e:
        LOG.error("Can't get format-extension map from file_processed schema. %s\n" % e)

    # processed file metadata
    output_files = []
    try:
        if 'arguments' in workflow_info:
            pf_meta = []
            for arg in workflow_info.get('arguments'):
                if (arg.get('argument_type') in
                   ['Output processed file', 'Output report file', 'Output QC file']):

                    of = dict()
                    of['workflow_argument_name'] = arg.get('workflow_argument_name')
                    of['type'] = arg.get('argument_type')
                    if 'argument_format' in arg:
                        # These are not processed files but report or QC files.
                        pf = ff_utils.ProcessedFileMetadata(file_format=arg.get('argument_format'))
                        try:
                            resp = pf.post(key=tibanna.ff_keys)  # actually post processed file metadata here
                            resp = resp.get('@graph')[0]
                            of['upload_key'] = resp.get('upload_key')
                            of['value'] = resp.get('uuid')
                        except Exception as e:
                            LOG.error("Failed to post Processed file metadata. %s\n" % e)
                            LOG.error("resp" + str(resp) + "\n")
                            raise e
                        of['format'] = arg.get('argument_format')
                        of['extension'] = fe_map.get(arg.get('argument_format'))
                        pf_meta.append(pf)
                    output_files.append(of)

    except Exception as e:
        LOG.error("output_files = " + str(output_files) + "\n")
        LOG.error("Can't prepare output_files information. %s\n" % e)
        raise e

    # create the ff_meta output info
    input_files = []
    for input_file in input_file_list:
        for idx, uuid in enumerate(ensure_list(input_file['uuid'])):
            input_files.append({'workflow_argument_name': input_file['workflow_argument_name'],
                                'value': uuid, 'ordinal': idx + 1})
    LOG.info("input_files is %s" % input_files)

    ff_meta = ff_utils.create_ffmeta_awsem(workflow_uuid, app_name, input_files,
                                           run_url=tibanna.settings.get('url', ''),
                                           output_files=output_files)

    LOG.info("ff_meta is %s" % ff_meta.__dict__)

    # store metadata so we know the run has started
    ff_meta.post_plain_wrf(key=tibanna.ff_keys)

    # input file args for awsem
    args['input_files'] = dict()
    for input_file in input_file_list:
        if isinstance(input_file['uuid'], unicode):
            input_file['uuid'] = input_file['uuid'].encode('utf-8')
        if isinstance(input_file['object_key'], unicode):
            input_file['object_key'] = input_file['object_key'].encode('utf-8')
        if isinstance(input_file['uuid'], str) and isinstance(input_file['object_key'], str):
            object_key = input_file['uuid'] + '/' + input_file['object_key']
        elif (isinstance(input_file['uuid'], list) and
              isinstance(input_file['object_key'], list) and
              len(input_file['uuid']) == len(input_file['object_key'])):

            object_key = [a + '/' + b for a, b in zip(input_file['uuid'], input_file['object_key'])]
        else:
            raise Exception("input_file uuid and object_key should match in their type and length (if lists) : " +
                            "type{}{} length{}{}".format(type(input_file['uuid']), type(input_file['object_key']),
                                                         len(input_file['uuid']), len(input_file['object_key'])))
        args['input_files'].update({input_file['workflow_argument_name']: {
                                    'bucket_name': input_file['bucket_name'],
                                    'object_key': object_key}})

    LOG.info("input_file_args is %s" % args['input_files'])
    args['secondary_files'] = dict()   # temporary, later fill in based on the attachment information

    # parameters
    args['input_parameters'] = event.get('parameters')

    # output target
    args['output_target'] = dict()
    for of in ff_meta.output_files:
        arg_name = of.get('workflow_argument_name')
        if of.get('type') == 'Output processed file':
            args['output_target'][arg_name] = of.get('upload_key')
        else:
            args['output_target'][arg_name] = ff_meta.uuid + '/' + arg_name

    # output bucket
    args['output_S3_bucket'] = event.get('output_bucket')

    # let's not pass keys in plain text parameters
    return {"input_file_args": input_file_list,
            "ff_meta": ff_meta.as_dict(),
            'pf_meta': [meta.as_dict() for meta in pf_meta],
            "_tibanna": tibanna.as_dict(),
            "config": event.get("config"),
            "args": args
            }
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# encoding: utf-8
#
# © 2017 Benjamin Mintz
# https://bmintz.mit-license.org/@2017
#

"""
techmeme.py: class that turns videos into dank technical may-mays
"""

from moviepy.video.compositing.CompositeVideoClip import CompositeVideoClip
from moviepy.video.io.VideoFileClip import VideoFileClip
from moviepy.video.fx.speedx import speedx

from .config import TechnicalMemeConfig

class TechnicalMeme(CompositeVideoClip):
	def __init__(self, *, source_filename, config):
		self.source_video = VideoFileClip(source_filename)
		self.config = config
		self._fix_up_timestamps()

		# write em up
		self._write_all_subclips()

	def _fix_up_timestamps(self):
		self.config.timestamps = [0] + self.config.timestamps + [self.source_video.end]

	def _get_subclip(self, timestamp_number):
		"""get a subclip at the timestamp given by self.config.timestamps[timestamp_number]"""

		if timestamp_number < len(self.config.timestamps):
			return self.source_video.subclip(*self.config.timestamps[timestamp_number:timestamp_number+2])
		else:
			raise ValueError("timestamp number too large")

	def _get_sped_up_subclip(self, timestamp_number):
		try:
			return self._get_subclip(timestamp_number)\
				.speedx(self.config.multiplier**timestamp_number)
		except ValueError:
			raise

	def _write_subclip(self, timestamp_number):
		try:
			self._get_sped_up_subclip(timestamp_number)\
				.write_videofile("TMP_techmeme_{}.mp4"
					.format(timestamp_number))
		except ValueError:
			raise

	def _write_all_subclips(self):
		for timestamp_number in range(len(self.config.timestamps)):
			try:
				self._write_subclip(timestamp_number)
			except ValueError:
				raise
<commit_msg>TechMeme: don't require the user to also import TechMemeConfig<commit_after>#!/usr/bin/env python3
# encoding: utf-8
#
# © 2017 Benjamin Mintz
# https://bmintz.mit-license.org/@2017
#

"""
techmeme.py: class that turns videos into dank technical may-mays
"""

from moviepy.video.compositing.CompositeVideoClip import CompositeVideoClip
from moviepy.video.io.VideoFileClip import VideoFileClip
from moviepy.video.fx.speedx import speedx

from .config import TechnicalMemeConfig

class TechnicalMeme(CompositeVideoClip):
	def __init__(self, *, source_filename, config_filename):
		self.source_video = VideoFileClip(source_filename)
		self.config = TechnicalMemeConfig(config_filename)
		self._fix_up_timestamps()

		# write em up
		self._write_all_subclips()

	def _fix_up_timestamps(self):
		self.config.timestamps = [0] + self.config.timestamps + [self.source_video.end]

	def _get_subclip(self, timestamp_number):
		"""get a subclip at the timestamp given by self.config.timestamps[timestamp_number]"""

		if timestamp_number < len(self.config.timestamps):
			return self.source_video.subclip(*self.config.timestamps[timestamp_number:timestamp_number+2])
		else:
			raise ValueError("timestamp number too large")

	def _get_sped_up_subclip(self, timestamp_number):
		try:
			return self._get_subclip(timestamp_number)\
				.speedx(self.config.multiplier**timestamp_number)
		except ValueError:
			raise

	def _write_subclip(self, timestamp_number):
		try:
			self._get_sped_up_subclip(timestamp_number)\
				.write_videofile("TMP_techmeme_{}.mp4"
					.format(timestamp_number))
		except ValueError:
			raise

	def _write_all_subclips(self):
		for timestamp_number in range(len(self.config.timestamps)):
			try:
				self._write_subclip(timestamp_number)
			except ValueError:
				raise
<|endoftext|>


<commit_before>import json
from django.core.cache import cache
from django.utils.translation import ugettext as _

from couchdbkit.exceptions import ResourceNotFound
from couchforms.analytics import get_form_analytics_metadata
from dimagi.utils.couch import get_cached_property, IncompatibleDocument, safe_index
from dimagi.utils.couch.database import get_db
from dimagi.utils.decorators.memoized import memoized

from corehq.apps.users.models import CouchUser
from corehq.const import USER_DATETIME_FORMAT_WITH_SEC
from corehq.util.dates import iso_string_to_datetime
from corehq.util.soft_assert import soft_assert
from corehq.util.timezones.conversions import ServerTime, PhoneTime
from corehq.util.view_utils import absolute_reverse

class StringWithAttributes(unicode):
    def replace(self, *args):
        string = super(StringWithAttributes, self).replace(*args)
        return StringWithAttributes(string)

class FormDisplay(object):
    def __init__(self, form_doc, report):
        self.form = form_doc
        self.report = report

    @property
    def form_data_link(self):
        return "%(text)s" % {
            "url": absolute_reverse('render_form_data', args=[self.report.domain, self.form['_id']]),
            "text": _("View Form")
        }

    @property
    def username(self):
        uid = self.form["form"]["meta"]["userID"]
        username = self.form["form"]["meta"].get("username")
        try:
            if username not in ['demo_user', 'admin']:
                full_name = get_cached_property(CouchUser, uid, 'full_name', expiry=7*24*60*60)
                name = '"%s"' % full_name if full_name else ""
            else:
                name = ""
        except (ResourceNotFound, IncompatibleDocument):
            name = "[unregistered]"
        return (username or _('No data for username')) + (" %s" % name if name else "")

    @property
    def submission_or_completion_time(self):
        time = iso_string_to_datetime(safe_index(self.form, self.report.time_field.split('.')))
        if self.report.by_submission_time:
            user_time = ServerTime(time).user_time(self.report.timezone)
        else:
            user_time = PhoneTime(time, self.report.timezone).user_time(self.report.timezone)
        return user_time.ui_string(USER_DATETIME_FORMAT_WITH_SEC)

    @property
    def readable_form_name(self):
        return xmlns_to_name(self.report.domain, self.form.get("xmlns"), app_id=self.form.get("app_id"))

    @property
    def other_columns(self):
        return [self.form["form"].get(field) for field in self.report.other_fields]

class _FormType(object):
    def __init__(self, domain, xmlns, app_id=None):
        self.domain = domain
        self.xmlns = xmlns
        if app_id:
            self.app_id = app_id
        else:
            try:
                form = get_form_analytics_metadata(domain, xmlns, app_id)
                self.app_id = form['app']['id']
            except Exception:
                self.app_id = {}

    @property
    @memoized
    def _metadata(self):
        try:
            return get_form_analytics_metadata(self.domain, self.xmlns, self.app_id)
        except Exception, e:
            _assert = soft_assert('{}@{}'.format('brudolph', 'dimagi.com'), notify_admins=True)
            _assert(False, 'Failed on domain: {} | xmlns: {} | app_id: {} | error: {}'.format(
                self.domain,
                self.xmlns,
                self.app_id,
                e,
            ))
            return None

    def get_label(self):
        if self._metadata:
            form = self._metadata
            if form.get('app'):
                langs = form['app']['langs']
                app_name = form['app']['name']
                module_name = form_name = None
                if form.get('is_user_registration'):
                    form_name = "User Registration"
                    title = "%s > %s" % (app_name, form_name)
                else:
                    for lang in langs + form['module']['name'].keys():
                        module_name = form['module']['name'].get(lang)
                        if module_name is not None:
                            break
                    for lang in langs + form['form']['name'].keys():
                        form_name = form['form']['name'].get(lang)
                        if form_name is not None:
                            break
                    if module_name is None:
                        module_name = "?"
                    if form_name is None:
                        form_name = "?"
                    title = "%s > %s > %s" % (app_name, module_name, form_name)

                if form.get('app_deleted'):
                    title += ' [Deleted]'
                if form.get('duplicate'):
                    title += " [Multiple Forms]"
                name = title
            else:
                name = self.xmlns
        else:
            name = self.xmlns
        return name

def xmlns_to_name(domain, xmlns, app_id):
    return _FormType(domain, xmlns, app_id).get_label()
<commit_msg>remove _FormType._metadata<commit_after>import json
from django.core.cache import cache
from django.utils.translation import ugettext as _

from couchdbkit.exceptions import ResourceNotFound
from couchforms.analytics import get_form_analytics_metadata
from dimagi.utils.couch import get_cached_property, IncompatibleDocument, safe_index
from dimagi.utils.couch.database import get_db
from dimagi.utils.decorators.memoized import memoized

from corehq.apps.users.models import CouchUser
from corehq.const import USER_DATETIME_FORMAT_WITH_SEC
from corehq.util.dates import iso_string_to_datetime
from corehq.util.soft_assert import soft_assert
from corehq.util.timezones.conversions import ServerTime, PhoneTime
from corehq.util.view_utils import absolute_reverse

class StringWithAttributes(unicode):
    def replace(self, *args):
        string = super(StringWithAttributes, self).replace(*args)
        return StringWithAttributes(string)

class FormDisplay(object):
    def __init__(self, form_doc, report):
        self.form = form_doc
        self.report = report

    @property
    def form_data_link(self):
        return "%(text)s" % {
            "url": absolute_reverse('render_form_data', args=[self.report.domain, self.form['_id']]),
            "text": _("View Form")
        }

    @property
    def username(self):
        uid = self.form["form"]["meta"]["userID"]
        username = self.form["form"]["meta"].get("username")
        try:
            if username not in ['demo_user', 'admin']:
                full_name = get_cached_property(CouchUser, uid, 'full_name', expiry=7*24*60*60)
                name = '"%s"' % full_name if full_name else ""
            else:
                name = ""
        except (ResourceNotFound, IncompatibleDocument):
            name = "[unregistered]"
        return (username or _('No data for username')) + (" %s" % name if name else "")

    @property
    def submission_or_completion_time(self):
        time = iso_string_to_datetime(safe_index(self.form, self.report.time_field.split('.')))
        if self.report.by_submission_time:
            user_time = ServerTime(time).user_time(self.report.timezone)
        else:
            user_time = PhoneTime(time, self.report.timezone).user_time(self.report.timezone)
        return user_time.ui_string(USER_DATETIME_FORMAT_WITH_SEC)

    @property
    def readable_form_name(self):
        return xmlns_to_name(self.report.domain, self.form.get("xmlns"), app_id=self.form.get("app_id"))

    @property
    def other_columns(self):
        return [self.form["form"].get(field) for field in self.report.other_fields]

class _FormType(object):
    def __init__(self, domain, xmlns, app_id=None):
        self.domain = domain
        self.xmlns = xmlns
        if app_id:
            self.app_id = app_id
        else:
            form = get_form_analytics_metadata(domain, xmlns, app_id)
            try:
                self.app_id = form['app']['id']
            except KeyError:
                self.app_id = {}

    def get_label(self):
        form = get_form_analytics_metadata(self.domain, self.xmlns, self.app_id)
        if form and form.get('app'):
            langs = form['app']['langs']
            app_name = form['app']['name']
            module_name = form_name = None
            if form.get('is_user_registration'):
                form_name = "User Registration"
                title = "%s > %s" % (app_name, form_name)
            else:
                for lang in langs + form['module']['name'].keys():
                    module_name = form['module']['name'].get(lang)
                    if module_name is not None:
                        break
                for lang in langs + form['form']['name'].keys():
                    form_name = form['form']['name'].get(lang)
                    if form_name is not None:
                        break
                if module_name is None:
                    module_name = "?"
                if form_name is None:
                    form_name = "?"
                title = "%s > %s > %s" % (app_name, module_name, form_name)

            if form.get('app_deleted'):
                title += ' [Deleted]'
            if form.get('duplicate'):
                title += " [Multiple Forms]"
            name = title
        else:
            name = self.xmlns
        return name

def xmlns_to_name(domain, xmlns, app_id):
    return _FormType(domain, xmlns, app_id).get_label()
<|endoftext|>


<commit_before>import hashlib
from sqlagg import SumWhen
import sqlalchemy
from django.conf import settings
from sqlalchemy.exc import IntegrityError
from corehq.db import Session
from corehq.apps.reports.sqlreport import DatabaseColumn
from dimagi.utils.decorators.memoized import memoized
from fluff.util import get_column_type

metadata = sqlalchemy.MetaData()

class IndicatorSqlAdapter(object):

    def __init__(self, engine, config):
        self.engine = engine
        self.config = config

    @memoized
    def get_table(self):
        return get_indicator_table(self.config)

    def rebuild_table(self):
        rebuild_table(self.engine, self.get_table())

    def drop_table(self):
        with self.engine.begin() as connection:
            self.get_table().drop(connection, checkfirst=True)

    def save(self, doc):
        indicator_rows = self.config.get_all_values(doc)
        if indicator_rows:
            table = self.get_table()
            for indicator_row in indicator_rows:
                with self.engine.begin() as connection:
                    # delete all existing rows for this doc to ensure we aren't left with stale data
                    delete = table.delete(table.c.doc_id == doc['_id'])
                    connection.execute(delete)
                    all_values = {i.column.id: i.value for i in indicator_row}
                    insert = table.insert().values(**all_values)
                    try:
                        connection.execute(insert)
                    except IntegrityError:
                        # Someone beat us to it. Concurrent inserts can happen
                        # when a doc is processed by the celery rebuild task
                        # at the same time as the pillow.
                        pass

    def delete(self, doc):
        table = self.get_table()
        with self.engine.begin() as connection:
            delete = table.delete(table.c.doc_id == doc['_id'])
            connection.execute(delete)

def get_indicator_table(indicator_config, custom_metadata=None):
    sql_columns = [column_to_sql(col) for col in indicator_config.get_columns()]
    table_name = get_table_name(indicator_config.domain, indicator_config.table_id)
    # todo: needed to add extend_existing=True to support multiple calls to this function for the same table.
    # is that valid?
    return sqlalchemy.Table(
        table_name,
        custom_metadata or metadata,
        extend_existing=True,
        *sql_columns
    )

def column_to_sql(column):
    return sqlalchemy.Column(
        column.id,
        get_column_type(column.datatype),
        nullable=column.is_nullable,
        primary_key=column.is_primary_key,
    )

def get_engine():
    return sqlalchemy.create_engine(settings.SQL_REPORTING_DATABASE_URL)

def rebuild_table(engine, table):
    with engine.begin() as connection:
        table.drop(connection, checkfirst=True)
        table.create(connection)
    engine.dispose()

def get_column_name(path):
    parts = path.split("/")

    def _hash(parts):
        front = "/".join(parts[:-1])
        end = parts[-1]
        return hashlib.sha1('{}_{}'.format(hashlib.sha1(front).hexdigest(), end)).hexdigest()[:8]

    return "_".join(parts + [_hash(parts)])

def get_table_name(domain, table_id):
    def _hash(domain, table_id):
        return hashlib.sha1('{}_{}'.format(hashlib.sha1(domain).hexdigest(), table_id)).hexdigest()[:8]

    return 'config_report_{0}_{1}_{2}'.format(domain, table_id, _hash(domain, table_id))

def get_expanded_columns(data_source_configuration, column_config):

    session = Session()
    try:
        connection = session.connection()
        table = get_indicator_table(data_source_configuration)
        column = table.c[column_config.field]

        query = sqlalchemy.select([column]).distinct()
        result = connection.execute(query).fetchall()
        distinct_values = [x[0] for x in result]
    except:
        session.rollback()
        raise

    columns = []
    for val in distinct_values:
        columns.append(DatabaseColumn(
            u"{}-{}".format(column_config.display, val),
            SumWhen(
                column_config.field,
                whens={val: 1},
                else_=0,
                alias=u"{}-{}".format(column_config.report_column_id, val),
            ),
            sortable=False,
            data_slug=u"{}-{}".format(column_config.report_column_id, val),
            format_fn=column_config.get_format_fn(),
            help_text=column_config.description
        ))
    return columns
<commit_msg>Close the session<commit_after>import hashlib
from sqlagg import SumWhen
import sqlalchemy
from django.conf import settings
from sqlalchemy.exc import IntegrityError
from corehq.db import Session
from corehq.apps.reports.sqlreport import DatabaseColumn
from dimagi.utils.decorators.memoized import memoized
from fluff.util import get_column_type

metadata = sqlalchemy.MetaData()

class IndicatorSqlAdapter(object):

    def __init__(self, engine, config):
        self.engine = engine
        self.config = config

    @memoized
    def get_table(self):
        return get_indicator_table(self.config)

    def rebuild_table(self):
        rebuild_table(self.engine, self.get_table())

    def drop_table(self):
        with self.engine.begin() as connection:
            self.get_table().drop(connection, checkfirst=True)

    def save(self, doc):
        indicator_rows = self.config.get_all_values(doc)
        if indicator_rows:
            table = self.get_table()
            for indicator_row in indicator_rows:
                with self.engine.begin() as connection:
                    # delete all existing rows for this doc to ensure we aren't left with stale data
                    delete = table.delete(table.c.doc_id == doc['_id'])
                    connection.execute(delete)
                    all_values = {i.column.id: i.value for i in indicator_row}
                    insert = table.insert().values(**all_values)
                    try:
                        connection.execute(insert)
                    except IntegrityError:
                        # Someone beat us to it. Concurrent inserts can happen
                        # when a doc is processed by the celery rebuild task
                        # at the same time as the pillow.
                        pass

    def delete(self, doc):
        table = self.get_table()
        with self.engine.begin() as connection:
            delete = table.delete(table.c.doc_id == doc['_id'])
            connection.execute(delete)

def get_indicator_table(indicator_config, custom_metadata=None):
    sql_columns = [column_to_sql(col) for col in indicator_config.get_columns()]
    table_name = get_table_name(indicator_config.domain, indicator_config.table_id)
    # todo: needed to add extend_existing=True to support multiple calls to this function for the same table.
    # is that valid?
    return sqlalchemy.Table(
        table_name,
        custom_metadata or metadata,
        extend_existing=True,
        *sql_columns
    )

def column_to_sql(column):
    return sqlalchemy.Column(
        column.id,
        get_column_type(column.datatype),
        nullable=column.is_nullable,
        primary_key=column.is_primary_key,
    )

def get_engine():
    return sqlalchemy.create_engine(settings.SQL_REPORTING_DATABASE_URL)

def rebuild_table(engine, table):
    with engine.begin() as connection:
        table.drop(connection, checkfirst=True)
        table.create(connection)
    engine.dispose()

def get_column_name(path):
    parts = path.split("/")

    def _hash(parts):
        front = "/".join(parts[:-1])
        end = parts[-1]
        return hashlib.sha1('{}_{}'.format(hashlib.sha1(front).hexdigest(), end)).hexdigest()[:8]

    return "_".join(parts + [_hash(parts)])

def get_table_name(domain, table_id):
    def _hash(domain, table_id):
        return hashlib.sha1('{}_{}'.format(hashlib.sha1(domain).hexdigest(), table_id)).hexdigest()[:8]

    return 'config_report_{0}_{1}_{2}'.format(domain, table_id, _hash(domain, table_id))

def get_expanded_columns(data_source_configuration, column_config):

    session = Session()
    try:
        connection = session.connection()
        table = get_indicator_table(data_source_configuration)
        column = table.c[column_config.field]

        query = sqlalchemy.select([column]).distinct()
        result = connection.execute(query).fetchall()
        distinct_values = [x[0] for x in result]
    except:
        session.rollback()
        raise
    finally:
        session.close()

    columns = []
    for val in distinct_values:
        columns.append(DatabaseColumn(
            u"{}-{}".format(column_config.display, val),
            SumWhen(
                column_config.field,
                whens={val: 1},
                else_=0,
                alias=u"{}-{}".format(column_config.report_column_id, val),
            ),
            sortable=False,
            data_slug=u"{}-{}".format(column_config.report_column_id, val),
            format_fn=column_config.get_format_fn(),
            help_text=column_config.description
        ))
    return columns
<|endoftext|>


<commit_before>from django.contrib import admin
from telegrambot.models import Message, Chat, Update, User, Bot

admin.site.register(Message)
admin.site.register(Chat)
admin.site.register(User)
admin.site.register(Update)
admin.site.register(Bot)<commit_msg>add AuthToken to admin<commit_after>from django.contrib import admin
from telegrambot.models import Message, Chat, Update, User, Bot, AuthToken

admin.site.register(Message)
admin.site.register(Chat)
admin.site.register(User)
admin.site.register(Update)
admin.site.register(Bot)
admin.site.register(AuthToken)<|endoftext|>


<commit_before>import logging
logger = logging.getLogger(__name__)
import struct

import bitcoin

def address_scriptpubkey(address):
    try:
        bech32 = bitcoin.bech32.CBech32Data(address)
        return b''.join([b'\x00\x14', bech32.to_bytes()])
    except Exception as e:
        bs58 = bitcoin.base58.decode(address)[1:-4]
        return b''.join([b'\x76\xa9\x14', bs58, b'\x88\xac'])

def pack(address):
    """
    Converts a base58 bitcoin address into a 21 byte bytes object
    """
    from .util import enabled # Here to account for test mock changes

    if enabled('segwit_support'):
        try:
            bech32 = bitcoin.bech32.CBech32Data(address)
            witver = (0x80 + bech32.witver).to_bytes(1, byteorder='big') # mark the first byte for segwit
            witprog = bech32.to_bytes()
            if len(witprog) > 20:
                raise Exception('p2wsh still not supported for sending')
            return b''.join([witver, witprog])
        except Exception as ne:
            try:
                script.validate(address) #This will check if the address is valid
                short_address_bytes = bitcoin.base58.decode(address)[:-4]
                return short_address_bytes
            except bitcoin.base58.InvalidBase58Error as e:
                raise e
            except Exception as e:
                raise Exception(('The address {} is not a valid bitcoin address ({})').format(address,'testnet' if config.TESTNET or config.REGTEST else 'mainnet'))
    else:
        try:
            short_address_bytes = bitcoin.base58.decode(address)[:-4]
            return short_address_bytes
        except bitcoin.base58.InvalidBase58Error as e:
            raise e

# retuns both the message type id and the remainder of the message data
def unpack(short_address_bytes):
    """
    Converts a 21 byte prefix and public key hash into a full base58 bitcoin address
    """
    from .util import enabled # Here to account for test mock changes

    if enabled('segwit_support') and short_address_bytes[0] >= 0x80 and short_address_bytes[0] <= 0x8F:
        # we have a segwit address here
        witver = short_address_bytes[0] - 0x80
        witprog = short_address_bytes[1:]
        return str(bitcoin.bech32.CBech32Data.from_bytes(witver, witprog))
    else:
        check = bitcoin.core.Hash(short_address_bytes)[0:4]
        return bitcoin.base58.encode(short_address_bytes + check)
<commit_msg>- Added config import to address.py<commit_after>import logging
logger = logging.getLogger(__name__)
import struct

import bitcoin
from counterpartylib.lib import config

def address_scriptpubkey(address):
    try:
        bech32 = bitcoin.bech32.CBech32Data(address)
        return b''.join([b'\x00\x14', bech32.to_bytes()])
    except Exception as e:
        bs58 = bitcoin.base58.decode(address)[1:-4]
        return b''.join([b'\x76\xa9\x14', bs58, b'\x88\xac'])

def pack(address):
    """
    Converts a base58 bitcoin address into a 21 byte bytes object
    """
    from .util import enabled # Here to account for test mock changes

    if enabled('segwit_support'):
        try:
            bech32 = bitcoin.bech32.CBech32Data(address)
            witver = (0x80 + bech32.witver).to_bytes(1, byteorder='big') # mark the first byte for segwit
            witprog = bech32.to_bytes()
            if len(witprog) > 20:
                raise Exception('p2wsh still not supported for sending')
            return b''.join([witver, witprog])
        except Exception as ne:
            try:
                script.validate(address) #This will check if the address is valid
                short_address_bytes = bitcoin.base58.decode(address)[:-4]
                return short_address_bytes
            except bitcoin.base58.InvalidBase58Error as e:
                raise e
            except Exception as e:
                raise Exception(('The address {} is not a valid bitcoin address ({})').format(address,'testnet' if config.TESTNET or config.REGTEST else 'mainnet'))
    else:
        try:
            short_address_bytes = bitcoin.base58.decode(address)[:-4]
            return short_address_bytes
        except bitcoin.base58.InvalidBase58Error as e:
            raise e

# retuns both the message type id and the remainder of the message data
def unpack(short_address_bytes):
    """
    Converts a 21 byte prefix and public key hash into a full base58 bitcoin address
    """
    from .util import enabled # Here to account for test mock changes

    if enabled('segwit_support') and short_address_bytes[0] >= 0x80 and short_address_bytes[0] <= 0x8F:
        # we have a segwit address here
        witver = short_address_bytes[0] - 0x80
        witprog = short_address_bytes[1:]
        return str(bitcoin.bech32.CBech32Data.from_bytes(witver, witprog))
    else:
        check = bitcoin.core.Hash(short_address_bytes)[0:4]
        return bitcoin.base58.encode(short_address_bytes + check)
<|endoftext|>


<commit_before>from datetime import datetime
import logging
from time import time

from flowpipe import logger
from flowpipe.node import INode, function_to_node
from flowpipe.plug import InputPlug, OutputPlug
from flowpipe.graph import Graph

@function_to_node(outputs=['time'])
def CurrentTime():
    return {'time': time()}

class ConvertTime(INode):

    def __init__(self, time=None, timezone=0, city=None):
        super(ConvertTime, self).__init__()
        InputPlug('time', self)
        InputPlug('timezone', self, timezone)
        InputPlug('city', self, city)
        OutputPlug('converted_time', self)

    def compute(self, time, timezone, city):
        return {'converted_time': [time + timezone * 60 * 60, city]}

@function_to_node()
def WorldClock(time1, time2, time3):
    print('-- World Clock -------------------')
    print('It is now {0} in {1}'.format(
        datetime.fromtimestamp(time1[0]).strftime("%H:%M"), time1[1]))
    print('It is now {0} in {1}'.format(
        datetime.fromtimestamp(time2[0]).strftime("%H:%M"), time2[1]))
    print('It is now {0} in {1}'.format(
        datetime.fromtimestamp(time3[0]).strftime("%H:%M"), time3[1]))
    print('----------------------------------')

def test_a():
    """@todo documentation for test_a."""
    logger.setLevel(logging.DEBUG)

    current_time = CurrentTime()
    van = ConvertTime(city='Vancouver', timezone=-8)
    ldn = ConvertTime(city='London', timezone=0)
    muc = ConvertTime(city='Munich', timezone=1)
    world_clock = WorldClock()

    current_time.outputs['time'] >> van.inputs['time']
    current_time.outputs['time'] >> ldn.inputs['time']
    current_time.outputs['time'] >> muc.inputs['time']
    van.outputs['converted_time'] >> world_clock.inputs['time1']
    ldn.outputs['converted_time'] >> world_clock.inputs['time2']
    muc.outputs['converted_time'] >> world_clock.inputs['time3']

    graph = Graph(name="WorldClockGraph", nodes=[current_time, van, ldn, muc, world_clock])

    # print(graph)

    graph.evaluate()
<commit_msg>Removing unnecessary file<commit_after><|endoftext|>


<commit_before># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import sys
import shutil
import tempfile
from functools import wraps
from nose.plugins.skip import SkipTest

if sys.version_info[0:2] < (2, 7):
    import unittest2 as unittest
else:
    import unittest

def skip_on_ci(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        if "BUILDBOT_UUID" in os.environ or "TRAVIS" in os.environ:
            raise SkipTest
        return func(*args, **kwargs)
    return wrapper

class TestCase(unittest.TestCase):
    TEMPDIR_PREFIX = ""
    BUILDBOT_UUID = os.environ.get("BUILDBOT_UUID")
    ORIGINAL_ENVIRONMENT = {}
    temp_directories = set()

    @classmethod
    def remove(cls, path):
        assert isinstance(path, basestring), "expected a string for `path`"

        if os.path.isfile(path):
            delete = os.remove
        elif os.path.isdir(path):
            delete = shutil.rmtree
        else:
            delete = lambda path: None

        # delete the path
        try:
            delete(path)

        except (OSError, IOError):
            pass

        else:
            if path in cls.temp_directories:
                cls.temp_directories.remove(path)

    @classmethod
    def setUpClass(cls):
        cls.ORIGINAL_ENVIRONMENT = os.environ.copy()

    @classmethod
    def mktempdir(cls):
        tempdir = tempfile.mkdtemp(prefix=cls.TEMPDIR_PREFIX)
        cls.temp_directories.add(tempdir)
        return tempdir

    def setUp(self):
        self.tempdir = self.mktempdir()
        os.environ.clear()
        os.environ.update(self.ORIGINAL_ENVIRONMENT)

    def tearDown(self):
        self.remove(self.tempdir)
        map(self.remove, self.temp_directories.copy())<commit_msg>silence root logger<commit_after># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import sys
import shutil
import tempfile
from functools import wraps
from nose.plugins.skip import SkipTest

if sys.version_info[0:2] < (2, 7):
    import unittest2 as unittest
else:
    import unittest

def skip_on_ci(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        if "BUILDBOT_UUID" in os.environ or "TRAVIS" in os.environ:
            raise SkipTest
        return func(*args, **kwargs)
    return wrapper

# silence PyFarm's logger
try:
    from logging import NullHandler
except ImportError:
    from pyfarm.core.backports import NullHandler

from pyfarm.core.logger import root as pyfarm_logger
del pyfarm_logger.handlers[:]
pyfarm_logger.addHandler(NullHandler())

class TestCase(unittest.TestCase):
    TEMPDIR_PREFIX = ""
    BUILDBOT_UUID = os.environ.get("BUILDBOT_UUID")
    ORIGINAL_ENVIRONMENT = {}
    temp_directories = set()

    @classmethod
    def remove(cls, path):
        assert isinstance(path, basestring), "expected a string for `path`"

        if os.path.isfile(path):
            delete = os.remove
        elif os.path.isdir(path):
            delete = shutil.rmtree
        else:
            delete = lambda path: None

        # delete the path
        try:
            delete(path)

        except (OSError, IOError):
            pass

        else:
            if path in cls.temp_directories:
                cls.temp_directories.remove(path)

    @classmethod
    def setUpClass(cls):
        cls.ORIGINAL_ENVIRONMENT = os.environ.copy()

    @classmethod
    def mktempdir(cls):
        tempdir = tempfile.mkdtemp(prefix=cls.TEMPDIR_PREFIX)
        cls.temp_directories.add(tempdir)
        return tempdir

    def setUp(self):
        self.tempdir = self.mktempdir()
        os.environ.clear()
        os.environ.update(self.ORIGINAL_ENVIRONMENT)

    def tearDown(self):
        self.remove(self.tempdir)
        map(self.remove, self.temp_directories.copy())<|endoftext|>


<commit_before># Copyright (C) 2010 Samuel Abels.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2, as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
import re
from simpleparse.dispatchprocessor import DispatchProcessor, getString, singleMap
from Function                      import Function
from Grammar                       import Grammar
from WhenStatement                 import WhenStatement
from MatchStatement                import MatchStatement
from MatchFieldList                import MatchFieldList
from MatchList                     import MatchList
from Number                        import Number
from Regex                         import Regex
from String                        import String
from Context                       import Context

"""
Indent handling:
    o NEWLINE:
        - If the amount of indent matches the previous line, parse the \n
          and skip all indent.
        - If the amount of indent does NOT match the previous line, parse
          the \n and stay at the beginning of the new line to let INDENT
          or DEDENT figure it out.
    o INDENT: Skips all indent, then looks backward to update the indent
      count. Checks to make sure that the indent was increased.
    o DEDENT: Like INDENT, except it does not check for errors.
"""
class SyntaxCompiler(DispatchProcessor):
    """
    Processor sub-class defining processing functions for the productions.
    """
    def __init__(self):
        self.context = None

    def reset(self):
        self.context = Context()

    def _regex(self, (tag, left, right, sublist), buffer):
        regex      = Regex()
        regex.data = getString(sublist[0], buffer)
        return regex

    def _string(self, (tag, left, right, sublist), buffer):
        string = getString(sublist[0], buffer)
        return String(self.context, string)

    def _varname(self, token, buffer):
        varname = getString(token, buffer)
        return self.context.lexicon[varname]

    def _number(self, token, buffer):
        number = getString(token, buffer)
        return Number(int(number))

    def _expression(self, token, buffer):
        tag = token[0]
        if tag == 'string':
            return self._string(token, buffer)
        elif tag == 'regex':
            return self._regex(token, buffer)
        elif tag == 'varname':
            return self._varname(token, buffer)
        elif tag == 'number':
            return self._number(token, buffer)
        else:
            raise Exception('BUG: invalid token %s' % tag)

    def _match_field_list(self, (tag, left, right, sublist), buffer, flags):
        field_list = MatchFieldList(flags)
        for field in sublist:
            expression = self._expression(field, buffer)
            field_list.expressions.append(expression)
        return field_list

    def _match_list(self, (tag, left, right, sublist), buffer, flags):
        matchlist = MatchList()
        for field_list in sublist:
            field_list = self._match_field_list(field_list, buffer, flags)
            matchlist.field_lists.append(field_list)
        return matchlist

    def _match_stmt(self, (tag, left, right, sublist), buffer, flags = 0):
        matcher            = MatchStatement()
        matcher.matchlist  = self._match_list(sublist[0], buffer, flags)
        matcher.statements = self._suite(sublist[1], buffer)
        return matcher

    def _when_stmt(self, (tag, left, right, sublist), buffer):
        matcher            = WhenStatement()
        matcher.matchlist  = self._match_list(sublist[0], buffer, None)
        matcher.statements = self._suite(sublist[1], buffer)
        return matcher

    def _function(self, (tag, left, right, sublist), buffer):
        function      = Function()
        function.name = getString(sublist[0], buffer)
        if len(sublist) == 1:
            return function
        for arg in sublist[1][3]:
            expression = self._expression(arg, buffer)
            function.args.append(expression)
        return function

    def _inherit(self, (tag, left, right, sublist), buffer):
        return getString(sublist[0], buffer)

    def _suite(self, (tag, left, right, sublist), buffer):
        statements = []
        for token in sublist:
            tag = token[0]
            if tag == 'match_stmt':
                statement = self._match_stmt(token, buffer)
            elif tag == 'imatch_stmt':
                statement = self._match_stmt(token, buffer, re.I)
            elif tag == 'when_stmt':
                statement = self._when_stmt(token, buffer)
            elif tag == 'function':
                statement = self._function(token, buffer)
            else:
                raise Exception('BUG: invalid token %s' % tag)
            statements.append(statement)
        return statements

    def define_stmt(self, (tag, left, right, sublist), buffer):
        name_tup, value_tup = sublist
        value_tag           = value_tup[0]
        name                = getString(name_tup,   buffer)
        value               = getString(value_tup,  buffer)
        if value_tag == 'regex':
            value = self._regex(value_tup, buffer)
        elif value_tag == 'varname':
            if not self.context.lexicon.has_key(value):
                _error(buffer, value_tup[1], 'no such variable')
            value = self.context.lexicon[value]
        else:
            raise Exception('BUG: invalid token %s' % value_tag)
        self.context.lexicon[name] = value

    def grammar_stmt(self, (tag, left, right, sublist), buffer):
        map                = singleMap(sublist)
        grammar            = Grammar()
        grammar.name       = getString(map['varname'], buffer)
        grammar.statements = self._suite(map['suite'], buffer)
        if map.has_key('inherit'):
            grammar.inherit = self._inherit(map['inherit'], buffer)
        self.context.grammars[grammar.name] = grammar
<commit_msg>fix: last commit broke the 'when' statement.<commit_after># Copyright (C) 2010 Samuel Abels.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2, as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
import re
from simpleparse.dispatchprocessor import DispatchProcessor, getString, singleMap
from Function                      import Function
from Grammar                       import Grammar
from WhenStatement                 import WhenStatement
from MatchStatement                import MatchStatement
from MatchFieldList                import MatchFieldList
from MatchList                     import MatchList
from Number                        import Number
from Regex                         import Regex
from String                        import String
from Context                       import Context

"""
Indent handling:
    o NEWLINE:
        - If the amount of indent matches the previous line, parse the \n
          and skip all indent.
        - If the amount of indent does NOT match the previous line, parse
          the \n and stay at the beginning of the new line to let INDENT
          or DEDENT figure it out.
    o INDENT: Skips all indent, then looks backward to update the indent
      count. Checks to make sure that the indent was increased.
    o DEDENT: Like INDENT, except it does not check for errors.
"""
class SyntaxCompiler(DispatchProcessor):
    """
    Processor sub-class defining processing functions for the productions.
    """
    def __init__(self):
        self.context = None

    def reset(self):
        self.context = Context()

    def _regex(self, (tag, left, right, sublist), buffer):
        regex      = Regex()
        regex.data = getString(sublist[0], buffer)
        return regex

    def _string(self, (tag, left, right, sublist), buffer):
        string = getString(sublist[0], buffer)
        return String(self.context, string)

    def _varname(self, token, buffer):
        varname = getString(token, buffer)
        return self.context.lexicon[varname]

    def _number(self, token, buffer):
        number = getString(token, buffer)
        return Number(int(number))

    def _expression(self, token, buffer):
        tag = token[0]
        if tag == 'string':
            return self._string(token, buffer)
        elif tag == 'regex':
            return self._regex(token, buffer)
        elif tag == 'varname':
            return self._varname(token, buffer)
        elif tag == 'number':
            return self._number(token, buffer)
        else:
            raise Exception('BUG: invalid token %s' % tag)

    def _match_field_list(self, (tag, left, right, sublist), buffer, flags):
        field_list = MatchFieldList(flags)
        for field in sublist:
            expression = self._expression(field, buffer)
            field_list.expressions.append(expression)
        return field_list

    def _match_list(self, (tag, left, right, sublist), buffer, flags):
        matchlist = MatchList()
        for field_list in sublist:
            field_list = self._match_field_list(field_list, buffer, flags)
            matchlist.field_lists.append(field_list)
        return matchlist

    def _match_stmt(self, (tag, left, right, sublist), buffer, flags = 0):
        matcher            = MatchStatement()
        matcher.matchlist  = self._match_list(sublist[0], buffer, flags)
        matcher.statements = self._suite(sublist[1], buffer)
        return matcher

    def _when_stmt(self, (tag, left, right, sublist), buffer, flags = 0):
        matcher            = WhenStatement()
        matcher.matchlist  = self._match_list(sublist[0], buffer, flags)
        matcher.statements = self._suite(sublist[1], buffer)
        return matcher

    def _function(self, (tag, left, right, sublist), buffer):
        function      = Function()
        function.name = getString(sublist[0], buffer)
        if len(sublist) == 1:
            return function
        for arg in sublist[1][3]:
            expression = self._expression(arg, buffer)
            function.args.append(expression)
        return function

    def _inherit(self, (tag, left, right, sublist), buffer):
        return getString(sublist[0], buffer)

    def _suite(self, (tag, left, right, sublist), buffer):
        statements = []
        for token in sublist:
            tag = token[0]
            if tag == 'match_stmt':
                statement = self._match_stmt(token, buffer)
            elif tag == 'imatch_stmt':
                statement = self._match_stmt(token, buffer, re.I)
            elif tag == 'when_stmt':
                statement = self._when_stmt(token, buffer)
            elif tag == 'function':
                statement = self._function(token, buffer)
            else:
                raise Exception('BUG: invalid token %s' % tag)
            statements.append(statement)
        return statements

    def define_stmt(self, (tag, left, right, sublist), buffer):
        name_tup, value_tup = sublist
        value_tag           = value_tup[0]
        name                = getString(name_tup,   buffer)
        value               = getString(value_tup,  buffer)
        if value_tag == 'regex':
            value = self._regex(value_tup, buffer)
        elif value_tag == 'varname':
            if not self.context.lexicon.has_key(value):
                _error(buffer, value_tup[1], 'no such variable')
            value = self.context.lexicon[value]
        else:
            raise Exception('BUG: invalid token %s' % value_tag)
        self.context.lexicon[name] = value

    def grammar_stmt(self, (tag, left, right, sublist), buffer):
        map                = singleMap(sublist)
        grammar            = Grammar()
        grammar.name       = getString(map['varname'], buffer)
        grammar.statements = self._suite(map['suite'], buffer)
        if map.has_key('inherit'):
            grammar.inherit = self._inherit(map['inherit'], buffer)
        self.context.grammars[grammar.name] = grammar
<|endoftext|>