<commit_before>import unittest, sys
from lxml.tests.common_imports import doctest, make_doctest, HelperTestCase
from lxml import html
from lxml import cssselect
import os

doc_fn = os.path.join(os.path.dirname(__file__),
                      'css_shakespear.html')

try:
    basestring
except NameError:
    basestring = (str, bytes)

# Data borrowed from http://mootools.net/slickspeed/

class CSSTestCase(HelperTestCase):

    selectors = [
        ## Changed from original; probably because I'm only searching the body
        #('*', 252),
        ('*', 246),
        ('div:only-child', 22), # ?
        ## Changed from original, because the original doesn't make sense.
        ## There really aren't that many occurrances of 'celia'
        #('div:contains(CELIA)', 243),
        ('div:contains(CELIA)', 30),
        ('div:nth-child(even)', 106),
        ('div:nth-child(2n)', 106),
        ('div:nth-child(odd)', 137),
        ('div:nth-child(2n+1)', 137),
        ('div:nth-child(n)', 243),
        ('div:last-child', 53),
        ('div:first-child', 51),
        ('div > div', 242),
        ('div + div', 190),
        ('div ~ div', 190),
        ('body', 1),
        ('body div', 243),
        ('div', 243),
        ('div div', 242),
        ('div div div', 241),
        ('div, div, div', 243),
        ('div, a, span', 243),
        ('.dialog', 51),
        ('div.dialog', 51),
        ('div .dialog', 51),
        ('div.character, div.dialog', 99),
        ('div.direction.dialog', 0),
        ('div.dialog.direction', 0),
        ('div.dialog.scene', 1),
        ('div.scene.scene', 1),
        ('div.scene .scene', 0),
        ('div.direction .dialog ', 0),
        ('div .dialog .direction', 4),
        ('div.dialog .dialog .direction', 4),
        ('#speech5', 1),
        ('div#speech5', 1),
        ('div #speech5', 1),
        ('div.scene div.dialog', 49),
        ('div#scene1 div.dialog div', 142),
        ('#scene1 #speech1', 1),
        ('div[class]', 103),
        ('div[class=dialog]', 50),
        ('div[class^=dia]', 51),
        ('div[class$=log]', 50),
        ('div[class*=sce]', 1),
        ('div[class|=dialog]', 50), # ? Seems right
        ('div[class!=madeup]', 243), # ? Seems right
        ('div[class~=dialog]', 51), # ? Seems right
        ]

    def __init__(self, index):
        self.index = index
        super(HelperTestCase, self).__init__()

    def all(cls):
        for i in range(len(cls.selectors)):
            yield cls(i)
    all = classmethod(all)

    def runTest(self):
        f = open(doc_fn, 'rb')
        c = f.read()
        f.close()
        doc = html.document_fromstring(c)
        body = doc.xpath('//body')[0]
        bad = []
        selector, count = self.selectors[self.index]
        xpath = cssselect.css_to_xpath(cssselect.parse(selector))
        try:
            results = body.xpath(xpath)
        except Exception:
            e = sys.exc_info()[1]
            e.args = ("%s for xpath %r" % (e, xpath))
            raise
        found = {}
        for item in results:
            if item in found:
                assert 0, (
                    "Element shows up multiple times: %r" % item)
            found[item] = None
        if isinstance(results, basestring):
            assert 0, (
                "Got string result (%r), not element, for xpath %r"
                % (results[:20], str(xpath)))
        if len(results) != count:
            #if self.shortDescription() == 'div.character, div.dialog':
            #    import pdb; pdb.set_trace()
            assert 0, (
                "Did not get expected results (%s) instead %s for xpath %r"
                % (count, len(results), str(xpath)))

    def shortDescription(self):
        return self.selectors[self.index][0]

def unique(s):
    found = {}
    result = []
    for item in s:
        if item in found:
            continue
        found[item] = None
        result.append(s)
    return result

def test_suite():
    suite = unittest.TestSuite()
    if sys.version_info >= (2,4):
        suite.addTests([make_doctest('test_css_select.txt')])
    suite.addTests([make_doctest('test_css.txt')])
    suite.addTests(doctest.DocTestSuite(cssselect))
    suite.addTests(list(CSSTestCase.all()))
    return suite
<commit_msg>e.args is intended to be a tuple with a single item.<commit_after>import unittest, sys
from lxml.tests.common_imports import doctest, make_doctest, HelperTestCase
from lxml import html
from lxml import cssselect
import os

doc_fn = os.path.join(os.path.dirname(__file__),
                      'css_shakespear.html')

try:
    basestring
except NameError:
    basestring = (str, bytes)

# Data borrowed from http://mootools.net/slickspeed/

class CSSTestCase(HelperTestCase):

    selectors = [
        ## Changed from original; probably because I'm only searching the body
        #('*', 252),
        ('*', 246),
        ('div:only-child', 22), # ?
        ## Changed from original, because the original doesn't make sense.
        ## There really aren't that many occurrances of 'celia'
        #('div:contains(CELIA)', 243),
        ('div:contains(CELIA)', 30),
        ('div:nth-child(even)', 106),
        ('div:nth-child(2n)', 106),
        ('div:nth-child(odd)', 137),
        ('div:nth-child(2n+1)', 137),
        ('div:nth-child(n)', 243),
        ('div:last-child', 53),
        ('div:first-child', 51),
        ('div > div', 242),
        ('div + div', 190),
        ('div ~ div', 190),
        ('body', 1),
        ('body div', 243),
        ('div', 243),
        ('div div', 242),
        ('div div div', 241),
        ('div, div, div', 243),
        ('div, a, span', 243),
        ('.dialog', 51),
        ('div.dialog', 51),
        ('div .dialog', 51),
        ('div.character, div.dialog', 99),
        ('div.direction.dialog', 0),
        ('div.dialog.direction', 0),
        ('div.dialog.scene', 1),
        ('div.scene.scene', 1),
        ('div.scene .scene', 0),
        ('div.direction .dialog ', 0),
        ('div .dialog .direction', 4),
        ('div.dialog .dialog .direction', 4),
        ('#speech5', 1),
        ('div#speech5', 1),
        ('div #speech5', 1),
        ('div.scene div.dialog', 49),
        ('div#scene1 div.dialog div', 142),
        ('#scene1 #speech1', 1),
        ('div[class]', 103),
        ('div[class=dialog]', 50),
        ('div[class^=dia]', 51),
        ('div[class$=log]', 50),
        ('div[class*=sce]', 1),
        ('div[class|=dialog]', 50), # ? Seems right
        ('div[class!=madeup]', 243), # ? Seems right
        ('div[class~=dialog]', 51), # ? Seems right
        ]

    def __init__(self, index):
        self.index = index
        super(HelperTestCase, self).__init__()

    def all(cls):
        for i in range(len(cls.selectors)):
            yield cls(i)
    all = classmethod(all)

    def runTest(self):
        f = open(doc_fn, 'rb')
        c = f.read()
        f.close()
        doc = html.document_fromstring(c)
        body = doc.xpath('//body')[0]
        bad = []
        selector, count = self.selectors[self.index]
        xpath = cssselect.css_to_xpath(cssselect.parse(selector))
        try:
            results = body.xpath(xpath)
        except Exception:
            e = sys.exc_info()[1]
            e.args = ("%s for xpath %r" % (e, xpath),)
            raise
        found = {}
        for item in results:
            if item in found:
                assert 0, (
                    "Element shows up multiple times: %r" % item)
            found[item] = None
        if isinstance(results, basestring):
            assert 0, (
                "Got string result (%r), not element, for xpath %r"
                % (results[:20], str(xpath)))
        if len(results) != count:
            #if self.shortDescription() == 'div.character, div.dialog':
            #    import pdb; pdb.set_trace()
            assert 0, (
                "Did not get expected results (%s) instead %s for xpath %r"
                % (count, len(results), str(xpath)))

    def shortDescription(self):
        return self.selectors[self.index][0]

def unique(s):
    found = {}
    result = []
    for item in s:
        if item in found:
            continue
        found[item] = None
        result.append(s)
    return result

def test_suite():
    suite = unittest.TestSuite()
    if sys.version_info >= (2,4):
        suite.addTests([make_doctest('test_css_select.txt')])
    suite.addTests([make_doctest('test_css.txt')])
    suite.addTests(doctest.DocTestSuite(cssselect))
    suite.addTests(list(CSSTestCase.all()))
    return suite
<|endoftext|>


<commit_before>import flask
import flask.ext.pymongo
import util
import warnings

import pymongo

class FlaskPyMongoConfigTest(util.FlaskRequestTest):

    def setUp(self):
        self.app = flask.Flask('test')
        self.context = self.app.test_request_context('/')
        self.context.push()

    def tearDown(self):
        self.context.pop()

    def test_default_config_prefix(self):
        self.app.config['MONGO_DBNAME'] = 'flask_pymongo_test_db'
        self.app.config['MONGO_HOST'] = 'localhost'
        self.app.config['MONGO_PORT'] = 27017

        mongo = flask.ext.pymongo.PyMongo(self.app)
        assert mongo.db.name == 'flask_pymongo_test_db', 'wrong dbname: %s' % mongo.db.name
        assert mongo.cx.host == 'localhost'
        assert mongo.cx.port == 27017

    def test_custom_config_prefix(self):
        self.app.config['CUSTOM_DBNAME'] = 'flask_pymongo_test_db'
        self.app.config['CUSTOM_HOST'] = 'localhost'
        self.app.config['CUSTOM_PORT'] = 27017

        mongo = flask.ext.pymongo.PyMongo(self.app, 'CUSTOM')
        assert mongo.db.name == 'flask_pymongo_test_db', 'wrong dbname: %s' % mongo.db.name
        assert mongo.cx.host == 'localhost'
        assert mongo.cx.port == 27017

    def test_converts_str_to_int(self):
        self.app.config['MONGO_DBNAME'] = 'flask_pymongo_test_db'
        self.app.config['MONGO_HOST'] = 'localhost'
        self.app.config['MONGO_PORT'] = '27017'

        mongo = flask.ext.pymongo.PyMongo(self.app)
        assert mongo.db.name == 'flask_pymongo_test_db', 'wrong dbname: %s' % mongo.db.name
        assert mongo.cx.host == 'localhost'
        assert mongo.cx.port == 27017

    def test_rejects_invalid_string(self):
        self.app.config['MONGO_PORT'] = '27017x'

        self.assertRaises(TypeError, flask.ext.pymongo.PyMongo, self.app)

    def test_sets_after_request_in_21(self):
        # fake PyMongo version 2.1; 2.2 uses auto_start_request
        original_tuple = pymongo.version_tuple
        original_version = pymongo.version

        pymongo.version_tuple = (2, 1)
        pymongo.version = '2.1'

        flask.ext.pymongo.PyMongo(self.app)

        pymongo.version_tuple = original_tuple
        pymongo.version = original_version

        print pymongo
        assert len(self.app.after_request_funcs.get(None, [])) == 1, 'did not set after_request handler'

    def test_does_not_set_after_request_in_22(self):
        # fake PyMongo version 2.2, which uses auto_start_request
        original_tuple = pymongo.version_tuple
        original_version = pymongo.version

        pymongo.version_tuple = (2, 2)
        pymongo.version = '2.2'

        flask.ext.pymongo.PyMongo(self.app)

        pymongo.version_tuple = original_tuple
        pymongo.version = original_version

        assert None not in self.app.after_request_funcs, 'set unneeded after_request handler'

    def test_multiple_pymongos(self):
        for prefix in ('ONE', 'TWO'):
            self.app.config['%s_DBNAME' % prefix] = prefix

        for prefix in ('ONE', 'TWO'):
            flask.ext.pymongo.PyMongo(self.app, config_prefix=prefix)

        # this test passes if it raises no exceptions

    def test_config_with_uri(self):
        self.app.config['MONGO_URI'] = 'mongodb://localhost:27017/flask_pymongo_test_db'

        with warnings.catch_warnings():
            # URI connections without a username and password
            # work, but warn that auth should be supplied
            warnings.simplefilter('ignore')
            mongo = flask.ext.pymongo.PyMongo(self.app)
        assert mongo.db.name == 'flask_pymongo_test_db', 'wrong dbname: %s' % mongo.db.name
        assert mongo.cx.host == 'localhost'
        assert mongo.cx.port == 27017

    def test_config_with_uri_no_port(self):
        self.app.config['MONGO_URI'] = 'mongodb://localhost/flask_pymongo_test_db'

        with warnings.catch_warnings():
            # URI connections without a username and password
            # work, but warn that auth should be supplied
            warnings.simplefilter('ignore')
            mongo = flask.ext.pymongo.PyMongo(self.app)
        assert mongo.db.name == 'flask_pymongo_test_db', 'wrong dbname: %s' % mongo.db.name
        assert mongo.cx.host == 'localhost'
        assert mongo.cx.port == 27017
<commit_msg>Remove redundant tests<commit_after>import flask
import flask.ext.pymongo
import util
import warnings

class FlaskPyMongoConfigTest(util.FlaskRequestTest):

    def setUp(self):
        self.app = flask.Flask('test')
        self.context = self.app.test_request_context('/')
        self.context.push()

    def tearDown(self):
        self.context.pop()

    def test_default_config_prefix(self):
        self.app.config['MONGO_DBNAME'] = 'flask_pymongo_test_db'
        self.app.config['MONGO_HOST'] = 'localhost'
        self.app.config['MONGO_PORT'] = 27017

        mongo = flask.ext.pymongo.PyMongo(self.app)
        assert mongo.db.name == 'flask_pymongo_test_db', 'wrong dbname: %s' % mongo.db.name
        assert mongo.cx.host == 'localhost'
        assert mongo.cx.port == 27017

    def test_custom_config_prefix(self):
        self.app.config['CUSTOM_DBNAME'] = 'flask_pymongo_test_db'
        self.app.config['CUSTOM_HOST'] = 'localhost'
        self.app.config['CUSTOM_PORT'] = 27017

        mongo = flask.ext.pymongo.PyMongo(self.app, 'CUSTOM')
        assert mongo.db.name == 'flask_pymongo_test_db', 'wrong dbname: %s' % mongo.db.name
        assert mongo.cx.host == 'localhost'
        assert mongo.cx.port == 27017

    def test_converts_str_to_int(self):
        self.app.config['MONGO_DBNAME'] = 'flask_pymongo_test_db'
        self.app.config['MONGO_HOST'] = 'localhost'
        self.app.config['MONGO_PORT'] = '27017'

        mongo = flask.ext.pymongo.PyMongo(self.app)
        assert mongo.db.name == 'flask_pymongo_test_db', 'wrong dbname: %s' % mongo.db.name
        assert mongo.cx.host == 'localhost'
        assert mongo.cx.port == 27017

    def test_rejects_invalid_string(self):
        self.app.config['MONGO_PORT'] = '27017x'

        self.assertRaises(TypeError, flask.ext.pymongo.PyMongo, self.app)

    def test_multiple_pymongos(self):
        for prefix in ('ONE', 'TWO'):
            self.app.config['%s_DBNAME' % prefix] = prefix

        for prefix in ('ONE', 'TWO'):
            flask.ext.pymongo.PyMongo(self.app, config_prefix=prefix)

        # this test passes if it raises no exceptions

    def test_config_with_uri(self):
        self.app.config['MONGO_URI'] = 'mongodb://localhost:27017/flask_pymongo_test_db'

        with warnings.catch_warnings():
            # URI connections without a username and password
            # work, but warn that auth should be supplied
            warnings.simplefilter('ignore')
            mongo = flask.ext.pymongo.PyMongo(self.app)
        assert mongo.db.name == 'flask_pymongo_test_db', 'wrong dbname: %s' % mongo.db.name
        assert mongo.cx.host == 'localhost'
        assert mongo.cx.port == 27017

    def test_config_with_uri_no_port(self):
        self.app.config['MONGO_URI'] = 'mongodb://localhost/flask_pymongo_test_db'

        with warnings.catch_warnings():
            # URI connections without a username and password
            # work, but warn that auth should be supplied
            warnings.simplefilter('ignore')
            mongo = flask.ext.pymongo.PyMongo(self.app)
        assert mongo.db.name == 'flask_pymongo_test_db', 'wrong dbname: %s' % mongo.db.name
        assert mongo.cx.host == 'localhost'
        assert mongo.cx.port == 27017
<|endoftext|>


<commit_before>import unittest
from plexcleaner.exception import PlexCleanerException
from plexcleaner.database import Database

__author__ = 'Jean-Bernard Ratte - jean.bernard.ratte@unary.ca'

class TestDatabase(unittest.TestCase):
    def get_db(self):
        db = Database(database_override=':memory:')
        db._cursor.execute('CREATE TABLE media_parts(id, file)')
        db._cursor.executemany('INSERT INTO media_parts(id, file) VALUES (?, ?)', [(1, '/test/fail'),
                                                                                (2, '/test/fail'),
                                                                                (3, '/test/fail')])
        return db

    def test_database_exception(self):
        with self.assertRaises(PlexCleanerException) as e:
            db = Database(database_override='./path/to/com.plexapp.plugins.library.db')
        self.assertTrue('Could not connect' in e.exception.message)

    def test_update(self):
        db = self.get_db()
        db.update_row(1, '/test/success')
        row = db._cursor.execute('SELECT file FROM media_parts WHERE id = ?', (1,)).fetchone()
        self.assertEqual(row[0], '/test/success')

    def test_update_many(self):
        db = self.get_db()
        db.update_many_row([('/test/success', 1), ('/test/success', 2), ('/test/success', 3)])
        for row in db._cursor.execute('SELECT file FROM media_parts').fetchall():
            self.assertEqual(row[0], '/test/success')

    def test_get_rows(self):
        db = Database(database_override='./test/database/com.plexapp.plugins.library.db')
        rows = db.get_rows().fetchall()
        self.assertEqual(len(rows), 98)
<commit_msg>added test for rollback<commit_after>import unittest
from plexcleaner.exception import PlexCleanerException
from plexcleaner.database import Database

__author__ = 'Jean-Bernard Ratte - jean.bernard.ratte@unary.ca'

class TestDatabase(unittest.TestCase):
    def get_db(self):
        db = Database(database_override=':memory:')
        db._cursor.execute('CREATE TABLE media_parts(id, file)')
        db._cursor.executemany('INSERT INTO media_parts(id, file) VALUES (?, ?)', [(1, '/test/fail'),
                                                                                (2, '/test/fail'),
                                                                                (3, '/test/fail')])
        db.commit()
        return db

    def test_database_exception(self):
        with self.assertRaises(PlexCleanerException) as e:
            db = Database(database_override='./path/to/com.plexapp.plugins.library.db')
        self.assertTrue('Could not connect' in e.exception.message)

    def test_update(self):
        db = self.get_db()
        db.update_row(1, '/test/success')
        row = db._cursor.execute('SELECT file FROM media_parts WHERE id = ?', (1,)).fetchone()
        self.assertEqual(row[0], '/test/success')

    def test_update_many(self):
        db = self.get_db()
        db.update_many_row([('/test/success', 1), ('/test/success', 2), ('/test/success', 3)])
        for row in db._cursor.execute('SELECT file FROM media_parts').fetchall():
            self.assertEqual(row[0], '/test/success')

    def test_get_rows(self):
        db = Database(database_override='./test/database/com.plexapp.plugins.library.db')
        rows = db.get_rows().fetchall()
        self.assertEqual(len(rows), 98)

    def test_rollback(self):
        db = self.get_db()
        db.update_row(1, '/test/success')
        row = db._cursor.execute('SELECT file FROM media_parts WHERE id = ?', (1,)).fetchone()
        self.assertEqual(row[0], '/test/success')
        db.rollback()
        row = db._cursor.execute('SELECT file FROM media_parts WHERE id = ?', (1,)).fetchone()
        self.assertEqual(row[0], '/test/fail')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Functions and classes for adapter working

@author: Seva Zhidkov
@contact: zhidkovseva@gmail.com
@license: The MIT license

Copyright (C) 2015
"""

import importlib

from sheldon.utils import logger
from sheldon.config import parse_config

class Adapter:
    """
    Adapter class contains information about adapter:
    name, variables and module using to call adapter methods.
    All adapters inherited from Adapter class'
    """

    def __init__(self, name, module, config):
        """
        Init new Adapter object

        :param name: public name of adapter which used in
                     config/adapters directory
        :param module: imported adapter's module
        :param config: ModuleConfig, parsed adapter config
        """
        self.name = name
        self.module = module
        self.config = config

class Message:
    """
    Class for every message: incoming and outgoing.
    """
    def __init__(self, text, attachments, channel=None, variables={}):
        """
        Create new message.

        :param text: string, text of message
        :param attachments: list[Attachment] or Attachment object,
                                    attachments with message
        :param channel: Message's channel: channel in Slack, room in Hipchat etc.
        :param variables: dict, external parameters from/to adapter:
                          may be 'slack_username', 'slack_emoji', 'telegram_id'
                          Read about those in adapters' documentation.
                          Parameters should start from adapter name.
        """
        self.text = text
        # If attachment only one, convert it to list
        if type(attachments) == Attachment:
            self.attachments = [attachments]
        else:
            self.attachments = attachments
        self.channel = channel
        self.variables = variables

class IncomingMessage(Message):
    """
    Class for messages from user.
    """

    def __init__(self, sender, *args, **kwargs):
        """
        Create new message from user.

        :param sender: User object, sender of message
        """
        super().__init__(*args, **kwargs)
        self.sender = sender

class OutgoingMessage(Message):
    """
    Class for messages from bot.
    """

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

class Attachment:
    """
    Class for every attachment: incoming and outgoing
    """

    def __init__(self, attachment_type, attachment_path=None,
                 attachment_text='', attachment_id=None):
        """
        Init new attachment.

        :param attachment_type: 'photo', 'video', 'link'
                                and other things, supported by adapter
        :param attachment_path: path to photo, video, url etc.
        :param attachment_text: optional text of attachment - may
                                be сaption or something another
        :param attachment_id: int, not required id of attachment from adapter
        :return:
        """
        self.type = attachment_type.lower()
        self.path = attachment_path
        self.text = attachment_text
        self.id = attachment_id

def load_adapter(adapter_name):
    """
    Load adapter which set in bot config.

    :param adapter_name: name of adapter. May be local package in
                         adapters folder or package from PyPi.
    :return:
    """
    # Try to import adapter from project directory
    adapter_module = import_adapter('adapters.{}'.format(adapter_name))

    # If it failed, try to import it as PyPi module
    if adapter_module is None:
        adapter_module = import_adapter(adapter_name)

    # If it still fail, just return error
    if adapter_module is None:
        logger.critical_message('Problems with importing adapter')
        return None

    adapter_config = parse_config(adapter_module)

    adapter_object = Adapter(adapter_name,
                             adapter_module,
                             adapter_config)
    return adapter_object

def import_adapter(package_name):
    """
    Import adapter using importlib

    :param package_name: full name of adapter, ex. 'adapters.console'
    :return:
    """
    try:
        return importlib.import_module(package_name)
    except ImportError as error:
        logger.critical_message(error)
        return None
<commit_msg>Add default value to message text and attachments<commit_after># -*- coding: utf-8 -*-

"""
Functions and classes for adapter working

@author: Seva Zhidkov
@contact: zhidkovseva@gmail.com
@license: The MIT license

Copyright (C) 2015
"""

import importlib

from sheldon.utils import logger
from sheldon.config import parse_config

class Adapter:
    """
    Adapter class contains information about adapter:
    name, variables and module using to call adapter methods.
    All adapters inherited from Adapter class'
    """

    def __init__(self, name, module, config):
        """
        Init new Adapter object

        :param name: public name of adapter which used in
                     config/adapters directory
        :param module: imported adapter's module
        :param config: ModuleConfig, parsed adapter config
        """
        self.name = name
        self.module = module
        self.config = config

class Message:
    """
    Class for every message: incoming and outgoing.
    """
    def __init__(self, text='', attachments=[], channel=None, variables={}):
        """
        Create new message.

        :param text: string, text of message
        :param attachments: list[Attachment] or Attachment object,
                                    attachments with message
        :param channel: Message's channel: channel in Slack, room in Hipchat etc.
        :param variables: dict, external parameters from/to adapter:
                          may be 'slack_username', 'slack_emoji', 'telegram_id'
                          Read about those in adapters' documentation.
                          Parameters should start from adapter name.
        """
        self.text = text
        # If attachment only one, convert it to list
        if type(attachments) == Attachment:
            self.attachments = [attachments]
        else:
            self.attachments = attachments
        self.channel = channel
        self.variables = variables

class IncomingMessage(Message):
    """
    Class for messages from user.
    """

    def __init__(self, sender, *args, **kwargs):
        """
        Create new message from user.

        :param sender: User object, sender of message
        """
        super().__init__(*args, **kwargs)
        self.sender = sender

class OutgoingMessage(Message):
    """
    Class for messages from bot.
    """

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

class Attachment:
    """
    Class for every attachment: incoming and outgoing
    """

    def __init__(self, attachment_type, attachment_path=None,
                 attachment_text='', attachment_id=None):
        """
        Init new attachment.

        :param attachment_type: 'photo', 'video', 'link'
                                and other things, supported by adapter
        :param attachment_path: path to photo, video, url etc.
        :param attachment_text: optional text of attachment - may
                                be сaption or something another
        :param attachment_id: int, not required id of attachment from adapter
        :return:
        """
        self.type = attachment_type.lower()
        self.path = attachment_path
        self.text = attachment_text
        self.id = attachment_id

def load_adapter(adapter_name):
    """
    Load adapter which set in bot config.

    :param adapter_name: name of adapter. May be local package in
                         adapters folder or package from PyPi.
    :return:
    """
    # Try to import adapter from project directory
    adapter_module = import_adapter('adapters.{}'.format(adapter_name))

    # If it failed, try to import it as PyPi module
    if adapter_module is None:
        adapter_module = import_adapter(adapter_name)

    # If it still fail, just return error
    if adapter_module is None:
        logger.critical_message('Problems with importing adapter')
        return None

    adapter_config = parse_config(adapter_module)

    adapter_object = Adapter(adapter_name,
                             adapter_module,
                             adapter_config)
    return adapter_object

def import_adapter(package_name):
    """
    Import adapter using importlib

    :param package_name: full name of adapter, ex. 'adapters.console'
    :return:
    """
    try:
        return importlib.import_module(package_name)
    except ImportError as error:
        logger.critical_message(error)
        return None
<|endoftext|>


<commit_before><commit_msg>opti calcul obstacle<commit_after><|endoftext|>


<commit_before># Copyright (C) 2013 Google Inc., authors, and contributors <see AUTHORS file>
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
# Created By: dan@reciprocitylabs.com
# Maintained By: dan@reciprocitylabs.com

import ggrc.builder
from blinker import Namespace
from flask import request, render_template, current_app, url_for
from ggrc.rbac import permissions
from ggrc.services.common import ModelView, as_json
from werkzeug.exceptions import Forbidden

class BaseObjectView(ModelView):
  model_template = '{model_plural}/show.haml'
  base_template = 'base_objects/show.haml'

  signals = Namespace()
  extension_contributions = signals.signal('View Extension Contributions',
      """
      Gathers any extension contributions to be included into a template.
      Receiver functions must expect the following arguments:
      :sender: The model class of the object being rendered.
      :obj: the model instance being rendered.
      :context: A context for extensions to use in rendering the their
          contribution.
      """,
      )

  def dispatch_request(self, *args, **kwargs):
    method = request.method.lower()

    if method == 'get':
      if self.pk in kwargs and kwargs[self.pk] is not None:
        return self.get(*args, **kwargs)
      else:
        # No `pk` given; fallthrough for now
        pass
    else:
      # Method not supported; fallthrough for now
      pass

    raise NotImplementedError()

  def get_context_for_object(self, obj):
    return {
      'instance': obj,
      'controller': self,
      'instance_json':
        lambda: as_json({
            self.model._inflector.table_singular: ggrc.builder.json.publish(obj)
          })
      }

  def get_model_template_paths_for_object(self, obj):
    # Generate lookup paths for templates based on inheritance
    return [
      self.model_template.format(model_plural=model._inflector.table_plural)
        for model in self.model.mro() if hasattr(model, '__table__')]

  def render_template_for_object(self, obj):
    context = self.get_context_for_object(obj)
    template_paths =\
      self.get_model_template_paths_for_object(obj) + [self.base_template]
    return render_template(template_paths, **context)

  def extension_content(self, obj):
    contributions = self.extension_contributions.send(
        obj.__class__,
        obj=obj,
        context=self.get_context_for_object(obj),
        )
    return [template for func,template in contributions if template is not None]

  def get(self, id):
    obj = self.get_object(id)
    if obj is None:
      return self.not_found_response()
    if 'Accept' in self.request.headers and \
       'text/html' not in self.request.headers['Accept']:
      return current_app.make_response((
        'text/html', 406, [('Content-Type', 'text/plain')]))
    if not permissions.is_allowed_read(self.model.__name__, obj.context_id):
      raise Forbidden()

    rendered_template = self.render_template_for_object(obj)

    # FIXME: Etag based on rendered output, or object itself?
    #if 'If-None-Match' in self.request.headers and \
    #    self.request.headers['If-None-Match'] == self.etag(object_for_json):
    #  return current_app.make_response((
    #    '', 304, [('Etag', self.etag(object_for_json))]))

    return rendered_template

  @classmethod
  def add_to(cls, app, url, model_class=None, decorators=()):
    if model_class:
      cls_name = '{0}ObjectView'.format(model_class.__name__)
      view_class = type(
        cls_name,
        (BaseObjectView,),
        {
          '_model': model_class,
          'base_url_for': classmethod(lambda cls: url),
        })
      import ggrc.views
      setattr(ggrc.views, model_class.__name__, view_class)
    else:
      view_class = cls

    view_func = view_class.as_view(view_class.endpoint_name())
    view_func = cls.decorate_view_func(view_func, decorators)
    view_route = '{url}/<{type}:{pk}>'.format(
        url=url, type=cls.pk_type, pk=cls.pk)
    app.add_url_rule(view_route, cls.__name__,
      view_func=view_func,
      methods=['GET'])
<commit_msg>Fix endpoint name registration of views.<commit_after># Copyright (C) 2013 Google Inc., authors, and contributors <see AUTHORS file>
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
# Created By: dan@reciprocitylabs.com
# Maintained By: dan@reciprocitylabs.com

import ggrc.builder
from blinker import Namespace
from flask import request, render_template, current_app, url_for
from ggrc.rbac import permissions
from ggrc.services.common import ModelView, as_json
from werkzeug.exceptions import Forbidden

class BaseObjectView(ModelView):
  model_template = '{model_plural}/show.haml'
  base_template = 'base_objects/show.haml'

  signals = Namespace()
  extension_contributions = signals.signal('View Extension Contributions',
      """
      Gathers any extension contributions to be included into a template.
      Receiver functions must expect the following arguments:
      :sender: The model class of the object being rendered.
      :obj: the model instance being rendered.
      :context: A context for extensions to use in rendering the their
          contribution.
      """,
      )

  def dispatch_request(self, *args, **kwargs):
    method = request.method.lower()

    if method == 'get':
      if self.pk in kwargs and kwargs[self.pk] is not None:
        return self.get(*args, **kwargs)
      else:
        # No `pk` given; fallthrough for now
        pass
    else:
      # Method not supported; fallthrough for now
      pass

    raise NotImplementedError()

  def get_context_for_object(self, obj):
    return {
      'instance': obj,
      'controller': self,
      'instance_json':
        lambda: as_json({
            self.model._inflector.table_singular: ggrc.builder.json.publish(obj)
          })
      }

  def get_model_template_paths_for_object(self, obj):
    # Generate lookup paths for templates based on inheritance
    return [
      self.model_template.format(model_plural=model._inflector.table_plural)
        for model in self.model.mro() if hasattr(model, '__table__')]

  def render_template_for_object(self, obj):
    context = self.get_context_for_object(obj)
    template_paths =\
      self.get_model_template_paths_for_object(obj) + [self.base_template]
    return render_template(template_paths, **context)

  def extension_content(self, obj):
    contributions = self.extension_contributions.send(
        obj.__class__,
        obj=obj,
        context=self.get_context_for_object(obj),
        )
    return [template for func,template in contributions if template is not None]

  def get(self, id):
    obj = self.get_object(id)
    if obj is None:
      return self.not_found_response()
    if 'Accept' in self.request.headers and \
       'text/html' not in self.request.headers['Accept']:
      return current_app.make_response((
        'text/html', 406, [('Content-Type', 'text/plain')]))
    if not permissions.is_allowed_read(self.model.__name__, obj.context_id):
      raise Forbidden()

    rendered_template = self.render_template_for_object(obj)

    # FIXME: Etag based on rendered output, or object itself?
    #if 'If-None-Match' in self.request.headers and \
    #    self.request.headers['If-None-Match'] == self.etag(object_for_json):
    #  return current_app.make_response((
    #    '', 304, [('Etag', self.etag(object_for_json))]))

    return rendered_template

  @classmethod
  def add_to(cls, app, url, model_class=None, decorators=()):
    if model_class:
      cls_name = '{0}ObjectView'.format(model_class.__name__)
      view_class = type(
        cls_name,
        (BaseObjectView,),
        {
          '_model': model_class,
          'base_url_for': classmethod(lambda cls: url),
        })
      import ggrc.views
      setattr(ggrc.views, model_class.__name__, view_class)
    else:
      view_class = cls

    view_func = view_class.as_view(view_class.endpoint_name())
    view_func = cls.decorate_view_func(view_func, decorators)
    view_route = '{url}/<{type}:{pk}>'.format(
        url=url, type=cls.pk_type, pk=cls.pk)
    app.add_url_rule(view_route, view_class.endpoint_name(),
      view_func=view_func,
      methods=['GET'])
<|endoftext|>


<commit_before>#!/usr/bin/python

# written by andrewt@cse.unsw.edu.au September 2015
# as a starting point for COMP2041/9041 assignment 2
# http://cgi.cse.unsw.edu.au/~cs2041/assignments/bitter/

import cgi, cgitb, glob, os

class user(object):
    """Bitter user"""
    def __init__(self, user_dir):
        # super(user, self).__init__()
        # self.arg = arg
        self.details = {}
        self.pic = os.path.join(user_dir,"profile.jpg")
        with open(os.path.join(user_dir,"details.txt")) as f:
            # details = f.read()
            for line in f:
                field, _, value = line.rstrip().partition(": ")
                # if field == "listens":
                    # list?
                self.details[field] = value
        with open(os.path.join(user_dir,"bleats.txt")) as f:
            self.bleats = f.readlines()

    def details_full(self):
        details_formatted = ""
        # for field in vars(self):
        #     details += field + ": " + self.details[field]
        for field,_ in sorted(self.details.items()): # sorted
        # for field in self.details: # unsorted
            # details_formatted += field + ": " + self.details[field]
            details_formatted += "
%s: %s
\n" % (field,self.details[field])
        # return details # sorted(vars(self))
        return details_formatted

def main():
    print page_header()
    cgitb.enable()
    dataset_size = "small"
    users_dir = "dataset-%s/users"% dataset_size
    parameters = cgi.FieldStorage()
    print user_page(parameters, users_dir)
    print page_trailer(parameters)

#
# Show unformatted details for user "n".
# Increment parameter n and store it as a hidden variable
#
def user_page(parameters, users_dir):
    n = int(parameters.getvalue('n', 0))
    users = sorted(glob.glob(os.path.join(users_dir, "*")))
    user_to_show  = users[n % len(users)]
    curr_user = user(user_to_show)
    details = curr_user.details_full()
    # details_filename = os.path.join(user_to_show, "details.txt")
    # with open(details_filename) as f:
    #     details = f.read()
    return """

%s

<form method="POST" action="">
    <input type="hidden" name="n" value="%s">
    <input type="submit" value="Next user" class="btn btn-default">
</form>

Bleats

""" % (details, n + 1)

#
# HTML placed at the top of every page
#
def page_header():
    return """Content-Type: text/html

<!DOCTYPE html>

        <meta charset="utf-8">
        <meta http-equiv="X-UA-Compatible" content="IE=edge">
        <meta name="viewport" content="width=device-width, initial-scale=1">

        <title>Bitter</title>

        <link href="css/bootstrap.min.css" rel="stylesheet">

					<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
						Toggle nagivation

					</button>
					Bitter

Home(current)

New Bleat

					<form class="navbar-form navbar-left" role="search">

							<input type="text" class="form-control" placeholder="Search">

						<button type="submit" class="btn btn-default">
							<span class="glyphicon glyphicon-search" aria-hidden="true"
						</button>
					</form>

Log In

"""

#
# HTML placed at the bottom of every page
# It includes all supplied parameter values as a HTML comment
# if global variable debug is set
#
def page_trailer(parameters):
    html = ""
    if debug:
        html += "".join("\n" % (p, parameters.getvalue(p)) for p in parameters)
    html += """

    """
    html += "\n"
    return html

if __name__ == '__main__':
    debug = 1
    main()
<commit_msg>Add logged in user<commit_after>#!/usr/bin/python

# written by andrewt@cse.unsw.edu.au September 2015
# as a starting point for COMP2041/9041 assignment 2
# http://cgi.cse.unsw.edu.au/~cs2041/assignments/bitter/

import cgi, cgitb, glob, os

class user(object):
    """Bitter user"""
    def __init__(self, user_dir):
        # super(user, self).__init__()
        # self.arg = arg
        self.details = {}
        self.pic = os.path.join(user_dir,"profile.jpg")
        with open(os.path.join(user_dir,"details.txt")) as f:
            # details = f.read()
            for line in f:
                field, _, value = line.rstrip().partition(": ")
                # if field == "listens":
                    # list?
                self.details[field] = value
        with open(os.path.join(user_dir,"bleats.txt")) as f:
            self.bleats = f.readlines()

    def details_full(self):
        details_formatted = ""
        # for field in vars(self):
        #     details += field + ": " + self.details[field]
        for field,_ in sorted(self.details.items()): # sorted
        # for field in self.details: # unsorted
            # details_formatted += field + ": " + self.details[field]
            details_formatted += "
%s: %s
\n" % (field,self.details[field])
        # return details # sorted(vars(self))
        return details_formatted

def main():
    print page_header()
    cgitb.enable()
    dataset_size = "small"
    users_dir = "dataset-%s/users"% dataset_size
    parameters = cgi.FieldStorage()
    print user_page(parameters, users_dir)
    print page_trailer(parameters)

#
# Show unformatted details for user "n".
# Increment parameter n and store it as a hidden variable
#
def user_page(parameters, users_dir):
    n = int(parameters.getvalue('n', 0))
    users = sorted(glob.glob(os.path.join(users_dir, "*")))
    user_to_show  = users[n % len(users)]
    curr_user = user(user_to_show)
    details = curr_user.details_full()
    # details_filename = os.path.join(user_to_show, "details.txt")
    # with open(details_filename) as f:
    #     details = f.read()
    return """

%s

<form method="POST" action="">
    <input type="hidden" name="n" value="%s">
    <input type="submit" value="Next user" class="btn btn-default">
</form>

Bleats

""" % (details, n + 1)

#
# HTML placed at the top of every page
#
def page_header():
    return """Content-Type: text/html

<!DOCTYPE html>

        <meta charset="utf-8">
        <meta http-equiv="X-UA-Compatible" content="IE=edge">
        <meta name="viewport" content="width=device-width, initial-scale=1">

        <title>Bitter</title>

        <link href="css/bootstrap.min.css" rel="stylesheet">

					<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
						Toggle nagivation

					</button>
					Bitter

Home(current)

New Bleat

					<form class="navbar-form navbar-left" role="search">

							<input type="text" class="form-control" placeholder="Search">

						<button type="submit" class="btn btn-default">
							<span class="glyphicon glyphicon-search" aria-hidden="true"
						</button>
					</form>

Signed in as John Doe

Log Out

"""

#
# HTML placed at the bottom of every page
# It includes all supplied parameter values as a HTML comment
# if global variable debug is set
#
def page_trailer(parameters):
    html = ""
    if debug:
        html += "".join("\n" % (p, parameters.getvalue(p)) for p in parameters)
    html += """

    """
    html += "\n"
    return html

if __name__ == '__main__':
    debug = 1
    main()
<|endoftext|>


<commit_before>"""Serve nix store objects over HTTP."""
import os
from os.path import exists, isdir, join
import argparse
from flask import Flask, make_response, send_file
from subprocess import check_output
import re

_HASH_REGEX=re.compile(r"[a-z0-9]{32}")
_PATH_REGEX=re.compile(r"([a-z0-9]{32})-.*")

class NoSuchObject(IOError):
    """Raises when a store object can't be found."""
    def __init__(self, message):
        self.message = message

def check_output_str(command):
    """Call check_output and convert into a string."""
    result = check_output(command)
    if hasattr(result, "decode"):
        return result.decode("utf-8")
    else:
        return result

class NixServer(Flask):
    """Serves nix packages."""
    def __init__(self, nix_store_path, nix_bin_path, compression_type):
        self._nix_store_path = nix_store_path
        self._nix_bin_path = nix_bin_path
        self._compression_type = compression_type
        # Cache mapping object hashes to store paths.
        self._hashes_to_paths = {}
        # Cache mapping store paths to object info.
        self._paths_to_info = {}
        # A static string telling a nix client what this store serves.
        self._cache_info = "\n".join([
            "StoreDir: {}".format(self._nix_store_path),
            "WantMassQuery: 1",
            "Priority: 30"
        ])
        if self._compression_type == "bzip2":
            self._content_type = "application/x-bzip2"
            self._nar_extension = ".nar.bz2"
        else:
            self._content_type = "application/x-xz"
            self._nar_extension = ".nar.xz"

    def store_path_from_hash(self, store_object_hash):
        """Look up a store path using its hash.

        The name of every object in the nix store starts with a
        32-character hash. We can find the full path by finding an
        object that starts with this hash.

        :param store_object_hash: The 32-character hash prefix of the object.
        :type store_object_hash: ``str``

        :return: The full store path to the object.
        :rtype: ``str``

        :raises: :py:class:`NoSuchObject` if the object isn't in the store.
        """
        if store_object_hash in self._hashes_to_paths:
            return self._hashes_to_paths[store_object_hash]
        store_objects = os.listdir(self._nix_store_path)
        try:
            for obj in store_objects:
                if hasattr(obj, "decode"):
                    obj = obj.decode("utf-8")
                match = _PATH_REGEX.match(obj)
                if match is None:
                    continue
                full_path = join(self._nix_store_path, obj)
                _hash = match.group(1)
                self._hashes_to_paths[_hash] = full_path
                if _hash == store_object_hash:
                    return full_path
            raise NoSuchObject("No object with hash {} was found."
                               .format(store_object_hash))
        except TypeError as err:
            import pdb; pdb.set_trace()

    def get_object_info(self, store_path):
        """Given a store path, get some information about the path.

        :param store_path: Path to the object in the store.
        :type store_path: ``str``

        :return: A dictionary of store object information.
        :rtype: ``dict``
        """
        if store_path in self._paths_to_info:
            return self._paths_to_info[store_path]
        # Invoke nix-store with various queries to get package info.
        nix_store_q = lambda option: check_output_str([
            "{}/nix-store".format(self._nix_bin_path),
            "--query", option, store_path
        ]).strip()
        info = {
            "StorePath": store_path,
            "NarHash": nix_store_q("--hash"),
            "NarSize": nix_store_q("--size"),
        }
        references = nix_store_q("--references").replace("\n", " ")
        if references != "":
            info["References"] = references
        deriver = nix_store_q("--deriver")
        if deriver != "unknown-deriver":
            info["Deriver"] = deriver
        self._paths_to_info[store_path] = info
        return info

    def build_nar(self, store_path):
        """Build a nix archive (nar) and return the resulting path."""
        # Construct a nix expression which will produce a nar.
        nar_expr = "".join([
            "(import <nix/nar.nix> {",
            'storePath = "{}";'.format(store_path),
            'hashAlgo = "sha256";',
            'compressionType = "{}";'.format(self._compression_type),
            "})"])
        # Nix-build this expression, resulting in a store
        compressed_path = check_output_str([
            join(self._nix_bin_path, "nix-build"),
            "--expr", nar_expr
        ]).strip()

        # This path will contain a compressed file; return its path.
        contents = os.listdir(compressed_path)
        for filename in contents:
            if hasattr(filename, "decode"):
                filename = filename.decode("utf-8")
            if filename.endswith(self._nar_extension):
                return join(compressed_path, filename)

    def make_app(self):
        """Create a flask app and set up routes on it.

        :return: A flask app.
        :rtype: :py:class:`Flask`
        """
        app = Flask(__name__)

        @app.route('/')
        def hello_world():
            return 'Hello, World!'

        @app.route("/nix-cache-info")
        def nix_cache_info():
            """Return information about the binary cache."""
            return self._cache_info

        @app.route("/<obj_hash>.narinfo")
        def get_narinfo(obj_hash):
            if _HASH_REGEX.match(obj_hash) is None:
                 return ("Hash {} must match {}"
                         .format(obj_hash, _HASH_REGEX.pattern), 400)
            try:
                store_path = self.store_path_from_hash(obj_hash)
                store_info = self.get_object_info(store_path)
            except NoSuchObject as err:
                return (err.message, 404)
            # Add a few more keys to the store object, specific to the
            # compression type we're serving.
            store_info["Url"] = "nar/{}{}".format(obj_hash,
                                                  self._nar_extension)
            store_info["Compression"] = self._compression_type
            info_string = "\n".join("{}: {}".format(k, v)
                             for k, v in store_info.items()) + "\n"
            return make_response((info_string, 200,
                                 {"Content-Type": "text/x-nix-narinfo"}))

        @app.route("/nar/<obj_hash>{}".format(self._nar_extension))
        def serve_nar(obj_hash):
            """Return the compressed binary from the nix store."""
            try:
                store_path = self.store_path_from_hash(obj_hash)
            except NoSuchObject as err:
                return (err.message, 404)
            nar_path = self.build_nar(store_path)
            return send_file(nar_path, mimetype=self._content_type)

        @app.errorhandler(500)
        def h(error):
            import pdb; pdb.set_trace()

        return app

def _get_args():
    """Parse command-line arguments."""
    parser = argparse.ArgumentParser(prog="servenix")
    parser.add_argument("--port", type=int, default=5000,
                        help="Port to listen on.")
    parser.add_argument("--compression-type", default="xz",
                        choices=("xz", "bzip2"),
                        help="How served objects should be compressed.")
    return parser.parse_args()

def main():
    """Main entry point."""
    try:
        NIX_BIN_PATH = os.environ["NIX_BIN_PATH"]
        assert exists(join(NIX_BIN_PATH, "nix-store"))
        NIX_STORE_PATH = os.environ["NIX_STORE_PATH"]
        assert isdir(NIX_STORE_PATH)
    except KeyError as err:
        exit("Invalid environment: variable {} must be set.".format(err))
    args = _get_args()
    nixserver = NixServer(nix_store_path=NIX_STORE_PATH,
                          nix_bin_path=NIX_BIN_PATH,
                          compression_type=args.compression_type)
    app = nixserver.make_app()
    app.run(port=args.port)

if __name__ == "__main__":
    main()
<commit_msg>rm 500 error handler<commit_after>"""Serve nix store objects over HTTP."""
import os
from os.path import exists, isdir, join
import argparse
from flask import Flask, make_response, send_file
from subprocess import check_output
import re

_HASH_REGEX=re.compile(r"[a-z0-9]{32}")
_PATH_REGEX=re.compile(r"([a-z0-9]{32})-.*")

class NoSuchObject(IOError):
    """Raises when a store object can't be found."""
    def __init__(self, message):
        self.message = message

def check_output_str(command):
    """Call check_output and convert into a string."""
    result = check_output(command)
    if hasattr(result, "decode"):
        return result.decode("utf-8")
    else:
        return result

class NixServer(Flask):
    """Serves nix packages."""
    def __init__(self, nix_store_path, nix_bin_path, compression_type):
        self._nix_store_path = nix_store_path
        self._nix_bin_path = nix_bin_path
        self._compression_type = compression_type
        # Cache mapping object hashes to store paths.
        self._hashes_to_paths = {}
        # Cache mapping store paths to object info.
        self._paths_to_info = {}
        # A static string telling a nix client what this store serves.
        self._cache_info = "\n".join([
            "StoreDir: {}".format(self._nix_store_path),
            "WantMassQuery: 1",
            "Priority: 30"
        ])
        if self._compression_type == "bzip2":
            self._content_type = "application/x-bzip2"
            self._nar_extension = ".nar.bz2"
        else:
            self._content_type = "application/x-xz"
            self._nar_extension = ".nar.xz"

    def store_path_from_hash(self, store_object_hash):
        """Look up a store path using its hash.

        The name of every object in the nix store starts with a
        32-character hash. We can find the full path by finding an
        object that starts with this hash.

        :param store_object_hash: The 32-character hash prefix of the object.
        :type store_object_hash: ``str``

        :return: The full store path to the object.
        :rtype: ``str``

        :raises: :py:class:`NoSuchObject` if the object isn't in the store.
        """
        if store_object_hash in self._hashes_to_paths:
            return self._hashes_to_paths[store_object_hash]
        store_objects = os.listdir(self._nix_store_path)
        try:
            for obj in store_objects:
                if hasattr(obj, "decode"):
                    obj = obj.decode("utf-8")
                match = _PATH_REGEX.match(obj)
                if match is None:
                    continue
                full_path = join(self._nix_store_path, obj)
                _hash = match.group(1)
                self._hashes_to_paths[_hash] = full_path
                if _hash == store_object_hash:
                    return full_path
            raise NoSuchObject("No object with hash {} was found."
                               .format(store_object_hash))
        except TypeError as err:
            import pdb; pdb.set_trace()

    def get_object_info(self, store_path):
        """Given a store path, get some information about the path.

        :param store_path: Path to the object in the store.
        :type store_path: ``str``

        :return: A dictionary of store object information.
        :rtype: ``dict``
        """
        if store_path in self._paths_to_info:
            return self._paths_to_info[store_path]
        # Invoke nix-store with various queries to get package info.
        nix_store_q = lambda option: check_output_str([
            "{}/nix-store".format(self._nix_bin_path),
            "--query", option, store_path
        ]).strip()
        info = {
            "StorePath": store_path,
            "NarHash": nix_store_q("--hash"),
            "NarSize": nix_store_q("--size"),
        }
        references = nix_store_q("--references").replace("\n", " ")
        if references != "":
            info["References"] = references
        deriver = nix_store_q("--deriver")
        if deriver != "unknown-deriver":
            info["Deriver"] = deriver
        self._paths_to_info[store_path] = info
        return info

    def build_nar(self, store_path):
        """Build a nix archive (nar) and return the resulting path."""
        # Construct a nix expression which will produce a nar.
        nar_expr = "".join([
            "(import <nix/nar.nix> {",
            'storePath = "{}";'.format(store_path),
            'hashAlgo = "sha256";',
            'compressionType = "{}";'.format(self._compression_type),
            "})"])
        # Nix-build this expression, resulting in a store
        compressed_path = check_output_str([
            join(self._nix_bin_path, "nix-build"),
            "--expr", nar_expr
        ]).strip()

        # This path will contain a compressed file; return its path.
        contents = os.listdir(compressed_path)
        for filename in contents:
            if hasattr(filename, "decode"):
                filename = filename.decode("utf-8")
            if filename.endswith(self._nar_extension):
                return join(compressed_path, filename)

    def make_app(self):
        """Create a flask app and set up routes on it.

        :return: A flask app.
        :rtype: :py:class:`Flask`
        """
        app = Flask(__name__)

        @app.route('/')
        def hello_world():
            return 'Hello, World!'

        @app.route("/nix-cache-info")
        def nix_cache_info():
            """Return information about the binary cache."""
            return self._cache_info

        @app.route("/<obj_hash>.narinfo")
        def get_narinfo(obj_hash):
            if _HASH_REGEX.match(obj_hash) is None:
                 return ("Hash {} must match {}"
                         .format(obj_hash, _HASH_REGEX.pattern), 400)
            try:
                store_path = self.store_path_from_hash(obj_hash)
                store_info = self.get_object_info(store_path)
            except NoSuchObject as err:
                return (err.message, 404)
            # Add a few more keys to the store object, specific to the
            # compression type we're serving.
            store_info["Url"] = "nar/{}{}".format(obj_hash,
                                                  self._nar_extension)
            store_info["Compression"] = self._compression_type
            info_string = "\n".join("{}: {}".format(k, v)
                             for k, v in store_info.items()) + "\n"
            return make_response((info_string, 200,
                                 {"Content-Type": "text/x-nix-narinfo"}))

        @app.route("/nar/<obj_hash>{}".format(self._nar_extension))
        def serve_nar(obj_hash):
            """Return the compressed binary from the nix store."""
            try:
                store_path = self.store_path_from_hash(obj_hash)
            except NoSuchObject as err:
                return (err.message, 404)
            nar_path = self.build_nar(store_path)
            return send_file(nar_path, mimetype=self._content_type)

        return app

def _get_args():
    """Parse command-line arguments."""
    parser = argparse.ArgumentParser(prog="servenix")
    parser.add_argument("--port", type=int, default=5000,
                        help="Port to listen on.")
    parser.add_argument("--compression-type", default="xz",
                        choices=("xz", "bzip2"),
                        help="How served objects should be compressed.")
    return parser.parse_args()

def main():
    """Main entry point."""
    try:
        NIX_BIN_PATH = os.environ["NIX_BIN_PATH"]
        assert exists(join(NIX_BIN_PATH, "nix-store"))
        NIX_STORE_PATH = os.environ["NIX_STORE_PATH"]
        assert isdir(NIX_STORE_PATH)
    except KeyError as err:
        exit("Invalid environment: variable {} must be set.".format(err))
    args = _get_args()
    nixserver = NixServer(nix_store_path=NIX_STORE_PATH,
                          nix_bin_path=NIX_BIN_PATH,
                          compression_type=args.compression_type)
    app = nixserver.make_app()
    app.run(port=args.port)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import marshmallow as ma
from marshmallow_mongoengine import ModelSchema
from flask_mongoengine import Document, DynamicDocument
from mongoengine import fields, DynamicEmbeddedDocument, EmbeddedDocumentField
from mongoengine.queryset import DoesNotExist
from flask import Blueprint, jsonify, request
from flasgger import SwaggerView

provenances = Blueprint("provenances", __name__)

class Urls(DynamicEmbeddedDocument):
    main = fields.StringField(required=True)
    # TODO URL validation?
    # TODO make sure all fields show up in response

class Provenance(DynamicDocument):
    __project_regex__ = '^[a-zA-Z0-9_]+$'
    project = fields.StringField(
        min_length=3, max_length=30, required=True, unique=True,
        regex = __project_regex__,
        help_text="project name/slug (valid format: `{}`)".format(
            __project_regex__
        )
    )
    title = fields.StringField(
        min_length=5, max_length=30, required=True, unique=True,
        help_text='(short) title for the project/dataset'
    )
    authors = fields.StringField(
        required=True, help_text='comma-separated list of authors'
    )
    description = fields.StringField(
        min_length=5, max_length=1500, required=True,
        help_text='brief description of the project'
    )
    urls = EmbeddedDocumentField(
        Urls, required=True, help_text='list of URLs for references'
    )
    meta = {
        'collection': 'provenances', 'indexes': [{
            'fields': ['$title', "$description", "$authors"],
        }, 'project']
    }

    def __str__(self):
        return '<Provenance(project={self.project!r})>'.format(self=self)

class ProvenanceSchema(ModelSchema):
    class Meta:
        model = Provenance
        ordered = True

class ProvenancesView(SwaggerView):
    definitions = {'ProvenanceSchema': ProvenanceSchema}
    tags = ['provenances']

    def marshal(self, entries):
        return jsonify(ProvenanceSchema().dump(entries, many=True).data)

    def get(self):
        """Retrieve and search provenance information for all projects.
        If a string is provided  via the search parameter, `title`, \
        `description`, and `authors` are searched using a MongoEngine/MongoDB \
        text index. Provide a space-separated list to the search query \
        parameter to search for multiple words. For more, see \
        https://docs.mongodb.com/manual/text-search/.
        ---
        parameters:
            - name: search
              in: query
              type: string
              description: string to search for in title, authors, description
        responses:
            200:
                description: list of provenance entries
                schema:
                    $ref: '#/definitions/ProvenanceSchema'
                examples:
                    entries: |
                        [{
                            "authors": "P. Huck, K. Persson",
                            "description": "Bandgaps calculated ... electron affinity.",
                            "id": "5bef38d9aba702da481fe974",
                            "project": "dtu",
                            "title": "GLLB-SC Bandgaps",
                            "urls": { "main": "https://doi.org/10.1002/aenm.201400915" }
                        }, ...]
        """
        entries = Provenance.objects.search_text(request.args['search']) \
                if 'search' in request.args else Provenance.objects.all()
        return self.marshal(entries)

    def post(self):
        """Create a new provenance entry"""
        # validate ProvenanceSchema().load(args, partial=True)
        pass

class ProjectView(SwaggerView):
    definitions = {'ProvenanceSchema': ProvenanceSchema}
    tags = ['provenances']

    def marshal(self, entry):
        return jsonify(ProvenanceSchema().dump(entry).data)

    def get(self, project):
        """Retrieve a project/dataset's provenance information.
        ---
        parameters:
            - name: project
              in: path
              type: string
              pattern: '^[a-zA-Z0-9_]{3,30}$'
              required: true
              description: project identifier (name/slug)
        responses:
            200:
                description: provenance entry
                schema:
                    $ref: '#/definitions/ProvenanceSchema'
                examples:
                    entry: |
                        {
                            "authors": "P. Huck, K. Persson",
                            "description": "Bandgaps calculated ... electron affinity.",
                            "id": "5bef38d9aba702da481fe974",
                            "project": "dtu",
                            "title": "GLLB-SC Bandgaps",
                            "urls": { "main": "https://doi.org/10.1002/aenm.201400915" }
                        }
        """
        try:
            entry = Provenance.objects.get(project=project)
        except DoesNotExist:
            return jsonify({project: 'DoesNotExist'}), 404
        return self.marshal(entry)

    def put(self, project):
        """Update a project's provenance entry"""
        pass

    def patch(self, project):
        """Partially update a project's provenance entry"""
        pass

    def delete(self, project):
        """Delete a project's provenance entry"""
        pass

# Flask-RESTful/plus
# api.add_resource(UserAPI, '/<userId>', '/<userId>/<username>', endpoint = 'user')

# url_prefix added in register_blueprint
provenances.add_url_rule(
    '/', view_func=ProvenancesView.as_view('provenances'), methods=['GET', 'POST']
)
provenances.add_url_rule(
    '/<string:project>',
    view_func=ProjectView.as_view('project'),
    methods=['GET', 'DELETE', 'PATCH', 'PUT']
)
<commit_msg>api: stick to plural everywhere<commit_after>import marshmallow as ma
from marshmallow_mongoengine import ModelSchema
from flask_mongoengine import Document, DynamicDocument
from mongoengine import fields, DynamicEmbeddedDocument, EmbeddedDocumentField
from mongoengine.queryset import DoesNotExist
from flask import Blueprint, jsonify, request
from flasgger import SwaggerView

provenances = Blueprint("provenances", __name__)

class Urls(DynamicEmbeddedDocument):
    main = fields.StringField(required=True)
    # TODO URL validation?
    # TODO make sure all fields show up in response

class Provenances(DynamicDocument):
    __project_regex__ = '^[a-zA-Z0-9_]+$'
    project = fields.StringField(
        min_length=3, max_length=30, required=True, unique=True,
        regex = __project_regex__,
        help_text="project name/slug (valid format: `{}`)".format(
            __project_regex__
        )
    )
    title = fields.StringField(
        min_length=5, max_length=30, required=True, unique=True,
        help_text='(short) title for the project/dataset'
    )
    authors = fields.StringField(
        required=True, help_text='comma-separated list of authors'
    )
    description = fields.StringField(
        min_length=5, max_length=1500, required=True,
        help_text='brief description of the project'
    )
    urls = EmbeddedDocumentField(
        Urls, required=True, help_text='list of URLs for references'
    )
    meta = {
        'collection': 'provenances', 'indexes': [{
            'fields': ['$title', "$description", "$authors"],
        }, 'project']
    }

    def __str__(self):
        return '<Provenances(project={self.project!r})>'.format(self=self)

class ProvenancesSchema(ModelSchema):
    class Meta:
        model = Provenances
        ordered = True

class ProvenancesView(SwaggerView):
    definitions = {'ProvenancesSchema': ProvenancesSchema}
    tags = ['provenances']

    def marshal(self, entries):
        return jsonify(ProvenancesSchema().dump(entries, many=True).data)

    def get(self):
        """Retrieve and search provenance information for all projects.
        If a string is provided  via the search parameter, `title`, \
        `description`, and `authors` are searched using a MongoEngine/MongoDB \
        text index. Provide a space-separated list to the search query \
        parameter to search for multiple words. For more, see \
        https://docs.mongodb.com/manual/text-search/.
        ---
        parameters:
            - name: search
              in: query
              type: string
              description: string to search for in title, authors, description
        responses:
            200:
                description: list of provenance entries
                schema:
                    $ref: '#/definitions/ProvenancesSchema'
                examples:
                    entries: |
                        [{
                            "authors": "P. Huck, K. Persson",
                            "description": "Bandgaps calculated ... electron affinity.",
                            "id": "5bef38d9aba702da481fe974",
                            "project": "dtu",
                            "title": "GLLB-SC Bandgaps",
                            "urls": { "main": "https://doi.org/10.1002/aenm.201400915" }
                        }, ...]
        """
        entries = Provenances.objects.search_text(request.args['search']) \
                if 'search' in request.args else Provenances.objects.all()
        return self.marshal(entries)

    def post(self):
        """Create a new provenance entry"""
        # validate ProvenancesSchema().load(args, partial=True)
        pass

class ProjectsView(SwaggerView):
    definitions = {'ProvenancesSchema': ProvenancesSchema}
    tags = ['provenances']

    def marshal(self, entry):
        return jsonify(ProvenancesSchema().dump(entry).data)

    def get(self, project):
        """Retrieve a project/dataset's provenance information.
        ---
        parameters:
            - name: project
              in: path
              type: string
              pattern: '^[a-zA-Z0-9_]{3,30}$'
              required: true
              description: project identifier (name/slug)
        responses:
            200:
                description: provenance entry
                schema:
                    $ref: '#/definitions/ProvenancesSchema'
                examples:
                    entry: |
                        {
                            "authors": "P. Huck, K. Persson",
                            "description": "Bandgaps calculated ... electron affinity.",
                            "id": "5bef38d9aba702da481fe974",
                            "project": "dtu",
                            "title": "GLLB-SC Bandgaps",
                            "urls": { "main": "https://doi.org/10.1002/aenm.201400915" }
                        }
        """
        try:
            entry = Provenances.objects.get(project=project)
        except DoesNotExist:
            return jsonify({project: 'DoesNotExist'}), 404
        return self.marshal(entry)

    def put(self, project):
        """Update a project's provenance entry"""
        pass

    def patch(self, project):
        """Partially update a project's provenance entry"""
        pass

    def delete(self, project):
        """Delete a project's provenance entry"""
        pass

# Flask-RESTful/plus
# api.add_resource(UserAPI, '/<userId>', '/<userId>/<username>', endpoint = 'user')

# url_prefix added in register_blueprint
provenances.add_url_rule(
    '/', view_func=ProvenancesView.as_view(ProvenancesView.__name__),
    methods=['GET', 'POST']
)
provenances.add_url_rule(
    '/<string:project>',
    view_func=ProjectsView.as_view(ProjectsView.__name__),
    methods=['GET', 'DELETE', 'PATCH', 'PUT']
)
<|endoftext|>


<commit_before>import sys
import unittest

from skeletor.config import Config

from .helpers import nostdout

class ConfigTests(unittest.TestCase):
    """ Argument Passing & Config Tests. """

    def setUp(self):
        self._old_sys_argv = sys.argv
        sys.argv = [self._old_sys_argv[0].replace('nosetests', 'skeletor')]

    def tearDown(self):
        sys.argv = self._old_sys_argv

    def should_exit_with_no_arguments(self):
        try:
            with nostdout():
                Config()
        except SystemExit:
            assert True

    def ensure_valid_project_name(self):
        with nostdout():
            sys.argv = ['', '-n', 'this_is_valid']
            c = Config()
        self.assertEquals(c.project_name, 'this_is_valid')
        with nostdout():
            sys.argv = ['', '-n', 'Thisisvalid']
            c = Config()
        self.assertEquals(c.project_name, 'Thisisvalid')

    def should_exit_on_invalid_name(self):
        try:
            with nostdout():
                sys.argv = ['', '-n', 'not-valid']
                Config()
        except SystemExit:
            assert True
        try:
            with nostdout():
                sys.argv = ['', '-n', 'not valid']
                Config()
        except SystemExit:
            assert True
        try:
            with nostdout():
                sys.argv = ['', '-n', 'not_valid-*']
                Config()
        except SystemExit:
            assert True
<commit_msg>Refactoring name tests, code reuse ftw<commit_after>import sys
import unittest

from skeletor.config import Config

from .helpers import nostdout

class ConfigTests(unittest.TestCase):
    """ Argument Passing & Config Tests. """

    def setUp(self):
        self._old_sys_argv = sys.argv
        sys.argv = [self._old_sys_argv[0].replace('nosetests', 'skeletor')]

    def tearDown(self):
        sys.argv = self._old_sys_argv

    def _set_project_name(self, name):
        with nostdout():
            sys.argv = ['', '-n', name]
            self.config = Config()

    def should_exit_with_no_arguments(self):
        try:
            with nostdout():
                Config()
        except SystemExit:
            assert True

    def ensure_valid_project_name(self):
        self._set_project_name('this_is_valid')
        self.assertEquals(self.config.project_name, 'this_is_valid')
        self._set_project_name('Thisisvalid')
        self.assertEquals(self.config.project_name, 'Thisisvalid')

    def should_exit_on_invalid_name(self):
        try:
            self._set_project_name('this_is_not-valid')
        except SystemExit:
            assert True
        try:
            self._set_project_name('this_is not_valid')
        except SystemExit:
            assert True
        try:
            self._set_project_name('*this_is_not_valid')
        except SystemExit:
            assert True
<|endoftext|>


<commit_before><commit_msg>the language field of works stores iso_code_3<commit_after><|endoftext|>


<commit_before>import datetime
import re

from lxml import etree

class Foxpath(object):
    def __init__(self, tests, codelists=None):
        self.tests = {
            test['id']: self.parse(test['expression'], codelists)
            for test in tests
        }

    def parse(self, test, codelists):
        def xpath(activity, groups, **kwargs):
            # [1:-1] gets rid of the backticks
            return activity.xpath(groups[0][1:-1])

        def integer(activity, groups, **kwargs):
            return int(groups[0])

        def code(activity, groups, **kwargs):
            return groups[0]

        def codelist(activity, groups, codelists, **kwargs):
            # [:-9] gets rid of ' codelist'
            codelist = codelists.get(groups[0][:-9])
            if not codelist:
                raise Exception('That codelist doesn\'t exist!')
            return codelist

        def for_any(activity, groups, **kwargs):
            return any([groups[1](exp) for exp in groups[0](activity)])

        def either(activity, groups, **kwargs):
            return groups[0](activity) or groups[1](activity)

        def both(activity, groups, **kwargs):
            return groups[0](activity) and groups[1](activity)

        def if_then(activity, groups, **kwargs):
            if groups[0](activity):
                return groups[1](activity)
            else:
                # if the condition fails,
                # return not relevant
                return None

        # defaults to true
        def is_not(activity, groups, **kwargs):
            vals = groups[0](activity)
            const = groups[1](activity)
            return not any([val == str(const) for val in vals])

        # defaults to true
        def is_at_least(activity, groups, **kwargs):
            vals = groups[0](activity)
            const = groups[1](activity)
            if len(vals) < 1:
                print('note: is_at_least was expecting to find one and only one element. Found 0')
                return True
            if len(vals) > 1:
                print('note: is_at_least was expecting to find one and only one element. Found {}'.format(len(vals)))
            return any([int(val) >= const for val in vals])

        def exists(activity, groups, **kwargs):
            return any([val != '' for val in groups[0](activity)])

        # defaults to true
        def is_on_list(activity, groups, **kwargs):
            def check_list(vals, codelist):
                for v in vals:
                    v = str(v)
                    versions = [v, v.lower(), v.upper()]
                    yield any([v in codelist for v in versions])
            vals = groups[0](activity)
            return all(check_list(vals, groups[1](activity)))

        # defaults to false
        def is_more_than_x_characters(activity, groups, **kwargs):
            exp = groups[0](activity)
            val = groups[1](activity)
            return any([len(x) > val for x in exp])

        # defaults to false
        def starts_with(activity, groups, **kwargs):
            x = groups[0](activity)
            y = groups[1](activity)
            if x == [] or y == []:
                return False
            return x[0].startswith(str(y[0]))

        # defaults to false
        def is_less_than_x_months_ago(activity, groups, **kwargs):
            current_date = datetime.date.today()
            def less_than_x_months_ago(date_str, months_ago):
                date = datetime.datetime.strptime(date_str, '%Y-%m-%d')
                year_diff = current_date.year - date.year
                month_diff = 12 * year_diff + current_date.month - date.month
                if month_diff == months_ago:
                    return date.day > current_date.day
                return month_diff < months_ago

            dates = groups[0](activity)
            months_ago = groups[1](activity)
            return any([less_than_x_months_ago(date_str, months_ago) for date_str in dates])

        def forward_budget(activity, els, period):
            def mkdate(date_str):
                dt = datetime.datetime.strptime(date_str, '%Y-%m-%d')
                return dt.date()

            def max_date(dates, default):
                try:
                    return mkdate(max(dates))
                except ValueError:
                    return default

            # Window start is from today onwards. We're only interested in budgets
            # that start or end after today.
            today = datetime.date.today()

            # Window period is for the next 365 days. We don't want to look later
            # than that; we're only interested in budgets that end before then.
            #
            # We get the latest date for end and start; 365 days fwd
            # if there are no dates
            one_years_time = today.replace(today.year + 1)
            end_dates = activity.xpath('activity-date[@type="end-planned"]/@iso-date|activity-date[@type="end-actual"]/@iso-date|activity-date[@type="3"]/@iso-date|activity-date[@type="4"]/@iso-date')
            end_date = max_date(end_dates, one_years_time)

            # If the activity ends earlier than the end of the window period,
            # don't penalise the donor for not having a budget
            escape_end_date = today + datetime.timedelta(days=177)
            if end_date < escape_end_date:
                return None

            def check_after(element, today):
                dates = element.xpath('period-start/@iso-date|period-end/@iso-date')
                return any([mkdate(date) >= today for date in dates])

            def max_budget_length(element, max_budget_length):
                # NB this will error if there's no period-end/@iso-date
                start = mkdate(element.xpath('period-start/@iso-date')[0])
                end = mkdate(element.xpath('period-end/@iso-date')[0])
                return ((end-start).days <= max_budget_length)

            # We set a maximum number of days for which a budget can last,
            # depending on the number of quarters that should be covered.
            if period == 'annual':
                max_days = 370
            elif period == 'quarterly':
                max_days = 94

            # A budget has to be:
            # 1) period-end after today
            # 2) a maximum number of days, depending on # of qtrs.
            for element in els:
                after_today = check_after(element, today)
                within_length = max_budget_length(element, max_days)
                if after_today and within_length:
                    return True
            return False

        def is_available_forward(activity, groups, **kwargs):
            els = groups[0](activity)
            return forward_budget(activity, els, 'annual')

        def is_available_forward_by_qtrs(activity, groups, **kwargs):
            els = groups[0](activity)
            return forward_budget(activity, els, 'quarterly')

        mappings = (
            (re.compile(r'^if (.*) then (.*)$'), if_then, 'if_then'),
            (re.compile(r'\S* codelist$'), codelist, 'codelist'),
            (re.compile(r'`[^`]+`$'), xpath, 'xpath'),
            (re.compile(r'^\d+$'), integer, 'integer'),
            (re.compile(r'[A-Z]+\d+$'), code, 'code'),
            (re.compile(r'^for any (`[^`]+`), (.*)$'), for_any, 'for_any'),
            (re.compile(r'^(.*) or (.*)$'), either, 'or'),
            (re.compile(r'^(.*) and (.*)$'), both, 'and'),
            (re.compile(r'^(`[^`]+`) is not (\S*)$'), is_not, 'is_not'),
            (re.compile(r'^(`[^`]+`) is at least (\d+)$'), is_at_least, 'is_at_least'),
            (re.compile(r'^(`[^`]+`) should be present$'), exists, 'exists'),
            (re.compile(r'^(`[^`]+`) should start with (`[^`]+`)$'), starts_with, 'starts_with'),
            (re.compile(r'^every (`[^`]+`) should be on the (\S* codelist)$'), is_on_list, 'is_on_list'),
            (re.compile(r'^(`[^`]+`) should have more than (\d+) characters$'), is_more_than_x_characters, 'is_more_than_x_characters'),
            (re.compile(r'^(.*) is less than (\d+) months ago$'), is_less_than_x_months_ago, 'is_less_than_x_months_ago'),
            (re.compile(r'^(`[^`]+`) should be available forward$'), is_available_forward, 'is_available_forward'),
            (re.compile(r'^(`[^`]+`) should be available forward by quarters$'), is_available_forward_by_qtrs, 'is_available_forward_by_qtrs'),
        )
        for regex, fn, name in mappings:
            r = regex.match(test)
            if r:
                if r.groups():
                    groups = [self.parse(g, codelists) for g in r.groups()]
                else:
                    groups = [r.group()]
                return lambda x: fn(x, groups=groups, codelists=codelists)
        raise Exception('I don\'t understand {}'.format(test))

    def test_activity(self, activity):
        def translate_result(result_value):
            results = {
                False: 'fail',
                True: 'pass',
                None: 'not-relevant',
            }
            return results[result_value]

        # hierarchy = activity.xpath("@hierarchy")
        # hierarchy = hierarchy[0] if hierarchy != [] else ""
        # try:
        #     iati_identifier = activity.xpath('iati-identifier/text()')[0]
        # except Exception:
        #     iati_identifier = "Unknown"
        return {
            test_id: translate_result(test_fn(activity))
            for test_id, test_fn in self.tests.items()
        }
        # return {
        #     'results': results,
        #     'iati-identifier': iati_identifier,
        #     'hierarchy': hierarchy,
        # }

    def test_activities(self, activities):
        return [self.test_activity(activity) for activity in activities]

    def test_doc(self, filepath):
        doc = etree.parse(filepath)
        activities = doc.xpath('//iati-activity')
        return self.test_activities(activities)

    def summarize_results(self, activities_results):
        summary = {}
        scores = {
            'fail': 0,
            'pass': 0,
            'not-relevant': 0,
        }
        for activity_results in activities_results:
            for test_id, result in activity_results.items():
                if test_id not in summary:
                    summary[test_id] = scores.copy()
                summary[test_id][result] += 1
        return summary
<commit_msg>Make 'at least one … is on list' work<commit_after>import datetime
import re

from lxml import etree

class Foxpath(object):
    def __init__(self, tests, codelists=None):
        self.tests = {
            test['id']: self.parse(test['expression'], codelists)
            for test in tests
        }

    def parse(self, test, codelists):
        def xpath(activity, groups, **kwargs):
            # [1:-1] gets rid of the backticks
            return activity.xpath(groups[0][1:-1])

        def integer(activity, groups, **kwargs):
            return int(groups[0])

        def code(activity, groups, **kwargs):
            return groups[0]

        def codelist(activity, groups, codelists, **kwargs):
            # [:-9] gets rid of ' codelist'
            codelist = codelists.get(groups[0][:-9])
            if not codelist:
                raise Exception('That codelist doesn\'t exist!')
            return codelist

        def for_any(activity, groups, **kwargs):
            return any([groups[1](exp) for exp in groups[0](activity)])

        def either(activity, groups, **kwargs):
            return groups[0](activity) or groups[1](activity)

        def both(activity, groups, **kwargs):
            return groups[0](activity) and groups[1](activity)

        def if_then(activity, groups, **kwargs):
            if groups[0](activity):
                return groups[1](activity)
            else:
                # if the condition fails,
                # return not relevant
                return None

        # defaults to true
        def is_not(activity, groups, **kwargs):
            vals = groups[0](activity)
            const = groups[1](activity)
            return not any([val == str(const) for val in vals])

        # defaults to true
        def is_at_least(activity, groups, **kwargs):
            vals = groups[0](activity)
            const = groups[1](activity)
            if len(vals) < 1:
                print('note: is_at_least was expecting to find one and only one element. Found 0')
                return True
            if len(vals) > 1:
                print('note: is_at_least was expecting to find one and only one element. Found {}'.format(len(vals)))
            return any([int(val) >= const for val in vals])

        def exists(activity, groups, **kwargs):
            return any([val != '' for val in groups[0](activity)])

        def is_on_list(activity, groups, **kwargs):
            def check_list(vals, codelist):
                for v in vals:
                    v = str(v)
                    versions = [v, v.lower(), v.upper()]
                    yield any([v in codelist for v in versions])
            vals = groups[1](activity)
            on_list = check_list(vals, groups[2](activity))
            if groups[0] == 'every':
                return all(on_list)
            else:
                return any(on_list)

        # defaults to false
        def is_more_than_x_characters(activity, groups, **kwargs):
            exp = groups[0](activity)
            val = groups[1](activity)
            return any([len(x) > val for x in exp])

        # defaults to false
        def starts_with(activity, groups, **kwargs):
            x = groups[0](activity)
            y = groups[1](activity)
            if x == [] or y == []:
                return False
            return x[0].startswith(str(y[0]))

        # defaults to false
        def is_less_than_x_months_ago(activity, groups, **kwargs):
            current_date = datetime.date.today()
            def less_than_x_months_ago(date_str, months_ago):
                date = datetime.datetime.strptime(date_str, '%Y-%m-%d')
                year_diff = current_date.year - date.year
                month_diff = 12 * year_diff + current_date.month - date.month
                if month_diff == months_ago:
                    return date.day > current_date.day
                return month_diff < months_ago

            dates = groups[0](activity)
            months_ago = groups[1](activity)
            return any([less_than_x_months_ago(date_str, months_ago) for date_str in dates])

        def forward_budget(activity, els, period):
            def mkdate(date_str):
                dt = datetime.datetime.strptime(date_str, '%Y-%m-%d')
                return dt.date()

            def max_date(dates, default):
                try:
                    return mkdate(max(dates))
                except ValueError:
                    return default

            # Window start is from today onwards. We're only interested in budgets
            # that start or end after today.
            today = datetime.date.today()

            # Window period is for the next 365 days. We don't want to look later
            # than that; we're only interested in budgets that end before then.
            #
            # We get the latest date for end and start; 365 days fwd
            # if there are no dates
            one_years_time = today.replace(today.year + 1)
            end_dates = activity.xpath('activity-date[@type="end-planned"]/@iso-date|activity-date[@type="end-actual"]/@iso-date|activity-date[@type="3"]/@iso-date|activity-date[@type="4"]/@iso-date')
            end_date = max_date(end_dates, one_years_time)

            # If the activity ends earlier than the end of the window period,
            # don't penalise the donor for not having a budget
            escape_end_date = today + datetime.timedelta(days=177)
            if end_date < escape_end_date:
                return None

            def check_after(element, today):
                dates = element.xpath('period-start/@iso-date|period-end/@iso-date')
                return any([mkdate(date) >= today for date in dates])

            def max_budget_length(element, max_budget_length):
                # NB this will error if there's no period-end/@iso-date
                start = mkdate(element.xpath('period-start/@iso-date')[0])
                end = mkdate(element.xpath('period-end/@iso-date')[0])
                return ((end-start).days <= max_budget_length)

            # We set a maximum number of days for which a budget can last,
            # depending on the number of quarters that should be covered.
            if period == 'annual':
                max_days = 370
            elif period == 'quarterly':
                max_days = 94

            # A budget has to be:
            # 1) period-end after today
            # 2) a maximum number of days, depending on # of qtrs.
            for element in els:
                after_today = check_after(element, today)
                within_length = max_budget_length(element, max_days)
                if after_today and within_length:
                    return True
            return False

        def is_available_forward(activity, groups, **kwargs):
            els = groups[0](activity)
            return forward_budget(activity, els, 'annual')

        def is_available_forward_by_qtrs(activity, groups, **kwargs):
            els = groups[0](activity)
            return forward_budget(activity, els, 'quarterly')

        ignored_constants = [
            'at least one',
            'every',
        ]
        if test in ignored_constants:
            return test

        mappings = (
            (re.compile(r'^if (.*) then (.*)$'), if_then, 'if_then'),
            (re.compile(r'\S* codelist$'), codelist, 'codelist'),
            (re.compile(r'`[^`]+`$'), xpath, 'xpath'),
            (re.compile(r'^\d+$'), integer, 'integer'),
            (re.compile(r'[A-Z]+\d+$'), code, 'code'),
            (re.compile(r'^for any (`[^`]+`), (.*)$'), for_any, 'for_any'),
            (re.compile(r'^(.*) or (.*)$'), either, 'or'),
            (re.compile(r'^(.*) and (.*)$'), both, 'and'),
            (re.compile(r'^(`[^`]+`) is not (\S*)$'), is_not, 'is_not'),
            (re.compile(r'^(`[^`]+`) is at least (\d+)$'), is_at_least, 'is_at_least'),
            (re.compile(r'^(`[^`]+`) should be present$'), exists, 'exists'),
            (re.compile(r'^(`[^`]+`) should start with (`[^`]+`)$'), starts_with, 'starts_with'),
            (re.compile(r'^(at least one|every) (`[^`]+`) should be on the (\S* codelist)$'), is_on_list, 'is_on_list'),
            (re.compile(r'^(`[^`]+`) should have more than (\d+) characters$'), is_more_than_x_characters, 'is_more_than_x_characters'),
            (re.compile(r'^(.*) is less than (\d+) months ago$'), is_less_than_x_months_ago, 'is_less_than_x_months_ago'),
            (re.compile(r'^(`[^`]+`) should be available forward$'), is_available_forward, 'is_available_forward'),
            (re.compile(r'^(`[^`]+`) should be available forward by quarters$'), is_available_forward_by_qtrs, 'is_available_forward_by_qtrs'),
        )
        for regex, fn, name in mappings:
            r = regex.match(test)
            if r:
                if r.groups():
                    groups = [self.parse(g, codelists) for g in r.groups()]
                else:
                    groups = [r.group()]
                return lambda x: fn(x, groups=groups, codelists=codelists)
        raise Exception('I don\'t understand {}'.format(test))

    def test_activity(self, activity):
        def translate_result(result_value):
            results = {
                False: 'fail',
                True: 'pass',
                None: 'not-relevant',
            }
            return results[result_value]

        # hierarchy = activity.xpath("@hierarchy")
        # hierarchy = hierarchy[0] if hierarchy != [] else ""
        # try:
        #     iati_identifier = activity.xpath('iati-identifier/text()')[0]
        # except Exception:
        #     iati_identifier = "Unknown"
        return {
            test_id: translate_result(test_fn(activity))
            for test_id, test_fn in self.tests.items()
        }
        # return {
        #     'results': results,
        #     'iati-identifier': iati_identifier,
        #     'hierarchy': hierarchy,
        # }

    def test_activities(self, activities):
        return [self.test_activity(activity) for activity in activities]

    def test_doc(self, filepath):
        doc = etree.parse(filepath)
        activities = doc.xpath('//iati-activity')
        return self.test_activities(activities)

    def summarize_results(self, activities_results):
        summary = {}
        scores = {
            'fail': 0,
            'pass': 0,
            'not-relevant': 0,
        }
        for activity_results in activities_results:
            for test_id, result in activity_results.items():
                if test_id not in summary:
                    summary[test_id] = scores.copy()
                summary[test_id][result] += 1
        return summary
<|endoftext|>