<commit_before>'''
ARCHES - a program developed to inventory and manage immovable cultural heritage.
Copyright (C) 2013 J. Paul Getty Trust and World Monuments Fund
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
'''

import uuid
import json
import couchdb
from copy import copy, deepcopy
from django.db import transaction
from arches.app.models import models
from arches.app.models.tile import Tile
from arches.app.models.graph import Graph
from arches.app.models.system_settings import settings
from django.http import HttpRequest
from arches.app.utils.couch import Couch
from arches.app.utils.betterJSONSerializer import JSONSerializer, JSONDeserializer
import arches.app.views.search as search
from django.utils.translation import ugettext as _

class MobileSurvey(models.MobileSurveyModel):
    """
    Used for mapping complete mobile survey objects to and from the database
    """

    class Meta:
        proxy = True

    def __init__(self, *args, **kwargs):
        super(MobileSurvey, self).__init__(*args, **kwargs)
        # from models.MobileSurveyModel
        # self.id = models.UUIDField(primary_key=True, default=uuid.uuid1)
        # self.name = models.TextField(null=True)
        # self.active = models.BooleanField(default=False)
        # self.createdby = models.ForeignKey(User, related_name='createdby')
        # self.lasteditedby = models.ForeignKey(User, related_name='lasteditedby')
        # self.users = models.ManyToManyField(to=User, through='MobileSurveyXUser')
        # self.groups = models.ManyToManyField(to=Group, through='MobileSurveyXGroup')
        # self.startdate = models.DateField(blank=True, null=True)
        # self.enddate = models.DateField(blank=True, null=True)
        # self.description = models.TextField(null=True)
        # self.bounds = models.MultiPolygonField(null=True)
        # self.tilecache = models.TextField(null=True)
        # self.datadownloadconfig = JSONField(blank=True, null=True, default='{"download":false, "count":1000, "resources":[]}')
        # end from models.MobileSurvey

        self.couch = Couch()

    def save(self):
        super(MobileSurvey, self).save()
        db = self.couch.create_db('project_' + str(self.id))

        survey = self.serialize()
        survey['type'] = 'metadata'
        self.couch.update_doc(db, survey, 'metadata')
        self.load_data_into_couch()
        return db

    def delete(self):
        self.couch.delete_db('project_' + str(self.id))
        super(MobileSurvey, self).delete()

    def serialize(self, fields=None, exclude=None):
        """
        serialize to a different form then used by the internal class structure
        used to append additional values (like parent ontology properties) that
        internal objects (like models.Nodes) don't support
        """
        serializer = JSONSerializer()
        serializer.geom_format = 'geojson'
        obj = serializer.handle_model(self)
        ordered_cards = self.get_ordered_cards()
        ret = JSONSerializer().serializeToPython(obj)
        graphs = []
        graphids = []
        for card in self.cards.all():
            if card.graph_id not in graphids:
                graphids.append(card.graph_id)
                #we may want the full proxy model at some point, but for now just the root node color
                # graphs.append(Graph.objects.get(pk=card.graph_id))
                graph = JSONSerializer().serializeToPython(card.graph, exclude=['functions','disable_instance_creation','deploymentdate','deploymentfile'])
                graph['color'] = card.graph.color
                graph['ontology_id']=str(graph['ontology_id'])
                graphs.append(graph)
        ret['graphs'] = graphs
        ret['cards'] = ordered_cards
        try:
            ret['bounds'] = json.loads(ret['bounds'])
        except TypeError as e:
            print 'Could not parse', ret['bounds'], e

        return ret

    def get_ordered_cards(self):
        ordered_cards = models.MobileSurveyXCard.objects.filter(mobile_survey=self).order_by('sortorder')
        ordered_card_ids = [unicode(mpc.card_id) for mpc in ordered_cards]
        return ordered_card_ids

    def push_edits_to_db(self):
        # read all docs that have changes
        # save back to postgres db
        db = self.couch.create_db('project_' + str(self.id))
        ret = []
        for row in db.view('_all_docs', include_docs=True):
            ret.append(row)
            if 'tileid' in row.doc:
                tile = Tile(row.doc)
                #if tile.filter_by_perm(request.user, 'write_nodegroup'):
                with transaction.atomic():
                    tile.save()
                #tile = models.TileModel.objects.get(pk=row.doc.tileid).update(**row.doc)
        return ret

    def collect_resource_instances_for_couch(self):
        """
        Uses the data definition configs of a mobile survey object to search for
        resource instances relevant to a mobile survey. Takes a user object which
        is required for search.
        """
        query = self.datadownloadconfig['custom']
        resource_types = self.datadownloadconfig['resources']
        instances = {}
        if query in ('', None) and len(resource_types) == 0:
            print "No resources or data query defined"
        else:
            request = HttpRequest()
            request.user = self.lasteditedby
            request.GET['mobiledownload'] = True
            if query in ('', None):
                if len(self.bounds.coords) == 0:
                    default_bounds = settings.DEFAULT_BOUNDS
                    default_bounds['features'][0]['properties']['inverted'] = False
                    request.GET['mapFilter'] = json.dumps(default_bounds)
                else:
                    request.GET['mapFilter'] = json.dumps({u'type': u'FeatureCollection', 'features':[{'geometry': json.loads(self.bounds.json)}]})
                request.GET['typeFilter'] = json.dumps([{'graphid': resourceid, 'inverted': False } for resourceid in self.datadownloadconfig['resources']])
            else:
                parsed = urlparse.urlparse(query)
                urlparams = urlparse.parse_qs(parsed.query)
                for k, v in urlparams.iteritems():
                    request.GET[k] = v[0]
            search_res_json = search.search_results(request)
            search_res = JSONDeserializer().deserialize(search_res_json.content)
            try:
                instances = {hit['_source']['resourceinstanceid']: hit['_source'] for hit in search_res['results']['hits']['hits']}
            except KeyError:
                print 'no instances found in', search_res
        return instances

    def load_tiles_into_couch(self, instances):
        """
        Takes a mobile survey object, a couch database instance, and a dictionary
        of resource instances to identify eligible tiles and load them into the
        database instance
        """
        db = self.couch.create_db('project_' + str(self.id))
        cards = self.cards.all()
        for card in cards:
            tiles = models.TileModel.objects.filter(nodegroup=card.nodegroup_id)
            tiles_serialized = json.loads(JSONSerializer().serialize(tiles))
            for tile in tiles_serialized:
                if str(tile['resourceinstance_id']) in instances:
                    try:
                        tile['type'] = 'tile'
                        couch_record = db.get(tile['tileid'])
                        if couch_record == None:
                            db[tile['tileid']] = tile
                        else:
                            if couch_record['data'] != tile['data']:
                                couch_record['data'] = tile['data']
                                db[tile['tileid']] = couch_record
                    except Exception as e:
                        print e, tile

    def load_instances_into_couch(self, instances):
        """
        Takes a mobile survey object, a couch database instance, and a dictionary
        of resource instances and loads them into the database instance.
        """
        db = self.couch.create_db('project_' + str(self.id))
        for instanceid, instance in instances.iteritems():
            try:
                instance['type'] = 'resource'
                couch_record = db.get(instanceid)
                if couch_record == None:
                    db[instanceid] = instance
            except Exception as e:
                print e, instance

    def load_data_into_couch(self):
        """
        Takes a mobile survey, a couch database intance and a django user and loads
        tile and resource instance data into the couch instance.
        """
        instances = self.collect_resource_instances_for_couch()
        self.load_tiles_into_couch(instances)
        self.load_instances_into_couch(instances)
<commit_msg>Utilize couch update_doc in writing instances and tiles RE: 3246<commit_after>'''
ARCHES - a program developed to inventory and manage immovable cultural heritage.
Copyright (C) 2013 J. Paul Getty Trust and World Monuments Fund
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
'''

import uuid
import json
import couchdb
from copy import copy, deepcopy
from django.db import transaction
from arches.app.models import models
from arches.app.models.tile import Tile
from arches.app.models.graph import Graph
from arches.app.models.system_settings import settings
from django.http import HttpRequest
from arches.app.utils.couch import Couch
from arches.app.utils.betterJSONSerializer import JSONSerializer, JSONDeserializer
import arches.app.views.search as search
from django.utils.translation import ugettext as _

class MobileSurvey(models.MobileSurveyModel):
    """
    Used for mapping complete mobile survey objects to and from the database
    """

    class Meta:
        proxy = True

    def __init__(self, *args, **kwargs):
        super(MobileSurvey, self).__init__(*args, **kwargs)
        # from models.MobileSurveyModel
        # self.id = models.UUIDField(primary_key=True, default=uuid.uuid1)
        # self.name = models.TextField(null=True)
        # self.active = models.BooleanField(default=False)
        # self.createdby = models.ForeignKey(User, related_name='createdby')
        # self.lasteditedby = models.ForeignKey(User, related_name='lasteditedby')
        # self.users = models.ManyToManyField(to=User, through='MobileSurveyXUser')
        # self.groups = models.ManyToManyField(to=Group, through='MobileSurveyXGroup')
        # self.startdate = models.DateField(blank=True, null=True)
        # self.enddate = models.DateField(blank=True, null=True)
        # self.description = models.TextField(null=True)
        # self.bounds = models.MultiPolygonField(null=True)
        # self.tilecache = models.TextField(null=True)
        # self.datadownloadconfig = JSONField(blank=True, null=True, default='{"download":false, "count":1000, "resources":[]}')
        # end from models.MobileSurvey

        self.couch = Couch()

    def save(self):
        super(MobileSurvey, self).save()
        db = self.couch.create_db('project_' + str(self.id))

        survey = self.serialize()
        survey['type'] = 'metadata'
        self.couch.update_doc(db, survey, 'metadata')
        self.load_data_into_couch()
        return db

    def delete(self):
        self.couch.delete_db('project_' + str(self.id))
        super(MobileSurvey, self).delete()

    def serialize(self, fields=None, exclude=None):
        """
        serialize to a different form then used by the internal class structure
        used to append additional values (like parent ontology properties) that
        internal objects (like models.Nodes) don't support
        """
        serializer = JSONSerializer()
        serializer.geom_format = 'geojson'
        obj = serializer.handle_model(self)
        ordered_cards = self.get_ordered_cards()
        ret = JSONSerializer().serializeToPython(obj)
        graphs = []
        graphids = []
        for card in self.cards.all():
            if card.graph_id not in graphids:
                graphids.append(card.graph_id)
                #we may want the full proxy model at some point, but for now just the root node color
                # graphs.append(Graph.objects.get(pk=card.graph_id))
                graph = JSONSerializer().serializeToPython(card.graph, exclude=['functions','disable_instance_creation','deploymentdate','deploymentfile'])
                graph['color'] = card.graph.color
                graph['ontology_id']=str(graph['ontology_id'])
                graphs.append(graph)
        ret['graphs'] = graphs
        ret['cards'] = ordered_cards
        try:
            ret['bounds'] = json.loads(ret['bounds'])
        except TypeError as e:
            print 'Could not parse', ret['bounds'], e

        return ret

    def get_ordered_cards(self):
        ordered_cards = models.MobileSurveyXCard.objects.filter(mobile_survey=self).order_by('sortorder')
        ordered_card_ids = [unicode(mpc.card_id) for mpc in ordered_cards]
        return ordered_card_ids

    def push_edits_to_db(self):
        # read all docs that have changes
        # save back to postgres db
        db = self.couch.create_db('project_' + str(self.id))
        ret = []
        for row in db.view('_all_docs', include_docs=True):
            ret.append(row)
            if 'tileid' in row.doc:
                tile = Tile(row.doc)
                #if tile.filter_by_perm(request.user, 'write_nodegroup'):
                with transaction.atomic():
                    tile.save()
                #tile = models.TileModel.objects.get(pk=row.doc.tileid).update(**row.doc)
        return ret

    def collect_resource_instances_for_couch(self):
        """
        Uses the data definition configs of a mobile survey object to search for
        resource instances relevant to a mobile survey. Takes a user object which
        is required for search.
        """
        query = self.datadownloadconfig['custom']
        resource_types = self.datadownloadconfig['resources']
        instances = {}
        if query in ('', None) and len(resource_types) == 0:
            print "No resources or data query defined"
        else:
            request = HttpRequest()
            request.user = self.lasteditedby
            request.GET['mobiledownload'] = True
            if query in ('', None):
                if len(self.bounds.coords) == 0:
                    default_bounds = settings.DEFAULT_BOUNDS
                    default_bounds['features'][0]['properties']['inverted'] = False
                    request.GET['mapFilter'] = json.dumps(default_bounds)
                else:
                    request.GET['mapFilter'] = json.dumps({u'type': u'FeatureCollection', 'features':[{'geometry': json.loads(self.bounds.json)}]})
                request.GET['typeFilter'] = json.dumps([{'graphid': resourceid, 'inverted': False } for resourceid in self.datadownloadconfig['resources']])
            else:
                parsed = urlparse.urlparse(query)
                urlparams = urlparse.parse_qs(parsed.query)
                for k, v in urlparams.iteritems():
                    request.GET[k] = v[0]
            search_res_json = search.search_results(request)
            search_res = JSONDeserializer().deserialize(search_res_json.content)
            try:
                instances = {hit['_source']['resourceinstanceid']: hit['_source'] for hit in search_res['results']['hits']['hits']}
            except KeyError:
                print 'no instances found in', search_res
        return instances

    def load_tiles_into_couch(self, instances):
        """
        Takes a mobile survey object, a couch database instance, and a dictionary
        of resource instances to identify eligible tiles and load them into the
        database instance
        """
        db = self.couch.create_db('project_' + str(self.id))
        cards = self.cards.all()
        for card in cards:
            tiles = models.TileModel.objects.filter(nodegroup=card.nodegroup_id)
            tiles_serialized = json.loads(JSONSerializer().serialize(tiles))
            for tile in tiles_serialized:
                if str(tile['resourceinstance_id']) in instances:
                    try:
                        tile['type'] = 'tile'
                        self.couch.update_doc(db, tile, tile['tileid'])
                        # couch_record = db.get(tile['tileid'])
                        # if couch_record == None:
                        #     db[tile['tileid']] = tile
                        # else:
                        #     if couch_record['data'] != tile['data']:
                        #         couch_record['data'] = tile['data']
                        #         db[tile['tileid']] = couch_record
                    except Exception as e:
                        print e, tile

    def load_instances_into_couch(self, instances):
        """
        Takes a mobile survey object, a couch database instance, and a dictionary
        of resource instances and loads them into the database instance.
        """
        db = self.couch.create_db('project_' + str(self.id))
        for instanceid, instance in instances.iteritems():
            try:
                instance['type'] = 'resource'
                self.couch.update_doc(db, instance, instanceid)
            except Exception as e:
                print e, instance

    def load_data_into_couch(self):
        """
        Takes a mobile survey, a couch database intance and a django user and loads
        tile and resource instance data into the couch instance.
        """
        instances = self.collect_resource_instances_for_couch()
        self.load_tiles_into_couch(instances)
        self.load_instances_into_couch(instances)
<|endoftext|>


<commit_before>#
# Copyright (C) 2011 - 2014 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
from anyconfig.globals import LOGGER as logging
import anyconfig.backend.base as Base

SUPPORTED = True
try:
    # First, try lxml compatible with elementtree and looks faster a lot.
    # see also: http://getpython3.com/diveintopython3/xml.html
    from lxml2 import etree
except ImportError:
    try:
        import xml.etree.ElementTree as etree
    except ImportError:
        try:
            import elementtree.ElementTree as etree
        except ImportError:
            logging.warn(
                "ElementTree module is not available. Disabled XML support."
            )
            SUPPORTED = False

if SUPPORTED:
    def etree_getroot_fromstring(s):
        return etree.ElementTree(etree.fromstring(s)).getroot()

    def etree_getroot_fromfile(f):
        return etree.parse(f).getroot()
else:
    def _dummy_fun(*args, **kwargs):
        logging.warn(
            "Return {} as no any XML modules used, are not available."
        )
        return {}

    etree_getroot_fromstring = etree_getroot_fromfile = _dummy_fun

def etree_to_container(root, container):
    """
    Convert XML ElementTree to a collection of container objects.
    """
    tree = container()

    if len(root):  # It has children.
        # FIXME: Configuration item cannot have both attributes and
        # values (list) at the same time in current implementation:
        tree[root.tag] = container()
        tree[root.tag]["children"] = [etree_to_container(c, container) for c
                                      in root]
        tree[root.tag]["attributes"] = container(**root.attrib)

        if root.text.strip():
            tree[root.tag]["text"] = root.text.strip()
    else:
        tree[root.tag]["attributes"] = container(**root.attrib)
        if root.text.strip():
            tree[root.tag]["text"] = root.text.strip()

    return tree

class XmlConfigParser(Base.ConfigParser):

    _type = "xml"
    _extensions = ["xml"]
    _supported = SUPPORTED

    @classmethod
    def loads(cls, config_content, **kwargs):
        """
        :param config_content:  Config file content
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: cls.container() object holding config parameters
        """
        root = etree_getroot_fromstring(config_content)
        return etree_to_container(root, cls.container())

    @classmethod
    def load(cls, config_path, **kwargs):
        """
        :param config_path:  Config file path
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: cls.container() object holding config parameters
        """
        root = etree_getroot_fromfile(config_path)
        return etree_to_container(root, cls.container())

    @classmethod
    def dumps_impl(cls, data, **kwargs):
        """
        :param data: Data to dump :: dict
        :param kwargs: backend-specific optional keyword parameters :: dict

        :return: string represents the configuration
        """
        raise NotImplementedError("XML dumper not implemented yet!")

# vim:sw=4:ts=4:et:
<commit_msg>just re-format logging messages in anyconfig.backend.xml_<commit_after>#
# Copyright (C) 2011 - 2014 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
from anyconfig.globals import LOGGER as logging
import anyconfig.backend.base as Base

SUPPORTED = True
try:
    # First, try lxml compatible with elementtree and looks faster a lot.
    # see also: http://getpython3.com/diveintopython3/xml.html
    from lxml2 import etree
except ImportError:
    try:
        import xml.etree.ElementTree as etree
    except ImportError:
        try:
            import elementtree.ElementTree as etree
        except ImportError:
            logging.warn("ElementTree module is not available. Disabled "
                         "XML support.")
            SUPPORTED = False

if SUPPORTED:
    def etree_getroot_fromstring(s):
        return etree.ElementTree(etree.fromstring(s)).getroot()

    def etree_getroot_fromfile(f):
        return etree.parse(f).getroot()
else:
    def _dummy_fun(*args, **kwargs):
        logging.warn("Return {} as no any XML modules are not "
                     "available.")
        return {}

    etree_getroot_fromstring = etree_getroot_fromfile = _dummy_fun

def etree_to_container(root, container):
    """
    Convert XML ElementTree to a collection of container objects.
    """
    tree = container()

    if len(root):  # It has children.
        # FIXME: Configuration item cannot have both attributes and
        # values (list) at the same time in current implementation:
        tree[root.tag] = container()
        tree[root.tag]["children"] = [etree_to_container(c, container) for c
                                      in root]
        tree[root.tag]["attributes"] = container(**root.attrib)

        if root.text.strip():
            tree[root.tag]["text"] = root.text.strip()
    else:
        tree[root.tag]["attributes"] = container(**root.attrib)
        if root.text.strip():
            tree[root.tag]["text"] = root.text.strip()

    return tree

class XmlConfigParser(Base.ConfigParser):

    _type = "xml"
    _extensions = ["xml"]
    _supported = SUPPORTED

    @classmethod
    def loads(cls, config_content, **kwargs):
        """
        :param config_content:  Config file content
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: cls.container() object holding config parameters
        """
        root = etree_getroot_fromstring(config_content)
        return etree_to_container(root, cls.container())

    @classmethod
    def load(cls, config_path, **kwargs):
        """
        :param config_path:  Config file path
        :param kwargs: optional keyword parameters to be sanitized :: dict

        :return: cls.container() object holding config parameters
        """
        root = etree_getroot_fromfile(config_path)
        return etree_to_container(root, cls.container())

    @classmethod
    def dumps_impl(cls, data, **kwargs):
        """
        :param data: Data to dump :: dict
        :param kwargs: backend-specific optional keyword parameters :: dict

        :return: string represents the configuration
        """
        raise NotImplementedError("XML dumper not implemented yet!")

# vim:sw=4:ts=4:et:
<|endoftext|>


<commit_before>"""sample implementations for IntegrationPlugin"""
from plugins.integration.integration import SettingsMixin, UrlsMixin, NavigationMixin, IntegrationPlugin

from django.http import HttpResponse
from django.utils.translation import ugettext_lazy as _
from django.conf.urls import url, include

class SampleIntegrationPlugin(SettingsMixin, UrlsMixin, NavigationMixin, IntegrationPlugin):
    """
    An full integration plugin
    """

    PLUGIN_NAME = "SampleIntegrationPlugin"

    def view_test(self, request):
        """very basic view"""
        return HttpResponse(f'Hi there {request.user.username} this works')

    def setup_urls(self):
        he = [
            url(r'^he/', self.view_test, name='he'),
            url(r'^ha/', self.view_test, name='ha'),
        ]

        return [
            url(r'^hi/', self.view_test, name='hi'),
            url(r'^ho/', include(he), name='ho'),
        ]

    SETTINGS = {
        'PO_FUNCTION_ENABLE': {
            'name': _('Enable PO'),
            'description': _('Enable PO functionality in InvenTree interface'),
            'default': True,
            'validator': bool,
        },
    }

    NAVIGATION = [
        {'name': 'SampleIntegration', 'link': 'plugin:SampleIntegrationPlugin:hi'},
    ]
<commit_msg>refactor name<commit_after>"""sample implementations for IntegrationPlugin"""
from plugins.integration.integration import SettingsMixin, UrlsMixin, NavigationMixin, IntegrationPlugin

from django.http import HttpResponse
from django.utils.translation import ugettext_lazy as _
from django.conf.urls import url, include

class SampleIntegrationPlugin(SettingsMixin, UrlsMixin, NavigationMixin, IntegrationPlugin):
    """
    An full integration plugin
    """

    PLUGIN_NAME = "SampleIntegrationPlugin"

    def view_test(self, request):
        """very basic view"""
        return HttpResponse(f'Hi there {request.user.username} this works')

    def setup_urls(self):
        he_urls = [
            url(r'^he/', self.view_test, name='he'),
            url(r'^ha/', self.view_test, name='ha'),
        ]

        return [
            url(r'^hi/', self.view_test, name='hi'),
            url(r'^ho/', include(he_urls), name='ho'),
        ]

    SETTINGS = {
        'PO_FUNCTION_ENABLE': {
            'name': _('Enable PO'),
            'description': _('Enable PO functionality in InvenTree interface'),
            'default': True,
            'validator': bool,
        },
    }

    NAVIGATION = [
        {'name': 'SampleIntegration', 'link': 'plugin:SampleIntegrationPlugin:hi'},
    ]
<|endoftext|>


<commit_before>#
# Copyright (C) 2015 Satoru SATOH <ssato at redhat.com>
# License: MIT
#
# pylint: disable=missing-docstring, invalid-name
from __future__ import absolute_import

import unittest
import anyconfig.schema as TT

from anyconfig.tests.common import dicts_equal

class Test_00_Base(unittest.TestCase):

    obj = {'a': 1}
    schema = {"type": "object",
              "properties": {"a": {"type": "integer"}}}

    obj2 = dict(a=1, b=[1, 2], c=dict(d="aaa", e=0.1))
    ref_scm = {'properties': {'a': {'type': 'integer'},
                              'b': {'items': {'type': 'integer'},
                                    'type': 'array'},
                              'c': {'properties': {'d': {'type': 'string'},
                                                   'e': {'type':
                                                         'number'}},
                                    'type': 'object'}},
               'type': 'object'}

    opts = dict(ac_schema_typemap=TT._SIMPLETYPE_MAP)

class Test_00_Functions(Test_00_Base):

    def test_20_array_to_schema(self):
        scm = TT.array_to_schema([1])
        ref = dict(items=dict(type="integer"), type="array")
        self.assertTrue(dicts_equal(scm, ref), scm)

    def test_22_array_to_schema__empty_array(self):
        scm = TT.array_to_schema([])
        ref = dict(items=dict(type="string"), type="array")
        self.assertTrue(dicts_equal(scm, ref), scm)

    def test_30_object_to_schema_nodes_iter(self):
        scm = TT.object_to_schema({'a': 1})
        ref = dict(type="object", properties=dict(a=dict(type="integer")))
        self.assertTrue(dicts_equal(scm, ref), scm)

class Test_10_Validation(Test_00_Base):

    def test_10_validate(self):
        (ret, msg) = TT.validate(self.obj, self.schema)
        self.assertFalse(msg)
        self.assertTrue(ret)

    def test_12_validate__ng(self):
        (ret, msg) = TT.validate({'a': "aaa"}, self.schema)
        self.assertTrue(msg)
        self.assertFalse(ret)

class Test_20_GenSchema(Test_00_Base):

    def test_40_gen_schema__primitive_types(self):
        self.assertEqual(TT.gen_schema(None), {'type': 'null'})
        self.assertEqual(TT.gen_schema(0), {'type': 'integer'})
        self.assertEqual(TT.gen_schema("aaa"), {'type': 'string'})

        scm = TT.gen_schema([1])
        ref_scm = {'items': {'type': 'integer'}, 'type': 'array'}
        self.assertTrue(dicts_equal(scm, ref_scm))

        scm = TT.gen_schema({'a': 1})
        ref_scm = {'properties': {'a': {'type': 'integer'}}, 'type': 'object'}
        self.assertTrue(dicts_equal(scm, ref_scm))

    def test_42_gen_schema_and_validate(self):
        scm = TT.gen_schema(self.obj)
        self.assertTrue(TT.validate(self.obj, scm))

    def test_44_gen_schema__complex_types(self):
        scm = TT.gen_schema(self.obj2)
        self.assertTrue(dicts_equal(scm, self.ref_scm))

    def test_46_gen_schema_and_validate__complex_types(self):
        scm = TT.gen_schema(self.obj2)
        self.assertTrue(TT.validate(self.obj2, scm))

def _gen_scm(val):
    return TT.gen_schema(val, ac_schema_strict=True)

class Test_30_GenStrictSchema(Test_00_Base):

    schema = {"type": "object",
              "properties": {"a": {"type": "integer"}},
              "required": ["a"]}

    ref_scm = {'properties': {'a': {'type': 'integer'},
                              'b': {'items': {'type': 'integer'},
                                    'type': 'array',
                                    'minItems': 2, 'uniqueItems': True},
                              'c': {'properties': {'d': {'type': 'string'},
                                                   'e': {'type':
                                                         'number'}},
                                    'type': 'object',
                                    'required': ['d', 'e']}},
               'type': 'object',
               'required': ['a', 'b', 'c']}

    def test_40_gen_schema__primitive_types(self):
        self.assertEqual(_gen_scm(None), {'type': 'null'})
        self.assertEqual(_gen_scm(0), {'type': 'integer'})
        self.assertEqual(_gen_scm("aaa"), {'type': 'string'})

        scm = _gen_scm([1])
        ref_scm = {'items': {'type': 'integer'}, 'type': 'array',
                   'minItems': 1, 'uniqueItems': True}
        self.assertTrue(dicts_equal(scm, ref_scm))

        scm = _gen_scm(["aaa", "bbb", "aaa"])
        ref_scm = {'items': {'type': 'string'}, 'type': 'array',
                   'minItems': 3, 'uniqueItems': False}
        self.assertTrue(dicts_equal(scm, ref_scm))

        scm = _gen_scm({'a': 1})
        ref_scm = {'properties': {'a': {'type': 'integer'}},
                   'type': 'object', 'required': ['a']}
        self.assertTrue(dicts_equal(scm, ref_scm))

    def test_42_gen_schema_and_validate(self):
        scm = _gen_scm(self.obj)
        self.assertTrue(TT.validate(self.obj, scm))

    def test_44_gen_schema__complex_types(self):
        scm = _gen_scm(self.obj2)
        self.assertTrue(dicts_equal(scm, self.ref_scm))

    def test_46_gen_schema_and_validate__complex_types(self):
        scm = _gen_scm(self.obj2)
        self.assertTrue(TT.validate(self.obj2, scm))

# vim:sw=4:ts=4:et:
<commit_msg>change: suppress a warning about access to a protected member<commit_after>#
# Copyright (C) 2015 Satoru SATOH <ssato at redhat.com>
# License: MIT
#
# pylint: disable=missing-docstring, invalid-name, protected-access
from __future__ import absolute_import

import unittest
import anyconfig.schema as TT

from anyconfig.tests.common import dicts_equal

class Test_00_Base(unittest.TestCase):

    obj = {'a': 1}
    schema = {"type": "object",
              "properties": {"a": {"type": "integer"}}}

    obj2 = dict(a=1, b=[1, 2], c=dict(d="aaa", e=0.1))
    ref_scm = {'properties': {'a': {'type': 'integer'},
                              'b': {'items': {'type': 'integer'},
                                    'type': 'array'},
                              'c': {'properties': {'d': {'type': 'string'},
                                                   'e': {'type':
                                                         'number'}},
                                    'type': 'object'}},
               'type': 'object'}

    opts = dict(ac_schema_typemap=TT._SIMPLETYPE_MAP)

class Test_00_Functions(Test_00_Base):

    def test_20_array_to_schema(self):
        scm = TT.array_to_schema([1])
        ref = dict(items=dict(type="integer"), type="array")
        self.assertTrue(dicts_equal(scm, ref), scm)

    def test_22_array_to_schema__empty_array(self):
        scm = TT.array_to_schema([])
        ref = dict(items=dict(type="string"), type="array")
        self.assertTrue(dicts_equal(scm, ref), scm)

    def test_30_object_to_schema_nodes_iter(self):
        scm = TT.object_to_schema({'a': 1})
        ref = dict(type="object", properties=dict(a=dict(type="integer")))
        self.assertTrue(dicts_equal(scm, ref), scm)

class Test_10_Validation(Test_00_Base):

    def test_10_validate(self):
        (ret, msg) = TT.validate(self.obj, self.schema)
        self.assertFalse(msg)
        self.assertTrue(ret)

    def test_12_validate__ng(self):
        (ret, msg) = TT.validate({'a': "aaa"}, self.schema)
        self.assertTrue(msg)
        self.assertFalse(ret)

class Test_20_GenSchema(Test_00_Base):

    def test_40_gen_schema__primitive_types(self):
        self.assertEqual(TT.gen_schema(None), {'type': 'null'})
        self.assertEqual(TT.gen_schema(0), {'type': 'integer'})
        self.assertEqual(TT.gen_schema("aaa"), {'type': 'string'})

        scm = TT.gen_schema([1])
        ref_scm = {'items': {'type': 'integer'}, 'type': 'array'}
        self.assertTrue(dicts_equal(scm, ref_scm))

        scm = TT.gen_schema({'a': 1})
        ref_scm = {'properties': {'a': {'type': 'integer'}}, 'type': 'object'}
        self.assertTrue(dicts_equal(scm, ref_scm))

    def test_42_gen_schema_and_validate(self):
        scm = TT.gen_schema(self.obj)
        self.assertTrue(TT.validate(self.obj, scm))

    def test_44_gen_schema__complex_types(self):
        scm = TT.gen_schema(self.obj2)
        self.assertTrue(dicts_equal(scm, self.ref_scm))

    def test_46_gen_schema_and_validate__complex_types(self):
        scm = TT.gen_schema(self.obj2)
        self.assertTrue(TT.validate(self.obj2, scm))

def _gen_scm(val):
    return TT.gen_schema(val, ac_schema_strict=True)

class Test_30_GenStrictSchema(Test_00_Base):

    schema = {"type": "object",
              "properties": {"a": {"type": "integer"}},
              "required": ["a"]}

    ref_scm = {'properties': {'a': {'type': 'integer'},
                              'b': {'items': {'type': 'integer'},
                                    'type': 'array',
                                    'minItems': 2, 'uniqueItems': True},
                              'c': {'properties': {'d': {'type': 'string'},
                                                   'e': {'type':
                                                         'number'}},
                                    'type': 'object',
                                    'required': ['d', 'e']}},
               'type': 'object',
               'required': ['a', 'b', 'c']}

    def test_40_gen_schema__primitive_types(self):
        self.assertEqual(_gen_scm(None), {'type': 'null'})
        self.assertEqual(_gen_scm(0), {'type': 'integer'})
        self.assertEqual(_gen_scm("aaa"), {'type': 'string'})

        scm = _gen_scm([1])
        ref_scm = {'items': {'type': 'integer'}, 'type': 'array',
                   'minItems': 1, 'uniqueItems': True}
        self.assertTrue(dicts_equal(scm, ref_scm))

        scm = _gen_scm(["aaa", "bbb", "aaa"])
        ref_scm = {'items': {'type': 'string'}, 'type': 'array',
                   'minItems': 3, 'uniqueItems': False}
        self.assertTrue(dicts_equal(scm, ref_scm))

        scm = _gen_scm({'a': 1})
        ref_scm = {'properties': {'a': {'type': 'integer'}},
                   'type': 'object', 'required': ['a']}
        self.assertTrue(dicts_equal(scm, ref_scm))

    def test_42_gen_schema_and_validate(self):
        scm = _gen_scm(self.obj)
        self.assertTrue(TT.validate(self.obj, scm))

    def test_44_gen_schema__complex_types(self):
        scm = _gen_scm(self.obj2)
        self.assertTrue(dicts_equal(scm, self.ref_scm))

    def test_46_gen_schema_and_validate__complex_types(self):
        scm = _gen_scm(self.obj2)
        self.assertTrue(TT.validate(self.obj2, scm))

# vim:sw=4:ts=4:et:
<|endoftext|>


<commit_before>INITALQCFINISHEDLIB = {'Description':'All processes included in the initial qc protocol for finished libraries, except the aggregation step.',
    '24' : 'Customer Gel QC',
    '62' : 'qPCR QC (Library Validation) 4.0',
    '64' : 'Quant-iT QC (Library Validation) 4.0',
    '67' : 'Qubit QC (Library Validation) 4.0',
    '20' : 'CaliperGX QC (DNA)',
    '17' : 'Bioanalyzer QC (Library Validation) 4.0'}
INITALQC ={'Description':'All processes included in the initial qc protocol, except the aggrigation step.',
    '63' : 'Quant-iT QC (DNA) 4.0',
    '65' : 'Quant-iT QC (RNA) 4.0',
    '66' : 'Qubit QC (DNA) 4.0',
    '68' : 'Qubit QC (RNA) 4.0',
    '24' : 'Customer Gel QC',
    '20' : 'CaliperGX QC (DNA)',
    '16' : 'Bioanalyzer QC (DNA) 4.0',
    '18' : 'Bioanalyzer QC (RNA) 4.0',
    '116' : 'CaliperGX QC (RNA)',
    '504' : 'Volume Measurement QC'}
AGRINITQC = {'Description':'Aggregation step of the initial qc protocol',
    '7' : 'Aggregate QC (DNA) 4.0',
    '9' : 'Aggregate QC (RNA) 4.0'}
PREPREPSTART = {'Description':'Process/processes that can be defined as a start of the library preparation protocol. If the work flow involves two library preparation protocols, such as for exome captue, only the steps of the first protocol should be given here.',
    '304' : "Ligate 3' adapters (TruSeq small RNA) 1.0"}
POOLING = {'Description':'To identify the reagent labels (indexes) of each sample. If multiple pooling steps, the first pooling step after indexing should be specified',
    '42': "Library Pooling (Illumina SBS) 4.0",
    '43': "Library Pooling (MiSeq) 4.0",
    '44': "Library Pooling (TruSeq Amplicon) 4.0",
    '45': "Library Pooling (TruSeq Exome) 4.0",
    '58': "Pooling For Multiplexed Sequencing (SS XT) 4.0",
    '255': "Library Pooling (Finished Libraries) 4.0",
    '308': "Library Pooling (TruSeq Small RNA) 1.0",
    '404': "Pre-Pooling (Illumina SBS) 4.0",
    '506': "Pre-Pooling (MiSeq) 4.0",
    '508': "Applications Pre-Pooling"}
PREPSTART = {'Description':'Process/processes that can be defined as a start of the library preparation protocol. The first one of these that are run in lims is used to set the prep start date. If the work flow involves two library preparation protocols, such as for exome capture, the prep start step of the second protocol should be given here. ',
    '10' : 'Aliquot Libraries for Hybridization (SS XT)',
    '47' : 'mRNA Purification, Fragmentation & cDNA synthesis (TruSeq RNA) 4.0',
    '33' : 'Fragment DNA (TruSeq DNA) 4.0',
    '407' : 'Fragment DNA (ThruPlex)',
    '308': 'Library Pooling (TruSeq Small RNA) 1.0',
    '117' : 'Applications Generic Process',
    '405' : 'RiboZero depletion'}
PREPEND = {'Description':'Process that can be defined as a end of the library preparation. If more than one library preparation protocol is included in the work flow, only the prep end step of the second protocol should be given here. Used to set the prep finished date.',
    '157': 'Applications Finish Prep',
    '109' : 'CA Purification',
    '456' : 'Purification (ThruPlex)',
    '111' : 'Amplify Captured Libraries to Add Index Tags (SS XT) 4.0',
    '406' : 'End repair, size selection, A-tailing and adapter ligation (TruSeq PCR-free DNA) 4.0',
    '311': 'Sample Placement (Size Selection)'}
LIBVAL = {'Description':'All processes included in the library validation protocol, except the aggregation step. If the work flow involves two library preparation protocols, such as for exome capture, only the steps of the second protocol should be given here.',
    '62' : 'qPCR QC (Library Validation) 4.0',
    '64' : 'Quant-iT QC (Library Validation) 4.0',
    '67' : 'Qubit QC (Library Validation) 4.0',
    '20' : 'CaliperGX QC (DNA)',
    '17' : 'Bioanalyzer QC (Library Validation) 4.0'}
LIBVALFINISHEDLIB = {'Description':'',
    '62' : 'qPCR QC (Library Validation) 4.0',
    '64' : 'Quant-iT QC (Library Validation) 4.0',
    '67' : 'Qubit QC (Library Validation) 4.0',
    '20' : 'CaliperGX QC (DNA)',
    '17' : 'Bioanalyzer QC (Library Validation) 4.0',
    '24' : 'Customer Gel QC'}
AGRLIBVAL = {'Description':'The aggregation step of the library validation protocol',
    '8': 'Aggregate QC (Library Validation) 4.0'}
SEQSTART = {'Description':'These processes are used to set the sequencing_start_date',
    '23':'Cluster Generation (Illumina SBS) 4.0',
    '26':'Denature, Dilute and Load Sample (MiSeq) 4.0'}
DILSTART = {'Description':'These processes are used to set the dilution_and_pooling_start_date',
    '40' : 'Library Normalization (MiSeq) 4.0',
    '39' : 'Library Normalization (Illumina SBS) 4.0'}
SEQUENCING = {'Description':'Sequencing',
    '38' : 'Illumina Sequencing (Illumina SBS) 4.0',
    '46' : 'MiSeq Run (MiSeq) 4.0'}
WORKSET = {'Description':'To identify the work sets on which the samples has been run. The process used to define a workset for the protocol. ',
    '204' : 'Setup Workset/Plate'}
SUMMARY = {'Description':'',
    '356' : 'Project Summary 1.3'}
DEMULTIPLEX={'Description':'',
    '13' : 'Bcl Conversion & Demultiplexing (Illumina SBS) 4.0'}
CALIPER = {'Description':'',
    '20' : 'CaliperGX QC (DNA)',
    '116' : 'CaliperGX QC (RNA)'}

FINLIB = ['Finished library', 'Amplicon']
PROJ_UDF_EXCEPTIONS = ['customer_reference','uppnex_id','reference_genome','application']
SAMP_UDF_EXCEPTIONS = ['customer_name','reads_requested_(millions)','min_reads',
    'm_reads','dup_rm','status_auto','status_manual','average_size_bp','incoming_qc_status']

PROCESSCATEGORIES = {'INITALQCFINISHEDLIB' : INITALQCFINISHEDLIB,
                     'INITALQC':INITALQC,
                     'AGRINITQC':AGRINITQC,
                     'PREPREPSTART':PREPREPSTART,
                     'POOLING':POOLING,
                     'PREPSTART':PREPSTART,
                     'PREPEND':PREPEND,
                     'LIBVAL':LIBVAL,
                     'LIBVALFINISHEDLIB':LIBVALFINISHEDLIB,
                     'AGRLIBVAL':AGRLIBVAL,
                     'SEQSTART':SEQSTART,
                     'DILSTART':DILSTART,
                     'SEQUENCING':SEQUENCING,
                     'WORKSET':WORKSET,
                     'SUMMARY':SUMMARY,
                     'DEMULTIPLEX':DEMULTIPLEX,
                     'CALIPER':CALIPER}
<commit_msg>added missing step<commit_after>INITALQCFINISHEDLIB = {'Description':'All processes included in the initial qc protocol for finished libraries, except the aggregation step.',
    '24' : 'Customer Gel QC',
    '62' : 'qPCR QC (Library Validation) 4.0',
    '64' : 'Quant-iT QC (Library Validation) 4.0',
    '67' : 'Qubit QC (Library Validation) 4.0',
    '20' : 'CaliperGX QC (DNA)',
    '17' : 'Bioanalyzer QC (Library Validation) 4.0'}
INITALQC ={'Description':'All processes included in the initial qc protocol, except the aggrigation step.',
    '63' : 'Quant-iT QC (DNA) 4.0',
    '65' : 'Quant-iT QC (RNA) 4.0',
    '66' : 'Qubit QC (DNA) 4.0',
    '68' : 'Qubit QC (RNA) 4.0',
    '24' : 'Customer Gel QC',
    '20' : 'CaliperGX QC (DNA)',
    '16' : 'Bioanalyzer QC (DNA) 4.0',
    '18' : 'Bioanalyzer QC (RNA) 4.0',
    '116' : 'CaliperGX QC (RNA)',
    '504' : 'Volume Measurement QC'}
AGRINITQC = {'Description':'Aggregation step of the initial qc protocol',
    '7' : 'Aggregate QC (DNA) 4.0',
    '9' : 'Aggregate QC (RNA) 4.0'}
PREPREPSTART = {'Description':'Process/processes that can be defined as a start of the library preparation protocol. If the work flow involves two library preparation protocols, such as for exome captue, only the steps of the first protocol should be given here.',
    '304' : "Ligate 3' adapters (TruSeq small RNA) 1.0"}
POOLING = {'Description':'To identify the reagent labels (indexes) of each sample. If multiple pooling steps, the first pooling step after indexing should be specified',
    '42': "Library Pooling (Illumina SBS) 4.0",
    '43': "Library Pooling (MiSeq) 4.0",
    '44': "Library Pooling (TruSeq Amplicon) 4.0",
    '45': "Library Pooling (TruSeq Exome) 4.0",
    '58': "Pooling For Multiplexed Sequencing (SS XT) 4.0",
    '255': "Library Pooling (Finished Libraries) 4.0",
    '308': "Library Pooling (TruSeq Small RNA) 1.0",
    '404': "Pre-Pooling (Illumina SBS) 4.0",
    '506': "Pre-Pooling (MiSeq) 4.0",
    '508': "Applications Pre-Pooling"}
PREPSTART = {'Description':'Process/processes that can be defined as a start of the library preparation protocol. The first one of these that are run in lims is used to set the prep start date. If the work flow involves two library preparation protocols, such as for exome capture, the prep start step of the second protocol should be given here. ',
    '10' : 'Aliquot Libraries for Hybridization (SS XT)',
    '47' : 'mRNA Purification, Fragmentation & cDNA synthesis (TruSeq RNA) 4.0',
    '33' : 'Fragment DNA (TruSeq DNA) 4.0',
    '407' : 'Fragment DNA (ThruPlex)',
    '308': 'Library Pooling (TruSeq Small RNA) 1.0',
    '117' : 'Applications Generic Process',
    '454' : 'ThruPlex template preparation and synthesis',
    '405' : 'RiboZero depletion'}
PREPEND = {'Description':'Process that can be defined as a end of the library preparation. If more than one library preparation protocol is included in the work flow, only the prep end step of the second protocol should be given here. Used to set the prep finished date.',
    '157': 'Applications Finish Prep',
    '109' : 'CA Purification',
    '456' : 'Purification (ThruPlex)',
    '111' : 'Amplify Captured Libraries to Add Index Tags (SS XT) 4.0',
    '406' : 'End repair, size selection, A-tailing and adapter ligation (TruSeq PCR-free DNA) 4.0',
    '311': 'Sample Placement (Size Selection)'}
LIBVAL = {'Description':'All processes included in the library validation protocol, except the aggregation step. If the work flow involves two library preparation protocols, such as for exome capture, only the steps of the second protocol should be given here.',
    '62' : 'qPCR QC (Library Validation) 4.0',
    '64' : 'Quant-iT QC (Library Validation) 4.0',
    '67' : 'Qubit QC (Library Validation) 4.0',
    '20' : 'CaliperGX QC (DNA)',
    '17' : 'Bioanalyzer QC (Library Validation) 4.0'}
LIBVALFINISHEDLIB = {'Description':'',
    '62' : 'qPCR QC (Library Validation) 4.0',
    '64' : 'Quant-iT QC (Library Validation) 4.0',
    '67' : 'Qubit QC (Library Validation) 4.0',
    '20' : 'CaliperGX QC (DNA)',
    '17' : 'Bioanalyzer QC (Library Validation) 4.0',
    '24' : 'Customer Gel QC'}
AGRLIBVAL = {'Description':'The aggregation step of the library validation protocol',
    '8': 'Aggregate QC (Library Validation) 4.0'}
SEQSTART = {'Description':'These processes are used to set the sequencing_start_date',
    '23':'Cluster Generation (Illumina SBS) 4.0',
    '26':'Denature, Dilute and Load Sample (MiSeq) 4.0'}
DILSTART = {'Description':'These processes are used to set the dilution_and_pooling_start_date',
    '40' : 'Library Normalization (MiSeq) 4.0',
    '39' : 'Library Normalization (Illumina SBS) 4.0'}
SEQUENCING = {'Description':'Sequencing',
    '38' : 'Illumina Sequencing (Illumina SBS) 4.0',
    '46' : 'MiSeq Run (MiSeq) 4.0'}
WORKSET = {'Description':'To identify the work sets on which the samples has been run. The process used to define a workset for the protocol. ',
    '204' : 'Setup Workset/Plate'}
SUMMARY = {'Description':'',
    '356' : 'Project Summary 1.3'}
DEMULTIPLEX={'Description':'',
    '13' : 'Bcl Conversion & Demultiplexing (Illumina SBS) 4.0'}
CALIPER = {'Description':'',
    '20' : 'CaliperGX QC (DNA)',
    '116' : 'CaliperGX QC (RNA)'}

FINLIB = ['Finished library', 'Amplicon']
PROJ_UDF_EXCEPTIONS = ['customer_reference','uppnex_id','reference_genome','application']
SAMP_UDF_EXCEPTIONS = ['customer_name','reads_requested_(millions)','min_reads',
    'm_reads','dup_rm','status_auto','status_manual','average_size_bp','incoming_qc_status']

PROCESSCATEGORIES = {'INITALQCFINISHEDLIB' : INITALQCFINISHEDLIB,
                     'INITALQC':INITALQC,
                     'AGRINITQC':AGRINITQC,
                     'PREPREPSTART':PREPREPSTART,
                     'POOLING':POOLING,
                     'PREPSTART':PREPSTART,
                     'PREPEND':PREPEND,
                     'LIBVAL':LIBVAL,
                     'LIBVALFINISHEDLIB':LIBVALFINISHEDLIB,
                     'AGRLIBVAL':AGRLIBVAL,
                     'SEQSTART':SEQSTART,
                     'DILSTART':DILSTART,
                     'SEQUENCING':SEQUENCING,
                     'WORKSET':WORKSET,
                     'SUMMARY':SUMMARY,
                     'DEMULTIPLEX':DEMULTIPLEX,
                     'CALIPER':CALIPER}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import unicode_literals

EXTENSIONS = {
    'apinotes': {'text', 'apinotes'},
    'asar': {'binary', 'asar'},
    'bash': {'text', 'shell', 'bash'},
    'bat': {'text', 'batch'},
    'bmp': {'binary', 'image', 'bitmap'},
    'bz2': {'binary', 'bzip2'},
    'c': {'text', 'c'},
    'cc': {'text', 'c++'},
    'cfg': {'text'},
    'cmake': {'text', 'cmake'},
    'cnf': {'text'},
    'coffee': {'text', 'coffee'},
    'conf': {'text'},
    'cpp': {'text', 'c++'},
    'crt': {'text', 'pem'},
    'cson': {'text', 'cson'},
    'css': {'text', 'css'},
    'csv': {'text', 'csv'},
    'cxx': {'text', 'c++'},
    'def': {'text', 'def'},
    'dtd': {'text', 'dtd'},
    'ear': {'binary', 'zip', 'jar'},
    'ejs': {'text', 'ejs'},
    'eot': {'binary', 'eot'},
    'eps': {'binary', 'eps'},
    'erb': {'text', 'erb'},
    'exe': {'binary'},
    'eyaml': {'text', 'yaml'},
    'feature': {'text', 'gherkin'},
    'fish': {'text', 'fish'},
    'gemspec': {'text', 'ruby'},
    'gif': {'binary', 'image', 'gif'},
    'go': {'text', 'go'},
    'gotmpl': {'text', 'gotmpl'},
    'gradle': {'text', 'groovy'},
    'groovy': {'text', 'groovy'},
    'gyb': {'text', 'gyb'},
    'gyp': {'text', 'gyp', 'python'},
    'gypi': {'text', 'gyp', 'python'},
    'gz': {'binary', 'gzip'},
    'h': {'text', 'header', 'c', 'c++'},
    'hpp': {'text', 'header', 'c++'},
    'htm': {'text', 'html'},
    'html': {'text', 'html'},
    'hxx': {'text', 'header', 'c++'},
    'icns': {'binary', 'icns'},
    'ico': {'binary', 'icon'},
    'idl': {'text', 'idl'},
    'inc': {'text', 'inc'},
    'ini': {'text', 'ini'},
    'jade': {'text', 'jade'},
    'jar': {'binary', 'zip', 'jar'},
    'java': {'text', 'java'},
    'jenkinsfile': {'text', 'groovy'},
    'jinja': {'text', 'jinja'},
    'jinja2': {'text', 'jinja'},
    'jpeg': {'binary', 'image', 'jpeg'},
    'jpg': {'binary', 'image', 'jpeg'},
    'js': {'text', 'javascript'},
    'json': {'text', 'json'},
    'jsx': {'text', 'jsx'},
    'key': {'text', 'pem'},
    'kt': {'text', 'kotlin'},
    'less': {'text', 'less'},
    'm': {'text', 'c', 'objective-c'},
    'manifest': {'text', 'manifest'},
    'map': {'text', 'map'},
    'markdown': {'text', 'markdown'},
    'md': {'text', 'markdown'},
    'mk': {'text', 'makefile'},
    'mm': {'text', 'c++', 'objective-c++'},
    'modulemap': {'text', 'modulemap'},
    'ngdoc': {'text', 'ngdoc'},
    'otf': {'binary', 'otf'},
    'p12': {'binary', 'p12'},
    'patch': {'text', 'diff'},
    'pdf': {'binary', 'pdf'},
    'pem': {'text', 'pem'},
    'php': {'text', 'php'},
    'php4': {'text', 'php'},
    'php5': {'text', 'php'},
    'pl': {'text', 'perl'},
    'plist': {'text', 'plist'},
    'png': {'binary', 'image', 'png'},
    'po': {'text', 'pofile'},
    'pp': {'text', 'puppet'},
    'proto': {'text', 'proto'},
    'purs': {'text', 'purescript'},
    'py': {'text', 'python'},
    'r': {'text', 'r'},
    'rb': {'text', 'ruby'},
    'rs': {'text', 'rust'},
    'rst': {'text', 'rst'},
    's': {'text', 'asm'},
    'scss': {'text', 'scss'},
    'sh': {'text', 'shell'},
    'sls': {'text', 'salt'},
    'so': {'binary'},
    'spec': {'text', 'spec'},
    'sql': {'text', 'sql'},
    'svg': {'text', 'svg'},
    'swf': {'binary', 'swf'},
    'swift': {'text', 'swift'},
    'swiftdeps': {'text', 'swiftdeps'},
    'tac': {'text', 'twisted', 'python'},
    'tar': {'binary', 'tar'},
    'tgz': {'binary', 'gzip'},
    'thrift': {'text', 'thrift'},
    'tiff': {'binary', 'image', 'tiff'},
    'toml': {'text', 'toml'},
    'tf': {'text', 'terraform'},
    'ts': {'text', 'ts'},
    'tsx': {'text', 'tsx'},
    'ttf': {'binary', 'ttf'},
    'txt': {'text', 'plain-text'},
    'vdx': {'text', 'vdx'},
    'vim': {'text', 'vim'},
    'vue': {'text', 'vue'},
    'war': {'binary', 'zip', 'jar'},
    'wav': {'binary', 'audio', 'wav'},
    'whl': {'binary', 'wheel', 'zip'},
    'woff': {'binary', 'woff'},
    'woff2': {'binary', 'woff2'},
    'wsgi': {'text', 'wsgi', 'python'},
    'xml': {'text', 'xml'},
    'xsd': {'text', 'xml', 'xsd'},
    'xsl': {'text', 'xml', 'xsl'},
    'yaml': {'text', 'yaml'},
    'yml': {'text', 'yaml'},
    'zip': {'binary', 'zip'},
    'zsh': {'text', 'shell', 'zsh'},
}

NAMES = {
    '.babelrc': {'text', 'json', 'babelrc'},
    '.bowerrc': {'text', 'json', 'bowerrc'},
    '.coveragerc': {'text', 'ini', 'coveragerc'},
    '.dockerignore': {'text', 'dockerignore'},
    '.editorconfig': {'text', 'editorconfig'},
    '.gitattributes': {'text', 'gitattributes'},
    '.gitignore': {'text', 'gitignore'},
    '.gitmodules': {'text', 'gitmodules'},
    '.jshintrc': {'text', 'json', 'jshintrc'},
    '.mailmap': {'text', 'mailmap'},
    '.npmignore': {'text', 'npmignore'},
    '.yamllint': {'text', 'yaml', 'yamllint'},
    'AUTHORS': EXTENSIONS['txt'],
    'COPYING': EXTENSIONS['txt'],
    'Dockerfile': {'text', 'dockerfile'},
    'Gemfile': EXTENSIONS['rb'],
    'Jenkinsfile': {'text', 'groovy'},
    'LICENSE': EXTENSIONS['txt'],
    'MAINTAINERS': EXTENSIONS['txt'],
    'Makefile': EXTENSIONS['mk'],
    'NOTICE': EXTENSIONS['txt'],
    'PATENTS': EXTENSIONS['txt'],
    'README': EXTENSIONS['txt'],
    'Rakefile': EXTENSIONS['rb'],
    'setup.cfg': EXTENSIONS['ini'],
}
<commit_msg>Add Lua file extension<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import unicode_literals

EXTENSIONS = {
    'apinotes': {'text', 'apinotes'},
    'asar': {'binary', 'asar'},
    'bash': {'text', 'shell', 'bash'},
    'bat': {'text', 'batch'},
    'bmp': {'binary', 'image', 'bitmap'},
    'bz2': {'binary', 'bzip2'},
    'c': {'text', 'c'},
    'cc': {'text', 'c++'},
    'cfg': {'text'},
    'cmake': {'text', 'cmake'},
    'cnf': {'text'},
    'coffee': {'text', 'coffee'},
    'conf': {'text'},
    'cpp': {'text', 'c++'},
    'crt': {'text', 'pem'},
    'cson': {'text', 'cson'},
    'css': {'text', 'css'},
    'csv': {'text', 'csv'},
    'cxx': {'text', 'c++'},
    'def': {'text', 'def'},
    'dtd': {'text', 'dtd'},
    'ear': {'binary', 'zip', 'jar'},
    'ejs': {'text', 'ejs'},
    'eot': {'binary', 'eot'},
    'eps': {'binary', 'eps'},
    'erb': {'text', 'erb'},
    'exe': {'binary'},
    'eyaml': {'text', 'yaml'},
    'feature': {'text', 'gherkin'},
    'fish': {'text', 'fish'},
    'gemspec': {'text', 'ruby'},
    'gif': {'binary', 'image', 'gif'},
    'go': {'text', 'go'},
    'gotmpl': {'text', 'gotmpl'},
    'gradle': {'text', 'groovy'},
    'groovy': {'text', 'groovy'},
    'gyb': {'text', 'gyb'},
    'gyp': {'text', 'gyp', 'python'},
    'gypi': {'text', 'gyp', 'python'},
    'gz': {'binary', 'gzip'},
    'h': {'text', 'header', 'c', 'c++'},
    'hpp': {'text', 'header', 'c++'},
    'htm': {'text', 'html'},
    'html': {'text', 'html'},
    'hxx': {'text', 'header', 'c++'},
    'icns': {'binary', 'icns'},
    'ico': {'binary', 'icon'},
    'idl': {'text', 'idl'},
    'inc': {'text', 'inc'},
    'ini': {'text', 'ini'},
    'jade': {'text', 'jade'},
    'jar': {'binary', 'zip', 'jar'},
    'java': {'text', 'java'},
    'jenkinsfile': {'text', 'groovy'},
    'jinja': {'text', 'jinja'},
    'jinja2': {'text', 'jinja'},
    'jpeg': {'binary', 'image', 'jpeg'},
    'jpg': {'binary', 'image', 'jpeg'},
    'js': {'text', 'javascript'},
    'json': {'text', 'json'},
    'jsx': {'text', 'jsx'},
    'key': {'text', 'pem'},
    'kt': {'text', 'kotlin'},
    'less': {'text', 'less'},
    'lua': {'text', 'lua'},
    'm': {'text', 'c', 'objective-c'},
    'manifest': {'text', 'manifest'},
    'map': {'text', 'map'},
    'markdown': {'text', 'markdown'},
    'md': {'text', 'markdown'},
    'mk': {'text', 'makefile'},
    'mm': {'text', 'c++', 'objective-c++'},
    'modulemap': {'text', 'modulemap'},
    'ngdoc': {'text', 'ngdoc'},
    'otf': {'binary', 'otf'},
    'p12': {'binary', 'p12'},
    'patch': {'text', 'diff'},
    'pdf': {'binary', 'pdf'},
    'pem': {'text', 'pem'},
    'php': {'text', 'php'},
    'php4': {'text', 'php'},
    'php5': {'text', 'php'},
    'pl': {'text', 'perl'},
    'plist': {'text', 'plist'},
    'png': {'binary', 'image', 'png'},
    'po': {'text', 'pofile'},
    'pp': {'text', 'puppet'},
    'proto': {'text', 'proto'},
    'purs': {'text', 'purescript'},
    'py': {'text', 'python'},
    'r': {'text', 'r'},
    'rb': {'text', 'ruby'},
    'rs': {'text', 'rust'},
    'rst': {'text', 'rst'},
    's': {'text', 'asm'},
    'scss': {'text', 'scss'},
    'sh': {'text', 'shell'},
    'sls': {'text', 'salt'},
    'so': {'binary'},
    'spec': {'text', 'spec'},
    'sql': {'text', 'sql'},
    'svg': {'text', 'svg'},
    'swf': {'binary', 'swf'},
    'swift': {'text', 'swift'},
    'swiftdeps': {'text', 'swiftdeps'},
    'tac': {'text', 'twisted', 'python'},
    'tar': {'binary', 'tar'},
    'tgz': {'binary', 'gzip'},
    'thrift': {'text', 'thrift'},
    'tiff': {'binary', 'image', 'tiff'},
    'toml': {'text', 'toml'},
    'tf': {'text', 'terraform'},
    'ts': {'text', 'ts'},
    'tsx': {'text', 'tsx'},
    'ttf': {'binary', 'ttf'},
    'txt': {'text', 'plain-text'},
    'vdx': {'text', 'vdx'},
    'vim': {'text', 'vim'},
    'vue': {'text', 'vue'},
    'war': {'binary', 'zip', 'jar'},
    'wav': {'binary', 'audio', 'wav'},
    'whl': {'binary', 'wheel', 'zip'},
    'woff': {'binary', 'woff'},
    'woff2': {'binary', 'woff2'},
    'wsgi': {'text', 'wsgi', 'python'},
    'xml': {'text', 'xml'},
    'xsd': {'text', 'xml', 'xsd'},
    'xsl': {'text', 'xml', 'xsl'},
    'yaml': {'text', 'yaml'},
    'yml': {'text', 'yaml'},
    'zip': {'binary', 'zip'},
    'zsh': {'text', 'shell', 'zsh'},
}

NAMES = {
    '.babelrc': {'text', 'json', 'babelrc'},
    '.bowerrc': {'text', 'json', 'bowerrc'},
    '.coveragerc': {'text', 'ini', 'coveragerc'},
    '.dockerignore': {'text', 'dockerignore'},
    '.editorconfig': {'text', 'editorconfig'},
    '.gitattributes': {'text', 'gitattributes'},
    '.gitignore': {'text', 'gitignore'},
    '.gitmodules': {'text', 'gitmodules'},
    '.jshintrc': {'text', 'json', 'jshintrc'},
    '.mailmap': {'text', 'mailmap'},
    '.npmignore': {'text', 'npmignore'},
    '.yamllint': {'text', 'yaml', 'yamllint'},
    'AUTHORS': EXTENSIONS['txt'],
    'COPYING': EXTENSIONS['txt'],
    'Dockerfile': {'text', 'dockerfile'},
    'Gemfile': EXTENSIONS['rb'],
    'Jenkinsfile': {'text', 'groovy'},
    'LICENSE': EXTENSIONS['txt'],
    'MAINTAINERS': EXTENSIONS['txt'],
    'Makefile': EXTENSIONS['mk'],
    'NOTICE': EXTENSIONS['txt'],
    'PATENTS': EXTENSIONS['txt'],
    'README': EXTENSIONS['txt'],
    'Rakefile': EXTENSIONS['rb'],
    'setup.cfg': EXTENSIONS['ini'],
}
<|endoftext|>


<commit_before><commit_msg>replaced tiger.py cli w/ argparse<commit_after><|endoftext|>


<commit_before>import importlib
from ifind.search.query import Query
from ifind.search.cache import QueryCache
from ifind.search.engines import ENGINE_LIST
from ifind.search.exceptions import EngineLoadException
from ifind.search.exceptions import InvalidQueryException

class Engine(object):
    """
    Abstract class representing an ifind search engine.

    """
    def __init__(self, cache_type=None, proxies=None):
        """
        Engine constructor.

        Kwargs:
            cache_type (str): type of cache to use i.e.'instance' or 'engine'.
            proxies (dict): mapping of proxies to use i.e. {"http":"10.10.1.10:3128", "https":"10.10.1.10:1080"}.

        Attributes:
            cache (QueryCache): instance of QueryCache, instantiated by cache_type arg

        Raises:
            CacheException

        Usage:
            See EngineFactory.

        """
        self.name = self.__class__.__name__

        self.cache_type = cache_type
        if cache_type:
            self._cache = QueryCache(self)

        self.proxies = proxies # TODO engine proxies

    def search(self, query):
        """
        Public search method for an Engine instance, returning the results of a query argument.
        Caching handled here, true search implementation deferred to subclass '_search' method.

        Args:
            query (ifind Query): object encapsulating details of search query.

        Returns:
            ifind Response: object encapsulating a search request's results.

        Raises:
            CacheException, InvalidQueryException

        Usage:
            query = Query('hello world')
            engine = EngineFactory('wikipedia')
            response = engine.search(query)

        """
        if not isinstance(query, Query):
            raise InvalidQueryException('Engine', 'Expected type {}'
                                        .format("<class 'ifind.search.query.Query'>"))

        if self.cache_type:
            if query in self._cache:
                print "********************** cache"
                return self._cache.get(query)
            else:
                response = self._search(query)
                self._cache.store(query, response)
                print "********************** request"
                return response
        else:
            return self._search(query)

    def _search(self, query):
        """
        Abstract search method for an Engine instance, to be implemented by subclasses.
        Performs a search and retrieves the results as an ifind Response.

        Args:
            query (ifind Query): object encapsulating details of a search query.

        Returns:
            ifind Response: object encapsulating a search request's results.

        Raises:
            See subclasses.

        Usage:
            Private method.

        """
        pass

class EngineFactory(object):
    """
    Public class representing an ifind search engine factory.

    Instantiates and returns an Engine subclass, keyed by
    the 'engine' argument, or None otherwise.

    Args:
        engine (str): Name of Engine subclass to instantiate.

    Kwargs:
        Accepted kwargs defined by the Engine subclass being instantiated.

    Returns:
        ifind Engine object: Dynamically dispatched instance of Engine subclass.

    Raises:
        EngineLoadException.
        See subclasses.

    Usage:
        engine = EngineFactory('wikipedia')
        engine = EngineFactory('bing', cache_type='engine')
        engine_list = EngineFactory().engines()

        """
    def engines(self):
        """
        Returns list of available engines.

        """
        return ENGINE_LIST

    def __contains__(self, engine):
        """
        Special containment override for 'in' operator.

        Usage:
            print 'bing' in EngineFactory() --> True

        """
        return engine.lower() in ENGINE_LIST

    def __iter__(self):
        """
        Implements iterator for EngineFactory, returning a single engine at a time.

        Usage:
            for engine in EngineFactory():
                print result

        """
        for engine in ENGINE_LIST:
            yield engine

    def __new__(cls, engine="", **kwargs):
        """
        Overrides object construction so as to bypass __init__ so to dynamically
        dispatch 'engine' subclass. See class docstring.

        """
        # if engine in subclass list, return instantiation
        if engine.lower() in ENGINE_LIST:
            module = importlib.import_module('ifind.search.engines.{}'.format(engine.lower()))
            return getattr(module, engine.lower().title())(**kwargs)

        # if 'engine' defined but not in supported list
        elif engine:
            raise EngineLoadException("EngineFactory", "Engine '{}' not found".format(engine.lower().title()))

        # if 'engine' undefined ("" or None)
        else:
            return super(EngineFactory, cls).__new__(cls)<commit_msg>changed cache_type for better EngineFactory api, improved docstring too<commit_after>import importlib
from ifind.search.query import Query
from ifind.search.cache import QueryCache
from ifind.search.engines import ENGINE_LIST
from ifind.search.exceptions import EngineLoadException
from ifind.search.exceptions import InvalidQueryException

class Engine(object):
    """
    Abstract class representing an ifind search engine.

    """
    def __init__(self, cache=None, proxies=None):
        """
        Engine constructor.

        Kwargs:
            cache_type (str): type of cache to use i.e.'instance' or 'engine'.
            proxies (dict): mapping of proxies to use i.e. {"http":"10.10.1.10:3128", "https":"10.10.1.10:1080"}.

        Attributes:
            cache (QueryCache): instance of QueryCache, instantiated by cache_type arg

        Raises:
            CacheException

        Usage:
            See EngineFactory.

        """
        self.name = self.__class__.__name__

        self.cache_type = cache
        if cache:
            self._cache = QueryCache(self)

        self.proxies = proxies # TODO engine proxies

    def search(self, query):
        """
        Public search method for an Engine instance, returning the results of a query argument.
        Caching handled here, true search implementation deferred to subclass '_search' method.

        Args:
            query (ifind Query): object encapsulating details of search query.

        Returns:
            ifind Response: object encapsulating a search request's results.

        Raises:
            CacheException, InvalidQueryException

        Usage:
            query = Query('hello world')
            engine = EngineFactory('wikipedia')
            response = engine.search(query)

        """
        if not isinstance(query, Query):
            raise InvalidQueryException('Engine', 'Expected type {}'
                                        .format("<class 'ifind.search.query.Query'>"))

        if self.cache_type:
            if query in self._cache:
                print "********************** cache"
                return self._cache.get(query)
            else:
                response = self._search(query)
                self._cache.store(query, response)
                print "********************** request"
                return response
        else:
            return self._search(query)

    def _search(self, query):
        """
        Abstract search method for an Engine instance, to be implemented by subclasses.
        Performs a search and retrieves the results as an ifind Response.

        Args:
            query (ifind Query): object encapsulating details of a search query.

        Returns:
            ifind Response: object encapsulating a search request's results.

        Raises:
            See subclasses.

        Usage:
            Private method.

        """
        pass

class EngineFactory(object):
    """
    Public class representing an ifind search engine factory.

    Instantiates and returns an Engine subclass, keyed by
    the 'engine' argument, or None otherwise.

    Args:
        engine (str): Name of Engine subclass to instantiate.

    Kwargs:
        cache (str): Type of cache to associate with engine.
                     'engine' is persistent across engines
                     'instance' is valid only for that instance

    Returns:
        ifind Engine object: Dynamically dispatched instance of Engine subclass.

    Raises:
        EngineLoadException.
        See subclasses.

    Usage:
        engine = EngineFactory('govuk')
        engine = EngineFactory('bing', cache='engine')
        engine_list = EngineFactory().engines()

        """
    def engines(self):
        """
        Returns list of available engines.

        """
        return ENGINE_LIST

    def __contains__(self, engine):
        """
        Special containment override for 'in' operator.

        Usage:
            print 'bing' in EngineFactory() --> True

        """
        return engine.lower() in ENGINE_LIST

    def __iter__(self):
        """
        Implements iterator for EngineFactory, returning a single engine at a time.

        Usage:
            for engine in EngineFactory():
                print result

        """
        for engine in ENGINE_LIST:
            yield engine

    def __new__(cls, engine="", **kwargs):
        """
        Overrides object construction so as to bypass __init__ so to dynamically
        dispatch 'engine' subclass. See class docstring.

        """
        # if engine in subclass list, return instantiation
        if engine.lower() in ENGINE_LIST:
            module = importlib.import_module('ifind.search.engines.{}'.format(engine.lower()))
            return getattr(module, engine.lower().title())(**kwargs)

        # if 'engine' defined but not in supported list
        elif engine:
            raise EngineLoadException("EngineFactory", "Engine '{}' not found".format(engine.lower().title()))

        # if 'engine' undefined ("" or None)
        else:
            return super(EngineFactory, cls).__new__(cls)<|endoftext|>


<commit_before>#!/usr/bin/env python
# Software License Agreement (BSD License)
#
# Copyright (c) 2014, Philipp Wagner <bytefish[at]gmx[dot]de>.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above
#    copyright notice, this list of conditions and the following
#    disclaimer in the documentation and/or other materials provided
#    with the distribution.
#  * Neither the name of the author nor the names of its
#    contributors may be used to endorse or promote products derived
#    from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import sys
# append facerec
sys.path.append("../..")

from facerec.feature import ChainOperator, Fisherfaces
from facerec.preprocessing import Resize
from facerec.distance import EuclideanDistance
from facerec.classifier import NearestNeighbor
from facerec.model import PredictableModel
from facerec.validation import KFoldCrossValidation
from facerec.serialization import save_model, load_model

# This is the face recognition module for the RESTful Webservice.
#
# The current implementation uses a fixed model defined in code.
# A simple wrapper works around a limitation of the current framework,
# because as time of writing this only integer labels can be passed into
# the classifiers of the facerec framework.
#
#
#
# First of all define a model, that supports updating itself. This
# is necessary, so we don't need to retrain the entire model for each
# input image. This is not suitable for all models, it may be limited
# to Local Binary Patterns for the current framework.
#
# Moreover it hides the complexity of dealing with integer labels for
# the faces (or objects) to identify.
#
class PredictableModelWrapper(object):

    def __init__(self, model):
        self.model = model
        self.numeric_dataset = NumericDataSet()

    def compute(self):
        X,y = self.numeric_dataset.get()
        self.model.compute(X,y)

    def set_data(self, numeric_dataset):
        self.numeric_dataset = numeric_dataset

    def predict(self, image):
        class_label = self.model.predict(image)
        return self.numeric_dataset.resolve_by_num(class_label)

    def update(self, name, image):
        self.numeric_dataset.add(name, image)
        class_label = self.numeric_dataset.resolve_by_str(name)
        extracted_feature = self.feature.extract(image)
        self.classifier.update(extracted_feature, class_label)

# To abstract the dirty things away, we are going to use a
# new class, which we call a NumericDataSet. This NumericDataSet
# allows us to add images and turn them into a facerec compatible
# representation.
class NumericDataSet(object):
    def __init__(self):
        self.data = {}
        self.str_to_num_mapping = {}
        self.num_to_str_mapping = {}

    def add(self, identifier, image):
        numeric_identifier = self.__resolve_subject_id(identifier)
        try:
            self.data[identifier].append(image)
        except:
            self.data[identifier] = [image]
            numerical_identifier = len(self.str_to_num_mapping)
            # Store in mapping tables:
            self.str_to_num_mapping[identifier] = numerical_identifier
            self.num_to_str_mapping[numerical_identifier] = identifier

    def get(self):
        X = []
        y = []
        for name, num in self.str_to_num_mapping:
            for image in self.data[name]:
                X.append(image)
                y.append(num)
        return X,y

    def resolve_by_str(self, identifier):
        return self.str_num_mapping[identifier]

    def resolve_by_num(self, numerical_identifier):
        return self.num_to_str_mapping[numerical_identifier]

    def length(self):
        return len(self.data)

    def __repr__(self):
        print "TODO"
<commit_msg>A first shot...<commit_after>#!/usr/bin/env python
# Software License Agreement (BSD License)
#
# Copyright (c) 2014, Philipp Wagner <bytefish[at]gmx[dot]de>.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above
#    copyright notice, this list of conditions and the following
#    disclaimer in the documentation and/or other materials provided
#    with the distribution.
#  * Neither the name of the author nor the names of its
#    contributors may be used to endorse or promote products derived
#    from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import sys
# append facerec
sys.path.append("../..")

from facerec.feature import ChainOperator, Fisherfaces
from facerec.preprocessing import Resize
from facerec.distance import EuclideanDistance
from facerec.classifier import NearestNeighbor
from facerec.model import PredictableModel
from facerec.validation import KFoldCrossValidation
from facerec.serialization import save_model, load_model

# This is the face recognition module for the RESTful Webservice.
#
# The current implementation uses a fixed model defined in code.
# A simple wrapper works around a limitation of the current framework,
# because as time of writing this only integer labels can be passed into
# the classifiers of the facerec framework.

# First of all define a model, that supports updating itself. This
# is necessary, so we don't need to retrain the entire model for each
# input image. This is not suitable for all models, it may be limited
# to Local Binary Patterns for the current framework.
#
# Moreover it hides the complexity of dealing with integer labels for
# the faces (or objects) to identify.
#
class PredictableModelWrapper(object):

    def __init__(self, model):
        self.model = model
        self.numeric_dataset = NumericDataSet()

    def compute(self):
        X,y = self.numeric_dataset.get()
        self.model.compute(X,y)

    def set_data(self, numeric_dataset):
        self.numeric_dataset = numeric_dataset

    def predict(self, image):
        class_label = self.model.predict(image)
        return self.numeric_dataset.resolve_by_num(class_label)

    def update(self, name, image):
        self.numeric_dataset.add(name, image)
        class_label = self.numeric_dataset.resolve_by_str(name)
        extracted_feature = self.feature.extract(image)
        self.classifier.update(extracted_feature, class_label)

# Now define a method to get a model trained on a NumericDataSet,
# which should also store the model into a file if filename is given.
def get_model(numeric_dataset, model_filename=None):
    feature = ChainOperator(Resize(128,128), Fisherfaces())
    classifier = NearestNeighbor(dist_metric=EuclideanDistance(), k=1)
    model = PredictableModelWrapper(feature=feature, classifier=classifier)
    model.set_data(numeric_dataset)
    model.compute()
    if not model_filename is None:
        save_model(model_filename, model)

# Now a method to read images from a folder. It's pretty simple,
# since we can pass a numeric_dataset into the read_images  method
# and just add the files as we read them.
def read_images(path, identifier, numeric_dataset):
    for filename in os.listdir(subject_path):
        try:
            img = Image.open(os.path.join(subject_path, filename))
            img = img.convert("L")
            numeric_dataset.add(identifier, img)
        except IOError, (errno, strerror):
            print "I/O error({0}): {1}".format(errno, strerror)
        except:
            print "Unexpected error:", sys.exc_info()[0]
            raise

# read_csv is a tiny little method, that takes a csv file defined
# like this:
#
#   Philipp Wagner;D:/images/philipp
#   Another Name;D:/images/another_name
#   ...
#
def read_from_csv(filename):
    numeric_dataset = NumericDataSet()
    with open(filename, 'rb') as csvfile:
        datasetfilename = csv.reader(csvfile, delimiter=';', quotechar='#')
        for row in spamreader:
            identifier = row[0]
            path = row[1]
            read_images(path, identifier, numeric_dataset)
    return numeric_dataset

# Just some sugar on top...
def get_model_from_csv(filename, out_model_filename):
    numeric_dataset = read_from_csv(filename)
    return get_model(numeric_dataset, out_model_filename)
<|endoftext|>


<commit_before>__author__ = 'Thomas Rueckstiess, ruecksti@in.tum.de'

from pybrain.rl.learners.learner import ExploringLearner, DataSetLearner, OntogeneticLearner
from pybrain.rl.explorers.discrete.egreedy import EpsilonGreedyExplorer

class ValueBasedLearner(ExploringLearner, DataSetLearner, EpisodicLearner):
    """ An RL algorithm based on estimating a value-function."""

    #: Does the algorithm work on-policy or off-policy?
    offPolicy = False

    #: Does the algorithm run in batch mode or online?
    batchMode = True

    _module = None
    _explorer = None

    def __init__(self):
        """ Create a default explorer for discrete learning tasks. """
        self.explorer = EpsilonGreedyExplorer()

    def _setModule(self, module):
        """ Set module and tell explorer about the module. """
        self.explorer.module = module
        self._module = module

    def _getModule(self):
        """ Return the internal module. """
        return self._module

    module = property(_getModule, _setModule)

    def _setExplorer(self, explorer):
        """ Set explorer and tell it the module, if already available. """
        self._explorer = explorer
        if self.module:
            self._explorer.module = self.module

    def _getExplorer(self):
        """ Return the internal explorer. """
        return self._explorer

    explorer = property(_getExplorer, _setExplorer)

<commit_msg>removed OntogeneticLearner from import list and added EpisodicLearner.<commit_after>__author__ = 'Thomas Rueckstiess, ruecksti@in.tum.de'

from pybrain.rl.learners.learner import ExploringLearner, DataSetLearner, EpisodicLearner
from pybrain.rl.explorers.discrete.egreedy import EpsilonGreedyExplorer

class ValueBasedLearner(ExploringLearner, DataSetLearner, EpisodicLearner):
    """ An RL algorithm based on estimating a value-function."""

    #: Does the algorithm work on-policy or off-policy?
    offPolicy = False

    #: Does the algorithm run in batch mode or online?
    batchMode = True

    _module = None
    _explorer = None

    def __init__(self):
        """ Create a default explorer for discrete learning tasks. """
        self.explorer = EpsilonGreedyExplorer()

    def _setModule(self, module):
        """ Set module and tell explorer about the module. """
        self.explorer.module = module
        self._module = module

    def _getModule(self):
        """ Return the internal module. """
        return self._module

    module = property(_getModule, _setModule)

    def _setExplorer(self, explorer):
        """ Set explorer and tell it the module, if already available. """
        self._explorer = explorer
        if self.module:
            self._explorer.module = self.module

    def _getExplorer(self):
        """ Return the internal explorer. """
        return self._explorer

    explorer = property(_getExplorer, _setExplorer)

<|endoftext|>


<commit_before>""" Module contains small functions that are used to format, transform or classify the date-time structures """

__author__ = 'Bohdan Mushkevych'

import calendar

from datetime import datetime, timedelta
from system.process_context import ProcessContext

SYNERGY_DATE_PATTERN = '%Y%m%d%H'
SYNERGY_SESSION_PATTERN = '%Y%m%d%H%M%S'

def define_pattern(timeperiod):
    hasYear = False
    hasMonth = False
    hasDay = False
    hasHour = False

    for index in range(0, len(timeperiod)):
        if 0 <= index < 4 and timeperiod[index] != '0':
            hasYear = True
        elif 4 <= index < 6 and timeperiod[index] != '0':
            hasMonth = True
        elif 6 <= index < 8 and timeperiod[index] != '0':
            hasDay = True
        elif index >= 8 and timeperiod[index] != '0':
            hasHour = True

    pattern = ''
    if hasYear:
        pattern += '%Y'
    else:
        pattern += '0000'
    if hasMonth:
        pattern += '%m'
    else:
        pattern += '00'
    if hasDay:
        pattern += '%d'
    else:
        pattern += '00'
    if hasHour:
        pattern += '%H'
    else:
        pattern += '00'

    return pattern

def raw_to_session(timestamp):
    """@param timestamp: as float from time.time() output
    @return string in YYYYMMDD_HHMMSS format"""
    t = datetime.utcfromtimestamp(timestamp)
    return t.strftime(SYNERGY_SESSION_PATTERN)

def session_to_hour(timestamp):
    """@param timestamp: as string in YYYYMMDD_HHMMSS format
    @return string in YYYYMMDDHH format"""
    t = datetime.strptime(timestamp, SYNERGY_SESSION_PATTERN)
    return t.strftime(SYNERGY_DATE_PATTERN)

def hour_to_day(timeperiod):
    """@param timeperiod: as string in YYYYMMDDHH format
    @return string in YYYYMMDD00 format"""
    t = datetime.strptime(timeperiod, SYNERGY_DATE_PATTERN)
    return t.strftime('%Y%m%d00')

def day_to_month(timeperiod):
    """@param timeperiod: as string in YYYYMMDD00 format
    @return string in YYYYMM0000 format"""
    t = datetime.strptime(timeperiod, '%Y%m%d00')
    return t.strftime('%Y%m0000')

def month_to_year(timeperiod):
    """@param timeperiod: as string in YYYYMM0000 format
    @return string in YYYY000000 format"""
    t = datetime.strptime(timeperiod, '%Y%m0000')
    return t.strftime('%Y000000')

def actual_timeperiod(time_qualifier):
    """ method receives process' time qualifier (hourly/daily/monthly/yearly)
    @return: string representing current datetime (utc now) in proper Synergy Date format """
    return datetime_to_synergy(time_qualifier, datetime.utcnow())

def increment_timeperiod(time_qualifier, timeperiod):
    """ method is used by Scheduler to define <<next>> time period.
    For hourly, it is next hour: 2010010119 -> 2010010120
    For month - next month:      2010010000 -> 2010020000, etc"""

    pattern = define_pattern(timeperiod)
    t = datetime.strptime(timeperiod, pattern)

    if time_qualifier == ProcessContext.QUALIFIER_HOURLY:
        t = t + timedelta(hours=1)
        return t.strftime('%Y%m%d%H')

    elif time_qualifier == ProcessContext.QUALIFIER_DAILY:
        t = t + timedelta(days=1)
        return t.strftime('%Y%m%d00')

    elif time_qualifier == ProcessContext.QUALIFIER_MONTHLY:
        if t.month + 1 > 12:
            new_month = 1
            new_year = t.year + 1
            t = t.replace(year=new_year, month=new_month)
        else:
            t = t.replace(month=t.month + 1)
        return t.strftime('%Y%m0000')

    elif time_qualifier == ProcessContext.QUALIFIER_YEARLY:
        t = t.replace(year=t.year + 1)
        return t.strftime('%Y000000')
    else:
        raise ValueError('unknown time qualifier: %s' % time_qualifier)

def cast_to_time_qualifier(time_qualifier, timeperiod):
    """ method is used to cast synergy_time accordingly to process' time qualifier.
    For example: for QUALIFIER_HOURLY, it can be either 20100101_19 or 20100101_193412 """

    date_format = None
    if time_qualifier == ProcessContext.QUALIFIER_HOURLY:
        if len(timeperiod) > 10:
            t = datetime.strptime(timeperiod, SYNERGY_SESSION_PATTERN)
            return t.strftime(SYNERGY_DATE_PATTERN)
        else:
            return timeperiod

    elif time_qualifier == ProcessContext.QUALIFIER_DAILY:
        date_format = '%Y%m%d00'
    elif time_qualifier == ProcessContext.QUALIFIER_MONTHLY:
        date_format = '%Y%m0000'
    elif time_qualifier == ProcessContext.QUALIFIER_YEARLY:
        date_format = '%Y000000'

    pattern = define_pattern(timeperiod)
    t = datetime.strptime(timeperiod, pattern)

    if date_format is not None:
        return t.strftime(date_format)
    else:
        raise ValueError('unknown time qualifier: %s' % time_qualifier)

def datetime_to_synergy(time_qualifier, dt):
    """ method parses datetime and returns Synergy Date"""
    if time_qualifier == ProcessContext.QUALIFIER_HOURLY:
        date_format = SYNERGY_DATE_PATTERN
    elif time_qualifier == ProcessContext.QUALIFIER_DAILY:
        date_format = '%Y%m%d00'
    elif time_qualifier == ProcessContext.QUALIFIER_MONTHLY:
        date_format = '%Y%m0000'
    elif time_qualifier == ProcessContext.QUALIFIER_YEARLY:
        date_format = '%Y000000'
    elif time_qualifier == ProcessContext.QUALIFIER_REAL_TIME:
        date_format = SYNERGY_SESSION_PATTERN
    else:
        raise ValueError('unknown time qualifier: %s' % time_qualifier)
    return dt.strftime(date_format)

def synergy_to_datetime(time_qualifier, timeperiod):
    """ method receives timeperiod in Synergy format YYYYMMDD_HH and convert it to _naive_ datetime"""
    if time_qualifier == ProcessContext.QUALIFIER_HOURLY:
        date_format = SYNERGY_DATE_PATTERN
    elif time_qualifier == ProcessContext.QUALIFIER_DAILY:
        date_format = '%Y%m%d00'
    elif time_qualifier == ProcessContext.QUALIFIER_MONTHLY:
        date_format = '%Y%m0000'
    elif time_qualifier == ProcessContext.QUALIFIER_YEARLY:
        date_format = '%Y000000'
    elif time_qualifier == ProcessContext.QUALIFIER_REAL_TIME:
        date_format = SYNERGY_SESSION_PATTERN
    else:
        raise ValueError('unknown time qualifier: %s' % time_qualifier)
    return datetime.strptime(timeperiod, date_format).replace(tzinfo=None)

def session_to_epoch(timestamp):
    """ converts Synergy Timestamp for session to UTC zone seconds since epoch """
    utc_timetuple = datetime.strptime(timestamp, SYNERGY_SESSION_PATTERN).replace(tzinfo=None).utctimetuple()
    return calendar.timegm(utc_timetuple)

if __name__ == '__main__':
    pass<commit_msg>Added ability to shift timeperiod by delta<commit_after>""" Module contains small functions that are used to format, transform or classify the date-time structures """

__author__ = 'Bohdan Mushkevych'

import calendar

from datetime import datetime, timedelta
from system.process_context import ProcessContext

SYNERGY_DATE_PATTERN = '%Y%m%d%H'
SYNERGY_SESSION_PATTERN = '%Y%m%d%H%M%S'

def define_pattern(timeperiod):
    has_year = False
    has_month = False
    has_day = False
    has_hour = False

    for index in range(0, len(timeperiod)):
        if 0 <= index < 4 and timeperiod[index] != '0':
            has_year = True
        elif 4 <= index < 6 and timeperiod[index] != '0':
            has_month = True
        elif 6 <= index < 8 and timeperiod[index] != '0':
            has_day = True
        elif index >= 8 and timeperiod[index] != '0':
            has_hour = True

    pattern = ''
    if has_year:
        pattern += '%Y'
    else:
        pattern += '0000'
    if has_month:
        pattern += '%m'
    else:
        pattern += '00'
    if has_day:
        pattern += '%d'
    else:
        pattern += '00'
    if has_hour:
        pattern += '%H'
    else:
        pattern += '00'

    return pattern

def raw_to_session(timestamp):
    """@param timestamp: as float from time.time() output
    @return string in YYYYMMDD_HHMMSS format"""
    t = datetime.utcfromtimestamp(timestamp)
    return t.strftime(SYNERGY_SESSION_PATTERN)

def session_to_hour(timestamp):
    """@param timestamp: as string in YYYYMMDD_HHMMSS format
    @return string in YYYYMMDDHH format"""
    t = datetime.strptime(timestamp, SYNERGY_SESSION_PATTERN)
    return t.strftime(SYNERGY_DATE_PATTERN)

def hour_to_day(timeperiod):
    """@param timeperiod: as string in YYYYMMDDHH format
    @return string in YYYYMMDD00 format"""
    t = datetime.strptime(timeperiod, SYNERGY_DATE_PATTERN)
    return t.strftime('%Y%m%d00')

def day_to_month(timeperiod):
    """@param timeperiod: as string in YYYYMMDD00 format
    @return string in YYYYMM0000 format"""
    t = datetime.strptime(timeperiod, '%Y%m%d00')
    return t.strftime('%Y%m0000')

def month_to_year(timeperiod):
    """@param timeperiod: as string in YYYYMM0000 format
    @return string in YYYY000000 format"""
    t = datetime.strptime(timeperiod, '%Y%m0000')
    return t.strftime('%Y000000')

def actual_timeperiod(time_qualifier):
    """ method receives process' time qualifier (hourly/daily/monthly/yearly)
    @return: string representing current datetime (utc now) in proper Synergy Date format """
    return datetime_to_synergy(time_qualifier, datetime.utcnow())

def increment_timeperiod(time_qualifier, timeperiod, delta=1):
    """ method is used by Scheduler to define <<next>> time period.
    For hourly, it is next hour: 2010010119 -> 2010010120
    For month - next month:      2010010000 -> 2010020000, etc"""

    pattern = define_pattern(timeperiod)
    t = datetime.strptime(timeperiod, pattern)

    if time_qualifier == ProcessContext.QUALIFIER_HOURLY:
        t = t + timedelta(hours=delta)
        return t.strftime('%Y%m%d%H')

    elif time_qualifier == ProcessContext.QUALIFIER_DAILY:
        t = t + timedelta(days=delta)
        return t.strftime('%Y%m%d00')

    elif time_qualifier == ProcessContext.QUALIFIER_MONTHLY:
        if delta not in [-1, 1]:
            raise ValueError('For QUALIFIER_MONTHLY delta can be only +/- 1')

        if t.month + delta > 12:
            new_month = 1
            new_year = t.year + 1
            t = t.replace(year=new_year, month=new_month)
        elif t.month + delta < 1:
            new_month = 12
            new_year = t.year - 1
            t = t.replace(year=new_year, month=new_month)
        else:
            t = t.replace(month=t.month + delta)
        return t.strftime('%Y%m0000')

    elif time_qualifier == ProcessContext.QUALIFIER_YEARLY:
        t = t.replace(year=t.year + delta)
        return t.strftime('%Y000000')
    else:
        raise ValueError('unknown time qualifier: %s' % time_qualifier)

def cast_to_time_qualifier(time_qualifier, timeperiod):
    """ method is used to cast synergy_time accordingly to process' time qualifier.
    For example: for QUALIFIER_HOURLY, it can be either 20100101_19 or 20100101_193412 """

    date_format = None
    if time_qualifier == ProcessContext.QUALIFIER_HOURLY:
        if len(timeperiod) > 10:
            t = datetime.strptime(timeperiod, SYNERGY_SESSION_PATTERN)
            return t.strftime(SYNERGY_DATE_PATTERN)
        else:
            return timeperiod

    elif time_qualifier == ProcessContext.QUALIFIER_DAILY:
        date_format = '%Y%m%d00'
    elif time_qualifier == ProcessContext.QUALIFIER_MONTHLY:
        date_format = '%Y%m0000'
    elif time_qualifier == ProcessContext.QUALIFIER_YEARLY:
        date_format = '%Y000000'

    pattern = define_pattern(timeperiod)
    t = datetime.strptime(timeperiod, pattern)

    if date_format is not None:
        return t.strftime(date_format)
    else:
        raise ValueError('unknown time qualifier: %s' % time_qualifier)

def datetime_to_synergy(time_qualifier, dt):
    """ method parses datetime and returns Synergy Date"""
    if time_qualifier == ProcessContext.QUALIFIER_HOURLY:
        date_format = SYNERGY_DATE_PATTERN
    elif time_qualifier == ProcessContext.QUALIFIER_DAILY:
        date_format = '%Y%m%d00'
    elif time_qualifier == ProcessContext.QUALIFIER_MONTHLY:
        date_format = '%Y%m0000'
    elif time_qualifier == ProcessContext.QUALIFIER_YEARLY:
        date_format = '%Y000000'
    elif time_qualifier == ProcessContext.QUALIFIER_REAL_TIME:
        date_format = SYNERGY_SESSION_PATTERN
    else:
        raise ValueError('unknown time qualifier: %s' % time_qualifier)
    return dt.strftime(date_format)

def synergy_to_datetime(time_qualifier, timeperiod):
    """ method receives timeperiod in Synergy format YYYYMMDD_HH and convert it to _naive_ datetime"""
    if time_qualifier == ProcessContext.QUALIFIER_HOURLY:
        date_format = SYNERGY_DATE_PATTERN
    elif time_qualifier == ProcessContext.QUALIFIER_DAILY:
        date_format = '%Y%m%d00'
    elif time_qualifier == ProcessContext.QUALIFIER_MONTHLY:
        date_format = '%Y%m0000'
    elif time_qualifier == ProcessContext.QUALIFIER_YEARLY:
        date_format = '%Y000000'
    elif time_qualifier == ProcessContext.QUALIFIER_REAL_TIME:
        date_format = SYNERGY_SESSION_PATTERN
    else:
        raise ValueError('unknown time qualifier: %s' % time_qualifier)
    return datetime.strptime(timeperiod, date_format).replace(tzinfo=None)

def session_to_epoch(timestamp):
    """ converts Synergy Timestamp for session to UTC zone seconds since epoch """
    utc_timetuple = datetime.strptime(timestamp, SYNERGY_SESSION_PATTERN).replace(tzinfo=None).utctimetuple()
    return calendar.timegm(utc_timetuple)

if __name__ == '__main__':
    pass
<|endoftext|>