<commit_before>import logging
import json
import uuid
from collections import defaultdict

import tornado.web
import tornado.httpclient
from tornado.platform.asyncio import to_asyncio_future
import pymongo
import motor

from rest_tools.client import RestClient
from iceprod.server.rest import RESTHandler, RESTHandlerSetup, authorization
from iceprod.server.util import nowstr, dataset_statuses, dataset_status_sort

logger = logging.getLogger('rest.datasets')

def setup(config, *args, **kwargs):
    """
    Setup method for Dataset REST API.

    Sets up any database connections or other prerequisites.

    Args:
        config (dict): an instance of :py:class:`iceprod.server.config`.

    Returns:
        list: Routes for dataset, which can be passed to :py:class:`tornado.web.Application`.
    """
    cfg_rest = config.get('rest',{}).get('datasets',{})
    db_cfg = cfg_rest.get('database',{})

    # add indexes
    db = pymongo.MongoClient(**db_cfg).datasets
    if 'dataset_id_index' not in db.datasets.index_information():
        db.datasets.create_index('dataset_id', name='dataset_id_index', unique=True)

    handler_cfg = RESTHandlerSetup(config, *args, **kwargs)
    handler_cfg.update({
        'database': motor.motor_tornado.MotorClient(**db_cfg).datasets,
    })

    return [
        (r'/datasets', MultiDatasetHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)', DatasetHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)/description', DatasetDescriptionHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)/status', DatasetStatusHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)/priority', DatasetPriorityHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)/jobs_submitted', DatasetJobsSubmittedHandler, handler_cfg),
        (r'/dataset_summaries/status', DatasetSummariesStatusHandler, handler_cfg),
    ]

class BaseHandler(RESTHandler):
    """
    Base handler for Dataset REST API.
    """
    def initialize(self, database=None, **kwargs):
        super(BaseHandler, self).initialize(**kwargs)
        self.db = database

class MultiDatasetHandler(BaseHandler):
    """
    Handle multi-group requests.
    """
    @authorization(roles=['admin','client','system','user']) #TODO: figure out how to do auth for each dataset in the list
    async def get(self):
        """
        Get a dict of datasets.

        Params (optional):
            status: | separated list of status filters
            groups: | separated list of groups to filter on
            users: | separated list of users to filter on
            keys: | separated list of keys to return for each dataset

        Returns:
            dict: {<dataset_id>: metadata}
        """
        query = {}
        status = self.get_argument('status', None)
        if status:
            query['status'] = {'$in': status.split('|')}
        groups = self.get_argument('groups', None)
        if groups:
            query['group'] = {'$in': groups.split('|')}
        users = self.get_argument('users', None)
        if users:
            query['username'] = {'$in': users.split('|')}

        projection = {'_id': False}
        keys = self.get_argument('keys', None)
        if keys:
            projection.update({x:True for x in keys.split('|') if x})

        ret = {}
        async for row in self.db.datasets.find(query, projection=projection):
            k = row['dataset_id']
            ret[k] = row
        self.write(ret)
        self.finish()

    @authorization(roles=['admin','user']) # anyone should be able to create a dataset
    async def post(self):
        """
        Add a dataset.

        Body should contain all necessary fields for a dataset.
        """
        data = json.loads(self.request.body)

        # validate first
        req_fields = {
            'description': str,
            'jobs_submitted': int,
            'tasks_submitted': int,
            'tasks_per_job': int,
            'group': str,
        }
        for k in req_fields:
            if k not in data:
                raise tornado.web.HTTPError(400, reason='missing key: '+k)
            if not isinstance(data[k], req_fields[k]):
                r = 'key "{}" should be of type {}'.format(k, req_fields[k].__name__)
                raise tornado.web.HTTPError(400, reason=r)

        opt_fields = {
            'priority': int,
            'debug': bool,
            'jobs_immutable': bool,
            'status': str,
        }
        for k in opt_fields:
            if k in data and not isinstance(data[k], opt_fields[k]):
                r = 'key "{}" should be of type {}'.format(k, opt_fields[k].__name__)
                raise tornado.web.HTTPError(400, reason=r)

        bad_fields = set(data).difference(set(opt_fields).union(req_fields))
        if bad_fields:
            r = 'invalid keys found'
            raise tornado.web.HTTPError(400, reason=r)

        if data['jobs_submitted'] == 0 and data['tasks_per_job'] <= 0:
            r = '"tasks_per_job" must be > 0'
            raise tornado.web.HTTPError(400, reason=r)
        elif data['tasks_submitted'] != 0 and data['tasks_submitted'] / data['jobs_submitted'] != data['tasks_per_job']:
            r = '"tasks_per_job" does not match "tasks_submitted"/"jobs_submitted"'
            raise tornado.web.HTTPError(400, reason=r)

        # generate dataset number
        ret = await self.db.settings.find_one_and_update(
            {'name': 'dataset_num'},
            {'$inc': {'num': 1}},
            projection={'num': True, '_id': False},
            upsert=True,
            return_document=pymongo.ReturnDocument.AFTER)
        dataset_num = ret['num']

        # set some fields
        data['dataset_id'] = uuid.uuid1().hex
        data['dataset'] = dataset_num
        if 'status' not in data:
            data['status'] = 'processing'
        data['start_date'] = nowstr()
        data['username'] = self.auth_data['username']
        if 'priority' not in data:
            data['priority'] = 0.5
        if 'debug' not in data:
            data['debug'] = False
        if 'jobs_immutable' not in data:
            data['jobs_immutable'] = False

        # insert
        ret = await self.db.datasets.insert_one(data)

        # set auth rules
        url = '/auths/'+data['dataset_id']
        http_client = RestClient(self.auth_url, token=self.module_auth_key)
        auth_data = {
            'read_groups':[data['group'],'users'],
            'write_groups':[data['group']],
        }
        logger.info('Authorization header: %s', 'bearer '+self.module_auth_key)
        await http_client.request('PUT', url, auth_data)

        # return success
        self.set_status(201)
        self.set_header('Location', '/datasets/'+data['dataset_id'])
        self.write({'result': '/datasets/'+data['dataset_id']})
        self.finish()

class DatasetHandler(BaseHandler):
    """
    Handle dataset requests.
    """
    @authorization(roles=['admin','client','system','pilot'], attrs=['dataset_id:read'])
    async def get(self, dataset_id):
        """
        Get a dataset.

        Args:
            dataset_id (str): the dataset

        Returns:
            dict: dataset metadata
        """
        ret = await self.db.datasets.find_one({'dataset_id':dataset_id},
                projection={'_id':False})
        if not ret:
            self.send_error(404, reason="Dataset not found")
        else:
            self.write(ret)
            self.finish()

class DatasetDescriptionHandler(BaseHandler):
    """
    Handle dataset description updates.
    """
    @authorization(roles=['admin'], attrs=['dataset_id:write'])
    async def put(self, dataset_id):
        """
        Set a dataset description.

        Args:
            dataset_id (str): the dataset

        Returns:
            dict: empty dict
        """
        data = json.loads(self.request.body)
        if 'description' not in data:
            raise tornado.web.HTTPError(400, reason='missing description')
        elif not isinstance(data['description'],str):
            raise tornado.web.HTTPError(400, reason='bad description')

        ret = await self.db.datasets.find_one_and_update({'dataset_id':dataset_id},
                {'$set':{'description': data['description']}},
                projection=['_id'])
        if not ret:
            self.send_error(404, reason="Dataset not found")
        else:
            self.write({})
            self.finish()

class DatasetStatusHandler(BaseHandler):
    """
    Handle dataset status updates.
    """
    @authorization(roles=['admin','system','client'], attrs=['dataset_id:write'])
    async def put(self, dataset_id):
        """
        Set a dataset status.

        Args:
            dataset_id (str): the dataset

        Returns:
            dict: empty dict
        """
        data = json.loads(self.request.body)
        if 'status' not in data:
            raise tornado.web.HTTPError(400, reason='missing status')
        elif data['status'] not in dataset_statuses:
            raise tornado.web.HTTPError(400, reason='bad status')

        ret = await self.db.datasets.find_one_and_update({'dataset_id':dataset_id},
                {'$set':{'status': data['status']}},
                projection=['_id'])
        if not ret:
            self.send_error(404, reason="Dataset not found")
        else:
            self.write({})
            self.finish()

class DatasetPriorityHandler(BaseHandler):
    """
    Handle dataset priority updates.
    """
    @authorization(roles=['admin','system','client'], attrs=['dataset_id:write'])
    async def put(self, dataset_id):
        """
        Set a dataset priority.

        Args:
            dataset_id (str): the dataset

        Returns:
            dict: empty dict
        """
        data = json.loads(self.request.body)
        if 'priority' not in data:
            raise tornado.web.HTTPError(400, reason='missing priority')
        elif not isinstance(data['priority'], (int, float)):
            raise tornado.web.HTTPError(400, reason='priority is not a number')

        ret = await self.db.datasets.find_one_and_update({'dataset_id':dataset_id},
                {'$set':{'priority': data['priority']}},
                projection=['_id'])
        if not ret:
            self.send_error(404, reason="Dataset not found")
        else:
            self.write({})
            self.finish()

class DatasetJobsSubmittedHandler(BaseHandler):
    """
    Handle dataset jobs_submitted updates.
    """
    @authorization(roles=['admin'], attrs=['dataset_id:write'])
    async def put(self, dataset_id):
        """
        Set a dataset's jobs_submitted.

        Only allows increases, if the jobs_immutable flag is not set.

        Args:
            dataset_id (str): the dataset

        Json body:
            jobs_submitted (int): the number of jobs submitted

        Returns:
            dict: empty dict
        """
        data = json.loads(self.request.body)
        if 'jobs_submitted' not in data:
            raise tornado.web.HTTPError(400, reason='missing jobs_submitted')
        try:
            jobs_submitted = int(data['jobs_submitted'])
        except Exception:
            raise tornado.web.HTTPError(400, reason='jobs_submitted is not an int')

        ret = await self.db.datasets.find_one({'dataset_id':dataset_id})
        if not ret:
            raise tornado.web.HTTPError(404, reason='Dataset not found')
        if ret['jobs_immutable']:
            raise tornado.web.HTTPError(400, reason='jobs_submitted is immutable')
        if ret['jobs_submitted'] > jobs_submitted:
            raise tornado.web.HTTPError(400, reason='jobs_submitted must be larger than before')
        if 'tasks_per_job' not in ret or ret['tasks_per_job'] <= 0:
            raise tornado.web.HTTPError(400, reason='tasks_per_job not valid')

        ret = await self.db.datasets.find_one_and_update({'dataset_id':dataset_id},
                {'$set':{
                    'jobs_submitted': jobs_submitted,
                    'tasks_submitted': int(jobs_submitted*ret['tasks_per_job']),
                }},
                projection=['_id'])
        if not ret:
            self.send_error(404, reason="Dataset not found")
        else:
            self.write({})
            self.finish()

class DatasetSummariesStatusHandler(BaseHandler):
    """
    Handle dataset summary grouping by status.
    """
    @authorization(roles=['admin','system','client','user']) #TODO: figure out how to do auth for each dataset in the list
    async def get(self):
        """
        Get the dataset summary for all datasets, group by status.

        Returns:
            dict: {<status>: [<dataset_id>,]}
        """
        cursor = self.db.datasets.find(
                projection={'_id':False,'status':True,'dataset_id':True})
        ret = defaultdict(list)
        async for row in cursor:
            ret[row['status']].append(row['dataset_id'])
        ret2 = {}
        for k in sorted(ret, key=dataset_status_sort):
            ret2[k] = ret[k]
        self.write(ret2)
        self.finish()
<commit_msg>allow admin group to read/write all datasets<commit_after>import logging
import json
import uuid
from collections import defaultdict

import tornado.web
import tornado.httpclient
from tornado.platform.asyncio import to_asyncio_future
import pymongo
import motor

from rest_tools.client import RestClient
from iceprod.server.rest import RESTHandler, RESTHandlerSetup, authorization
from iceprod.server.util import nowstr, dataset_statuses, dataset_status_sort

logger = logging.getLogger('rest.datasets')

def setup(config, *args, **kwargs):
    """
    Setup method for Dataset REST API.

    Sets up any database connections or other prerequisites.

    Args:
        config (dict): an instance of :py:class:`iceprod.server.config`.

    Returns:
        list: Routes for dataset, which can be passed to :py:class:`tornado.web.Application`.
    """
    cfg_rest = config.get('rest',{}).get('datasets',{})
    db_cfg = cfg_rest.get('database',{})

    # add indexes
    db = pymongo.MongoClient(**db_cfg).datasets
    if 'dataset_id_index' not in db.datasets.index_information():
        db.datasets.create_index('dataset_id', name='dataset_id_index', unique=True)

    handler_cfg = RESTHandlerSetup(config, *args, **kwargs)
    handler_cfg.update({
        'database': motor.motor_tornado.MotorClient(**db_cfg).datasets,
    })

    return [
        (r'/datasets', MultiDatasetHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)', DatasetHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)/description', DatasetDescriptionHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)/status', DatasetStatusHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)/priority', DatasetPriorityHandler, handler_cfg),
        (r'/datasets/(?P<dataset_id>\w+)/jobs_submitted', DatasetJobsSubmittedHandler, handler_cfg),
        (r'/dataset_summaries/status', DatasetSummariesStatusHandler, handler_cfg),
    ]

class BaseHandler(RESTHandler):
    """
    Base handler for Dataset REST API.
    """
    def initialize(self, database=None, **kwargs):
        super(BaseHandler, self).initialize(**kwargs)
        self.db = database

class MultiDatasetHandler(BaseHandler):
    """
    Handle multi-group requests.
    """
    @authorization(roles=['admin','client','system','user']) #TODO: figure out how to do auth for each dataset in the list
    async def get(self):
        """
        Get a dict of datasets.

        Params (optional):
            status: | separated list of status filters
            groups: | separated list of groups to filter on
            users: | separated list of users to filter on
            keys: | separated list of keys to return for each dataset

        Returns:
            dict: {<dataset_id>: metadata}
        """
        query = {}
        status = self.get_argument('status', None)
        if status:
            query['status'] = {'$in': status.split('|')}
        groups = self.get_argument('groups', None)
        if groups:
            query['group'] = {'$in': groups.split('|')}
        users = self.get_argument('users', None)
        if users:
            query['username'] = {'$in': users.split('|')}

        projection = {'_id': False}
        keys = self.get_argument('keys', None)
        if keys:
            projection.update({x:True for x in keys.split('|') if x})

        ret = {}
        async for row in self.db.datasets.find(query, projection=projection):
            k = row['dataset_id']
            ret[k] = row
        self.write(ret)
        self.finish()

    @authorization(roles=['admin','user']) # anyone should be able to create a dataset
    async def post(self):
        """
        Add a dataset.

        Body should contain all necessary fields for a dataset.
        """
        data = json.loads(self.request.body)

        # validate first
        req_fields = {
            'description': str,
            'jobs_submitted': int,
            'tasks_submitted': int,
            'tasks_per_job': int,
            'group': str,
        }
        for k in req_fields:
            if k not in data:
                raise tornado.web.HTTPError(400, reason='missing key: '+k)
            if not isinstance(data[k], req_fields[k]):
                r = 'key "{}" should be of type {}'.format(k, req_fields[k].__name__)
                raise tornado.web.HTTPError(400, reason=r)

        opt_fields = {
            'priority': int,
            'debug': bool,
            'jobs_immutable': bool,
            'status': str,
        }
        for k in opt_fields:
            if k in data and not isinstance(data[k], opt_fields[k]):
                r = 'key "{}" should be of type {}'.format(k, opt_fields[k].__name__)
                raise tornado.web.HTTPError(400, reason=r)

        bad_fields = set(data).difference(set(opt_fields).union(req_fields))
        if bad_fields:
            r = 'invalid keys found'
            raise tornado.web.HTTPError(400, reason=r)

        if data['jobs_submitted'] == 0 and data['tasks_per_job'] <= 0:
            r = '"tasks_per_job" must be > 0'
            raise tornado.web.HTTPError(400, reason=r)
        elif data['tasks_submitted'] != 0 and data['tasks_submitted'] / data['jobs_submitted'] != data['tasks_per_job']:
            r = '"tasks_per_job" does not match "tasks_submitted"/"jobs_submitted"'
            raise tornado.web.HTTPError(400, reason=r)

        # generate dataset number
        ret = await self.db.settings.find_one_and_update(
            {'name': 'dataset_num'},
            {'$inc': {'num': 1}},
            projection={'num': True, '_id': False},
            upsert=True,
            return_document=pymongo.ReturnDocument.AFTER)
        dataset_num = ret['num']

        # set some fields
        data['dataset_id'] = uuid.uuid1().hex
        data['dataset'] = dataset_num
        if 'status' not in data:
            data['status'] = 'processing'
        data['start_date'] = nowstr()
        data['username'] = self.auth_data['username']
        if 'priority' not in data:
            data['priority'] = 0.5
        if 'debug' not in data:
            data['debug'] = False
        if 'jobs_immutable' not in data:
            data['jobs_immutable'] = False

        # insert
        ret = await self.db.datasets.insert_one(data)

        # set auth rules
        url = '/auths/'+data['dataset_id']
        http_client = RestClient(self.auth_url, token=self.module_auth_key)
        auth_data = {
            'read_groups':['admin',data['group'],'users'],
            'write_groups':['admin',data['group']],
        }
        logger.info('Authorization header: %s', 'bearer '+self.module_auth_key)
        await http_client.request('PUT', url, auth_data)

        # return success
        self.set_status(201)
        self.set_header('Location', '/datasets/'+data['dataset_id'])
        self.write({'result': '/datasets/'+data['dataset_id']})
        self.finish()

class DatasetHandler(BaseHandler):
    """
    Handle dataset requests.
    """
    @authorization(roles=['admin','client','system','pilot'], attrs=['dataset_id:read'])
    async def get(self, dataset_id):
        """
        Get a dataset.

        Args:
            dataset_id (str): the dataset

        Returns:
            dict: dataset metadata
        """
        ret = await self.db.datasets.find_one({'dataset_id':dataset_id},
                projection={'_id':False})
        if not ret:
            self.send_error(404, reason="Dataset not found")
        else:
            self.write(ret)
            self.finish()

class DatasetDescriptionHandler(BaseHandler):
    """
    Handle dataset description updates.
    """
    @authorization(roles=['admin'], attrs=['dataset_id:write'])
    async def put(self, dataset_id):
        """
        Set a dataset description.

        Args:
            dataset_id (str): the dataset

        Returns:
            dict: empty dict
        """
        data = json.loads(self.request.body)
        if 'description' not in data:
            raise tornado.web.HTTPError(400, reason='missing description')
        elif not isinstance(data['description'],str):
            raise tornado.web.HTTPError(400, reason='bad description')

        ret = await self.db.datasets.find_one_and_update({'dataset_id':dataset_id},
                {'$set':{'description': data['description']}},
                projection=['_id'])
        if not ret:
            self.send_error(404, reason="Dataset not found")
        else:
            self.write({})
            self.finish()

class DatasetStatusHandler(BaseHandler):
    """
    Handle dataset status updates.
    """
    @authorization(roles=['admin','system','client'], attrs=['dataset_id:write'])
    async def put(self, dataset_id):
        """
        Set a dataset status.

        Args:
            dataset_id (str): the dataset

        Returns:
            dict: empty dict
        """
        data = json.loads(self.request.body)
        if 'status' not in data:
            raise tornado.web.HTTPError(400, reason='missing status')
        elif data['status'] not in dataset_statuses:
            raise tornado.web.HTTPError(400, reason='bad status')

        ret = await self.db.datasets.find_one_and_update({'dataset_id':dataset_id},
                {'$set':{'status': data['status']}},
                projection=['_id'])
        if not ret:
            self.send_error(404, reason="Dataset not found")
        else:
            self.write({})
            self.finish()

class DatasetPriorityHandler(BaseHandler):
    """
    Handle dataset priority updates.
    """
    @authorization(roles=['admin','system','client'], attrs=['dataset_id:write'])
    async def put(self, dataset_id):
        """
        Set a dataset priority.

        Args:
            dataset_id (str): the dataset

        Returns:
            dict: empty dict
        """
        data = json.loads(self.request.body)
        if 'priority' not in data:
            raise tornado.web.HTTPError(400, reason='missing priority')
        elif not isinstance(data['priority'], (int, float)):
            raise tornado.web.HTTPError(400, reason='priority is not a number')

        ret = await self.db.datasets.find_one_and_update({'dataset_id':dataset_id},
                {'$set':{'priority': data['priority']}},
                projection=['_id'])
        if not ret:
            self.send_error(404, reason="Dataset not found")
        else:
            self.write({})
            self.finish()

class DatasetJobsSubmittedHandler(BaseHandler):
    """
    Handle dataset jobs_submitted updates.
    """
    @authorization(roles=['admin'], attrs=['dataset_id:write'])
    async def put(self, dataset_id):
        """
        Set a dataset's jobs_submitted.

        Only allows increases, if the jobs_immutable flag is not set.

        Args:
            dataset_id (str): the dataset

        Json body:
            jobs_submitted (int): the number of jobs submitted

        Returns:
            dict: empty dict
        """
        data = json.loads(self.request.body)
        if 'jobs_submitted' not in data:
            raise tornado.web.HTTPError(400, reason='missing jobs_submitted')
        try:
            jobs_submitted = int(data['jobs_submitted'])
        except Exception:
            raise tornado.web.HTTPError(400, reason='jobs_submitted is not an int')

        ret = await self.db.datasets.find_one({'dataset_id':dataset_id})
        if not ret:
            raise tornado.web.HTTPError(404, reason='Dataset not found')
        if ret['jobs_immutable']:
            raise tornado.web.HTTPError(400, reason='jobs_submitted is immutable')
        if ret['jobs_submitted'] > jobs_submitted:
            raise tornado.web.HTTPError(400, reason='jobs_submitted must be larger than before')
        if 'tasks_per_job' not in ret or ret['tasks_per_job'] <= 0:
            raise tornado.web.HTTPError(400, reason='tasks_per_job not valid')

        ret = await self.db.datasets.find_one_and_update({'dataset_id':dataset_id},
                {'$set':{
                    'jobs_submitted': jobs_submitted,
                    'tasks_submitted': int(jobs_submitted*ret['tasks_per_job']),
                }},
                projection=['_id'])
        if not ret:
            self.send_error(404, reason="Dataset not found")
        else:
            self.write({})
            self.finish()

class DatasetSummariesStatusHandler(BaseHandler):
    """
    Handle dataset summary grouping by status.
    """
    @authorization(roles=['admin','system','client','user']) #TODO: figure out how to do auth for each dataset in the list
    async def get(self):
        """
        Get the dataset summary for all datasets, group by status.

        Returns:
            dict: {<status>: [<dataset_id>,]}
        """
        cursor = self.db.datasets.find(
                projection={'_id':False,'status':True,'dataset_id':True})
        ret = defaultdict(list)
        async for row in cursor:
            ret[row['status']].append(row['dataset_id'])
        ret2 = {}
        for k in sorted(ret, key=dataset_status_sort):
            ret2[k] = ret[k]
        self.write(ret2)
        self.finish()
<|endoftext|>


<commit_before>from identifier import Result

AU_PATTERNS = [
	'2E 72 6E 64'
]

class AuResolver:
	def identify(self, stream):
		return Result('AU')

def load(hound):
	hound.add_matches(AU_PATTERNS, AuResolver())
<commit_msg>Update audio_identifier.py<commit_after>from identifier import Result

AU_PATTERNS = [
	'2E 72 6E 64',

]

AIFF_PATTERNS = [
	'46 4F 52 4D 00'
]

class AuResolver:
	def identify(self, stream):
		return Result('AU')

class AiffResolver:
	def identify(self, stream):
		return Result('AIFF')

def load(hound):
	hound.add_matches(AU_PATTERNS, AuResolver())
	hound.add_matches(AIFF_PATTERNS, AiffResolver())
<|endoftext|>


<commit_before>from django.contrib.flatpages.admin import FlatpageForm, FlatPageAdmin
from django.contrib.flatpages.models import FlatPage
from tinymce.widgets import TinyMCE

class PageForm(FlatpageForm):

    class Meta:
        model = FlatPage
        widgets = {
            'content' : TinyMCE(attrs={'cols': 100, 'rows': 15}),
        }

class PageAdmin(FlatPageAdmin):
    """
    Page Admin
    """
    form = PageForm

admin.site.unregister(FlatPage)
admin.site.register(FlatPage, PageAdmin)
<commit_msg>Added swappable widget<commit_after>from django.core import settings
from django.contrib import admin
from django.contrib.flatpages.admin import FlatpageForm, FlatPageAdmin
from django.contrib.flatpages.models import FlatPage
from django.utils.module_loading import import_string

FLATPAGE_WIDGET = getattr(settings, 'FLATPAGE_WIDGET', 'tinymce.widgets.TinyMCE')

FLATPAGE_WIDGET_KWARGS = getattr(settings, 'FLATPAGE_WIDGET_KWARGS',
                                 default={'attrs': {'cols': 100, 'rows': 15}})

class PageForm(FlatpageForm):

    class Meta:
        model = FlatPage
        widgets = {
            'content': import_string(FLATPAGE_WIDGET)(**FLATPAGE_WIDGET_KWARGS),
        }

class PageAdmin(FlatPageAdmin):
    """
    Page Admin
    """
    form = PageForm

admin.site.unregister(FlatPage)
admin.site.register(FlatPage, PageAdmin)
<|endoftext|>


<commit_before>import numpy as np
from keras import backend as K

from . import patches
from .core import content_loss

def find_analogy_patches(a, a_prime, b, patch_size=3, patch_stride=1):
    '''This is for precalculating the analogy_loss

    Since A, A', and B never change we only need to calculate the patch matches once.
    '''
    # extract patches from feature maps
    a_patches, a_patches_norm = patches.make_patches(K.variable(a), patch_size, patch_stride)
    a_prime_patches, a_prime_patches_norm = patches.make_patches(K.variable(a_prime), patch_size, patch_stride)
    b_patches, b_patches_norm = patches.make_patches(K.variable(b), patch_size, patch_stride)
    # find best patches and calculate loss
    p = patches.find_patch_matches(b_patches, b_patches_norm, a_patches / a_patches_norm)
    #best_patches = a_prime_patches[p]
    best_patches = K.reshape(a_prime_patches[p], K.shape(b_patches))
    f = K.function([], best_patches)
    best_patches = f([])
    return best_patches

def analogy_loss(a, a_prime, b, b_prime, patch_size=3, patch_stride=1, use_full_analogy=False):
    '''http://www.mrl.nyu.edu/projects/image-analogies/index.html'''
    best_a_prime_patches = find_analogy_patches(a, a_prime, b, patch_size=patch_size, patch_stride=patch_stride)
    if use_full_analogy:  # combine all the patches into a single image
        b_prime_patches, _ = make_patches(b_prime, patch_size, patch_stride)
        loss = content_loss(best_a_prime_patches, b_prime_patches) / patch_size ** 2
    else:
        bs = b.shape
        b_analogy = patches.combine_patches(best_a_prime_patches, (bs[1], bs[2], bs[0]))
        loss = content_loss(np.expand_dims(b_analogy, 0), b_prime)
    return loss
<commit_msg>Changed orphaned make_patches method call to patches.make_patches in order to reference appropriate module.<commit_after>import numpy as np
from keras import backend as K

from . import patches
from .core import content_loss

def find_analogy_patches(a, a_prime, b, patch_size=3, patch_stride=1):
    '''This is for precalculating the analogy_loss

    Since A, A', and B never change we only need to calculate the patch matches once.
    '''
    # extract patches from feature maps
    a_patches, a_patches_norm = patches.make_patches(K.variable(a), patch_size, patch_stride)
    a_prime_patches, a_prime_patches_norm = patches.make_patches(K.variable(a_prime), patch_size, patch_stride)
    b_patches, b_patches_norm = patches.make_patches(K.variable(b), patch_size, patch_stride)
    # find best patches and calculate loss
    p = patches.find_patch_matches(b_patches, b_patches_norm, a_patches / a_patches_norm)
    #best_patches = a_prime_patches[p]
    best_patches = K.reshape(a_prime_patches[p], K.shape(b_patches))
    f = K.function([], best_patches)
    best_patches = f([])
    return best_patches

def analogy_loss(a, a_prime, b, b_prime, patch_size=3, patch_stride=1, use_full_analogy=False):
    '''http://www.mrl.nyu.edu/projects/image-analogies/index.html'''
    best_a_prime_patches = find_analogy_patches(a, a_prime, b, patch_size=patch_size, patch_stride=patch_stride)
    if use_full_analogy:  # combine all the patches into a single image
        b_prime_patches, _ = patches.make_patches(b_prime, patch_size, patch_stride)
        loss = content_loss(best_a_prime_patches, b_prime_patches) / patch_size ** 2
    else:
        bs = b.shape
        b_analogy = patches.combine_patches(best_a_prime_patches, (bs[1], bs[2], bs[0]))
        loss = content_loss(np.expand_dims(b_analogy, 0), b_prime)
    return loss
<|endoftext|>


<commit_before>import time
from datetime import timedelta
from datetime import date
from datetime import datetime
import requests
import iso8601
import os
import logging
import pytz

logger = logging.getLogger("impactstoryanalytics.widgets")

class Widget:
    def get_name(self):
        return self.__class__.__name__.lower()

    def get_data(self):
        raise NotImplementedError

    def make_day_bins(self, num_days):
        pass

class Gmail(Widget):

    query_url = "https://api.keen.io/3.0/projects/51df37f0897a2c7fcd000000/queries/minimum?api_key=b915f0ca9fcbe1cc4760640adf9f09fa1d330f74c763bfd1aa867d6148f528055a3f97afc6b111e8905ef78bfe7f97d1d2dd2b7ddbb0f9ed8e586fd69d79f12f2215d06298924631d8ccfa7a12845dde94921855ae223c69ad26789dca2ec5fd26296a80af72c3a014df5554948bac8e&event_collection=Inbox%20check&timeframe=this_48_hours&timezone=-25200&target_property=thread_count&group_by=userId&interval=hourly"

    def get_data(self):
        return self.inbox_threads()

    def get_raw_data(self):
        raw_data = requests.get(self.query_url).json()["result"]
        return raw_data

    def inbox_threads(self):

        raw_data = self.get_raw_data()
        lines = {
            "Heather": [],
            "Jason": []
        }

        for this_bin in raw_data:
            bin_start_time = iso8601.parse_date(this_bin["timeframe"]["start"])
            adj_start_time = bin_start_time

            for val in this_bin["value"]:
                if val["result"] is not None:
                    datapoint = {
                        "x": int(time.mktime(adj_start_time.timetuple())),
                        "y": val["result"]
                    }

                    lines[val["userId"]].append(datapoint)

        return lines

class Rescuetime(Widget):
    def get_data(self):
        heather_and_jason_activity = {}
        for user in ["Heather", "Jason"]:
            raw_data = self.get_raw_data(user)
            heather_and_jason_activity[user] = self.list_activity_by_day(raw_data)

        return heather_and_jason_activity

    def get_raw_data(self, user):

        env_key = "RESCUETIME_KEY_" + user.upper()
        rescuetime_api_key = os.getenv(env_key)

        params = {
            "key": rescuetime_api_key,
            "format": "json",
            "perspective": "interval",
            "resolution_time": "day",
            "restrict_kind": "category",
            "restrict_begin": date.today() - timedelta(days=6),
            "restrict_end": date.today() + timedelta(days=1)
        }
        url = "https://www.rescuetime.com/anapi/data"

        data = requests.get(url, params=params).json()["rows"]
        return data

    def is_code_category(self, category):
        code_categories = [
            "general software development",
            "design & planning",
            "editing & ides",
            "quality assurance",
            "systems operations",
            "data modelling and analysis"
        ]

        if category.lower() in code_categories:
            return True
        else:
            return False

    def list_activity_by_day(self, data):

        days = {}
        for row in data:
            datestring = row[0]
            time_spent = float(row[1]) / 3600  # in hours
            category = row[3]

            # add this day if we don't have it yet
            if datestring not in days:
                # crazy hack to fix rescuetime rickshaw axis
                adj_data = iso8601.parse_date(datestring)

                timestamp = int(time.mktime(adj_data.timetuple()))
                days[datestring] = {
                    "total": 0,
                    "email": 0,
                    "code": 0,
                    "timestamp": timestamp
                }

            # add to the time counts for this day
            days[datestring]["total"] += time_spent
            if category == "Email":
                days[datestring]["email"] += time_spent
            elif self.is_code_category(category):
                days[datestring]["code"] += time_spent

        # now that all the data is in, calculate the "other" category
        for k, v in days.iteritems():
            v["other"] = v["total"] - (v["email"] + v["code"])

        # easier to work with a list
        dayslist = []
        for k in sorted(days.keys()):
            dayslist.append(days[k])

        # convert to the format Rickshaw likes
        categories = {"other": [], "email": [], "code": []}
        for category_name, category_values in categories.iteritems():
            for index, day in enumerate(dayslist):
                datapoint = {
                    "x": index,
                    "y": day[category_name]
                }
                category_values.append(datapoint)

        return categories

class Github(Widget):
    issue_q_url_template = "https://api.github.com/repos/total-impact/total-impact-{NAME}/issues"
    num_days = 31
    repo_names = ["webapp", "core"]

    def get_data(self):
        lines = {}
        for repo_name in self.repo_names:
            issues_list = self.get_raw_data_repo(repo_name)
            line = self.make_line(issues_list)
            lines[repo_name] = line.values()

        return lines

    def get_both_closed_and_open_issues(self, q_url):
        issues = []
        tz = pytz.timezone(os.getenv("TZ"))
        begin = (datetime.now(tz) - timedelta(days=self.num_days))
        params = {
            "since": begin.isoformat()
        }
        logger.info("querying github with url " + q_url)
        for state in ["closed", "open"]:
            params["state"] = state
            issues += requests.get(q_url, params=params).json()

        logger.debug("these are the issues we got back", str(issues))
        return issues

    def get_raw_data_repo(self, repo_name):
        q_url = self.issue_q_url_template.format(NAME=repo_name)
        return self.get_both_closed_and_open_issues(q_url)

    def beginning_of_day_ts(self, datetime_obj):
        d = datetime_obj.replace(hour=0, minute=0, second=0)
        return str(d.isoformat()[0:10])

    def make_line(self, issues_list):
        d = datetime.utcnow()
        days_list = {}
        for _ in xrange(0, 50):
            days_list[self.beginning_of_day_ts(d)] = 0
            d -= timedelta(days=1)

        logger.info("days list is this: " + str(sorted(days_list.keys())))

        for issue in issues_list:
            d = iso8601.parse_date(issue["created_at"])

            try:
                days_list[self.beginning_of_day_ts(d)] += 1
            except KeyError:
                continue  # it's not in our window of interest

        return days_list
<commit_msg>return the two repos' issues as a zipped-together list<commit_after>import time
from datetime import timedelta
from datetime import date
from datetime import datetime
import requests
import iso8601
import os
import logging
import pytz

logger = logging.getLogger("impactstoryanalytics.widgets")

class Widget:
    def get_name(self):
        return self.__class__.__name__.lower()

    def get_data(self):
        raise NotImplementedError

    def make_day_bins(self, num_days):
        pass

class Gmail(Widget):

    query_url = "https://api.keen.io/3.0/projects/51df37f0897a2c7fcd000000/queries/minimum?api_key=b915f0ca9fcbe1cc4760640adf9f09fa1d330f74c763bfd1aa867d6148f528055a3f97afc6b111e8905ef78bfe7f97d1d2dd2b7ddbb0f9ed8e586fd69d79f12f2215d06298924631d8ccfa7a12845dde94921855ae223c69ad26789dca2ec5fd26296a80af72c3a014df5554948bac8e&event_collection=Inbox%20check&timeframe=this_48_hours&timezone=-25200&target_property=thread_count&group_by=userId&interval=hourly"

    def get_data(self):
        return self.inbox_threads()

    def get_raw_data(self):
        raw_data = requests.get(self.query_url).json()["result"]
        return raw_data

    def inbox_threads(self):

        raw_data = self.get_raw_data()
        lines = {
            "Heather": [],
            "Jason": []
        }

        for this_bin in raw_data:
            bin_start_time = iso8601.parse_date(this_bin["timeframe"]["start"])
            adj_start_time = bin_start_time

            for val in this_bin["value"]:
                if val["result"] is not None:
                    datapoint = {
                        "x": int(time.mktime(adj_start_time.timetuple())),
                        "y": val["result"]
                    }

                    lines[val["userId"]].append(datapoint)

        return lines

class Rescuetime(Widget):
    def get_data(self):
        heather_and_jason_activity = {}
        for user in ["Heather", "Jason"]:
            raw_data = self.get_raw_data(user)
            heather_and_jason_activity[user] = self.list_activity_by_day(raw_data)

        return heather_and_jason_activity

    def get_raw_data(self, user):

        env_key = "RESCUETIME_KEY_" + user.upper()
        rescuetime_api_key = os.getenv(env_key)

        params = {
            "key": rescuetime_api_key,
            "format": "json",
            "perspective": "interval",
            "resolution_time": "day",
            "restrict_kind": "category",
            "restrict_begin": date.today() - timedelta(days=6),
            "restrict_end": date.today() + timedelta(days=1)
        }
        url = "https://www.rescuetime.com/anapi/data"

        data = requests.get(url, params=params).json()["rows"]
        return data

    def is_code_category(self, category):
        code_categories = [
            "general software development",
            "design & planning",
            "editing & ides",
            "quality assurance",
            "systems operations",
            "data modelling and analysis"
        ]

        if category.lower() in code_categories:
            return True
        else:
            return False

    def list_activity_by_day(self, data):

        days = {}
        for row in data:
            datestring = row[0]
            time_spent = float(row[1]) / 3600  # in hours
            category = row[3]

            # add this day if we don't have it yet
            if datestring not in days:
                # crazy hack to fix rescuetime rickshaw axis
                adj_data = iso8601.parse_date(datestring)

                timestamp = int(time.mktime(adj_data.timetuple()))
                days[datestring] = {
                    "total": 0,
                    "email": 0,
                    "code": 0,
                    "timestamp": timestamp
                }

            # add to the time counts for this day
            days[datestring]["total"] += time_spent
            if category == "Email":
                days[datestring]["email"] += time_spent
            elif self.is_code_category(category):
                days[datestring]["code"] += time_spent

        # now that all the data is in, calculate the "other" category
        for k, v in days.iteritems():
            v["other"] = v["total"] - (v["email"] + v["code"])

        # easier to work with a list
        dayslist = []
        for k in sorted(days.keys()):
            dayslist.append(days[k])

        # convert to the format Rickshaw likes
        categories = {"other": [], "email": [], "code": []}
        for category_name, category_values in categories.iteritems():
            for index, day in enumerate(dayslist):
                datapoint = {
                    "x": index,
                    "y": day[category_name]
                }
                category_values.append(datapoint)

        return categories

class Github(Widget):
    issue_q_url_template = "https://api.github.com/repos/total-impact/total-impact-{NAME}/issues"
    num_days = 31
    repo_names = ["webapp", "core"]

    def get_data(self):
        lines = {}
        for repo_name in self.repo_names:
            issues_list = self.get_raw_data_repo(repo_name)
            line = self.make_line(issues_list)
            lines[repo_name] = line.values()

        zipped = zip(*lines.values())
        return zipped

    def get_both_closed_and_open_issues(self, q_url):
        issues = []
        tz = pytz.timezone(os.getenv("TZ"))
        begin = (datetime.now(tz) - timedelta(days=self.num_days))
        params = {
            "since": begin.isoformat()
        }
        logger.info("querying github with url " + q_url)
        for state in ["closed", "open"]:
            params["state"] = state
            issues += requests.get(q_url, params=params).json()

        logger.debug("these are the issues we got back", str(issues))
        return issues

    def get_raw_data_repo(self, repo_name):
        q_url = self.issue_q_url_template.format(NAME=repo_name)
        return self.get_both_closed_and_open_issues(q_url)

    def beginning_of_day_ts(self, datetime_obj):
        d = datetime_obj.replace(hour=0, minute=0, second=0)
        return str(d.isoformat()[0:10])

    def make_line(self, issues_list):
        d = datetime.utcnow()
        days_list = {}
        for _ in xrange(0, 50):
            days_list[self.beginning_of_day_ts(d)] = 0
            d -= timedelta(days=1)

        logger.info("days list is this: " + str(sorted(days_list.keys())))

        for issue in issues_list:
            d = iso8601.parse_date(issue["created_at"])

            try:
                days_list[self.beginning_of_day_ts(d)] += 1
            except KeyError:
                continue  # it's not in our window of interest

        return days_list
<|endoftext|>


<commit_before># coding=utf-8
from flask import Flask, request
from flask.ext.restful import Resource, fields
from jormungandr import i_manager
from jormungandr.interfaces.parsers import true_false, option_value
from jormungandr.protobuf_to_dict import protobuf_to_dict
from jormungandr.find_extrem_datetimes import extremes
from flask.ext.restful import reqparse
from flask.ext.restful.types import boolean
from jormungandr.interfaces.argument import ArgumentDoc
from jormungandr.interfaces.parsers import depth_argument
from jormungandr.authentification import authentification_required

class Journeys(Resource):

    """ Compute journeys"""
    parsers = {}
    method_decorators = [authentification_required]

    def __init__(self, *args, **kwargs):
        types = ["all", "rapid"]
        self.parsers["get"] = reqparse.RequestParser(
            argument_class=ArgumentDoc)
        parser_get = self.parsers["get"]
        parser_get.add_argument("origin", type=str, required=True,
                                description="Origin's uri of your journey")
        parser_get.add_argument("destination", type=str, required=True,
                                description=
                                "Destination's uri of your journey")
        parser_get.add_argument("datetime", type=str, required=True,
                                description=
                                "Datetime's uri of your journey")
        parser_get.add_argument("clockwise", type=true_false, default=True,
                                description=
                                "Whether you want your journey to start"\
                                "after datetime or to end before datetime")
        parser_get.add_argument("max_duration", type=int, default=36000,
                                description="Maximum duration of your journey")
        parser_get.add_argument("max_transfers", type=int, default=10,
                                description=
                                "Maximum transfers you want in "\
                                "your journey")
        parser_get.add_argument("origin_mode",
                                type=option_value(["walking", "car", "bike",
                                                   "bss"]),
                                action="append", default=['walking', 'bike',
                                                          'car'],
                                description=
                                "The list of modes you want at the "\
                                "beggining of your journey")
        parser_get.add_argument("destination_mode",
                                type=option_value(["walking", "car", "bike",
                                                   "bss"]),
                                default=['walking', 'bike', 'car'],
                                action="append",
                                description=
                                "The list of modes you want at the"\
                                " end of your journey")
        parser_get.add_argument("max_duration_to_pt", type=int, default=10*60,
                                description=
                                "maximal duration of non public "\
                                "transport in second")
        parser_get.add_argument("walking_speed", type=float, default=1.68,
                                description=
                                "Walking speed in meter/second")
        parser_get.add_argument("bike_speed", type=float, default=8.8,
                                description="Bike speed in meter/second")
        parser_get.add_argument("bss_speed", type=float, default=8.8,
                                description="Bike rental speed in "\
                                "meter/second")
        parser_get.add_argument("car_speed", type=float, default=16.8,
                                description=
                                "Car speed in meter/second")
        parser_get.add_argument("forbidden_uris[]", type=str, action="append",
                                description="Uri you want to forbid")
        parser_get.add_argument("type", type=option_value(types),
                                default="all")
        parser_get.add_argument("wheelchair", type=boolean, default=False)
        parser_get.add_argument("count", type=int)
        parser_get.add_argument("debug", type=boolean, default=False,
                                hidden=True)

    def get(self, region=None):
        args = self.parsers["get"].parse_args()
        if region is None:
            region = i_manager.key_of_id(args["origin"])
        response = i_manager.dispatch(args, region, "journeys")
        if response.journeys:
            (before, after) = extremes(response, request)
            if before and after:
                response.prev = before
                response.next = after
        return protobuf_to_dict(response, use_enum_labels=True), 200

class Isochrone(Resource):

    """ Compute isochrones """
    method_decorators = [authentification_required]

    def __init__(self):
        self.parsers = {}
        self.parsers["get"] = reqparse.RequestParser()
        parser_get = self.parsers["get"]
        parser_get.add_argument("origin", type=str, required=True)
        parser_get.add_argument("datetime", type=str, required=True)
        parser_get.add_argument("clockwise", type=true_false, default=True)
        parser_get.add_argument("max_duration", type=int, default=3600)
        parser_get.add_argument("max_transfers", type=int, default=10)
        parser_get.add_argument("origin_mode",
                                type=option_value(["walking", "car", "bike",
                                                   "bss"]),
                                action="append", default=["walking"])
        parser_get.add_argument("max_duration_to_pt", type=int, default=10*60,
                                description="maximal duration of non public \
                                transport in second")
        parser_get.add_argument("walking_speed", type=float, default=1.68)
        parser_get.add_argument("bike_speed", type=float, default=8.8)
        parser_get.add_argument("bss_speed", type=float, default=8.8)
        parser_get.add_argument("car_speed", type=float, default=16.8)
        parser_get.add_argument("forbidden_uris[]", type=str, action="append")
        parser_get.add_argument("wheelchair", type=boolean, default=False)

    def get(self, region=None):
        args = self.parsers["get"].parse_args()
        if region is None:
            region = i_manager.key_of_id(args["origin"])
        response = i_manager.dispatch(args, region, "isochrone")
        if response.journeys:
            (before, after) = extremes(response, request)
            if before and after:
                response.prev = before
                response.next = after
        return protobuf_to_dict(response, use_enum_labels=True), 200
<commit_msg>Add debug parameter to v0 isochrone interface<commit_after># coding=utf-8
from flask import Flask, request
from flask.ext.restful import Resource, fields
from jormungandr import i_manager
from jormungandr.interfaces.parsers import true_false, option_value
from jormungandr.protobuf_to_dict import protobuf_to_dict
from jormungandr.find_extrem_datetimes import extremes
from flask.ext.restful import reqparse
from flask.ext.restful.types import boolean
from jormungandr.interfaces.argument import ArgumentDoc
from jormungandr.interfaces.parsers import depth_argument
from jormungandr.authentification import authentification_required

class Journeys(Resource):

    """ Compute journeys"""
    parsers = {}
    method_decorators = [authentification_required]

    def __init__(self, *args, **kwargs):
        types = ["all", "rapid"]
        self.parsers["get"] = reqparse.RequestParser(
            argument_class=ArgumentDoc)
        parser_get = self.parsers["get"]
        parser_get.add_argument("origin", type=str, required=True,
                                description="Origin's uri of your journey")
        parser_get.add_argument("destination", type=str, required=True,
                                description=
                                "Destination's uri of your journey")
        parser_get.add_argument("datetime", type=str, required=True,
                                description=
                                "Datetime's uri of your journey")
        parser_get.add_argument("clockwise", type=true_false, default=True,
                                description=
                                "Whether you want your journey to start"\
                                "after datetime or to end before datetime")
        parser_get.add_argument("max_duration", type=int, default=36000,
                                description="Maximum duration of your journey")
        parser_get.add_argument("max_transfers", type=int, default=10,
                                description=
                                "Maximum transfers you want in "\
                                "your journey")
        parser_get.add_argument("origin_mode",
                                type=option_value(["walking", "car", "bike",
                                                   "bss"]),
                                action="append", default=['walking', 'bike',
                                                          'car'],
                                description=
                                "The list of modes you want at the "\
                                "beggining of your journey")
        parser_get.add_argument("destination_mode",
                                type=option_value(["walking", "car", "bike",
                                                   "bss"]),
                                default=['walking', 'bike', 'car'],
                                action="append",
                                description=
                                "The list of modes you want at the"\
                                " end of your journey")
        parser_get.add_argument("max_duration_to_pt", type=int, default=10*60,
                                description=
                                "maximal duration of non public "\
                                "transport in second")
        parser_get.add_argument("walking_speed", type=float, default=1.68,
                                description=
                                "Walking speed in meter/second")
        parser_get.add_argument("bike_speed", type=float, default=8.8,
                                description="Bike speed in meter/second")
        parser_get.add_argument("bss_speed", type=float, default=8.8,
                                description="Bike rental speed in "\
                                "meter/second")
        parser_get.add_argument("car_speed", type=float, default=16.8,
                                description=
                                "Car speed in meter/second")
        parser_get.add_argument("forbidden_uris[]", type=str, action="append",
                                description="Uri you want to forbid")
        parser_get.add_argument("type", type=option_value(types),
                                default="all")
        parser_get.add_argument("wheelchair", type=boolean, default=False)
        parser_get.add_argument("count", type=int)
        parser_get.add_argument("debug", type=boolean, default=False,
                                hidden=True)

    def get(self, region=None):
        args = self.parsers["get"].parse_args()
        if region is None:
            region = i_manager.key_of_id(args["origin"])
        response = i_manager.dispatch(args, region, "journeys")
        if response.journeys:
            (before, after) = extremes(response, request)
            if before and after:
                response.prev = before
                response.next = after
        return protobuf_to_dict(response, use_enum_labels=True), 200

class Isochrone(Resource):

    """ Compute isochrones """
    method_decorators = [authentification_required]

    def __init__(self):
        self.parsers = {}
        self.parsers["get"] = reqparse.RequestParser()
        parser_get = self.parsers["get"]
        parser_get.add_argument("origin", type=str, required=True)
        parser_get.add_argument("datetime", type=str, required=True)
        parser_get.add_argument("clockwise", type=true_false, default=True)
        parser_get.add_argument("max_duration", type=int, default=3600)
        parser_get.add_argument("max_transfers", type=int, default=10)
        parser_get.add_argument("origin_mode",
                                type=option_value(["walking", "car", "bike",
                                                   "bss"]),
                                action="append", default=["walking"])
        parser_get.add_argument("max_duration_to_pt", type=int, default=10*60,
                                description="maximal duration of non public \
                                transport in second")
        parser_get.add_argument("walking_speed", type=float, default=1.68)
        parser_get.add_argument("bike_speed", type=float, default=8.8)
        parser_get.add_argument("bss_speed", type=float, default=8.8)
        parser_get.add_argument("car_speed", type=float, default=16.8)
        parser_get.add_argument("forbidden_uris[]", type=str, action="append")
        parser_get.add_argument("wheelchair", type=boolean, default=False)
        parser_get.add_argument("debug", type=boolean, default=False,
                                hidden=True)

    def get(self, region=None):
        args = self.parsers["get"].parse_args()
        if region is None:
            region = i_manager.key_of_id(args["origin"])
        response = i_manager.dispatch(args, region, "isochrone")
        if response.journeys:
            (before, after) = extremes(response, request)
            if before and after:
                response.prev = before
                response.next = after
        return protobuf_to_dict(response, use_enum_labels=True), 200
<|endoftext|>


<commit_before># -*- python -*-

# stdlib imports
import os
import os.path as osp
import sys

# waf imports ---
import waflib.Utils
import waflib.Logs as msg
import waflib.Configure
import waflib.Build
import waflib.Task
import waflib.Tools.ccroot
from waflib.Configure import conf
from waflib.TaskGen import feature, before_method, after_method, extension, after

_heptooldir = osp.dirname(osp.abspath(__file__))
# add this directory to sys.path to ease the loading of other hepwaf tools
if not _heptooldir in sys.path: sys.path.append(_heptooldir)

### ---------------------------------------------------------------------------
class symlink_tsk(waflib.Task.Task):
    """
    A task to install symlinks of binaries and libraries under the *build*
    install-area (to not require shaggy RPATH)
    this is needed e.g. for genconf and gencliddb.
    """
    color   = 'PINK'
    reentrant = True

    def run(self):
        import os
        try:
            os.remove(self.outputs[0].abspath())
        except OSError:
            pass
        return os.symlink(self.inputs[0].abspath(),
                          self.outputs[0].abspath())

@feature('symlink_tsk')
@after_method('apply_link')
def add_install_copy(self):
    link_tsk = getattr(self, 'link_task', None)
    if not link_tsk:
        return
    link_cls_name = self.link_task.__class__.__name__
    # FIXME: is there an API for this ?
    if link_cls_name.endswith('lib'):
        outdir = self.bld.path.make_node('.install_area').make_node('lib')
    else:
        outdir = self.bld.path.make_node('.install_area').make_node('bin')
    link_outputs = waflib.Utils.to_list(self.link_task.outputs)
    for out in link_outputs:
        if isinstance(out, str):
            n = out
        else:
            n = out.name
        out_sym = outdir.find_or_declare(n)
        #print("===> ", self.target, link_cls_name, out_sym.abspath())
        tsk = self.create_task('symlink_tsk',
                               out,
                               out_sym)
        self.source += tsk.outputs
        pass
    return

### -----------------------------------------------------------------------------
import waflib.Build
class InstallContext(waflib.Build.InstallContext):
    '''
    InstallContext class to be less verbose by default
    '''
    def __init__(self, **kw):
        super(InstallContext, self).__init__(**kw)
        return

    def execute_build(self):
        self.logger = msg
        lvl = msg.log.level
        if not msg.verbose:
            if lvl < msg.logging.ERROR:
                msg.log.setLevel(msg.logging.ERROR)
                pass
        try:
            ret = super(InstallContext, self).execute_build()
        finally:
            msg.log.setLevel(lvl)
        return ret
    pass # class InstallContext

### -----------------------------------------------------------------------------
import waflib.Build
class UninstallContext(waflib.Build.UninstallContext):
    '''
    UninstallContext class to be less verbose by default
    '''
    def __init__(self, **kw):
        super(UninstallContext, self).__init__(**kw)
        return

    def execute_build(self):
        self.logger = msg
        lvl = msg.log.level
        if not msg.verbose:
            if lvl < msg.logging.ERROR:
                msg.log.setLevel(msg.logging.ERROR)
        try:
            ret = super(UninstallContext, self).execute_build()
        finally:
            msg.log.setLevel(lvl)
        return ret
    pass # class UninstallContext

### ---------------------------------------------------------------------------
import os, sys
from waflib.TaskGen import feature, after_method
from waflib import Utils, Task, Logs, Options

@feature('hwaf_utest')
@after('symlink_tsk')
def make_test(self):
    """Create the unit test task. There can be only one unit test task by task generator."""
    if getattr(self, 'link_task', None):
        self.create_task('hwaf_utest', self.link_task.outputs)

g_testlock = Utils.threading.Lock()

class hwaf_utest(Task.Task):
    """
    Execute a unit test
    """
    color = 'CYAN'
    after = ['vnum', 'inst', 'symlink_tsk']
    vars = []

    def runnable_status(self):
        """
        Always execute the task if `waf --alltests` was used or no
                tests if ``waf --notests`` was used
        """
        if getattr(Options.options, 'no_tests', True):
            return Task.SKIP_ME

        if not self.generator.bld.cmd in ('build', 'check'):
            return Task.SKIP_ME

        ret = super(hwaf_utest, self).runnable_status()
        #print("%s: ret=%s" % (self.inputs[0].name, ret))
        if ret in (Task.SKIP_ME, Task.RUN_ME):
            if getattr(Options.options, 'all_tests', False):
                depends = waflib.Utils.to_list(getattr(self.generator, 'depends_on', []))
                #print("%s: deps=%s" % (self.inputs[0].name, depends))
                if depends:
                    results = getattr(self.generator.bld, 'hwaf_utest_results', [])
                    dep = 0
                    for tup in results:
                        fname = tup[0]
                        fname = osp.basename(fname)
                        #print("%s: fname=%s ?" % (self.inputs[0].name, fname))
                        if fname in depends:
                            #print("%s: fname=%s YES!" % (self.inputs[0].name, fname))
                            dep += 1
                    if dep == len(depends):
                        #print("%s: ALL DEPS OK!" % (self.inputs[0].name,))
                        return Task.RUN_ME
                    #print("%s: RUN LATER" % (self.inputs[0].name,))
                    return Task.ASK_LATER
                else:
                    return Task.RUN_ME
        return ret

    def run(self):
        """
        Execute the test. The execution is always successful, but the results
        are stored on ``self.generator.bld.hwaf_utest_results`` for postprocessing.
        """

        filename = self.inputs[0].abspath()
        self.ut_exec = getattr(self.generator, 'ut_exec', [filename])
        if getattr(self.generator, 'ut_fun', None):
            # FIXME waf 1.8 - add a return statement here?
            self.generator.ut_fun(self)

        try:
            fu = getattr(self.generator.bld, 'all_test_paths')
        except AttributeError:
            # this operation may be performed by at most #maxjobs
            fu = os.environ.copy()

            lst = []
            for g in self.generator.bld.groups:
                for tg in g:
                    if getattr(tg, 'link_task', None):
                        s = tg.link_task.outputs[0].parent.abspath()
                        if s not in lst:
                            lst.append(s)

            def add_path(dct, path, var):
                dct[var] = os.pathsep.join(Utils.to_list(path) + [os.environ.get(var, '')])

            if Utils.is_win32:
                add_path(fu, lst, 'PATH')
            elif Utils.unversioned_sys_platform() == 'darwin':
                add_path(fu, lst, 'DYLD_LIBRARY_PATH')
                add_path(fu, lst, 'LD_LIBRARY_PATH')
            else:
                add_path(fu, lst, 'LD_LIBRARY_PATH')
            self.generator.bld.all_test_paths = fu
            pass

        self.ut_exec = Utils.to_list(self.ut_exec)

        cwd = getattr(self.generator, 'ut_cwd', None) or self.inputs[0].parent.abspath()

        args = Utils.to_list(getattr(self.generator, 'ut_args', []))
        if args:
            self.ut_exec.extend(args)

        testcmd = getattr(Options.options, 'testcmd', False)
        if testcmd:
            self.ut_exec = (testcmd % self.ut_exec[0]).split(' ')

        #print(">>>> running %s..." % self.ut_exec[0])
        proc = Utils.subprocess.Popen(
            self.ut_exec,
            cwd=cwd,
            env=fu,
            stderr=Utils.subprocess.PIPE,
            stdout=Utils.subprocess.PIPE
            )
        (stdout, stderr) = proc.communicate()

        tup = (filename, proc.returncode, stdout, stderr)
        self.generator.utest_result = tup

        g_testlock.acquire()
        try:
            bld = self.generator.bld
            Logs.debug("ut: %r", tup)
            try:
                bld.hwaf_utest_results.append(tup)
            except AttributeError:
                bld.hwaf_utest_results = [tup]
        finally:
            #print(">>>> running %s... [done]" % self.ut_exec[0])
            g_testlock.release()

@waflib.Configure.conf
def hwaf_utest_summary(bld, *k, **kwargs):
    """
    Display an execution summary::

        def build(bld):
            bld(features='cxx cxxprogram test', source='main.c', target='app')
            bld.add_post_fun(hwaf_utest_summary)
    """
    lst = getattr(bld, 'hwaf_utest_results', [])
    if lst:
        Logs.pprint('CYAN', '='*80)
        Logs.pprint('CYAN', 'unit-tests execution summary')

        total = len(lst)
        tfail = len([x for x in lst if x[1]])
        val = 100 * (total - tfail) / (1.0 * total)
        Logs.pprint('CYAN', 'test report %3.0f%% success' % val)

        Logs.pprint('CYAN', '  tests that pass %d/%d' % (total-tfail, total))
        for (f, code, out, err) in lst:
            if not code:
                Logs.pprint('CYAN', '    %s' % f)
                pass
            pass

        Logs.pprint('CYAN', '  tests that fail %d/%d' % (tfail, total))
        for (f, code, out, err) in lst:
            if code:
                Logs.pprint('CYAN', '    %s (err=%d)' % (f,code))
                pass
            pass
        Logs.pprint('CYAN', '='*80)
        pass
    return

@waflib.Configure.conf
def hwaf_utest_set_exit_code(bld, *k):
    """
    If any of the tests fail waf will exit with that exit code.
    This is useful if you have an automated build system which need
    to report on errors from the tests.
    You may use it like this:

    def build(bld):
        bld(features='cxx cxxprogram test', source='main.c', target='app')
        bld.add_post_fun(hwaf_unit_set_exit_code)
    """
    lst = getattr(bld, 'hwaf_utest_results', [])
    if not lst:
        return

    msg = []
    for (f, code, out, err) in lst:
        if code:
            msg.append('=== %s === (err=%d)' % (f,code))
            if out: msg.append('stdout:%s%s' % (os.linesep, out.decode('utf-8')))
            if err: msg.append('stderr:%s%s' % (os.linesep, err.decode('utf-8')))
            pass
        pass

    if msg: bld.fatal(os.linesep.join(msg))
    return

def options(opt):
    """
    Provide the ``--alltests``, ``--notests`` and ``--testcmd`` command-line options.
    """
    opt.add_option(
        '--notests',
        action='store_true',
        default=False,
        help='Exec no unit tests',
        dest='no_tests')

    opt.add_option(
        '--alltests',
        action='store_true',
        default=False,
        help='Exec all unit tests',
        dest='all_tests')

    opt.add_option(
        '--testcmd',
        action='store',
        default=False,
        help = 'Run the unit tests using the test-cmd string'
               ' example "--test-cmd="valgrind --error-exitcode=1'
               ' %s" to run under valgrind', dest='testcmd')

## EOF ##
<commit_msg>utest: add ability to specify expected return-code value<commit_after># -*- python -*-

# stdlib imports
import os
import os.path as osp
import sys

# waf imports ---
import waflib.Utils
import waflib.Logs as msg
import waflib.Configure
import waflib.Build
import waflib.Task
import waflib.Tools.ccroot
from waflib.Configure import conf
from waflib.TaskGen import feature, before_method, after_method, extension, after

_heptooldir = osp.dirname(osp.abspath(__file__))
# add this directory to sys.path to ease the loading of other hepwaf tools
if not _heptooldir in sys.path: sys.path.append(_heptooldir)

### ---------------------------------------------------------------------------
class symlink_tsk(waflib.Task.Task):
    """
    A task to install symlinks of binaries and libraries under the *build*
    install-area (to not require shaggy RPATH)
    this is needed e.g. for genconf and gencliddb.
    """
    color   = 'PINK'
    reentrant = True

    def run(self):
        import os
        try:
            os.remove(self.outputs[0].abspath())
        except OSError:
            pass
        return os.symlink(self.inputs[0].abspath(),
                          self.outputs[0].abspath())

@feature('symlink_tsk')
@after_method('apply_link')
def add_install_copy(self):
    link_tsk = getattr(self, 'link_task', None)
    if not link_tsk:
        return
    link_cls_name = self.link_task.__class__.__name__
    # FIXME: is there an API for this ?
    if link_cls_name.endswith('lib'):
        outdir = self.bld.path.make_node('.install_area').make_node('lib')
    else:
        outdir = self.bld.path.make_node('.install_area').make_node('bin')
    link_outputs = waflib.Utils.to_list(self.link_task.outputs)
    for out in link_outputs:
        if isinstance(out, str):
            n = out
        else:
            n = out.name
        out_sym = outdir.find_or_declare(n)
        #print("===> ", self.target, link_cls_name, out_sym.abspath())
        tsk = self.create_task('symlink_tsk',
                               out,
                               out_sym)
        self.source += tsk.outputs
        pass
    return

### -----------------------------------------------------------------------------
import waflib.Build
class InstallContext(waflib.Build.InstallContext):
    '''
    InstallContext class to be less verbose by default
    '''
    def __init__(self, **kw):
        super(InstallContext, self).__init__(**kw)
        return

    def execute_build(self):
        self.logger = msg
        lvl = msg.log.level
        if not msg.verbose:
            if lvl < msg.logging.ERROR:
                msg.log.setLevel(msg.logging.ERROR)
                pass
        try:
            ret = super(InstallContext, self).execute_build()
        finally:
            msg.log.setLevel(lvl)
        return ret
    pass # class InstallContext

### -----------------------------------------------------------------------------
import waflib.Build
class UninstallContext(waflib.Build.UninstallContext):
    '''
    UninstallContext class to be less verbose by default
    '''
    def __init__(self, **kw):
        super(UninstallContext, self).__init__(**kw)
        return

    def execute_build(self):
        self.logger = msg
        lvl = msg.log.level
        if not msg.verbose:
            if lvl < msg.logging.ERROR:
                msg.log.setLevel(msg.logging.ERROR)
        try:
            ret = super(UninstallContext, self).execute_build()
        finally:
            msg.log.setLevel(lvl)
        return ret
    pass # class UninstallContext

### ---------------------------------------------------------------------------
import os, sys
from waflib.TaskGen import feature, after_method
from waflib import Utils, Task, Logs, Options

@feature('hwaf_utest')
@after('symlink_tsk')
def make_test(self):
    """Create the unit test task. There can be only one unit test task by task generator."""
    if getattr(self, 'link_task', None):
        self.create_task('hwaf_utest', self.link_task.outputs)

g_testlock = Utils.threading.Lock()

class hwaf_utest(Task.Task):
    """
    Execute a unit test.

    ctx(
        features='test cxx cxxprogram',
        source='foo.cxx',
        target='app',
        ut_cwd=None,           # optional dir for running test
        ut_args=['--verbose'], # optional additional arguments to test-binary
        ut_rc=0,               # optional expected return code
    )
    """
    color = 'CYAN'
    after = ['vnum', 'inst', 'symlink_tsk']
    vars = []

    def runnable_status(self):
        """
        Always execute the task if `waf --alltests` was used or no
                tests if ``waf --notests`` was used
        """
        if getattr(Options.options, 'no_tests', True):
            return Task.SKIP_ME

        if not self.generator.bld.cmd in ('build', 'check'):
            return Task.SKIP_ME

        ret = super(hwaf_utest, self).runnable_status()
        #print("%s: ret=%s" % (self.inputs[0].name, ret))
        if ret in (Task.SKIP_ME, Task.RUN_ME):
            if getattr(Options.options, 'all_tests', False):
                depends = waflib.Utils.to_list(getattr(self.generator, 'depends_on', []))
                #print("%s: deps=%s" % (self.inputs[0].name, depends))
                if depends:
                    results = getattr(self.generator.bld, 'hwaf_utest_results', [])
                    dep = 0
                    for tup in results:
                        fname = tup[0]
                        fname = osp.basename(fname)
                        #print("%s: fname=%s ?" % (self.inputs[0].name, fname))
                        if fname in depends:
                            #print("%s: fname=%s YES!" % (self.inputs[0].name, fname))
                            dep += 1
                    if dep == len(depends):
                        #print("%s: ALL DEPS OK!" % (self.inputs[0].name,))
                        return Task.RUN_ME
                    #print("%s: RUN LATER" % (self.inputs[0].name,))
                    return Task.ASK_LATER
                else:
                    return Task.RUN_ME
        return ret

    def run(self):
        """
        Execute the test. The execution is always successful, but the results
        are stored on ``self.generator.bld.hwaf_utest_results`` for postprocessing.
        """

        filename = self.inputs[0].abspath()
        self.ut_exec = getattr(self.generator, 'ut_exec', [filename])
        if getattr(self.generator, 'ut_fun', None):
            # FIXME waf 1.8 - add a return statement here?
            self.generator.ut_fun(self)

        try:
            fu = getattr(self.generator.bld, 'all_test_paths')
        except AttributeError:
            # this operation may be performed by at most #maxjobs
            fu = os.environ.copy()

            lst = []
            for g in self.generator.bld.groups:
                for tg in g:
                    if getattr(tg, 'link_task', None):
                        s = tg.link_task.outputs[0].parent.abspath()
                        if s not in lst:
                            lst.append(s)

            def add_path(dct, path, var):
                dct[var] = os.pathsep.join(Utils.to_list(path) + [os.environ.get(var, '')])

            if Utils.is_win32:
                add_path(fu, lst, 'PATH')
            elif Utils.unversioned_sys_platform() == 'darwin':
                add_path(fu, lst, 'DYLD_LIBRARY_PATH')
                add_path(fu, lst, 'LD_LIBRARY_PATH')
            else:
                add_path(fu, lst, 'LD_LIBRARY_PATH')
            self.generator.bld.all_test_paths = fu
            pass

        self.ut_exec = Utils.to_list(self.ut_exec)

        cwd = getattr(self.generator, 'ut_cwd', None) or self.inputs[0].parent.abspath()

        args = Utils.to_list(getattr(self.generator, 'ut_args', []))
        if args:
            self.ut_exec.extend(args)

        testcmd = getattr(Options.options, 'testcmd', False)
        if testcmd:
            self.ut_exec = (testcmd % self.ut_exec[0]).split(' ')

        returncode = getattr(self.generator, 'ut_rc', 0)
        if isinstance(returncode, (list,tuple)):
            returncode = int(returncode[0])

        #print(">>>> running %s..." % self.ut_exec[0])
        proc = Utils.subprocess.Popen(
            self.ut_exec,
            cwd=cwd,
            env=fu,
            stderr=Utils.subprocess.PIPE,
            stdout=Utils.subprocess.PIPE
            )
        (stdout, stderr) = proc.communicate()

        tup = (filename, proc.returncode, stdout, stderr, returncode)
        self.generator.utest_result = tup

        g_testlock.acquire()
        try:
            bld = self.generator.bld
            Logs.debug("ut: %r", tup)
            try:
                bld.hwaf_utest_results.append(tup)
            except AttributeError:
                bld.hwaf_utest_results = [tup]
        finally:
            #print(">>>> running %s... [done]" % self.ut_exec[0])
            g_testlock.release()

@waflib.Configure.conf
def hwaf_utest_summary(bld, *k, **kwargs):
    """
    Display an execution summary::

        def build(bld):
            bld(features='cxx cxxprogram test', source='main.c', target='app')
            bld.add_post_fun(hwaf_utest_summary)
    """
    lst = getattr(bld, 'hwaf_utest_results', [])
    if lst:
        Logs.pprint('CYAN', '='*80)
        Logs.pprint('CYAN', 'unit-tests execution summary')

        total = len(lst)
        tfail = len([x for x in lst if x[1] != x[4]])
        val = 100 * (total - tfail) / (1.0 * total)
        Logs.pprint('CYAN', 'test report %3.0f%% success' % val)

        Logs.pprint('CYAN', '  tests that pass %d/%d' % (total-tfail, total))
        for (f, code, out, err, exp_rc) in lst:
            if code == exp_rc:
                Logs.pprint('CYAN', '    %s' % f)
                pass
            pass

        Logs.pprint('CYAN', '  tests that fail %d/%d' % (tfail, total))
        for (f, code, out, err, exp_rc) in lst:
            if code != exp_rc:
                Logs.pprint('CYAN', '    %s (err=%d, exp=%d)' % (f,code, exp_rc))
                pass
            pass
        Logs.pprint('CYAN', '='*80)
        pass
    return

@waflib.Configure.conf
def hwaf_utest_set_exit_code(bld, *k):
    """
    If any of the tests fail waf will exit with that exit code.
    This is useful if you have an automated build system which need
    to report on errors from the tests.
    You may use it like this:

    def build(bld):
        bld(features='cxx cxxprogram test', source='main.c', target='app')
        bld.add_post_fun(hwaf_unit_set_exit_code)
    """
    lst = getattr(bld, 'hwaf_utest_results', [])
    if not lst:
        return

    msg = []
    for (f, code, out, err, exp_rc) in lst:
        if code != exp_rc:
            msg.append('=== %s === (err=%d expected=%d)' % (f,code, exp_rc))
            if out: msg.append('stdout:%s%s' % (os.linesep, out.decode('utf-8')))
            if err: msg.append('stderr:%s%s' % (os.linesep, err.decode('utf-8')))
            pass
        pass

    if msg: bld.fatal(os.linesep.join(msg))
    return

def options(opt):
    """
    Provide the ``--alltests``, ``--notests`` and ``--testcmd`` command-line options.
    """
    opt.add_option(
        '--notests',
        action='store_true',
        default=False,
        help='Exec no unit tests',
        dest='no_tests')

    opt.add_option(
        '--alltests',
        action='store_true',
        default=False,
        help='Exec all unit tests',
        dest='all_tests')

    opt.add_option(
        '--testcmd',
        action='store',
        default=False,
        help = 'Run the unit tests using the test-cmd string'
               ' example "--test-cmd="valgrind --error-exitcode=1'
               ' %s" to run under valgrind', dest='testcmd')

## EOF ##
<|endoftext|>


<commit_before># Copyright 2018 the authors.
# This file is part of Hy, which is free software licensed under the Expat
# license. See the LICENSE.

try:
    import __builtin__ as builtins
except ImportError:
    import builtins  # NOQA
try:
    from py_compile import MAGIC, wr_long
except ImportError:
    # py_compile.MAGIC removed and imp.get_magic() deprecated in Python 3.4
    from importlib.util import MAGIC_NUMBER as MAGIC  # NOQA

    def wr_long(f, x):
        """Internal; write a 32-bit int to a file in little-endian order."""
        f.write(bytes([x & 0xff,
                       (x >> 8) & 0xff,
                       (x >> 16) & 0xff,
                       (x >> 24) & 0xff]))
import sys, keyword

PY3 = sys.version_info[0] >= 3
PY35 = sys.version_info >= (3, 5)
PY36 = sys.version_info >= (3, 6)
PY37 = sys.version_info >= (3, 7)

str_type     = str   if PY3 else unicode      # NOQA
bytes_type   = bytes if PY3 else str          # NOQA
long_type    = int   if PY3 else long         # NOQA
string_types = str   if PY3 else basestring   # NOQA

if PY3:
    exec('def raise_empty(t, *args): raise t(*args) from None')
else:
    def raise_empty(t, *args):
        raise t(*args)

def isidentifier(x):
    if x in ('True', 'False', 'None', 'print'):
        # `print` is special-cased here because Python 2's
        # keyword.iskeyword will count it as a keyword, but we
        # use the __future__ feature print_function, which makes
        # it a non-keyword.
        return True
    if keyword.iskeyword(x):
        return False
    if PY3:
        return x.isidentifier()
    if x.rstrip() != x:
        return False
    import tokenize as T
    from StringIO import StringIO
    try:
        tokens = list(T.generate_tokens(StringIO(x).readline))
    except T.TokenError:
        return False
    return len(tokens) == 2 and tokens[0][0] == T.NAME
<commit_msg>Use io.StringIO instead of the StringIO module<commit_after># Copyright 2018 the authors.
# This file is part of Hy, which is free software licensed under the Expat
# license. See the LICENSE.

try:
    import __builtin__ as builtins
except ImportError:
    import builtins  # NOQA
try:
    from py_compile import MAGIC, wr_long
except ImportError:
    # py_compile.MAGIC removed and imp.get_magic() deprecated in Python 3.4
    from importlib.util import MAGIC_NUMBER as MAGIC  # NOQA

    def wr_long(f, x):
        """Internal; write a 32-bit int to a file in little-endian order."""
        f.write(bytes([x & 0xff,
                       (x >> 8) & 0xff,
                       (x >> 16) & 0xff,
                       (x >> 24) & 0xff]))
import sys, keyword

PY3 = sys.version_info[0] >= 3
PY35 = sys.version_info >= (3, 5)
PY36 = sys.version_info >= (3, 6)
PY37 = sys.version_info >= (3, 7)

str_type     = str   if PY3 else unicode      # NOQA
bytes_type   = bytes if PY3 else str          # NOQA
long_type    = int   if PY3 else long         # NOQA
string_types = str   if PY3 else basestring   # NOQA

if PY3:
    exec('def raise_empty(t, *args): raise t(*args) from None')
else:
    def raise_empty(t, *args):
        raise t(*args)

def isidentifier(x):
    if x in ('True', 'False', 'None', 'print'):
        # `print` is special-cased here because Python 2's
        # keyword.iskeyword will count it as a keyword, but we
        # use the __future__ feature print_function, which makes
        # it a non-keyword.
        return True
    if keyword.iskeyword(x):
        return False
    if PY3:
        return x.isidentifier()
    if x.rstrip() != x:
        return False
    import tokenize as T
    from io import StringIO
    try:
        tokens = list(T.generate_tokens(StringIO(x).readline))
    except T.TokenError:
        return False
    return len(tokens) == 2 and tokens[0][0] == T.NAME
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import sys

class VersionPart(object):
    @classmethod
    def from_dict(cls, dic):
        try:
            part_type = dic.pop('type')
        except KeyError:
            part_type = 'integer'

        class_name = part_type.title().replace("_", "") + 'VersionPart'
        part_class = getattr(sys.modules[__name__], class_name)

        return part_class(**dic)

class IntegerVersionPart(VersionPart):
    def __init__(self, name, value=None, start_value=None):
        self.name = name

        if start_value is None:
            self.start_value = 0
        else:
            self.start_value = start_value

        if value is None:
            self.value = self.start_value
        else:
            self.set(value)

    def inc(self):
        self.value = self.value + 1

    def set(self, value):
        self.value = int(value)

    def reset(self):
        self.value = self.start_value

    def copy(self):
        return IntegerVersionPart(self.name, self.value, self.start_value)

class ValueListVersionPart(VersionPart):
    def __init__(self, name, value, allowed_values):
        self.name = name
        self.allowed_values = allowed_values

        if value is None:
            self.value = allowed_values[0]
        else:
            self.set(value)

        # When copying this does not take the object itself
        self.values = [v for v in allowed_values]

    def set(self, value):
        if value not in self.allowed_values:
            raise ValueError("The given value {} is not allowed, the list of possible values is {}", value,
                             self.allowed_values)
        self.value = value

    def inc(self):
        idx = self.values.index(self.value)
        self.value = self.values[(idx + 1) % len(self.values)]

    def reset(self):
        self.value = self.values[0]

    def copy(self):
        return ValueListVersionPart(self.name, self.value, self.values)
<commit_msg>Added DateVersionPart<commit_after># -*- coding: utf-8 -*-

import sys
from datetime import datetime

def _strftime(fmt):
    return datetime.now().strftime(fmt)

def strftime(fmt):
    calver_syntax = {
        'YYYY': {
            'fmt': '%Y',
            'strip': False
        },
        'YY': {
            'fmt': '%y',
            'strip': False
        },
        '0M': {
            'fmt': '%m',
            'strip': False
        },
        '0D': {
            'fmt': '%d',
            'strip': False
        },
        'MM': {
            'fmt': '%m',
            'strip': True
        },
        'DD': {
            'fmt': '%d',
            'strip': True
        }
    }

    newfmt = calver_syntax.get(fmt, {'fmt': fmt, 'strip': False})
    value = _strftime(newfmt['fmt'])

    if newfmt['strip']:
        return value.strip('0')

    return value

class VersionPart(object):

    @classmethod
    def from_dict(cls, dic):
        try:
            part_type = dic.pop('type')
        except KeyError:
            part_type = 'integer'

        class_name = part_type.title().replace("_", "") + 'VersionPart'
        part_class = getattr(sys.modules[__name__], class_name)

        return part_class(**dic)

class IntegerVersionPart(VersionPart):

    def __init__(self, name, value=None, start_value=None):
        self.name = name

        if start_value is None:
            self.start_value = 0
        else:
            self.start_value = start_value

        if value is None:
            self.value = self.start_value
        else:
            self.set(value)

    def inc(self):
        self.value = self.value + 1

    def set(self, value):
        self.value = int(value)

    def reset(self):
        self.value = self.start_value

    def copy(self):
        return IntegerVersionPart(self.name, self.value, self.start_value)

class ValueListVersionPart(VersionPart):

    def __init__(self, name, value, allowed_values):
        self.name = name
        self.allowed_values = allowed_values

        if value is None:
            self.value = allowed_values[0]
        else:
            self.set(value)

        # When copying this does not take the object itself
        self.values = [v for v in allowed_values]

    def set(self, value):
        if value not in self.allowed_values:
            raise ValueError("The given value {} is not allowed, the list of possible values is {}", value,
                             self.allowed_values)
        self.value = value

    def inc(self):
        idx = self.values.index(self.value)
        self.value = self.values[(idx + 1) % len(self.values)]

    def reset(self):
        self.value = self.values[0]

    def copy(self):
        return ValueListVersionPart(self.name, self.value, self.values)

class DateVersionPart(VersionPart):

    def __init__(self, name, value, fmt):
        self.name = name
        self.fmt = fmt

        if value is None:
            self.value = strftime(fmt)
        else:
            self.value = value

    def reset(self):
        self.value = strftime(self.fmt)

    def inc(self):
        self.reset()

    def copy(self):
        return DateVersionPart(self.name, self.value, self.fmt)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

from flask import request
from flask_classy import route

from core.entities import Entity, Malware, TTP, Actor, ExploitKit, Exploit, \
    Campaign
from core.indicators import Indicator, Yara, Regex
from core.observables import Observable
from core.web.api.api import render
from core.web.api.crud import CrudApi
from core.web.helpers import requires_permissions

NODES_CLASSES = {
    'entity': Entity,
    'observable': Observable,
    'indicator': Indicator,
    'malware': Malware,
    'ttp': TTP,
    'actor': Actor,
    'campaign': Campaign,
    'exploit': Exploit,
    'exploitkit': ExploitKit,
    'regex': Regex,
    'yara': Yara,
}

class Neighbors(CrudApi):

    @requires_permissions('read')
    def get(self, klass, node_id):
        klass = NODES_CLASSES[klass.lower().split('.')[0]]
        node = klass.objects.get(id=node_id)

        result = {'links': list(), 'nodes': list()}

        result['nodes'].append(node.to_mongo())

        node_ids = set()
        links = list(set(list(node.incoming()) + list(node.outgoing())))

        for link, node in links:
            if node.id not in node_ids:
                node_ids.add(node)
                result['nodes'].append(node.to_mongo())

            result['links'].append(link.to_dict())

        return render(result)

    @route("/tuples/<klass>/<node_id>/<type_filter>", methods=["POST"])
    @requires_permissions('read')
    def tuples(self, klass, node_id, type_filter):
        query = request.get_json(silent=True) or {}
        fltr = query.get("filter", {})
        params = query.get("params", {})

        klass = NODES_CLASSES[klass.lower().split('.')[0]]
        filter_class = NODES_CLASSES[type_filter.lower().split('.')[0]]
        node = klass.objects.get(id=node_id)

        regex = bool(params.pop('regex', False))
        ignorecase = bool(params.pop('ignorecase', False))
        page = int(params.pop("page", 1)) - 1
        rng = int(params.pop("range", 50))

        print("[{}] Filter: {}".format(self.__class__.__name__, fltr))
        print(filter_class, fltr, regex, ignorecase, page, rng)
        neighbors = node.neighbors_advanced(
            filter_class, fltr, regex, ignorecase, page, rng)

        _all = []
        links = []
        objs = []

        for link, obj in neighbors:
            links.append(link)
            objs.append(obj)
            _all.append((link, obj))

        data = {"objs": objs, "links": links}
        if issubclass(filter_class, Entity):
            return render(data, template='entity_api.html')
        if issubclass(filter_class, Indicator):
            return render(data, template='indicator_api.html')
        if issubclass(filter_class, Observable):
            return render(data, template='observable_api.html')
<commit_msg>add company in dict<commit_after>from __future__ import unicode_literals

from flask import request
from flask_classy import route

from core.entities import Entity, Malware, TTP, Actor, ExploitKit, Exploit, \
    Campaign, Company
from core.indicators import Indicator, Yara, Regex
from core.observables import Observable
from core.web.api.api import render
from core.web.api.crud import CrudApi
from core.web.helpers import requires_permissions

NODES_CLASSES = {
    'entity': Entity,
    'observable': Observable,
    'indicator': Indicator,
    'malware': Malware,
    'ttp': TTP,
    'actor': Actor,
    'campaign': Campaign,
    'exploit': Exploit,
    'exploitkit': ExploitKit,
    'regex': Regex,
    'yara': Yara,
    'company': Company
}

class Neighbors(CrudApi):

    @requires_permissions('read')
    def get(self, klass, node_id):
        klass = NODES_CLASSES[klass.lower().split('.')[0]]
        node = klass.objects.get(id=node_id)

        result = {'links': list(), 'nodes': list()}

        result['nodes'].append(node.to_mongo())

        node_ids = set()
        links = list(set(list(node.incoming()) + list(node.outgoing())))

        for link, node in links:
            if node.id not in node_ids:
                node_ids.add(node)
                result['nodes'].append(node.to_mongo())

            result['links'].append(link.to_dict())

        return render(result)

    @route("/tuples/<klass>/<node_id>/<type_filter>", methods=["POST"])
    @requires_permissions('read')
    def tuples(self, klass, node_id, type_filter):
        query = request.get_json(silent=True) or {}
        fltr = query.get("filter", {})
        params = query.get("params", {})

        klass = NODES_CLASSES[klass.lower().split('.')[0]]
        filter_class = NODES_CLASSES[type_filter.lower().split('.')[0]]
        node = klass.objects.get(id=node_id)

        regex = bool(params.pop('regex', False))
        ignorecase = bool(params.pop('ignorecase', False))
        page = int(params.pop("page", 1)) - 1
        rng = int(params.pop("range", 50))

        print("[{}] Filter: {}".format(self.__class__.__name__, fltr))
        print(filter_class, fltr, regex, ignorecase, page, rng)
        neighbors = node.neighbors_advanced(
            filter_class, fltr, regex, ignorecase, page, rng)

        _all = []
        links = []
        objs = []

        for link, obj in neighbors:
            links.append(link)
            objs.append(obj)
            _all.append((link, obj))

        data = {"objs": objs, "links": links}
        if issubclass(filter_class, Entity):
            return render(data, template='entity_api.html')
        if issubclass(filter_class, Indicator):
            return render(data, template='indicator_api.html')
        if issubclass(filter_class, Observable):
            return render(data, template='observable_api.html')
<|endoftext|>