<commit_before># -*- coding: utf-8 -*-
#    Copyright (C) 2017-2018 CIRCL Computer Incident Response Center Luxembourg (smile gie)
#    Copyright (C) 2017-2018 Christian Studer
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys, json, os, time
import stix2
import pymisp
from stix2misp_mapping import *

class StixParser():
    def __init__(self):
        self.misp_event = pymisp.MISPEvent()
        self.event = []
        self.misp_event['Galaxy'] = []

    def loadEvent(self, args, pathname):
        try:
            filename = os.path.join(pathname, args[1])
            tempFile = open(filename, 'r')
            self.filename = filename
            event = stix2.get_dict(tempFile)
            for o in event.get('objects'):
                try:
                    self.event.append(stix2.parse(o))
                except:
                    self.parse_custom(o)
        except:
            print(json.dumps({'success': 0, 'message': 'The temporary STIX export file could not be read'}))
            sys.exit(1)

    def parse_custom(self, obj):
        custom_object_type = obj.pop('type')
        labels = obj['labels']
        try:
            @stix2.CustomObject(custom_object_type,[('id', stix2.properties.StringProperty(required=True)),
            ('x_misp_timestamp', stix2.properties.StringProperty(required=True)),
            ('labels', stix2.properties.ListProperty(labels, required=True)),
            ('x_misp_value', stix2.properties.StringProperty(required=True)),
            ('created_by_ref', stix2.properties.StringProperty(required=True)),
            ('x_misp_comment', stix2.properties.StringProperty()),
            ('x_misp_category', stix2.properties.StringProperty())
            ])
            class Custom(object):
                def __init__(self, **kwargs):
                    return
            custom = Custom(**obj)
        except:
            @stix2.CustomObject(custom_object_type,[('id', stix2.properties.StringProperty(required=True)),
            ('x_misp_timestamp', stix2.properties.StringProperty(required=True)),
            ('labels', stix2.properties.ListProperty(labels, required=True)),
            ('x_misp_values', stix2.properties.DictionaryProperty(required=True)),
            ('created_by_ref', stix2.properties.StringProperty(required=True)),
            ('x_misp_comment', stix2.properties.StringProperty()),
            ('x_misp_category', stix2.properties.StringProperty())
            ])
            class Custom(object):
                def __init__(self, **kwargs):
                    return
            custom = Custom(**obj)
        self.event.append(stix2.parse(custom))

    def handler(self):
        self.outputname = '{}.stix2'.format(self.filename)
        if self.from_misp():
            self.buildMispDict()
        else:
            self.buildExternalDict()

    def from_misp(self):
        for o in self.event:
            if o._type == 'report' and 'misp:tool="misp2stix2"' in o.get('labels'):
                index = self.event.index(o)
                self.report = self.event.pop(index)
                return True
        return False

    def buildMispDict(self):
        self.parse_identity()
        self.parse_report()
        for o in self.event:
            object_type = o._type
            labels = o.get('labels')
            if object_type in ('attack-pattern', 'course-of-action', 'intrusion-set', 'malware', 'threat-actor', 'tool'):
                self.parse_galaxy(o, labels)
            elif 'x-misp-object' in object_type:
                if 'from_object' in labels:
                    self.parse_custom_object(o)
                else:
                    self.parse_custom_attribute(o, labels)
            else:
                if 'from_object' in labels:
                    self.parse_object(o, labels)
                else:
                    self.parse_attribute(o, labels)

    def parse_identity(self):
        identity = self.event.pop(0)
        org = {'name': identity.get('name')}
        self.misp_event['Org'] = org

    def parse_report(self):
        report = self.report
        self.misp_event.info = report.get('name')
        if report.get('published'):
            self.misp_event.publish_timestamp = self.getTimestampfromDate(report.get('published'))
        if hasattr(report, 'labels'):
            labels = report['labels']
            for l in labels:
                self.misp_event.add_tag(l)
        if hasattr(report, 'external_references'):
            ext_refs = report['external_references']
            for e in ext_refs:
                link = {"type": "link"}
                comment = e.get('source_name').split('url - ')[1]
                if comment:
                    link['comment'] = comment
                link['value'] = e.get('url')
                self.misp_event.add_attribute(**link)

    def parse_galaxy(self, o, labels):
        galaxy_type = self.get_misp_type(labels)
        tag = labels[1]
        value = tag.split(':')[1].split('=')[1]
        galaxy_description, cluster_description = o.get('description').split('|')
        galaxy = {'type': galaxy_type, 'name': o.get('name'), 'description': galaxy_description,
                  'GalaxyCluster': [{'type': galaxy_type, 'value':value, 'tag_name': tag,
                                     'description': cluster_description}]}
        self.misp_event['Galaxy'].append(galaxy)

    def parse_custom_object(self, o):
        name = o.get('type').split('x-misp-object-')[1]
        timestamp = self.getTimestampfromDate(o.get('x_misp_timestamp'))
        category = o.get('category')
        attributes = []
        values = o.get('x_misp_values')
        for v in values:
            attribute_type, object_relation = v.split('_')
            attribute = {'type': attribute_type, 'value': values.get(v),
                         'object_relation': object_relation}
            attributes.append(attribute)
        misp_object = {'name': name, 'timestamp': timestamp, 'meta-category': category,
                       'Attribute': attributes}
        self.misp_event.add_object(**misp_object)

    def parse_custom_attribute(self, o, labels):
        attribute_type = o.get('type').split('x-misp-object-')[1]
        timestamp = self.getTimestampfromDate(o.get('x_misp_timestamp'))
        to_ids = bool(labels[1].split('=')[1])
        value = o.get('x_misp_value')
        category = self.get_misp_category(labels)
        attribute = {'type': attribute_type, 'timestamp': timestamp, 'to_ids': to_ids,
                     'value': value, 'category': category}
        self.misp_event.add_attribute(**attribute)

    def parse_object(self, o, labels):
        object_type = self.get_misp_type(labels)
        object_category = self.get_misp_category(labels)
        stix_type = o._type
        misp_object = pymisp.MISPObject(object_type)
        misp_object['meta-category'] = object_category
        if stix_type == 'indicator':
            pattern = o.get('pattern').replace('\\\\', '\\').split(' AND ')
            pattern[0] = pattern[0][2:]
            pattern[-1] = pattern[-1][:-2]
            attributes = self.parse_pattern_from_object(pattern, object_type)
        if stix_type == 'observed-data':
            observable = o.get('objects')
            attributes = self.parse_observable_from_object(observable, object_type)
        for attribute in attributes:
            misp_object.add_attribute(**attribute)
        misp_object.to_ids = bool(labels[1].split('=')[1])
        self.misp_event.add_object(**misp_object)

    def parse_attribute(self, o, labels):
        attribute_type = self.get_misp_type(labels)
        attribute_category = self.get_misp_category(labels)
        attribute = {'type': attribute_type, 'category': attribute_category}
        stix_type = o._type
        if stix_type == 'vulnerability':
            value = o.get('name')
        else:
            if stix_type == 'indicator':
                o_date = o.get('valid_from')
                pattern = o.get('pattern').replace('\\\\', '\\')
                value = self.parse_pattern(pattern)
                attribute['to_ids'] = True
            else:
                o_date = o.get('first_observed')
                observable = o.get('objects')
                try:
                    value = self.parse_observable(observable, attribute_type)
                except:
                    print('{}: {}'.format(attribute_type, observable))
                attribute['to_ids'] = False
            attribute['timestamp'] = self.getTimestampfromDate(o_date)
        if 'description' in o:
            attribute['comment'] = o.get('description')
        try:
            attribute['value'] = value
            self.misp_event.add_attribute(**attribute)
        except:
            pass

    def buildExternalDict(self):
        self.fetch_report()
        for o in self.event:
            object_type = o._type
            if object_type in ('relationship', 'report'):
                continue
            if object_type == 'indicator':
                attribute = {'type': 'stix2-pattern', 'object_relation': 'stix2-pattern', 'value': o.get('pattern')}
                misp_object = {'name': 'stix2-pattern', 'meta-category': 'stix2-pattern', 'Attribute': [attribute]}
                self.misp_event.add_object(**misp_object)

    def fetch_report(self):
        reports = []
        for o in self.event:
            if o._type == 'report':
                reports.append(o)
        if len(reports) == 1:
            self.report = reports[0]
            self.parse_report()

    def saveFile(self):
        eventDict = self.misp_event.to_json()
        outputfile = '{}.stix2'.format(self.filename)
        with open(outputfile, 'w') as f:
            f.write(eventDict)

    @staticmethod
    def getTimestampfromDate(stix_date):
        try:
            return int(stix_date.timestamp())
        except:
            return int(time.mktime(time.strptime(stix_date.split('+')[0], "%Y-%m-%d %H:%M:%S")))

    @staticmethod
    def get_misp_type(labels):
        return labels[0].split('=')[1][1:-1]

    @staticmethod
    def get_misp_category(labels):
        return labels[1].split('=')[1][1:-1]

    @staticmethod
    def parse_observable(observable, attribute_type):
        return misp_types_mapping[attribute_type](observable, attribute_type)

    @staticmethod
    def parse_pattern(pattern):
        if ' AND ' in pattern:
            pattern_parts = pattern.split(' AND ')
            if len(pattern_parts) == 3:
                _, value1 = pattern_parts[2].split(' = ')
                _, value2 = pattern_parts[0].split(' = ')
                return '{}|{}'.format(value1[1:-3], value2[1:-1])
            else:
                _, value1 = pattern_parts[0].split(' = ')
                _, value2 = pattern_parts[1].split(' = ')
                if value1 in ("'ipv4-addr'", "'ipv6-addr'"):
                    return value2[1:-3]
                return '{}|{}'.format(value1[1:-1], value2[1:-3])
        else:
            return pattern.split(' = ')[1][1:-3]

    @staticmethod
    def parse_observable_from_object(observable, object_type):
        return objects_mapping[object_type]['observable'](observable)

    @staticmethod
    def parse_pattern_from_object(pattern, object_type):
        return objects_mapping[object_type]['pattern'](pattern)

def main(args):
    pathname = os.path.dirname(args[0])
    stix_parser = StixParser()
    stix_parser.loadEvent(args, pathname)
    stix_parser.handler()
    stix_parser.saveFile()
    print(1)

if __name__ == "__main__":
    main(sys.argv)
<commit_msg>Importing vulnerabilities<commit_after># -*- coding: utf-8 -*-
#    Copyright (C) 2017-2018 CIRCL Computer Incident Response Center Luxembourg (smile gie)
#    Copyright (C) 2017-2018 Christian Studer
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys, json, os, time
import stix2
import pymisp
from stix2misp_mapping import *

class StixParser():
    def __init__(self):
        self.misp_event = pymisp.MISPEvent()
        self.event = []
        self.misp_event['Galaxy'] = []

    def loadEvent(self, args, pathname):
        try:
            filename = os.path.join(pathname, args[1])
            tempFile = open(filename, 'r')
            self.filename = filename
            event = stix2.get_dict(tempFile)
            for o in event.get('objects'):
                try:
                    self.event.append(stix2.parse(o))
                except:
                    self.parse_custom(o)
        except:
            print(json.dumps({'success': 0, 'message': 'The temporary STIX export file could not be read'}))
            sys.exit(1)

    def parse_custom(self, obj):
        custom_object_type = obj.pop('type')
        labels = obj['labels']
        try:
            @stix2.CustomObject(custom_object_type,[('id', stix2.properties.StringProperty(required=True)),
            ('x_misp_timestamp', stix2.properties.StringProperty(required=True)),
            ('labels', stix2.properties.ListProperty(labels, required=True)),
            ('x_misp_value', stix2.properties.StringProperty(required=True)),
            ('created_by_ref', stix2.properties.StringProperty(required=True)),
            ('x_misp_comment', stix2.properties.StringProperty()),
            ('x_misp_category', stix2.properties.StringProperty())
            ])
            class Custom(object):
                def __init__(self, **kwargs):
                    return
            custom = Custom(**obj)
        except:
            @stix2.CustomObject(custom_object_type,[('id', stix2.properties.StringProperty(required=True)),
            ('x_misp_timestamp', stix2.properties.StringProperty(required=True)),
            ('labels', stix2.properties.ListProperty(labels, required=True)),
            ('x_misp_values', stix2.properties.DictionaryProperty(required=True)),
            ('created_by_ref', stix2.properties.StringProperty(required=True)),
            ('x_misp_comment', stix2.properties.StringProperty()),
            ('x_misp_category', stix2.properties.StringProperty())
            ])
            class Custom(object):
                def __init__(self, **kwargs):
                    return
            custom = Custom(**obj)
        self.event.append(stix2.parse(custom))

    def handler(self):
        self.outputname = '{}.stix2'.format(self.filename)
        if self.from_misp():
            self.buildMispDict()
        else:
            self.buildExternalDict()

    def from_misp(self):
        for o in self.event:
            if o._type == 'report' and 'misp:tool="misp2stix2"' in o.get('labels'):
                index = self.event.index(o)
                self.report = self.event.pop(index)
                return True
        return False

    def buildMispDict(self):
        self.parse_identity()
        self.parse_report()
        for o in self.event:
            object_type = o._type
            labels = o.get('labels')
            if object_type in ('attack-pattern', 'course-of-action', 'intrusion-set', 'malware', 'threat-actor', 'tool'):
                self.parse_galaxy(o, labels)
            elif 'x-misp-object' in object_type:
                if 'from_object' in labels:
                    self.parse_custom_object(o)
                else:
                    self.parse_custom_attribute(o, labels)
            else:
                if 'from_object' in labels:
                    self.parse_object(o, labels)
                else:
                    self.parse_attribute(o, labels)

    def parse_identity(self):
        identity = self.event.pop(0)
        org = {'name': identity.get('name')}
        self.misp_event['Org'] = org

    def parse_report(self):
        report = self.report
        self.misp_event.info = report.get('name')
        if report.get('published'):
            self.misp_event.publish_timestamp = self.getTimestampfromDate(report.get('published'))
        if hasattr(report, 'labels'):
            labels = report['labels']
            for l in labels:
                self.misp_event.add_tag(l)
        if hasattr(report, 'external_references'):
            ext_refs = report['external_references']
            for e in ext_refs:
                link = {"type": "link"}
                comment = e.get('source_name').split('url - ')[1]
                if comment:
                    link['comment'] = comment
                link['value'] = e.get('url')
                self.misp_event.add_attribute(**link)

    def parse_galaxy(self, o, labels):
        galaxy_type = self.get_misp_type(labels)
        tag = labels[1]
        value = tag.split(':')[1].split('=')[1]
        galaxy_description, cluster_description = o.get('description').split('|')
        galaxy = {'type': galaxy_type, 'name': o.get('name'), 'description': galaxy_description,
                  'GalaxyCluster': [{'type': galaxy_type, 'value':value, 'tag_name': tag,
                                     'description': cluster_description}]}
        self.misp_event['Galaxy'].append(galaxy)

    def parse_custom_object(self, o):
        name = o.get('type').split('x-misp-object-')[1]
        timestamp = self.getTimestampfromDate(o.get('x_misp_timestamp'))
        category = o.get('category')
        attributes = []
        values = o.get('x_misp_values')
        for v in values:
            attribute_type, object_relation = v.split('_')
            attribute = {'type': attribute_type, 'value': values.get(v),
                         'object_relation': object_relation}
            attributes.append(attribute)
        misp_object = {'name': name, 'timestamp': timestamp, 'meta-category': category,
                       'Attribute': attributes}
        self.misp_event.add_object(**misp_object)

    def parse_custom_attribute(self, o, labels):
        attribute_type = o.get('type').split('x-misp-object-')[1]
        timestamp = self.getTimestampfromDate(o.get('x_misp_timestamp'))
        to_ids = bool(labels[1].split('=')[1])
        value = o.get('x_misp_value')
        category = self.get_misp_category(labels)
        attribute = {'type': attribute_type, 'timestamp': timestamp, 'to_ids': to_ids,
                     'value': value, 'category': category}
        self.misp_event.add_attribute(**attribute)

    def parse_object(self, o, labels):
        object_type = self.get_misp_type(labels)
        object_category = self.get_misp_category(labels)
        stix_type = o._type
        misp_object = pymisp.MISPObject(object_type)
        misp_object['meta-category'] = object_category
        if stix_type == 'indicator':
            pattern = o.get('pattern').replace('\\\\', '\\').split(' AND ')
            pattern[0] = pattern[0][2:]
            pattern[-1] = pattern[-1][:-2]
            attributes = self.parse_pattern_from_object(pattern, object_type)
        if stix_type == 'observed-data':
            observable = o.get('objects')
            attributes = self.parse_observable_from_object(observable, object_type)
        for attribute in attributes:
            misp_object.add_attribute(**attribute)
        misp_object.to_ids = bool(labels[1].split('=')[1])
        self.misp_event.add_object(**misp_object)

    def parse_attribute(self, o, labels):
        attribute_type = self.get_misp_type(labels)
        attribute_category = self.get_misp_category(labels)
        attribute = {'type': attribute_type, 'category': attribute_category}
        stix_type = o._type
        if stix_type == 'vulnerability':
            value = o.get('name')
        else:
            if stix_type == 'indicator':
                o_date = o.get('valid_from')
                pattern = o.get('pattern').replace('\\\\', '\\')
                value = self.parse_pattern(pattern)
                attribute['to_ids'] = True
            else:
                o_date = o.get('first_observed')
                observable = o.get('objects')
                try:
                    value = self.parse_observable(observable, attribute_type)
                except:
                    print('{}: {}'.format(attribute_type, observable))
                attribute['to_ids'] = False
            attribute['timestamp'] = self.getTimestampfromDate(o_date)
        if 'description' in o:
            attribute['comment'] = o.get('description')
        try:
            attribute['value'] = value
            self.misp_event.add_attribute(**attribute)
        except:
            pass

    def buildExternalDict(self):
        self.fetch_report()
        for o in self.event:
            object_type = o._type
            if object_type in ('relationship', 'report'):
                continue
            if object_type == 'vulnerability':
                attribute = {'type': 'vulnerability', 'value': o.get('name')}
                if 'description' in o:
                    attribute['comment'] = o.get('description')
                self.misp_event.add_attribute(**attribute)
            elif object_type == 'indicator':
                attribute = {'type': 'stix2-pattern', 'object_relation': 'stix2-pattern', 'value': o.get('pattern')}
                misp_object = {'name': 'stix2-pattern', 'meta-category': 'stix2-pattern', 'Attribute': [attribute]}
                self.misp_event.add_object(**misp_object)

    def fetch_report(self):
        reports = []
        for o in self.event:
            if o._type == 'report':
                reports.append(o)
        if len(reports) == 1:
            self.report = reports[0]
            self.parse_report()

    def saveFile(self):
        eventDict = self.misp_event.to_json()
        outputfile = '{}.stix2'.format(self.filename)
        with open(outputfile, 'w') as f:
            f.write(eventDict)

    @staticmethod
    def getTimestampfromDate(stix_date):
        try:
            return int(stix_date.timestamp())
        except:
            return int(time.mktime(time.strptime(stix_date.split('+')[0], "%Y-%m-%d %H:%M:%S")))

    @staticmethod
    def get_misp_type(labels):
        return labels[0].split('=')[1][1:-1]

    @staticmethod
    def get_misp_category(labels):
        return labels[1].split('=')[1][1:-1]

    @staticmethod
    def parse_observable(observable, attribute_type):
        return misp_types_mapping[attribute_type](observable, attribute_type)

    @staticmethod
    def parse_pattern(pattern):
        if ' AND ' in pattern:
            pattern_parts = pattern.split(' AND ')
            if len(pattern_parts) == 3:
                _, value1 = pattern_parts[2].split(' = ')
                _, value2 = pattern_parts[0].split(' = ')
                return '{}|{}'.format(value1[1:-3], value2[1:-1])
            else:
                _, value1 = pattern_parts[0].split(' = ')
                _, value2 = pattern_parts[1].split(' = ')
                if value1 in ("'ipv4-addr'", "'ipv6-addr'"):
                    return value2[1:-3]
                return '{}|{}'.format(value1[1:-1], value2[1:-3])
        else:
            return pattern.split(' = ')[1][1:-3]

    @staticmethod
    def parse_observable_from_object(observable, object_type):
        return objects_mapping[object_type]['observable'](observable)

    @staticmethod
    def parse_pattern_from_object(pattern, object_type):
        return objects_mapping[object_type]['pattern'](pattern)

def main(args):
    pathname = os.path.dirname(args[0])
    stix_parser = StixParser()
    stix_parser.loadEvent(args, pathname)
    stix_parser.handler()
    stix_parser.saveFile()
    print(1)

if __name__ == "__main__":
    main(sys.argv)
<|endoftext|>


<commit_before>import os
import tarfile
import zipfile
import redis
from flask import Blueprint, request, jsonify, current_app
from flask_recaptcha import ReCaptcha
from datetime import datetime
from werkzeug.utils import secure_filename
from ast import literal_eval
from modules.groupComparisons.frontend_queries import get_all_attribute_types, get_attribute_values, get_types
from routes.utility_functions import fetch_job, fix_uri, handle_tar, handle_zip
from modules.gc import blob_gc_enqueue
from modules.spfy import spfy

bp_ra = Blueprint('reactapp', __name__)

def handle_groupresults(jobs_dict):
    '''
    if we're grouping results for the Front-End
    take all jobs in jobs_dict and store it in Redis (not RQ)
    respond with a key to fetch this jobs_dict from Redis
    relies on a new check in /results/<job_id> which is able to poll RQ
    tasks on behalf of the front-end
        Args:
            jobs_dict (dict): a dictionary where job ids are the keys and
                the values are also dictionaries with keys 'analysis' and 'file'
        Return:
            (dict): a dictionary with key as novel jobid of form 'blob' + hash
                meant to be parsable by same code as for old, non-grouped ver
    '''
    # generate a novel job id
    # we prepend 'blob' so the /results path can tell its our custom object
    # and shouldnt be retrieved from RQ
    job_id = 'blob' + str(hash(str(jobs_dict)))
    # start a redis connection
    redis_url = current_app.config['REDIS_URL']
    redis_connection = redis.from_url(redis_url)
    #with redis.from_url(redis_url) as redis_connection:
    # set the job_id: jobs_dict pair in Redis
    redis_connection.set(job_id, jobs_dict)
    # create a similar structure to the old return
    d = {}
    d[job_id] = {}
    d[job_id]['analysis'] = "Subtyping"
    st = set()
    for key in jobs_dict:
        st.add(jobs_dict[key]['file'])
    s = ''
    for f in st:
        s += f + ' '
    d[job_id]['file'] = s
    return d

# the /api/v0 prefix is set to allow CORS for any postfix
# this is a modification of the old upload() methods in views.py
@bp_ra.route('/api/v0/upload', methods=['POST'])
def upload():
    recaptcha = ReCaptcha(app=current_app)
    if recaptcha.verify():
        form = request.form
        options = {}
        # defaults
        options['amr']=True
        options['vf']=True
        options['serotype']=True
        options['pi']=90
        # new to 4.2.0
        # we consider False as default as the front-end should override this
        # to use the new feature
        groupresults = False

        # processing form data
        for key, value in form.items():
            #we need to convert lower-case true/false in js to upper case in python
                #remember, we also have numbers
            if not value.isdigit():
                if value.lower() == 'false':
                    value = False
                else:
                    value = True
                if key == 'options.amr':
                    options['amr']=value
                if key == 'options.vf':
                    options['vf']=value
                if key == 'options.serotype':
                    options['serotype']=value
                if key == 'options.groupresults':
                    groupresults = value
            else:
                if key =='options.pi':
                    options['pi']=int(value)

        # get a list of files submitted
        uploaded_files = request.files.getlist("file")
        print uploaded_files

        #set up constants for identifying this sessions
        now = datetime.now()
        now = now.strftime("%Y-%m-%d-%H-%M-%S-%f")
        jobs_dict = {}

        for file in uploaded_files:
            if file:
                # for saving file
                filename = os.path.join(current_app.config[
                                        'DATASTORE'], now + '-' + secure_filename(file.filename))
                file.save(filename)
                print 'IVE SAVED YO FILE AT', str(filename)

                if tarfile.is_tarfile(filename):
                    # set filename to dir for spfy call
                    filename = handle_tar(filename, now)
                elif zipfile.is_zipfile(filename):
                    filename = handle_zip(filename, now)

                # for enqueing task
                jobs_enqueued = spfy(
                    {'i': filename, 'disable_serotype': False, 'disable_amr': False, 'disable_vf': False, 'pi':options['pi'], 'options':options})
                jobs_dict.update(jobs_enqueued)
        # new in 4.2.0
        if groupresults:
            return jsonify(handle_groupresults(jobs_dict))
        else:
            return jsonify(jobs_dict)
    else:
        return "Captcha Failed Verification", 500

# new to 4.2.0
def merge_job_results(jobs_dict):
    '''
    Appends all results together and returns it.
    We don't do this while retriving job statuses as in most checks, the jobs
    wont all be finished
    Note: written for lists atm. (ie. only for Subtyping)
    '''
    r = []
    for key in jobs_dict:
        job = fetch_job(key)
        if job.is_finished:
            res = job.result
            # we check for type of result as we're not returning
            # Quality Control or ID Reservation results
            # print type(res)
            if type(res) is list:
                r += job.result
        else:
            return 'ERROR: merge_job_results() was called when all jobs werent complete', 415
    return r

# new to 4.2.0
def job_status_reactapp_grouped(job_id):
    '''
    Retrieves a dictionary of job_id from Redis (not RQ) and checks
    status of all jobs
    Returns the complete result only if all jobs are finished
    '''
    # start a redis connection
    redis_url = current_app.config['REDIS_URL']
    redis_connection = redis.from_url(redis_url)
    # Retrieves jobs_dict
    jobs_dict = redis_connection.get(job_id)
    # redis-py returns a string by default
    # we cast this using ast.literal_eval()
    # the alt. is to set a response callback via redis_connection.set_response_callback()
    jobs_dict = literal_eval(jobs_dict)
    print jobs_dict
    # print type(jobs_dict)
    for key in jobs_dict:
        key = str(key)
        # print key
        job = fetch_job(key)
        # print job
        if job.is_failed:
            return jsonify(job.exc_info)
        elif not job.is_finished:
            return jsonify("pending")
    # if you've gotten to this point, then all jobs are finished
    return jsonify(merge_job_results(jobs_dict))

@bp_ra.route('/api/v0/results/<job_id>')
def job_status_reactapp(job_id):
    '''
    This provides an endpoint for the reactapp to poll results. We leave job_status() intact to maintain backwards compatibility with the AngularJS app.
    '''
    # new to 4.2.0
    # check if the job_id is of the new format and should be handled diff
    if job_id.startswith('blob'):
        return job_status_reactapp_grouped(job_id)
    else:
        # old code
        job = fetch_job(job_id)
        if job.is_finished:
            r = job.result
            # subtyping results come in the form of a list and must
            # be conv to json otherwise, you get a 500 error (isa)
            if type(r) is list:
                return jsonify(r)
            # fishers results come in the form of a df.to_json object
            # and should be returned directly
            else:
                return job.result
        elif job.is_failed:
            return jsonify(job.exc_info)
        else:
            return jsonify("pending")

@bp_ra.route('/api/v0/newgroupcomparison', methods=['POST'])
def handle_group_comparison_submission():
    query = request.json['groups']
    target = request.json['target']
    jobid = blob_gc_enqueue(query, target)
    return jobid

@bp_ra.route('/api/v0/get_attribute_values/type/<path:attributetype>')
def call_get_attribute_values(attributetype):
    '''
    Front-End API:
    Get all attribute values for a given attribute type.
    '''
    # workaround: Flask's path converter allows slashes, but only a SINGLE slash
    # this adds the second slash
    # also convert to a rdflib.URIRef object
    uri = fix_uri(attributetype)
    return jsonify(get_attribute_values(attributeTypeUri=uri))
    # set_attribute_types = set(get_all_attribute_types())
    # set_object_types = get_types() # get types returns a set by default
    # return jsonify(list(set_attribute_types.union(set_object_types)))

@bp_ra.route('/api/v0/get_all_types')
def combine_types():
    '''
    Returns all URIs that is either a attribute type or and object type.
    '''
    set_attribute_types = set(get_all_attribute_types())
    set_object_types = get_types() # get types returns a set by default
    return jsonify(list(set_attribute_types.union(set_object_types)))

@bp_ra.route('/api/v0/get_all_attribute_types')
def call_get_all_atribute_types():
    '''
    Front-End API:
    Get all possible attribute types.
    '''
    return jsonify(get_all_attribute_types())
<commit_msg>FIX: return string values for RQ statuses<commit_after>import os
import tarfile
import zipfile
import redis
from flask import Blueprint, request, jsonify, current_app
from flask_recaptcha import ReCaptcha
from datetime import datetime
from werkzeug.utils import secure_filename
from ast import literal_eval
from modules.groupComparisons.frontend_queries import get_all_attribute_types, get_attribute_values, get_types
from routes.utility_functions import fetch_job, fix_uri, handle_tar, handle_zip
from modules.gc import blob_gc_enqueue
from modules.spfy import spfy

bp_ra = Blueprint('reactapp', __name__)

def handle_groupresults(jobs_dict):
    '''
    if we're grouping results for the Front-End
    take all jobs in jobs_dict and store it in Redis (not RQ)
    respond with a key to fetch this jobs_dict from Redis
    relies on a new check in /results/<job_id> which is able to poll RQ
    tasks on behalf of the front-end
        Args:
            jobs_dict (dict): a dictionary where job ids are the keys and
                the values are also dictionaries with keys 'analysis' and 'file'
        Return:
            (dict): a dictionary with key as novel jobid of form 'blob' + hash
                meant to be parsable by same code as for old, non-grouped ver
    '''
    # generate a novel job id
    # we prepend 'blob' so the /results path can tell its our custom object
    # and shouldnt be retrieved from RQ
    job_id = 'blob' + str(hash(str(jobs_dict)))
    # start a redis connection
    redis_url = current_app.config['REDIS_URL']
    redis_connection = redis.from_url(redis_url)
    #with redis.from_url(redis_url) as redis_connection:
    # set the job_id: jobs_dict pair in Redis
    redis_connection.set(job_id, jobs_dict)
    # create a similar structure to the old return
    d = {}
    d[job_id] = {}
    d[job_id]['analysis'] = "Subtyping"
    st = set()
    for key in jobs_dict:
        st.add(jobs_dict[key]['file'])
    s = ''
    for f in st:
        s += f + ' '
    d[job_id]['file'] = s
    return d

# the /api/v0 prefix is set to allow CORS for any postfix
# this is a modification of the old upload() methods in views.py
@bp_ra.route('/api/v0/upload', methods=['POST'])
def upload():
    recaptcha = ReCaptcha(app=current_app)
    if recaptcha.verify():
        form = request.form
        options = {}
        # defaults
        options['amr']=True
        options['vf']=True
        options['serotype']=True
        options['pi']=90
        # new to 4.2.0
        # we consider False as default as the front-end should override this
        # to use the new feature
        groupresults = False

        # processing form data
        for key, value in form.items():
            #we need to convert lower-case true/false in js to upper case in python
                #remember, we also have numbers
            if not value.isdigit():
                if value.lower() == 'false':
                    value = False
                else:
                    value = True
                if key == 'options.amr':
                    options['amr']=value
                if key == 'options.vf':
                    options['vf']=value
                if key == 'options.serotype':
                    options['serotype']=value
                if key == 'options.groupresults':
                    groupresults = value
            else:
                if key =='options.pi':
                    options['pi']=int(value)

        # get a list of files submitted
        uploaded_files = request.files.getlist("file")
        print uploaded_files

        #set up constants for identifying this sessions
        now = datetime.now()
        now = now.strftime("%Y-%m-%d-%H-%M-%S-%f")
        jobs_dict = {}

        for file in uploaded_files:
            if file:
                # for saving file
                filename = os.path.join(current_app.config[
                                        'DATASTORE'], now + '-' + secure_filename(file.filename))
                file.save(filename)
                print 'IVE SAVED YO FILE AT', str(filename)

                if tarfile.is_tarfile(filename):
                    # set filename to dir for spfy call
                    filename = handle_tar(filename, now)
                elif zipfile.is_zipfile(filename):
                    filename = handle_zip(filename, now)

                # for enqueing task
                jobs_enqueued = spfy(
                    {'i': filename, 'disable_serotype': False, 'disable_amr': False, 'disable_vf': False, 'pi':options['pi'], 'options':options})
                jobs_dict.update(jobs_enqueued)
        # new in 4.2.0
        if groupresults:
            return jsonify(handle_groupresults(jobs_dict))
        else:
            return jsonify(jobs_dict)
    else:
        return "Captcha Failed Verification", 500

# new to 4.2.0
def merge_job_results(jobs_dict):
    '''
    Appends all results together and returns it.
    We don't do this while retriving job statuses as in most checks, the jobs
    wont all be finished
    Note: written for lists atm. (ie. only for Subtyping)
    '''
    r = []
    for key in jobs_dict:
        job = fetch_job(key)
        if job.is_finished:
            res = job.result
            # we check for type of result as we're not returning
            # Quality Control or ID Reservation results
            # print type(res)
            if type(res) is list:
                r += job.result
        else:
            return 'ERROR: merge_job_results() was called when all jobs werent complete', 415
    return r

# new to 4.2.0
def job_status_reactapp_grouped(job_id):
    '''
    Retrieves a dictionary of job_id from Redis (not RQ) and checks
    status of all jobs
    Returns the complete result only if all jobs are finished
    '''
    # start a redis connection
    redis_url = current_app.config['REDIS_URL']
    redis_connection = redis.from_url(redis_url)
    # Retrieves jobs_dict
    jobs_dict = redis_connection.get(job_id)
    # redis-py returns a string by default
    # we cast this using ast.literal_eval()
    # the alt. is to set a response callback via redis_connection.set_response_callback()
    jobs_dict = literal_eval(jobs_dict)
    print jobs_dict
    # print type(jobs_dict)
    for key in jobs_dict:
        key = str(key)
        # print key
        job = fetch_job(key)
        # print job
        if job.is_failed:
            return jsonify("failed")
        elif not job.is_finished:
            return jsonify("pending")
    # if you've gotten to this point, then all jobs are finished
    return jsonify(merge_job_results(jobs_dict))

@bp_ra.route('/api/v0/results/<job_id>')
def job_status_reactapp(job_id):
    '''
    This provides an endpoint for the reactapp to poll results. We leave job_status() intact to maintain backwards compatibility with the AngularJS app.
    '''
    # new to 4.2.0
    # check if the job_id is of the new format and should be handled diff
    if job_id.startswith('blob'):
        return job_status_reactapp_grouped(job_id)
    else:
        # old code
        job = fetch_job(job_id)
        if job.is_finished:
            r = job.result
            # subtyping results come in the form of a list and must
            # be conv to json otherwise, you get a 500 error (isa)
            if type(r) is list:
                return jsonify(r)
            # fishers results come in the form of a df.to_json object
            # and should be returned directly
            else:
                return job.result
        elif job.is_failed:
            return jsonify("failed")
        else:
            return jsonify("pending")

@bp_ra.route('/api/v0/newgroupcomparison', methods=['POST'])
def handle_group_comparison_submission():
    query = request.json['groups']
    target = request.json['target']
    jobid = blob_gc_enqueue(query, target)
    return jobid

@bp_ra.route('/api/v0/get_attribute_values/type/<path:attributetype>')
def call_get_attribute_values(attributetype):
    '''
    Front-End API:
    Get all attribute values for a given attribute type.
    '''
    # workaround: Flask's path converter allows slashes, but only a SINGLE slash
    # this adds the second slash
    # also convert to a rdflib.URIRef object
    uri = fix_uri(attributetype)
    return jsonify(get_attribute_values(attributeTypeUri=uri))
    # set_attribute_types = set(get_all_attribute_types())
    # set_object_types = get_types() # get types returns a set by default
    # return jsonify(list(set_attribute_types.union(set_object_types)))

@bp_ra.route('/api/v0/get_all_types')
def combine_types():
    '''
    Returns all URIs that is either a attribute type or and object type.
    '''
    set_attribute_types = set(get_all_attribute_types())
    set_object_types = get_types() # get types returns a set by default
    return jsonify(list(set_attribute_types.union(set_object_types)))

@bp_ra.route('/api/v0/get_all_attribute_types')
def call_get_all_atribute_types():
    '''
    Front-End API:
    Get all possible attribute types.
    '''
    return jsonify(get_all_attribute_types())
<|endoftext|>


<commit_before>import os
import tarfile
import zipfile
import redis
from flask import Blueprint, request, jsonify, current_app
from flask_recaptcha import ReCaptcha
from datetime import datetime
from werkzeug.utils import secure_filename
from modules.groupComparisons.frontend_queries import get_all_attribute_types, get_attribute_values, get_types
from routes.utility_functions import fetch_job, fix_uri, handle_tar, handle_zip
from modules.gc import blob_gc_enqueue
from modules.spfy import spfy

bp_ra = Blueprint('reactapp', __name__)

def handle_groupresults(jobs_dict):
    '''
    if we're grouping results for the Front-End
    take all jobs in jobs_dict and store it in Redis (not RQ)
    respond with a key to fetch this jobs_dict from Redis
    relies on a new check in /results/<job_id> which is able to poll RQ
    tasks on behalf of the front-end
        Args:
            jobs_dict (dict): a dictionary where job ids are the keys and
                the values are also dictionaries with keys 'analysis' and 'file'
        Return:
            (dict): a dictionary with key as novel jobid of form 'blob' + hash
                meant to be parsable by same code as for old, non-grouped ver
    '''
    # generate a novel job id
    # we prepend 'blob' so the /results path can tell its our custom object
    # and shouldnt be retrieved from RQ
    job_id = 'blob' + str(hash(str(jobs_dict)))
    # start a redis connection
    redis_url = current_app.config['REDIS_URL']
    redis_connection = redis.from_url(redis_url)
    #with redis.from_url(redis_url) as redis_connection:
    # set the job_id: jobs_dict pair in Redis
    redis_connection.set(job_id, jobs_dict)
    # create a similar structure to the old return
    d = {}
    d[job_id] = {}
    d[job_id]['analysis'] = 'Grouped Subtyping'
    s = ''
    for key in jobs_dict:
        s += jobs_dict[key]['file'] + ' '
    d[job_id]['file'] = s
    return d

# the /api/v0 prefix is set to allow CORS for any postfix
# this is a modification of the old upload() methods in views.py
@bp_ra.route('/api/v0/upload', methods=['POST'])
def upload():
    recaptcha = ReCaptcha(app=current_app)
    if recaptcha.verify():
        form = request.form
        options = {}
        # defaults
        options['amr']=True
        options['vf']=True
        options['serotype']=True
        options['pi']=90
        # new to 4.2.0
        # we consider False as default as the front-end should override this
        # to use the new feature
        groupresults = False

        # processing form data
        for key, value in form.items():
            #we need to convert lower-case true/false in js to upper case in python
                #remember, we also have numbers
            if not value.isdigit():
                if value.lower() == 'false':
                    value = False
                else:
                    value = True
                if key == 'options.amr':
                    options['amr']=value
                if key == 'options.vf':
                    options['vf']=value
                if key == 'options.serotype':
                    options['serotype']=value
                if key == 'options.groupresults':
                    groupresults = value
            else:
                if key =='options.pi':
                    options['pi']=int(value)

        # get a list of files submitted
        uploaded_files = request.files.getlist("file")
        print uploaded_files

        #set up constants for identifying this sessions
        now = datetime.now()
        now = now.strftime("%Y-%m-%d-%H-%M-%S-%f")
        jobs_dict = {}

        for file in uploaded_files:
            if file:
                # for saving file
                filename = os.path.join(current_app.config[
                                        'DATASTORE'], now + '-' + secure_filename(file.filename))
                file.save(filename)
                print 'IVE SAVED YO FILE AT', str(filename)

                if tarfile.is_tarfile(filename):
                    # set filename to dir for spfy call
                    filename = handle_tar(filename, now)
                elif zipfile.is_zipfile(filename):
                    filename = handle_zip(filename, now)

                # for enqueing task
                jobs_enqueued = spfy(
                    {'i': filename, 'disable_serotype': False, 'disable_amr': False, 'disable_vf': False, 'pi':options['pi'], 'options':options})
                jobs_dict.update(jobs_enqueued)
        # new in 4.2.0
        if groupresults:
            return jsonify(handle_groupresults(jobs_dict))
        else:
            return jsonify(jobs_dict)
    else:
        return "Captcha Failed Verification", 500

# new to 4.2.0
def merge_job_results(jobs_dict):
    '''
    Appends all results together and returns it.
    We don't do this while retriving job statuses as in most checks, the jobs
    wont all be finished
    Note: written for lists atm. (ie. only for Subtyping)
    '''
    r = []
    for key in jobs_dict:
        job = fetch_job(key)
        if job.is_finished:
            res = job.result
            # we check for type of result as we're not returning
            # Quality Control or ID Reservation results
            if type(res) is list:
                r += job.result
        else:
            return 'ERROR: merge_job_results() was called when all jobs werent complete', 415
    return r

# new to 4.2.0
def job_status_reactapp_grouped(job_id):
    '''
    Retrieves a dictionary of job_id from Redis (not RQ) and checks
    status of all jobs
    Returns the complete result only if all jobs are finished
    '''
    # start a redis connection
    redis_url = current_app.config['REDIS_URL']
    redis_connection = redis.from_url(redis_url)
    #with redis.from_url(redis_url) as redis_connection:
    # Retrieves jobs_dict
    jobs_dict = redis_connection.get(job_id)
    print jobs_dict
    for key in jobs_dict:
        key = str(key)
        print key
        job = fetch_job(key)
        print job
        if job.is_failed:
            return job.exc_info, 415
        elif not job.is_finished:
            return "pending", 204
    # if you've gotten to this point, then all jobs are finished
    return jsonify(merge_job_results(jobs_dict))

@bp_ra.route('/api/v0/results/<job_id>')
def job_status_reactapp(job_id):
    '''
    This provides an endpoint for the reactapp to poll results. We leave job_status() intact to maintain backwards compatibility with the AngularJS app.
    '''
    # new to 4.2.0
    # check if the job_id is of the new format and should be handled diff
    if job_id.startswith('blob'):
        return job_status_reactapp_grouped(job_id)
    else:
        # old code
        job = fetch_job(job_id)
        if job.is_finished:
            r = job.result
            # subtyping results come in the form of a list and must
            # be conv to json otherwise, you get a 500 error (isa)
            if type(r) is list:
                return jsonify(r)
            # fishers results come in the form of a df.to_json object
            # and should be returned directly
            else:
                return job.result
        elif job.is_failed:
            return job.exc_info, 415
        else:
            return "pending", 204

@bp_ra.route('/api/v0/newgroupcomparison', methods=['POST'])
def handle_group_comparison_submission():
    query = request.json['groups']
    target = request.json['target']
    jobid = blob_gc_enqueue(query, target)
    return jobid

@bp_ra.route('/api/v0/get_attribute_values/type/<path:attributetype>')
def call_get_attribute_values(attributetype):
    '''
    Front-End API:
    Get all attribute values for a given attribute type.
    '''
    # workaround: Flask's path converter allows slashes, but only a SINGLE slash
    # this adds the second slash
    # also convert to a rdflib.URIRef object
    uri = fix_uri(attributetype)
    return jsonify(get_attribute_values(attributeTypeUri=uri))
    # set_attribute_types = set(get_all_attribute_types())
    # set_object_types = get_types() # get types returns a set by default
    # return jsonify(list(set_attribute_types.union(set_object_types)))

@bp_ra.route('/api/v0/get_all_types')
def combine_types():
    '''
    Returns all URIs that is either a attribute type or and object type.
    '''
    set_attribute_types = set(get_all_attribute_types())
    set_object_types = get_types() # get types returns a set by default
    return jsonify(list(set_attribute_types.union(set_object_types)))

@bp_ra.route('/api/v0/get_all_attribute_types')
def call_get_all_atribute_types():
    '''
    Front-End API:
    Get all possible attribute types.
    '''
    return jsonify(get_all_attribute_types())
<commit_msg>FIX: use sets to avoid dupl file names in job metadata<commit_after>import os
import tarfile
import zipfile
import redis
from flask import Blueprint, request, jsonify, current_app
from flask_recaptcha import ReCaptcha
from datetime import datetime
from werkzeug.utils import secure_filename
from modules.groupComparisons.frontend_queries import get_all_attribute_types, get_attribute_values, get_types
from routes.utility_functions import fetch_job, fix_uri, handle_tar, handle_zip
from modules.gc import blob_gc_enqueue
from modules.spfy import spfy

bp_ra = Blueprint('reactapp', __name__)

def handle_groupresults(jobs_dict):
    '''
    if we're grouping results for the Front-End
    take all jobs in jobs_dict and store it in Redis (not RQ)
    respond with a key to fetch this jobs_dict from Redis
    relies on a new check in /results/<job_id> which is able to poll RQ
    tasks on behalf of the front-end
        Args:
            jobs_dict (dict): a dictionary where job ids are the keys and
                the values are also dictionaries with keys 'analysis' and 'file'
        Return:
            (dict): a dictionary with key as novel jobid of form 'blob' + hash
                meant to be parsable by same code as for old, non-grouped ver
    '''
    # generate a novel job id
    # we prepend 'blob' so the /results path can tell its our custom object
    # and shouldnt be retrieved from RQ
    job_id = 'blob' + str(hash(str(jobs_dict)))
    # start a redis connection
    redis_url = current_app.config['REDIS_URL']
    redis_connection = redis.from_url(redis_url)
    #with redis.from_url(redis_url) as redis_connection:
    # set the job_id: jobs_dict pair in Redis
    redis_connection.set(job_id, jobs_dict)
    # create a similar structure to the old return
    d = {}
    d[job_id] = {}
    d[job_id]['analysis'] = 'Grouped Subtyping'
    st = set()
    for key in jobs_dict:
        st.add(jobs_dict[key]['file'])
    s = ''
    for f in st:
        s += f + ' '
    d[job_id]['file'] = s
    return d

# the /api/v0 prefix is set to allow CORS for any postfix
# this is a modification of the old upload() methods in views.py
@bp_ra.route('/api/v0/upload', methods=['POST'])
def upload():
    recaptcha = ReCaptcha(app=current_app)
    if recaptcha.verify():
        form = request.form
        options = {}
        # defaults
        options['amr']=True
        options['vf']=True
        options['serotype']=True
        options['pi']=90
        # new to 4.2.0
        # we consider False as default as the front-end should override this
        # to use the new feature
        groupresults = False

        # processing form data
        for key, value in form.items():
            #we need to convert lower-case true/false in js to upper case in python
                #remember, we also have numbers
            if not value.isdigit():
                if value.lower() == 'false':
                    value = False
                else:
                    value = True
                if key == 'options.amr':
                    options['amr']=value
                if key == 'options.vf':
                    options['vf']=value
                if key == 'options.serotype':
                    options['serotype']=value
                if key == 'options.groupresults':
                    groupresults = value
            else:
                if key =='options.pi':
                    options['pi']=int(value)

        # get a list of files submitted
        uploaded_files = request.files.getlist("file")
        print uploaded_files

        #set up constants for identifying this sessions
        now = datetime.now()
        now = now.strftime("%Y-%m-%d-%H-%M-%S-%f")
        jobs_dict = {}

        for file in uploaded_files:
            if file:
                # for saving file
                filename = os.path.join(current_app.config[
                                        'DATASTORE'], now + '-' + secure_filename(file.filename))
                file.save(filename)
                print 'IVE SAVED YO FILE AT', str(filename)

                if tarfile.is_tarfile(filename):
                    # set filename to dir for spfy call
                    filename = handle_tar(filename, now)
                elif zipfile.is_zipfile(filename):
                    filename = handle_zip(filename, now)

                # for enqueing task
                jobs_enqueued = spfy(
                    {'i': filename, 'disable_serotype': False, 'disable_amr': False, 'disable_vf': False, 'pi':options['pi'], 'options':options})
                jobs_dict.update(jobs_enqueued)
        # new in 4.2.0
        if groupresults:
            return jsonify(handle_groupresults(jobs_dict))
        else:
            return jsonify(jobs_dict)
    else:
        return "Captcha Failed Verification", 500

# new to 4.2.0
def merge_job_results(jobs_dict):
    '''
    Appends all results together and returns it.
    We don't do this while retriving job statuses as in most checks, the jobs
    wont all be finished
    Note: written for lists atm. (ie. only for Subtyping)
    '''
    r = []
    for key in jobs_dict:
        job = fetch_job(key)
        if job.is_finished:
            res = job.result
            # we check for type of result as we're not returning
            # Quality Control or ID Reservation results
            if type(res) is list:
                r += job.result
        else:
            return 'ERROR: merge_job_results() was called when all jobs werent complete', 415
    return r

# new to 4.2.0
def job_status_reactapp_grouped(job_id):
    '''
    Retrieves a dictionary of job_id from Redis (not RQ) and checks
    status of all jobs
    Returns the complete result only if all jobs are finished
    '''
    # start a redis connection
    redis_url = current_app.config['REDIS_URL']
    redis_connection = redis.from_url(redis_url)
    #with redis.from_url(redis_url) as redis_connection:
    # Retrieves jobs_dict
    jobs_dict = redis_connection.get(job_id)
    print jobs_dict
    for key in jobs_dict:
        key = str(key)
        print key
        job = fetch_job(key)
        print job
        if job.is_failed:
            return job.exc_info, 415
        elif not job.is_finished:
            return "pending", 204
    # if you've gotten to this point, then all jobs are finished
    return jsonify(merge_job_results(jobs_dict))

@bp_ra.route('/api/v0/results/<job_id>')
def job_status_reactapp(job_id):
    '''
    This provides an endpoint for the reactapp to poll results. We leave job_status() intact to maintain backwards compatibility with the AngularJS app.
    '''
    # new to 4.2.0
    # check if the job_id is of the new format and should be handled diff
    if job_id.startswith('blob'):
        return job_status_reactapp_grouped(job_id)
    else:
        # old code
        job = fetch_job(job_id)
        if job.is_finished:
            r = job.result
            # subtyping results come in the form of a list and must
            # be conv to json otherwise, you get a 500 error (isa)
            if type(r) is list:
                return jsonify(r)
            # fishers results come in the form of a df.to_json object
            # and should be returned directly
            else:
                return job.result
        elif job.is_failed:
            return job.exc_info, 415
        else:
            return "pending", 204

@bp_ra.route('/api/v0/newgroupcomparison', methods=['POST'])
def handle_group_comparison_submission():
    query = request.json['groups']
    target = request.json['target']
    jobid = blob_gc_enqueue(query, target)
    return jobid

@bp_ra.route('/api/v0/get_attribute_values/type/<path:attributetype>')
def call_get_attribute_values(attributetype):
    '''
    Front-End API:
    Get all attribute values for a given attribute type.
    '''
    # workaround: Flask's path converter allows slashes, but only a SINGLE slash
    # this adds the second slash
    # also convert to a rdflib.URIRef object
    uri = fix_uri(attributetype)
    return jsonify(get_attribute_values(attributeTypeUri=uri))
    # set_attribute_types = set(get_all_attribute_types())
    # set_object_types = get_types() # get types returns a set by default
    # return jsonify(list(set_attribute_types.union(set_object_types)))

@bp_ra.route('/api/v0/get_all_types')
def combine_types():
    '''
    Returns all URIs that is either a attribute type or and object type.
    '''
    set_attribute_types = set(get_all_attribute_types())
    set_object_types = get_types() # get types returns a set by default
    return jsonify(list(set_attribute_types.union(set_object_types)))

@bp_ra.route('/api/v0/get_all_attribute_types')
def call_get_all_atribute_types():
    '''
    Front-End API:
    Get all possible attribute types.
    '''
    return jsonify(get_all_attribute_types())
<|endoftext|>


<commit_before>import pandas as pd
import pymongo
from bokeh.plotting import cursession, figure, show, output_server
import time

client = pymongo.MongoClient()
db = client['bitmicro']
predictions = db['btc_predictions']

cursor = predictions.find().limit(60*60).sort('_id', pymongo.DESCENDING)
data = pd.DataFrame(list(cursor))
data = data.set_index('_id')
data = data.sort_index(ascending=True)
x = pd.to_datetime(data.index, unit='s').to_series()
y = data.prediction*100*100

output_server('model_signal')
p = figure(title='30-second Prediction',
           x_axis_type='datetime',
           y_axis_label='Basis Points')
p.line(x, y, name='signal')
show(p)

renderer = p.select(dict(name='signal'))
ds = renderer[0].data_source

while True:
    cursor = predictions.find().limit(60*60).sort('_id', pymongo.DESCENDING)
    data = pd.DataFrame(list(cursor))
    data = data.set_index('_id')
    data = data.sort_index(ascending=True)
    ds.data['x'] = pd.to_datetime(data.index, unit='s').to_series()
    ds.data['y'] = data.prediction*100*100
    cursession().store_objects(ds)
    time.sleep(1)
<commit_msg>add x label<commit_after>import pandas as pd
import pymongo
from bokeh.plotting import cursession, figure, show, output_server
import time

client = pymongo.MongoClient()
db = client['bitmicro']
predictions = db['btc_predictions']

cursor = predictions.find().limit(60*60).sort('_id', pymongo.DESCENDING)
data = pd.DataFrame(list(cursor))
data = data.set_index('_id')
data = data.sort_index(ascending=True)
x = pd.to_datetime(data.index, unit='s').to_series()
y = data.prediction*100*100

output_server('model_signal')
p = figure(title='30-second Prediction',
           x_axis_type='datetime',
           x_axis_label='Greenwich Time',
           y_axis_label='Basis Points')
p.line(x, y, name='signal')
show(p)

renderer = p.select(dict(name='signal'))
ds = renderer[0].data_source

while True:
    cursor = predictions.find().limit(60*60).sort('_id', pymongo.DESCENDING)
    data = pd.DataFrame(list(cursor))
    data = data.set_index('_id')
    data = data.sort_index(ascending=True)
    ds.data['x'] = pd.to_datetime(data.index, unit='s').to_series()
    ds.data['y'] = data.prediction*100*100
    cursession().store_objects(ds)
    time.sleep(1)
<|endoftext|>


<commit_before>import logging
import numpy as np

from markers import markers_to_events

def sliding_window_indices(window_size, window_step, sig_len):
  '''Returns indices for a sliding window with shape [nwindows x window_size]'''
  nwindows = int(np.floor((sig_len - window_size + window_step) /
    float(window_step)))
  starts = np.arange(nwindows).reshape(nwindows, 1) * window_step
  return starts + np.arange(window_size)

def sliding_window(signal, window_size, window_step, win_func=None):
  '''Apply a sliding window to a 1D signal. Returns [#windows x window_size].'''
  if signal.ndim != 1:
    raise ValueError, 'Sliding window works on 1D arrays only!'
  if win_func != None:
    if win_func.size != window_size:
      raise ValueError, 'window_size (%d) does not match win_func.size (%d)' % (
        window_size, win_func.size)
  indices = sliding_window_indices(window_size, window_step, signal.shape[0])
  windows = signal.take(indices=indices)
  if win_func != None:
    windows = windows * win_func # broadcasting matches from last dim
  return windows

def stft(signal, nfft, stepsize):
  '''Calculate the short-time Fourier transform (STFT).
  Returns [windows x FFT coefficients]'''
  wins = sliding_window(signal, nfft, stepsize, win_func=np.hanning(nfft))
  return np.fft.rfft(wins, axis=1)

def spectrogram(signal, nfft, stepsize):
  '''
  Calculate a spectrogram using the STFT.
  Returns [windows x frequencies], in units related to power.
  Equivalent to power spectral density.
  '''
  spec = stft(signal, nfft, stepsize)

  # convert to power. The abs() is the magnitude of a complex number
  spec = np.abs(spec) ** 2 / nfft

  # compensate for missing negative frequencies
  spec[:, 1:-1] *= 2

  # correct for window
  spec /= np.mean(np.abs(np.hanning(nfft)) ** 2)

  # compensate for overlapping windows
  nwins = spec.shape[0]
  overlap = stepsize / float(nfft)
  spec *= (1 + (nwins - 1) * overlap) / nwins
  return spec

def get_samplerate(d):
  '''
  Derive the sample rate from the timestamps given in ``d.I``

  Parameters
  ----------

  d : :class:`golem.DataSet`
    The data to estimate the sample rate of. Must contain time stamps
    in ``d.I``

  Returns
  -------
  sample_rate : float
    The estimated samplerate.
  '''
  if d.ndX.ndim == 2:
    return np.round(1./np.median(np.diff(d.I[0])))
  elif d.ndX.ndim == 3:
    return np.round(1./np.median(np.diff([float(i) for i in d.feat_nd_lab[1]])))

def find_segments(events, event_indices, start_mark, end_mark):
  '''Helper to find matching start/end markers in an event array'''
  events, event_indices = np.asarray(events), np.asarray(event_indices)
  assert events.size == event_indices.size
  mask = (events==start_mark) | (events==end_mark)
  sevents, sevent_ids = events[mask], event_indices[mask]
  stack, result = [], []
  for si in range(sevent_ids.size):
    if sevents[si] == start_mark:
      stack.append(sevent_ids[si])
    else:
      assert stack != [], 'Missing start marker'
      result.append((stack.pop(), sevent_ids[si]))
  if not stack == []:
    logging.getLogger('psychic.utils.find_segments').warning(
      'Did not end start marker(s) at %s' % repr(stack))
  return result

def cut_segments(d, marker_tuples, offsets=[0, 0]):
  '''
  Cut a dataset into segments using (start_marker, end_marker) tuples.

  Parameters
  ----------
  d : :class:`golem.DataSet`
    Continuous data to cut into segments.
  marker_tuples : list of tuples
    A list of (start_marker, end_marker) marker codes delimiting each
    type of segment.

  Returns
  -------
  data : list of :class:`golem.DataSet`
    A list with datasets.
  '''
  start_off, end_off = offsets
  segments = []
  e, ei = markers_to_events(d.ys.flat)
  for (sm, em) in marker_tuples:
    segments.extend(find_segments(e, ei, sm, em))
  segments.sort()
  return [d[s + start_off:e + end_off] for (s, e) in segments]

def wolpaw_bitr(N, P):
  assert 0 <= P <= 1
  assert 2 <= N
  result = np.log2(N)
  if P > 0:
    result += P * np.log2(P)
  if P < 1:
    result += (1 - P) * np.log2((1 - P)/(N - 1.))
  return result
<commit_msg>added split_in_bins<commit_after>import logging
import numpy as np
from golem import DataSet

from markers import markers_to_events

def sliding_window_indices(window_size, window_step, sig_len):
  '''Returns indices for a sliding window with shape [nwindows x window_size]'''
  nwindows = int(np.floor((sig_len - window_size + window_step) /
    float(window_step)))
  starts = np.arange(nwindows).reshape(nwindows, 1) * window_step
  return starts + np.arange(window_size)

def sliding_window(signal, window_size, window_step, win_func=None):
  '''Apply a sliding window to a 1D signal. Returns [#windows x window_size].'''
  if signal.ndim != 1:
    raise ValueError, 'Sliding window works on 1D arrays only!'
  if win_func != None:
    if win_func.size != window_size:
      raise ValueError, 'window_size (%d) does not match win_func.size (%d)' % (
        window_size, win_func.size)
  indices = sliding_window_indices(window_size, window_step, signal.shape[0])
  windows = signal.take(indices=indices)
  if win_func != None:
    windows = windows * win_func # broadcasting matches from last dim
  return windows

def stft(signal, nfft, stepsize):
  '''Calculate the short-time Fourier transform (STFT).
  Returns [windows x FFT coefficients]'''
  wins = sliding_window(signal, nfft, stepsize, win_func=np.hanning(nfft))
  return np.fft.rfft(wins, axis=1)

def spectrogram(signal, nfft, stepsize):
  '''
  Calculate a spectrogram using the STFT.
  Returns [windows x frequencies], in units related to power.
  Equivalent to power spectral density.
  '''
  spec = stft(signal, nfft, stepsize)

  # convert to power. The abs() is the magnitude of a complex number
  spec = np.abs(spec) ** 2 / nfft

  # compensate for missing negative frequencies
  spec[:, 1:-1] *= 2

  # correct for window
  spec /= np.mean(np.abs(np.hanning(nfft)) ** 2)

  # compensate for overlapping windows
  nwins = spec.shape[0]
  overlap = stepsize / float(nfft)
  spec *= (1 + (nwins - 1) * overlap) / nwins
  return spec

def get_samplerate(d):
  '''
  Derive the sample rate from the timestamps given in ``d.I``

  Parameters
  ----------

  d : :class:`golem.DataSet`
    The data to estimate the sample rate of. Must contain time stamps
    in ``d.I``

  Returns
  -------
  sample_rate : float
    The estimated samplerate.
  '''
  if d.ndX.ndim == 2:
    return np.round(1./np.median(np.diff(d.I[0])))
  elif d.ndX.ndim == 3:
    return np.round(1./np.median(np.diff([float(i) for i in d.feat_nd_lab[1]])))

def find_segments(events, event_indices, start_mark, end_mark):
  '''Helper to find matching start/end markers in an event array'''
  events, event_indices = np.asarray(events), np.asarray(event_indices)
  assert events.size == event_indices.size
  mask = (events==start_mark) | (events==end_mark)
  sevents, sevent_ids = events[mask], event_indices[mask]
  stack, result = [], []
  for si in range(sevent_ids.size):
    if sevents[si] == start_mark:
      stack.append(sevent_ids[si])
    else:
      assert stack != [], 'Missing start marker'
      result.append((stack.pop(), sevent_ids[si]))
  if not stack == []:
    logging.getLogger('psychic.utils.find_segments').warning(
      'Did not end start marker(s) at %s' % repr(stack))
  return result

def cut_segments(d, marker_tuples, offsets=[0, 0]):
  '''
  Cut a dataset into segments using (start_marker, end_marker) tuples.

  Parameters
  ----------
  d : :class:`golem.DataSet`
    Continuous data to cut into segments.
  marker_tuples : list of tuples
    A list of (start_marker, end_marker) marker codes delimiting each
    type of segment.

  Returns
  -------
  data : list of :class:`golem.DataSet`
    A list with datasets.
  '''
  start_off, end_off = offsets
  segments = []
  e, ei = markers_to_events(d.ys.flat)
  for (sm, em) in marker_tuples:
    segments.extend(find_segments(e, ei, sm, em))
  segments.sort()
  return [d[s + start_off:e + end_off] for (s, e) in segments]

def wolpaw_bitr(N, P):
  assert 0 <= P <= 1
  assert 2 <= N
  result = np.log2(N)
  if P > 0:
    result += P * np.log2(P)
  if P < 1:
    result += (1 - P) * np.log2((1 - P)/(N - 1.))
  return result

def split_in_bins(data, order, n, legend=lambda i,b: 'slice %d' % i, ascending=True):
  idx = np.argsort(order)
  if not ascending:
    idx = idx[::-1]

  bin_size = int(len(order) / float(n))
  bins = [idx[i*bin_size:(i+1)*bin_size] for i in range(n)]

  Y = np.zeros((n, data.ninstances), dtype=np.bool)
  for i,b in enumerate(bins):
    Y[i, b] = True
    cl_lab = [legend(i, bins[i]) for i in range(n)]

  return (bins, DataSet(Y=Y, cl_lab=cl_lab, default=data))
<|endoftext|>


<commit_before>"""Construction of parameter spaces."""

import collections
import itertools

from six import string_types

from psyrun.utils.doc import inherit_docs

def dict_concat(args):
    """Concatenates elements with the same key in the passed dictionaries.

    Parameters
    ----------
    args : sequenece of dict
        Dictionaries with sequences to concatenate.

    Returns
    -------
    dict
        The dictionary with the union of all the keys of the dictionaries
        passed in and elements with the same key concatenated. Missing elements
        will be None.

    Examples
    --------
    >>> from pprint import pprint
    >>> pprint(dict_concat(({'a': 0, 'b': 0}, {'a': 1})))
    {'a': [0, 1], 'b': [0, None]}
    """
    keys = set()
    for a in args:
        keys = keys.union(a.keys())
    return {k: [a.get(k, None) for a in args] for k in keys}

class ParameterSpace(collections.Sized):
    """Abstract base class for objects representing a parameter space.

    Supports addition, subtraction, and multiplication operators to construct
    more complicated parameter spaces.

    Deriving classes are supposed to implement the `iterate` and ``__len__``
    methods.

    Parameters
    ----------
    keys : sequence of strings
        Parameter names.
    """

    def __init__(self, keys):
        self._keys = list(keys)

    def build(self):
        """Builds the parameter space into a dictionary of parameter lists.

        Returns
        -------
        dict
            A dictionary with the parameter names as keys and lists with the
            parameter values.

        Examples
        --------
        >>> from pprint import pprint
        >>> pprint(Param(a=[1, 2], b=[1, 2]).build())
        {'a': [1, 2], 'b': [1, 2]}
        """
        built = dict_concat(list(self.iterate()))
        for k in self.keys():
            built.setdefault(k, [])
        return built

    def iterate(self):
        """Iterates over the parameter assignments in the parameter space."""
        raise NotImplementedError()

    def keys(self):
        """Returns the parameter names."""
        return self._keys

    def __len__(self):
        raise NotImplementedError()

    def __add__(self, other):
        return Sum(self, other)

    def __mul__(self, other):
        return Product(self, other)

    def __sub__(self, other):
        return Difference(self, other)

    def __repr__(self):
        keys = sorted(self.keys())
        built = self.build()
        return "Param(**{{{params}}})".format(
            params=", ".join(
                "{k!r}: {v!r}".format(k=k, v=built[k]) for k in keys))

    def __str__(self):
        keys = sorted(self.keys())
        built = self.build()
        return "Param({params})".format(
            params=", ".join(
                "{k!s}={v!r}".format(k=k, v=built[k]) for k in keys))

@inherit_docs
class Param(ParameterSpace):
    """Constructs a simple parameter space from constructor arguments.

    Supports addition, subtraction, and multiplication operators to construct
    more complicated parameter spaces.

    Parameters
    ----------
    params :
        Each keyword argument defines a parameter with a sequence of parameter
        values for it. The length of all lists has to be equal. If a scalar
        instead of a sequence is passed in, it will be replicated to match the
        length of the other parameters. At least one keyword argument has to be
        a sequence.
    """
    def __init__(self, **params):
        super(Param, self).__init__(params.keys())
        self._params = params

        # Make sure strings etc are in a list.
        for k in self._params:
            if isinstance(self._params[k], string_types + (bytes,)):
                self._params[k] = [self._params[k]]

        self._len = None
        for v in self._params.values():
            try:
                l = len(v)
            except TypeError:
                pass
            else:
                if self._len is None:
                    self._len = l
                elif self._len != l:
                    raise ValueError("Parameter lists differ in length.")
        if self._len is None:
            self._len = 1 if len(self._params) > 0 else 0

    def iterate(self):
        for i in range(len(self)):
            yield {k: self.get_param(k, i) for k in self.keys()}

    def __len__(self):
        return self._len

    def get_param(self, key, i):
        """Return the *i*-th parameter assignment.

        Parameters
        ----------
        key : str
            Parameter name of parameter to retrieve.
        i : int
            Index of assigned value to return.
        """
        p = self._params[key]
        try:
            return p[i]
        except TypeError:
            return p

@inherit_docs
class Difference(ParameterSpace):
    """Implements the difference of two parameter spaces.

    Parameters
    ----------
    minuend : `ParameterSpace`
        Minuend (left operand).
    subtrahend : `ParameterSpace`
        Subtrahend (right operand).

    Attributes
    ----------
    minuend : `ParameterSpace`
        Minuend (left operand).
    subtrahend : `ParameterSpace`
        Subtrahend (right operand).

    Examples
    --------
    >>> from pprint import pprint
    >>> pprint(Difference(Param(a=[1, 2], b=[1, 2]), Param(a=[1])).build())
    {'a': [2], 'b': [2]}
    """

    def __init__(self, minuend, subtrahend):
        super(Difference, self).__init__(minuend.keys())
        for k in subtrahend.keys():
            if k not in self._keys:
                raise AmbiguousOperationError(
                    'Key `{0}` not existent in minuend.'.format(k))
        self.left = minuend
        self.right = subtrahend
        self._cached = None

    def iterate(self):
        if len(self.right) == 0:
            return self.left.iterate()
        if self._cached is None:
            exclude = self.right.build()
            self._cached = [item for item in self.left.iterate()
                            if not all(item[k] in exclude[k]
                                       for k in exclude.keys())]
        return iter(self._cached)

    def __len__(self):
        return sum(1 for item in self.iterate())

@inherit_docs
class Product(ParameterSpace):
    """Implements the Cartesian product of two parameter spaces.

    Parameters
    ----------
    left : `ParameterSpace`
        Left operand.
    right : `ParameterSpace`
        Right operand.

    Attributes
    ----------
    left : `ParameterSpace`
        Left operand.
    right : `ParameterSpace`
        Right operand.

    Examples
    --------
    >>> from pprint import pprint
    >>> pprint(Product(Param(a=[1, 2]), Param(b=[1, 2])).build())
    {'a': [1, 1, 2, 2], 'b': [1, 2, 1, 2]}
    """
    def __init__(self, left, right):
        shared_keys = set(left.keys()).intersection(set(right.keys()))
        if len(shared_keys) > 0:
            raise AmbiguousOperationError(
                'Duplicate param keys: {0}'.format(shared_keys))

        super(Product, self).__init__(list(left.keys()) + list(right.keys()))
        self.left = left
        self.right = right

    def iterate(self):
        if len(self.left.keys()) == 0:
            return self.right.iterate()
        elif len(self.right.keys()) == 0:
            return self.left.iterate()
        else:
            return (self._merge(*item) for item in itertools.product(
                self.left.iterate(), self.right.iterate()))

    @staticmethod
    def _merge(left, right):
        merged = {}
        merged.update(left)
        merged.update(right)
        return merged

    def __len__(self):
        if len(self.left.keys()) == 0:
            return len(self.right)
        elif len(self.right.keys()) == 0:
            return len(self.left)
        else:
            return len(self.left) * len(self.right)

@inherit_docs
class Sum(ParameterSpace):
    """Implements the concatenation of two parameter spaces.

    Parameters
    ----------
    left : `ParameterSpace`
        Left operand.
    right : `ParameterSpace`
        Right operand.

    Attributes
    ----------
    left : `ParameterSpace`
        Left operand.
    right : `ParameterSpace`
        Right operand.

    Examples
    --------
    >>> from pprint import pprint
    >>> pprint(Sum(Param(a=[1]), Param(a=[2])).build())
    {'a': [1, 2]}
    """
    def __init__(self, left, right):
        super(Sum, self).__init__(set(left.keys()).union(set(right.keys())))
        self.left = left
        self.right = right

    def iterate(self):
        return ({k: item.get(k, float('nan')) for k in self.keys()}
                for item in itertools.chain(
                    self.left.iterate(), self.right.iterate()))

    def __len__(self):
        return len(self.left) + len(self.right)

def missing(minuend, subtrahend):
    """Return set of parameter assignments missing from another set.

    This differs from a simple subtraction by allowing additional keys in the
    subtrahend, but no additional keys in the minuend.

    Parameters
    ----------
    minuend : `ParameterSpace`
        Parameter space with all assignments.
    subtrahend : :class:`Param`
        Parameter space with assignments to remove from the parameter space.

    Returns
    -------
    `ParameterSpace`
        The reduced parameter space.

    Examples
    --------
    >>> from pprint import pprint
    >>> pprint(missing(Param(a=[1, 2, 3]), Param(a=[2])).build())
    {'a': [1, 3]}
    """
    if len(subtrahend) <= 0:
        return minuend
    for k in minuend.keys():
        if k not in subtrahend.keys():
            raise AmbiguousOperationError()
    return minuend - Param(
        **{k: v for k, v in subtrahend.build().items() if k in minuend.keys()})

class AmbiguousOperationError(RuntimeError):
    """Attempt to combine two parameter spaces in an ambiguous way."""
    pass
<commit_msg>Limit Difference recursion depth.<commit_after>"""Construction of parameter spaces."""

import collections
import itertools

from six import string_types

from psyrun.utils.doc import inherit_docs

def dict_concat(args):
    """Concatenates elements with the same key in the passed dictionaries.

    Parameters
    ----------
    args : sequenece of dict
        Dictionaries with sequences to concatenate.

    Returns
    -------
    dict
        The dictionary with the union of all the keys of the dictionaries
        passed in and elements with the same key concatenated. Missing elements
        will be None.

    Examples
    --------
    >>> from pprint import pprint
    >>> pprint(dict_concat(({'a': 0, 'b': 0}, {'a': 1})))
    {'a': [0, 1], 'b': [0, None]}
    """
    keys = set()
    for a in args:
        keys = keys.union(a.keys())
    return {k: [a.get(k, None) for a in args] for k in keys}

class ParameterSpace(collections.Sized):
    """Abstract base class for objects representing a parameter space.

    Supports addition, subtraction, and multiplication operators to construct
    more complicated parameter spaces.

    Deriving classes are supposed to implement the `iterate` and ``__len__``
    methods.

    Parameters
    ----------
    keys : sequence of strings
        Parameter names.
    """

    def __init__(self, keys):
        self._keys = list(keys)

    def build(self):
        """Builds the parameter space into a dictionary of parameter lists.

        Returns
        -------
        dict
            A dictionary with the parameter names as keys and lists with the
            parameter values.

        Examples
        --------
        >>> from pprint import pprint
        >>> pprint(Param(a=[1, 2], b=[1, 2]).build())
        {'a': [1, 2], 'b': [1, 2]}
        """
        built = dict_concat(list(self.iterate()))
        for k in self.keys():
            built.setdefault(k, [])
        return built

    def iterate(self):
        """Iterates over the parameter assignments in the parameter space."""
        raise NotImplementedError()

    def keys(self):
        """Returns the parameter names."""
        return self._keys

    def __len__(self):
        raise NotImplementedError()

    def __add__(self, other):
        return Sum(self, other)

    def __mul__(self, other):
        return Product(self, other)

    def __sub__(self, other):
        return Difference(self, other)

    def __repr__(self):
        keys = sorted(self.keys())
        built = self.build()
        return "Param(**{{{params}}})".format(
            params=", ".join(
                "{k!r}: {v!r}".format(k=k, v=built[k]) for k in keys))

    def __str__(self):
        keys = sorted(self.keys())
        built = self.build()
        return "Param({params})".format(
            params=", ".join(
                "{k!s}={v!r}".format(k=k, v=built[k]) for k in keys))

@inherit_docs
class Param(ParameterSpace):
    """Constructs a simple parameter space from constructor arguments.

    Supports addition, subtraction, and multiplication operators to construct
    more complicated parameter spaces.

    Parameters
    ----------
    params :
        Each keyword argument defines a parameter with a sequence of parameter
        values for it. The length of all lists has to be equal. If a scalar
        instead of a sequence is passed in, it will be replicated to match the
        length of the other parameters. At least one keyword argument has to be
        a sequence.
    """
    def __init__(self, **params):
        super(Param, self).__init__(params.keys())
        self._params = params

        # Make sure strings etc are in a list.
        for k in self._params:
            if isinstance(self._params[k], string_types + (bytes,)):
                self._params[k] = [self._params[k]]

        self._len = None
        for v in self._params.values():
            try:
                l = len(v)
            except TypeError:
                pass
            else:
                if self._len is None:
                    self._len = l
                elif self._len != l:
                    raise ValueError("Parameter lists differ in length.")
        if self._len is None:
            self._len = 1 if len(self._params) > 0 else 0

    def iterate(self):
        for i in range(len(self)):
            yield {k: self.get_param(k, i) for k in self.keys()}

    def __len__(self):
        return self._len

    def get_param(self, key, i):
        """Return the *i*-th parameter assignment.

        Parameters
        ----------
        key : str
            Parameter name of parameter to retrieve.
        i : int
            Index of assigned value to return.
        """
        p = self._params[key]
        try:
            return p[i]
        except TypeError:
            return p

@inherit_docs
class Difference(ParameterSpace):
    """Implements the difference of two parameter spaces.

    Parameters
    ----------
    minuend : `ParameterSpace`
        Minuend (left operand).
    subtrahend : `ParameterSpace`
        Subtrahend (right operand).

    Attributes
    ----------
    minuend : `ParameterSpace`
        Minuend (left operand).
    subtrahend : `ParameterSpace`
        Subtrahend (right operand).

    Examples
    --------
    >>> from pprint import pprint
    >>> pprint(Difference(Param(a=[1, 2], b=[1, 2]), Param(a=[1])).build())
    {'a': [2], 'b': [2]}
    """

    def __init__(self, minuend, subtrahend):
        super(Difference, self).__init__(minuend.keys())
        for k in subtrahend.keys():
            if k not in self._keys:
                raise AmbiguousOperationError(
                    'Key `{0}` not existent in minuend.'.format(k))
        self.left = minuend
        self.right = subtrahend
        if len(self.right) == 0:
            self._cached = list(self.left.iterate())
        else:
            exclude = self.right.build()
            self._cached = [item for item in self.left.iterate()
                            if not all(item[k] in exclude[k]
                                       for k in exclude.keys())]

    def iterate(self):
        return iter(self._cached)

    def __len__(self):
        return sum(1 for item in self.iterate())

@inherit_docs
class Product(ParameterSpace):
    """Implements the Cartesian product of two parameter spaces.

    Parameters
    ----------
    left : `ParameterSpace`
        Left operand.
    right : `ParameterSpace`
        Right operand.

    Attributes
    ----------
    left : `ParameterSpace`
        Left operand.
    right : `ParameterSpace`
        Right operand.

    Examples
    --------
    >>> from pprint import pprint
    >>> pprint(Product(Param(a=[1, 2]), Param(b=[1, 2])).build())
    {'a': [1, 1, 2, 2], 'b': [1, 2, 1, 2]}
    """
    def __init__(self, left, right):
        shared_keys = set(left.keys()).intersection(set(right.keys()))
        if len(shared_keys) > 0:
            raise AmbiguousOperationError(
                'Duplicate param keys: {0}'.format(shared_keys))

        super(Product, self).__init__(list(left.keys()) + list(right.keys()))
        self.left = left
        self.right = right

    def iterate(self):
        if len(self.left.keys()) == 0:
            return self.right.iterate()
        elif len(self.right.keys()) == 0:
            return self.left.iterate()
        else:
            return (self._merge(*item) for item in itertools.product(
                self.left.iterate(), self.right.iterate()))

    @staticmethod
    def _merge(left, right):
        merged = {}
        merged.update(left)
        merged.update(right)
        return merged

    def __len__(self):
        if len(self.left.keys()) == 0:
            return len(self.right)
        elif len(self.right.keys()) == 0:
            return len(self.left)
        else:
            return len(self.left) * len(self.right)

@inherit_docs
class Sum(ParameterSpace):
    """Implements the concatenation of two parameter spaces.

    Parameters
    ----------
    left : `ParameterSpace`
        Left operand.
    right : `ParameterSpace`
        Right operand.

    Attributes
    ----------
    left : `ParameterSpace`
        Left operand.
    right : `ParameterSpace`
        Right operand.

    Examples
    --------
    >>> from pprint import pprint
    >>> pprint(Sum(Param(a=[1]), Param(a=[2])).build())
    {'a': [1, 2]}
    """
    def __init__(self, left, right):
        super(Sum, self).__init__(set(left.keys()).union(set(right.keys())))
        self.left = left
        self.right = right

    def iterate(self):
        return ({k: item.get(k, float('nan')) for k in self.keys()}
                for item in itertools.chain(
                    self.left.iterate(), self.right.iterate()))

    def __len__(self):
        return len(self.left) + len(self.right)

def missing(minuend, subtrahend):
    """Return set of parameter assignments missing from another set.

    This differs from a simple subtraction by allowing additional keys in the
    subtrahend, but no additional keys in the minuend.

    Parameters
    ----------
    minuend : `ParameterSpace`
        Parameter space with all assignments.
    subtrahend : :class:`Param`
        Parameter space with assignments to remove from the parameter space.

    Returns
    -------
    `ParameterSpace`
        The reduced parameter space.

    Examples
    --------
    >>> from pprint import pprint
    >>> pprint(missing(Param(a=[1, 2, 3]), Param(a=[2])).build())
    {'a': [1, 3]}
    """
    if len(subtrahend) <= 0:
        return minuend
    for k in minuend.keys():
        if k not in subtrahend.keys():
            raise AmbiguousOperationError()
    return minuend - Param(
        **{k: v for k, v in subtrahend.build().items() if k in minuend.keys()})

class AmbiguousOperationError(RuntimeError):
    """Attempt to combine two parameter spaces in an ambiguous way."""
    pass
<|endoftext|>


<commit_before>import torch
import numpy as np
import torch.nn as nn
import torch.nn.functional as F

def cross_entropy2d(input, target, weight=None, size_average=True):
    n, c, h, w = input.size()
    log_p = F.log_softmax(input, dim=1)
    log_p = log_p.transpose(1, 2).transpose(2, 3).contiguous().view(-1, c)
    log_p = log_p[target.view(n * h * w, 1).repeat(1, c) >= 0]
    log_p = log_p.view(-1, c)

    mask = target >= 0
    target = target[mask]
    loss = F.nll_loss(log_p, target, weight=weight, size_average=False)
    if size_average:
        loss /= mask.data.sum()
    return loss

def bootstrapped_cross_entropy2d(input, target, K, weight=None, size_average=True):
    n, c, h, w = input.size()
    log_p = F.log_softmax(input, dim=1)
    log_p = log_p.transpose(1, 2).transpose(2, 3).contiguous().view(-1, c)
    log_p = log_p[target.view(n * h * w, 1).repeat(1, c) >= 0]
    log_p = log_p.view(-1, c)

    loss = F.nll_loss(log_p, target, weight=weight, reduce=False)
    topk_loss, _ = loss.topk(K)
    reduced_topk_loss = topk_loss.sum() / K

    return reduced_topk_loss

<commit_msg>Ignore background in bootstrapped Xentropy<commit_after>import torch
import numpy as np
import torch.nn as nn
import torch.nn.functional as F

def cross_entropy2d(input, target, weight=None, size_average=True):
    n, c, h, w = input.size()
    log_p = F.log_softmax(input, dim=1)
    log_p = log_p.transpose(1, 2).transpose(2, 3).contiguous().view(-1, c)
    log_p = log_p[target.view(n * h * w, 1).repeat(1, c) >= 0]
    log_p = log_p.view(-1, c)

    mask = target >= 0
    target = target[mask]
    loss = F.nll_loss(log_p, target, weight=weight, size_average=False)
    if size_average:
        loss /= mask.data.sum()
    return loss

def bootstrapped_cross_entropy2d(input, target, K, weight=None, size_average=True):
    n, c, h, w = input.size()
    log_p = F.log_softmax(input, dim=1)
    log_p = log_p.transpose(1, 2).transpose(2, 3).contiguous().view(-1, c)
    log_p = log_p[target.view(n * h * w, 1).repeat(1, c) >= 0]
    log_p = log_p.view(-1, c)

    mask = target >= 0
    target = target[mask]
    loss = F.nll_loss(log_p, target, weight=weight, reduce=False, size_average=False)
    topk_loss, _ = loss.topk(K)
    reduced_topk_loss = topk_loss.sum() / K

    return reduced_topk_loss

<|endoftext|>


<commit_before>#-*- coding:utf-8 -*-
import datetime
import subprocess
import shlex

def execute_shell_cmd(cmd,timeout=60,bufsize=4096):
	assert isinstance(cmd,str)
	cmd=shlex.split(cmd)
	sub=subprocess.Popen(cmd,stdin=subprocess.PIPE,bufsize=bufsize)
	end_time=datetime.datetime.now()+datetime.timedelta(seconds=timeout)
	while sub.poll() is None:
		if timeout:
			if end_time<datetime.datetime.now():
				raise Exception("execution shell command '%s' failed"%(cmd))
	return sub.returncode
def shell_echo(echo_string,color):
	COLOR_CODE={
		'black':'0;30m',
		'red':'0;31m',
		'green':'0;32m',
		'brown':'0;33m',
		'blue':'0;34m'
	}
	cmd='echo -e "\e[{color}{string}\e[m"'.format(color=COLOR_CODE.get(color.lower(),'0;30m'),string=echo_string)
	execute_shell_cmd(cmd)

<commit_msg>add locale_translate(keys,**kw)  function to provide convienience for accessing localization item<commit_after>#-*- coding:utf-8 -*-
import datetime
import subprocess
import shlex
from tools.localization import Locale
def execute_shell_cmd(cmd,timeout=60,bufsize=4096):
	assert isinstance(cmd,str)
	cmd=shlex.split(cmd)
	sub=subprocess.Popen(cmd,stdin=subprocess.PIPE,bufsize=bufsize)
	end_time=datetime.datetime.now()+datetime.timedelta(seconds=timeout)
	while sub.poll() is None:
		if timeout:
			if end_time<datetime.datetime.now():
				raise Exception("execution shell command '%s' failed"%(cmd))
	return sub.returncode
def shell_echo(echo_string,color):
	COLOR_CODE={
		'black':'0;30m',
		'red':'0;31m',
		'green':'0;32m',
		'brown':'0;33m',
		'blue':'0;34m'
	}
	cmd='echo -e "\e[{color}{string}\e[m"'.format(color=COLOR_CODE.get(color.lower(),'0;30m'),string=echo_string)
	execute_shell_cmd(cmd)

def locale_translate(keys,**kw):
	if not  hasattr(trans,'__locale__'):
		trans.__locale__=Locale()
	return trans.__locale__.translate(keys,**kw)

if __name__=='__main__':
	pass
	r'''print(trans("message:register.username",username="huangbiao",default="hello"))'''
<|endoftext|>


<commit_before>from __future__ import print_function

import os
from sys import stderr as STDERR

from .json import load_component

class Error(Exception):
    pass

class LoadError(Error):
    def __init__(self, msg):
        self._msg = msg

    def __str__(self):
        return self._msg

def _component_names(palette_dir):
    names = set()
    for item in os.listdir(palette_dir):
        if os.path.isdir(os.path.join(palette_dir, item)):
            names.add(item)
    names = ['hydrotrend']
    return names

def _load_table(name, db='.'):
    path_to_table = os.path.join(db, name + '.db')
    try:
        with open(path_to_table, 'r') as opened:
            try:
                table = json.loads(opened.read())
            except ValueError as error:
                raise LoadError(str(error))
    except IOError as error:
        raise LoadError(str(error))

    return table

def _construct_component_from_db(path_to_db):
    desc = _load_table('info', db=path_to_db)

    for table in ['parameters', 'uses', 'provides']:
        desc[table] = _load_table(table, db=path_to_db)

    return desc

def load_palette(palette_dir):

    palette = {}
    for name in _component_names(palette_dir):
        path_to_db = os.path.join(palette_dir, name, 'db')
        try:
            palette[name] = _construct_component_from_db(path_to_db)
        except LoadError:
            raise
            print('%s: unable to load' % name, file=STDERR)

    return palette
<commit_msg>Fixed NameError when using json module.<commit_after>from __future__ import (absolute_import, print_function)

import os
from sys import stderr as STDERR

from .json import load_component

class Error(Exception):
    pass

class LoadError(Error):
    def __init__(self, msg):
        self._msg = msg

    def __str__(self):
        return self._msg

def _component_names(palette_dir):
    names = set()
    for item in os.listdir(palette_dir):
        if os.path.isdir(os.path.join(palette_dir, item)):
            names.add(item)
    return names

def _load_table(name, db='.'):
    import json

    path_to_table = os.path.join(db, name + '.json')
    try:
        with open(path_to_table, 'r') as opened:
            try:
                table = json.loads(opened.read())
            except ValueError as error:
                raise LoadError(str(error))
    except IOError as error:
        raise LoadError(str(error))

    return table

def _construct_component_from_db(path_to_db):
    desc = _load_table('info', db=path_to_db)

    for table in ['parameters', 'uses', 'provides']:
        desc[table] = _load_table(table, db=path_to_db)

    return desc

def load_palette(palette_dir):

    palette = {}
    for name in _component_names(palette_dir):
        path_to_db = os.path.join(palette_dir, name, 'db')
        try:
            palette[name] = _construct_component_from_db(path_to_db)
        except LoadError:
            print('%s: unable to load' % name, file=STDERR)

    return palette
<|endoftext|>


<commit_before>from django.conf.urls.defaults import *

from app_metrics.views import *

urlpatterns = patterns('',
        url(
            regex   = r'^reports/$',
            view    = metric_report_view,
            name    = 'app_metrics_reports',
            ),
    )
<commit_msg>Avoid deprecation warning<commit_after>from django.conf.urls import *

from app_metrics.views import *

urlpatterns = patterns('',
        url(
            regex   = r'^reports/$',
            view    = metric_report_view,
            name    = 'app_metrics_reports',
            ),
    )
<|endoftext|>