<commit_before>import boto
import json
import logging
import os
import requests
import sys
import time
import xmltodict

from boto.s3.key import Key
from csv import DictReader
from datetime import datetime
from django.conf import settings
from django.contrib.gis.geos import *
from django.contrib.gis.measure import D
from django.core.management.base import BaseCommand, CommandError

from the_el.models import Route, Shape, Stop, Trip
from the_el.utils import get_key

def get_train_metadata():
    return {
        'red': {
            'color': '#C60C30',
            'name': 'Red Line'
        },
        'blue': {
            'color': '#00A1DE',
            'name': 'Blue Line'
        },
        'brn': {
            'color': '#62361B',
            'name': 'Brown Line'
        },
        'g': {
            'color': '#009B3A',
            'name': 'Green Line'
        },
        'org': {
            'color': '#F9461C',
            'name': 'Orange Line'
        },
        'p': {
            'color': '#522398',
            'name': 'Purple Line'
        },
        'pink': {
            'color': '#E27EA6',
            'name': 'Pink Line'
        },
        'y': {
            'color': '#F9E300',
            'name': 'Yellow Line'
        }
    }

def get_train_positions():
    """
    Gets all current train positions from CTA API
    """
    metadata = get_train_metadata()
    cta_url = ('http://lapi.transitchicago.com/api/1.0/ttpositions.aspx?key=%s&rt='
               'red,blue,brn,g,org,p,pink,y' % get_key(settings.CTA_TRAIN_KEY_FILE))
    logging.info('Fetching trains')
    response = xmltodict.parse(requests.get(cta_url).text)
    trains = []
    for route in response['ctatt']['route']:
        for train in route.get('train', []):
            try:
                if train.get('lat', '') and train.get('lon', ''):
                    trains.append({
                        'route': metadata[route['@name']]['name'],
                        'run': train['rn'],
                        'type': 'train',
                        'dest': train['destNm'],
                        'lat': train['lat'],
                        'lon': train['lon'],
                        'color': metadata[route['@name']]['color'],
                        'heading': train['heading'],
                        'next': train['nextStaNm'],
                        'approaching': True if train['isApp'] else False,
                        'delayed': True if train['isDly'] else False
                        })
            except Exception:
                pass
    return trains

def get_bus_positions_for_selected_routes(routes):
    """
    Given a list of up to 10 routes, fetches and returns all bus positions from CTA API
    """
    logging.info('Fetching these routes: %s' % ', '.join(routes))
    vehicles_url = ('http://www.ctabustracker.com/bustime/api/v1/getvehicles?key=%s&rt=%s' %
                    (get_key(settings.CTA_BUS_KEY_FILE), ','.join(routes[:10])))
    response = xmltodict.parse(requests.get(vehicles_url).text)
    return response

def get_bus_positions():
    """
    Gets all current bus positions
    """
    routes_url = ('http://www.ctabustracker.com/bustime/api/v1/getroutes?key=%s'
                  % get_key(settings.CTA_BUS_KEY_FILE))
    logging.info('Fetching all routes')
    response = xmltodict.parse(requests.get(routes_url).text)
    routes = []
    all_buses = []
    buses_json = []
    route_meta = {}
    for route in response['bustime-response']['route']:
        if len(routes) == 10:
            buses = get_bus_positions_for_selected_routes(routes)
            for vehicle in buses['bustime-response'].get('vehicle', []):
                all_buses.append(vehicle)
            routes = []
        routes.append(route['rt'])
        route_meta[route['rt']] = {'name': route['rtnm'], 'color': route['rtclr']}
    buses = get_bus_positions_for_selected_routes(routes)
    for vehicle in buses['bustime-response'].get('vehicle', []):
        all_buses.append(vehicle)
    for bus in all_buses:
        try:
            buses_json.append({
                'route': '%s %s' % (bus['rt'], route_meta[bus['rt']]['name']),
                'run': bus['vid'],
                'type': 'bus',
                'dest': bus['des'],
                'lat': bus['lat'],
                'lon': bus['lon'],
                'color': route_meta[bus['rt']]['color'],
                'heading': bus['hdg'],
                'delayed': True if bus.get('dly', False) else False
                })
        except Exception, e:
            print e
    return buses_json

def upload_data_to_s3(s3_bucket, filename, directory_path, data_str):
    """
    Uploads the data_str as a file named filename to directory_path (not ending in /) in s3_bucket.
    """
    s3 = boto.connect_s3(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
    bucket = s3.get_bucket(s3_bucket)
    key = Key(bucket)
    key.key = '%s/%s' % (directory_path, filename)
    key.set_contents_from_string(data_str)
    key.make_public()

class Command(BaseCommand):
    def handle(self, *args, **options):
        # Check PID file, quit if we're still running. Not super robust but who cares.
        pid = str(os.getpid())
        pidfile = '/tmp/load_positions.pid'
        if os.path.isfile(pidfile):
            logging.error('%s already exists, exiting' % pidfile)
            sys.exit()
        else:
            file(pidfile, 'w').write(pid)

        runs = 0
        while runs < 3:
            trains = get_train_positions()
            buses = get_bus_positions()
            """
            upload_data_to_s3(
                settings.EL_S3_BUCKET, 'train_positions.json', 'static', json.dumps(trains))
            upload_data_to_s3(
                settings.EL_S3_BUCKET, 'bus_positions.json', 'static', json.dumps(buses))
            upload_data_to_s3(
                settings.EL_S3_BUCKET, 'all_positions.json', 'static', json.dumps(trains + buses))
            """
            with open(os.path.join(settings.EL_STATIC_DIR, 'train_positions.json'), 'w') as posfile:
                posfile.write(json.dumps(trains))
                posfile.close()
            with open(os.path.join(settings.EL_STATIC_DIR, 'bus_positions.json'), 'w') as posfile:
                posfile.write(json.dumps(buses))
                posfile.close()
            with open(os.path.join(settings.EL_STATIC_DIR, 'all_positions.json'), 'w') as posfile:
                posfile.write(json.dumps(trains + buses))
                posfile.close()
            #"""
            # Sleep enough to run 3 times, leave headroom for all runs to complete in under a minute
            time.sleep(12)
            runs += 1

        os.unlink(pidfile)
<commit_msg>Re-enabling s3 uploads<commit_after>import boto
import json
import logging
import os
import requests
import sys
import time
import xmltodict

from boto.s3.key import Key
from csv import DictReader
from datetime import datetime
from django.conf import settings
from django.contrib.gis.geos import *
from django.contrib.gis.measure import D
from django.core.management.base import BaseCommand, CommandError

from the_el.models import Route, Shape, Stop, Trip
from the_el.utils import get_key

def get_train_metadata():
    return {
        'red': {
            'color': '#C60C30',
            'name': 'Red Line'
        },
        'blue': {
            'color': '#00A1DE',
            'name': 'Blue Line'
        },
        'brn': {
            'color': '#62361B',
            'name': 'Brown Line'
        },
        'g': {
            'color': '#009B3A',
            'name': 'Green Line'
        },
        'org': {
            'color': '#F9461C',
            'name': 'Orange Line'
        },
        'p': {
            'color': '#522398',
            'name': 'Purple Line'
        },
        'pink': {
            'color': '#E27EA6',
            'name': 'Pink Line'
        },
        'y': {
            'color': '#F9E300',
            'name': 'Yellow Line'
        }
    }

def get_train_positions():
    """
    Gets all current train positions from CTA API
    """
    metadata = get_train_metadata()
    cta_url = ('http://lapi.transitchicago.com/api/1.0/ttpositions.aspx?key=%s&rt='
               'red,blue,brn,g,org,p,pink,y' % get_key(settings.CTA_TRAIN_KEY_FILE))
    logging.info('Fetching trains')
    response = xmltodict.parse(requests.get(cta_url).text)
    trains = []
    for route in response['ctatt']['route']:
        for train in route.get('train', []):
            try:
                if train.get('lat', '') and train.get('lon', ''):
                    trains.append({
                        'route': metadata[route['@name']]['name'],
                        'run': train['rn'],
                        'type': 'train',
                        'dest': train['destNm'],
                        'lat': train['lat'],
                        'lon': train['lon'],
                        'color': metadata[route['@name']]['color'],
                        'heading': train['heading'],
                        'next': train['nextStaNm'],
                        'approaching': True if train['isApp'] else False,
                        'delayed': True if train['isDly'] else False
                        })
            except Exception:
                pass
    return trains

def get_bus_positions_for_selected_routes(routes):
    """
    Given a list of up to 10 routes, fetches and returns all bus positions from CTA API
    """
    logging.info('Fetching these routes: %s' % ', '.join(routes))
    vehicles_url = ('http://www.ctabustracker.com/bustime/api/v1/getvehicles?key=%s&rt=%s' %
                    (get_key(settings.CTA_BUS_KEY_FILE), ','.join(routes[:10])))
    response = xmltodict.parse(requests.get(vehicles_url).text)
    return response

def get_bus_positions():
    """
    Gets all current bus positions
    """
    routes_url = ('http://www.ctabustracker.com/bustime/api/v1/getroutes?key=%s'
                  % get_key(settings.CTA_BUS_KEY_FILE))
    logging.info('Fetching all routes')
    response = xmltodict.parse(requests.get(routes_url).text)
    routes = []
    all_buses = []
    buses_json = []
    route_meta = {}
    for route in response['bustime-response']['route']:
        if len(routes) == 10:
            buses = get_bus_positions_for_selected_routes(routes)
            for vehicle in buses['bustime-response'].get('vehicle', []):
                all_buses.append(vehicle)
            routes = []
        routes.append(route['rt'])
        route_meta[route['rt']] = {'name': route['rtnm'], 'color': route['rtclr']}
    buses = get_bus_positions_for_selected_routes(routes)
    for vehicle in buses['bustime-response'].get('vehicle', []):
        all_buses.append(vehicle)
    for bus in all_buses:
        try:
            buses_json.append({
                'route': '%s %s' % (bus['rt'], route_meta[bus['rt']]['name']),
                'run': bus['vid'],
                'type': 'bus',
                'dest': bus['des'],
                'lat': bus['lat'],
                'lon': bus['lon'],
                'color': route_meta[bus['rt']]['color'],
                'heading': bus['hdg'],
                'delayed': True if bus.get('dly', False) else False
                })
        except Exception, e:
            print e
    return buses_json

def upload_data_to_s3(s3_bucket, filename, directory_path, data_str):
    """
    Uploads the data_str as a file named filename to directory_path (not ending in /) in s3_bucket.
    """
    s3 = boto.connect_s3(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
    bucket = s3.get_bucket(s3_bucket)
    key = Key(bucket)
    key.key = '%s/%s' % (directory_path, filename)
    key.set_contents_from_string(data_str)
    key.make_public()

class Command(BaseCommand):
    def handle(self, *args, **options):
        # Check PID file, quit if we're still running. Not super robust but who cares.
        pid = str(os.getpid())
        pidfile = '/tmp/load_positions.pid'
        if os.path.isfile(pidfile):
            logging.error('%s already exists, exiting' % pidfile)
            sys.exit()
        else:
            file(pidfile, 'w').write(pid)

        runs = 0
        while runs < 3:
            trains = get_train_positions()
            buses = get_bus_positions()

            upload_data_to_s3(
                settings.EL_S3_BUCKET, 'train_positions.json', 'static', json.dumps(trains))
            upload_data_to_s3(
                settings.EL_S3_BUCKET, 'bus_positions.json', 'static', json.dumps(buses))
            upload_data_to_s3(
                settings.EL_S3_BUCKET, 'all_positions.json', 'static', json.dumps(trains + buses))
            """
            with open(os.path.join(settings.EL_STATIC_DIR, 'train_positions.json'), 'w') as posfile:
                posfile.write(json.dumps(trains))
                posfile.close()
            with open(os.path.join(settings.EL_STATIC_DIR, 'bus_positions.json'), 'w') as posfile:
                posfile.write(json.dumps(buses))
                posfile.close()
            with open(os.path.join(settings.EL_STATIC_DIR, 'all_positions.json'), 'w') as posfile:
                posfile.write(json.dumps(trains + buses))
                posfile.close()
            """
            # Sleep enough to run 3 times, leave headroom for all runs to complete in under a minute
            time.sleep(12)
            runs += 1

        os.unlink(pidfile)
<|endoftext|>


<commit_before>import json
import re
import sys
from functools import wraps

import msgpack
import requests

try:
    from urllib.parse import urlencode
except ImportError:
    from urllib import urlencode
try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse

if sys.version_info.major >= 3:
    PY3 = True
else:
    PY3 = False

_d = json.loads    # shorthand for json decode
_e = json.dumps    # shorthand for json encode

class TornadoRequestHelper(object):
    def __init__(self,biothing_test_helpermixin_instance):
        self.testinst = biothing_test_helpermixin_instance
        # remove host part (http://localhost:8000) as test client require URLs
        # starting with "/..."
        self.testinst.api = self.testinst.api.replace(self.testinst.host,'')
        self.testinst.host = ''

    def request(self,url,method="GET",body=None,headers=None):#, body=None, headers=None, redirections=5,
        '''This simulates httplib2.Http.request() calls'''
        res = self.testinst.fetch(url,method=method,body=body,headers=headers)#,body=body,headers=headers)
        res.status = res.code
        return res,res.body

class HttpRequestHelper(object):
    def request(self,url,method="GET",body=None,headers=None):
        '''This simulates httplib2.Http.request() calls with requests library'''
        if method=='GET':
            r = requests.get(url, headers=headers)
        elif method=='POST':
            r = requests.post(url, data=body, headers=headers)
        elif method=='HEAD':
            r = requests.head(url, headers=headers)
        r.headers.status=r.status_code
        return r.headers, r.content

class BiothingTestHelperMixin(object):
    ''' Contains common functions to help facilitate testing.  Assumes that this class will be
    subclassed by a class that inherits from both this mixin and from unittest.TestCase.
    Also assumes that .host .api are set in the subclass. '''

    #def __init__(self):
    #    self.host = os.getenv(ns.nosetest_envar)
    #    if not self.host:
    #        self.host = ns.nosetest_default_url
    #    self.host = self.host.rstrip('/')
    #    self.api = self.host + '/' + ns.api_version
    #    self.h = httplib2.Http()

    def __init__(self):
        self.h = HttpRequestHelper()

    # TODO: Additional_fields, query_endpoint

    #############################################################
    # Helper functions                                          #
    #############################################################
    def encode_dict(self, d):
        '''urllib.urlencode (python 2.x) cannot take unicode string.
           encode as utf-8 first to get it around.
        '''
        if PY3:
            # no need to do anything
            return d
        else:
            def smart_encode(s):
                return s.encode('utf-8') if isinstance(s, unicode) else s   # noqa

            return dict([(key, smart_encode(val)) for key, val in d.items()])

    def truncate(self, s, limit):
        '''truncate a string.'''
        if len(s) <= limit:
            return s
        else:
            return s[:limit] + '...'

    def json_ok(self, s, checkerror=True):
        d = _d(s.decode('utf-8'))
        if checkerror:
            assert not (isinstance(d, dict) and 'error' in d), self.truncate(str(d), 100)
        return d

    def msgpack_ok(self, b, checkerror=True):
        d = msgpack.unpackb(b)
        if checkerror:
            assert not (isinstance(d, dict) and 'error' in d), self.truncate(str(d), 100)
        return d

    def get_ok(self, url):
        res, con = self.h.request((url))
        assert res.status == 200, "status {} != 200 for GET to url: {}".format(res.status, url)
        return con

    def get_status_code(self, url, status_code):
        res, con = self.h.request((url))
        assert res.status == status_code, "status {} != {} for GET to url: {}".format(res.status, status_code, url)

    def post_status_code(self, url, params, status_code):
        headers = {'Content-type': 'application/x-www-form-urlencoded'}
        res, con = self.h.request(url, 'POST', urlencode(self.encode_dict(params)), headers=headers)
        assert res.status == status_code, "status {} != {} for url: {}\nparams: {}".format(res.status, status_code, url, params)
        #return con

    def get_404(self, url):
        res, con = self.h.request((url))
        assert res.status == 404, "status {} != 404 for GET to url: {}".format(res.status, url)

    def get_405(self, url):
        res, con = self.h.request((url))
        assert res.status == 405, "status {} != 405 for GET to url: {}".format(res.status, url)

    def head_ok(self, url):
        res, con = self.h.request((url), 'HEAD')
        assert res.status == 200, "status {} != 200 for HEAD to url: {}".format(res.status, url)

    def post_ok(self, url, params):
        headers = {'Content-type': 'application/x-www-form-urlencoded'}
        res, con = self.h.request(url, 'POST', urlencode(self.encode_dict(params)), headers=headers)
        assert res.status == 200, "status {} != 200 for url: {}\nparams: {}".format(res.status, url, params)
        return con

    def query_has_hits(self, q, query_endpoint='query'):
        d = self.json_ok(self.get_ok(self.api + '/' + query_endpoint + '?q=' + q))
        assert d.get('total', 0) > 0 and len(d.get('hits', [])) > 0
        return d

    def parse_url(self, url, option):
        # parse the url string to see if option is specified, if so return it, if not return ''
        options = urlparse(url).query.split('&')
        for o in options:
            if o.split('=')[0] == option:
                return o.split('=')[1]
        return ''

    def convert_msgpack(self, d):
        ''' d is a msgpack decoded dict (strings are still byte objects).

            Traverse through d and decode all bytes into python strings.  Return the result.
        '''
        # convert d to a proper json object
        def convert_str(k):
            if isinstance(k, bytes):
                return k.decode('utf-8')
            else:
                return k

        def traverse(d):
            # decode the key
            if isinstance(d, list):
                return [traverse(i) for i in d]
            elif isinstance(d, dict):
                return dict([(convert_str(tk), traverse(tv)) for (tk,tv) in d.items()])
            else:
                return convert_str(d)

        return traverse(d)

    def check_fields(self, o, t, f, a=[]):
        ''' Tests the fields parameter.  Currently this takes these parameters:
                o is the dict with original request (with fields)
                t is the dict with total request (with fields = all)
                f is the list of requested fields e.g. ['cadd.gene']
                a is the list of additional fields that could be in a biothing object eg. _license
                    for cadd in myvariant

            expand_requested_fields: gives the list of all possible keys in an object,
            based on the users fields input, e.g.: cadd.gene means that cadd and cadd.gene
            should be in the output (if available), similarly, if clinvar is not in the total
            object, then it should not be in the output (even if the user specifies it).

            flatten_dict: this flattens a json objects keys using dot notation, returns a
            list of all dotted keys in the object, written poorly, so it leaves a . in front :)

            The test basically works like this: it flattens the total request (all possible fields),
            and then uses it in expand_requested_fields (as all_fields).  This returns the list of
            fields which should be in the o object (with fields f).  Next we flatten o, and make sure that
            flattened o is a subset of the list of fields which should be in the object + the root fields
            (_id, _version, query, _score).  Can do a more strict test with set equality,
            but this is fine for now....
        '''

        def expand_requested_fields(requested_fields, all_fields):
            # find all possible fields from the request,
            possible_fields = []
            if requested_fields[0] == 'all':
                return all_fields
            for field in requested_fields:
                possible_fields += [s for s in all_fields if s == field or s.startswith(field + '.')]
            # Go through and add parent nodes, which must be in the object....
            pfs = set(possible_fields)
            for f in possible_fields:
                tk = ''
                for path in f.split('.'):
                    tk = tk + '.' + path if tk else path
                    pfs.add(tk)
            return list(pfs)

        def flatten_dict(d, p, r):
            if isinstance(d, list):
                for i in d:
                    flatten_dict(i, p, r)
            elif isinstance(d, dict):
                # Add these keys
                for k in d.keys():
                    r[p + '.' + k] = 0
                    flatten_dict(d[k], p + '.' + k, r)

        possible_fields = {}
        flatten_dict(t, '', possible_fields)
        true_fields = expand_requested_fields(f, [x.lstrip('.') for x in possible_fields.keys()])
        actual_flattened_keys = {}
        flatten_dict(o, '', actual_flattened_keys)
        actual_flattened_keys = [x.lstrip('.') for x in actual_flattened_keys.keys()]
        additional_fields = ['_id', '_version', 'query', '_score'] + a
        # Make sure that all of the actual keys are among the set of requested fields
        assert set(actual_flattened_keys).issubset(set(true_fields + additional_fields)), "The returned keys of object {} have extra keys than expected, the offending keys are: {}".format(o['_id'], set(actual_flattened_keys).difference(set(true_fields + additional_fields)))

    def extract_results_from_callback(self, base_url):
        '''
            given a base query (with callback), the corresponding escaped GET url, test the response and
            return the JSON object inside
        '''
        c = self.get_ok(base_url).decode('utf-8')
        f = self.parse_url(base_url, 'callback')
        c = re.sub('\n', '', c)
        p = r'^(?P<callback>' + f + '\()(?P<res>\{.*\})\)$'
        m = re.search(p, c)
        assert m, 'JSONP object malformed'
        r = m.groupdict()
        return _d(r['res']) # get the json object out of the callback so we can test it

    def check_jsonld(self, d, jsonld_context):
        '''
            Traverse through d and assert that JSON-LD contexts are inserted and then remove them.
        should leave d a context-less JSON object...(no guarantee of this though, see below)
        '''
        #TODO:  Currently only tests that contexts in context file are in the object, and
        #removes them.  Maybe should add an else to the if not k: clause, and test that no
        #@context are in objects where they shouldn't be, to be really complete
        def traverse(d, k):
        # valid jsonld context?
            if isinstance(d, list):
                return [traverse(i, k) for i in d]
            elif isinstance(d, dict):
                if not k:
                    # Root
                    self.assertIn('@context', d, "JSON-LD context not found in root.  Expected: {}".format(jsonld_context['root']))
                    self.assertDictEqual(jsonld_context['root']['@context'], d['@context'])
                    del(d['@context'])
                    return dict([(tk, traverse(tv, tk)) for (tk, tv) in d.items()])
                elif k in jsonld_context['root']['@context'] and k not in jsonld_context:
                    # No context, but defined in root context
                    return dict([(tk, traverse(tv, k + '/' + tk)) for (tk, tv) in d.items()])
                elif k in jsonld_context:
                    # Context inserted, test it, and remove it
                    self.assertIn('@context', d, "JSON-LD context not found in {}.  Expected: {}".format(k, jsonld_context[k]))
                    self.assertDictEqual(jsonld_context[k]['@context'], d['@context'])
                    del(d['@context'])
                    return dict([(tk, traverse(tv, k + '/' + tk)) for (tk, tv) in d.items()])
            else:
                return d

        if 'root' not in jsonld_context:
            return d # can't check anything

        jsonld_context = jsonld_context # maybe not needed...
        traverse(d, '')

#############################################
# Decorators for easy parameterized testing
#############################################

PATTR = '%values'

# decorator to create a new test function
def feed(f, new_test_name, v):
    @wraps(f)
    def wrapper(self):
        return f(self, v)
    wrapper.__name__ = new_test_name
    if f.__doc__:
        try:
            wrapper.__doc__ = f.__doc__
        except:
            pass
    return wrapper

def add_test(cls, new_test_name, f, v):
    # add test case with new_test_name to class cls
    setattr(cls, new_test_name, feed(f, new_test_name, v))

# decorator to add parameter lists to tests in unittest.TestCase
def parameters(*values):
    def wrapper(f):
        setattr(f, PATTR, values)
        return f
    return wrapper

# class decorator to go through and actually add all parameterized test cases to class
def parameterized(cls):
    for name, f in list(cls.__dict__.items()):
        if hasattr(f, PATTR):
            for i, v in enumerate(getattr(f, PATTR)[0]):
                new_test_name = "{}_{}".format(name, i + 1)
                add_test(cls, new_test_name, f, v)
            delattr(cls, name)
    return cls
<commit_msg>Refactors test helper with pylint compliance<commit_after>'''
    Biothings Test Helper
'''
import json
import re
import sys
import unittest
from functools import wraps

import msgpack
import requests
import six
from tornado.testing import AsyncHTTPTestCase

try:
    from urllib.parse import urlencode
except ImportError:
    from urllib import urlencode
try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse

PY3 = bool(sys.version_info.major >= 3)

class TornadoTestServerMixin(AsyncHTTPTestCase):
    ''' Starts a tornado server to run tests against
        Accepts server settings by overriding get_app in sub-classes
    '''
    __test__ = False

    def __init__(self, methodName):
        super(TornadoTestServerMixin, self).__init__(methodName)
        # remove host part (http://localhost:8000)
        # as test client require URLs starting with "/..."
        self.api = self.api.replace(self.host, '')
        self.host = ''

    def get_app(self):
        """Should be overridden by subclasses to return a
        `tornado.web.Application` or other `.HTTPServer` callback.
        """
        raise NotImplementedError()

    def request(self, url, method="GET", body=None, headers=None):
        ''' Overrides http request client with Tornado's
            Function signature simulates httplib2.Http.request()'''
        res = self.fetch(url, method=method, body=body, headers=headers)
        res.status_code = res.code
        return res, res.body

class BiothingsTestHelper(unittest.TestCase):
    ''' Contains common functions to help facilitate testing.
        Assumes that .host .api are set in the subclass '''

    __test__ = False

    _d = staticmethod(json.loads)    # shorthand for json decode
    _e = staticmethod(json.dumps)    # shorthand for json encode

    # TODO: Additional_fields, query_endpoint

    #############################################################
    # Helper functions                                          #
    #############################################################

    @staticmethod
    def request(url, method="GET", body=None, headers=None):
        ''' Function signature simulates httplib2.Http.request() '''
        if method == 'GET':
            res = requests.get(url, headers=headers)
        elif method == 'POST':
            res = requests.post(url, data=body, headers=headers)
        elif method == 'HEAD':
            res = requests.head(url, headers=headers)
        return res, res.content

    @staticmethod
    def encode_dict(dic):
        '''urllib.urlencode (python 2.x) cannot take unicode string.
           encode as utf-8 first to get it around.
        '''
        if PY3:
            # no need to do anything
            return dic

        def smart_encode(string):
            return string.encode('utf-8') if isinstance(string, six.text_type) else string

        return {key: smart_encode(val) for key, val in dic.items()}

    @staticmethod
    def truncate(string, limit):
        ''' truncate a long string with a trailing ellipsis '''
        if len(string) <= limit:
            return string
        return string[:limit] + '...'

    @classmethod
    def json_ok(cls, byte_str, checkerror=True):
        ''' load utf-8 encoded json into a dict '''
        dic = cls._d(byte_str.decode('utf-8'))
        if checkerror:
            assert not (isinstance(dic, dict)
                        and 'error' in dic), cls.truncate(str(dic), 100)
        return dic

    @classmethod
    def msgpack_ok(cls, packed_bytes, checkerror=True):
        ''' load msgpack into a dict '''
        dic = msgpack.unpackb(packed_bytes)
        if checkerror:
            assert not (isinstance(dic, dict)
                        and 'error' in dic), cls.truncate(str(dic), 100)
        return dic

    def head_ok(self, url):
        ''' status code indicating the entity-header fields corresponding to
        the requested resource are sent in the response without any message-body '''
        res, _ = self.request(url, 'HEAD')
        assert res.status_code == 200, "status {} != 200 for HEAD to url: {}".format(
            res.status_code, url)

    def get_status_code(self, url, status_code):
        ''' make a get request and return the content if the status code matches the expectation'''
        res, con = self.request(url)
        assert res.status_code == status_code, "status {} != {} for GET to url: {}".format(
            res.status_code, status_code, url)
        return con

    def get_ok(self, url):
        ''' status code indicating the requested resource is sent in the response '''
        return self.get_status_code(url, 200)

    def get_404(self, url):
        ''' status code indicating the requested resource is not found '''
        return self.get_status_code(url, 404)

    def get_405(self, url):
        ''' status code indicating the GET is not allowed for the specified resource '''
        return self.get_status_code(url, 405)

    def post_status_code(self, url, params, status_code):
        ''' make a post request and return the content if the status code matches the expectation'''
        headers = {'Content-type': 'application/x-www-form-urlencoded'}
        res, con = self.request(url, 'POST', urlencode(
            self.encode_dict(params)), headers=headers)
        assert res.status_code == status_code, "status {} != {} for url: {}\nparams: {}".format(
            res.status_code, status_code, url, params)
        return con

    def post_ok(self, url, params):
        ''' status code indicating the request has succeeded and the server
        responded with an entity describing or containing the result of the action'''
        headers = {'Content-type': 'application/x-www-form-urlencoded'}
        res, con = self.request(url, 'POST', urlencode(
            self.encode_dict(params)), headers=headers)
        assert res.status_code == 200, "status {} != 200 for url: {}\nparams: {}".format(
            res.status, url, params)
        return con

    def query_has_hits(self, param_q, query_endpoint='query'):
        ''' make the specified query and examine if return is empty '''
        dic = self.json_ok(self.get_ok(
            self.api + '/' + query_endpoint + '?q=' + param_q))
        assert dic.get('total', 0) > 0 and dic.get('hits', [])
        return dic

    @staticmethod
    def parse_url(url, option):
        ''' parse the url string to see if option is specified,
        if so return the option param, if not return empty string'''
        options = urlparse(url).query.split('&')
        for opt in options:
            if opt.split('=')[0] == option:
                return opt.split('=')[1]
        return ''

    @staticmethod
    def convert_msgpack(obj):
        ''' obj is a msgpack decoded dict (strings are still byte objects).
            Traverse through obj and decode all bytes into python strings.  Return the result.
        '''
        def convert_str(k):
            if isinstance(k, bytes):
                return k.decode('utf-8')
            return k

        def traverse(obj):
            if isinstance(obj, list):
                return [traverse(i) for i in obj]
            if isinstance(obj, dict):
                return {convert_str(tk): traverse(tv) for (tk, tv) in obj.items()}
            return convert_str(obj)

        return traverse(obj)

    @staticmethod
    def check_fields(res_req_flds, res_all_flds, req_flds, add_flds=None):
        ''' Tests the fields parameter.  Currently this takes these parameters:
                res_req_flds is the dict with original request response (with fields)
                res_all_flds is the dict with total request response (with fields = all)
                req_flds is the list of requested fields e.g. ['cadd.gene']
                add_flds is the list of additional fields not already known as
                    a typical biothing object eg. _license for cadd in myvariant

            expand_requested_fields: gives the list of all possible keys in an object,
            based on the users fields input, e.g.: cadd.gene means that cadd and cadd.gene
            should be in the output (if available), similarly, if clinvar is not in the total
            object, then it should not be in the output (even if the user specifies it).

            flatten_dict: this flattens a json objects keys using dot notation, returns a
            list of all dotted keys in the object, written poorly, so it leaves a . in front :)

            The test basically works like this: it flattens the total request (all possible fields),
            and then uses it in expand_requested_fields (as all_fields).  This returns the list of
            fields which should be in the o object (with fields f).  Next we flatten o, and make
            sure that flattened o is a subset of the list of fields which should be in the object +
            the root fields (_id, _version, query, _score).  Can do a more strict test with set
            equality, but this is fine for now....
        '''

        def expand_requested_fields(requested_fields, all_fields):
            # find all possible fields from the request,
            possible_fields = []
            if requested_fields[0] == 'all':
                return all_fields
            for field in requested_fields:
                possible_fields += [s for s in all_fields if s ==
                                    field or s.startswith(field + '.')]
            # Go through and add parent nodes, which must be in the object....
            pfs = set(possible_fields)
            for fld in possible_fields:
                tk = ''
                for path in fld.split('.'):
                    tk = tk + '.' + path if tk else path
                    pfs.add(tk)
            return list(pfs)

        def flatten_dict(d, p, r):
            if isinstance(d, list):
                for i in d:
                    flatten_dict(i, p, r)
            elif isinstance(d, dict):
                # Add these keys
                for k in d.keys():
                    r[p + '.' + k] = 0
                    flatten_dict(d[k], p + '.' + k, r)

        if not add_flds:
            add_flds = []
        possible_fields = {}
        flatten_dict(res_all_flds, '', possible_fields)
        true_fields = expand_requested_fields(
            req_flds, [x.lstrip('.') for x in possible_fields])
        actual_flattened_keys = {}
        flatten_dict(res_req_flds, '', actual_flattened_keys)
        actual_flattened_keys = [x.lstrip('.')
                                 for x in actual_flattened_keys]
        additional_fields = ['_id', '_version', 'query', '_score'] + add_flds
        # Make sure that all of the actual keys are among the set of requested fields
        assert set(actual_flattened_keys).issubset(set(true_fields + additional_fields)), \
            "The returned keys of object {} have extra keys than expected, the offending keys \
            are: {}".format(res_req_flds['_id'], set(actual_flattened_keys).difference(
                set(true_fields + additional_fields)))

    def extract_results_from_callback(self, base_url):
        '''
            given a base query (with callback), the corresponding escaped GET url,
            test the response and return the JSON object inside
        '''
        c = self.get_ok(base_url).decode('utf-8')
        f = self.parse_url(base_url, 'callback')
        c = re.sub('\n', '', c)
        p = r'^(?P<callback>' + f + '\()(?P<res>\{.*\})\)$'
        m = re.search(p, c)
        assert m, 'JSONP object malformed'
        r = m.groupdict()
        # get the json object out of the callback so we can test it
        return self._d(r['res'])

    def check_jsonld(self, d, jsonld_context):
        '''
            Traverse through d and assert that JSON-LD contexts are inserted and then remove them.
        should leave d a context-less JSON object...(no guarantee of this though, see below)
        '''
        # TODO:  Currently only tests that contexts in context file are in the object, and
        # removes them.  Maybe should add an else to the if not k: clause, and test that no
        # @context are in objects where they shouldn't be, to be really complete
        def traverse(d, k):
            # valid jsonld context?
            if isinstance(d, list):
                return [traverse(i, k) for i in d]
            elif isinstance(d, dict):
                if not k:
                    # Root
                    self.assertIn(
                        '@context', d, "JSON-LD context not found in root.  Expected: {}"
                        .format(jsonld_context['root']))
                    self.assertDictEqual(
                        jsonld_context['root']['@context'], d['@context'])
                    del(d['@context'])
                    return dict([(tk, traverse(tv, tk)) for (tk, tv) in d.items()])
                elif k in jsonld_context['root']['@context'] and k not in jsonld_context:
                    # No context, but defined in root context
                    return dict([(tk, traverse(tv, k + '/' + tk)) for (tk, tv) in d.items()])
                elif k in jsonld_context:
                    # Context inserted, test it, and remove it
                    self.assertIn(
                        '@context', d, "JSON-LD context not found in {}.  Expected: {}"
                        .format(k, jsonld_context[k]))
                    self.assertDictEqual(
                        jsonld_context[k]['@context'], d['@context'])
                    del(d['@context'])
                    return dict([(tk, traverse(tv, k + '/' + tk)) for (tk, tv) in d.items()])
            else:
                return d

        if 'root' not in jsonld_context:
            return d  # can't check anything

        jsonld_context = jsonld_context  # maybe not needed...
        traverse(d, '')

#############################################
# Decorators for easy parameterized testing
#############################################

PATTR = '%values'

def feed(f, new_test_name, v):
    ''' decorator to create a new test function '''
    @wraps(f)
    def wrapper(self):
        return f(self, v)
    wrapper.__name__ = new_test_name
    if f.__doc__:
        try:
            wrapper.__doc__ = f.__doc__
        except:
            pass
    return wrapper

def add_test(cls, new_test_name, f, v):
    ''' add test case with new_test_name to class cls '''
    setattr(cls, new_test_name, feed(f, new_test_name, v))

def parameters(*values):
    ''' decorator to add parameter lists to tests in unittest.TestCase '''
    def wrapper(f):
        setattr(f, PATTR, values)
        return f
    return wrapper

def parameterized(cls):
    ''' class decorator to go through and actually add all parameterized test cases to class '''
    for name, f in list(cls.__dict__.items()):
        if hasattr(f, PATTR):
            for i, v in enumerate(getattr(f, PATTR)[0]):
                new_test_name = "{}_{}".format(name, i + 1)
                add_test(cls, new_test_name, f, v)
            delattr(cls, name)
    return cls
<|endoftext|>


<commit_before>import RPi.GPIO as GPIO
import time
import subprocess

def main():
    GPIO.setwarnings(False)
    GPIO.setmode(GPIO.BOARD)
    GPIO.setup(11, GPIO.IN)         #Read output from PIR motion sensor
    GPIO.setup(3, GPIO.OUT)         #LED output pin
    tvstatus = 1                    #tv starts on
    count = 0
    while True:
        i=GPIO.input(11)
        if i==0:                 #When output from motion sensor is LOW
            print "No intruders",i
            if tvstatus==1 and count==200:  #tv stays on for about 30 sec. adjust count to change
                turn_off()
                tvstatus=0
            count=count+1
            time.sleep(0.1)
        elif i==1:               #When output from motion sensor is HIGH
            print "Intruder detected",i
            if tvstatus==0:
                turn_on()
                tvstatus=1
            count=0
            time.sleep(0.1)

def turn_on():
    subprocess.call("sh /usr/local/sbin/SmartMirror/piPower/power.sh on", shell=True)

def turn_off():
    subprocess.call("sh /usr/local/sbin/SmartMirror/piPower/power.sh off", shell=True)

if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        io.cleanup()
<commit_msg>edit power scripts because poweroff functionality<commit_after>import RPi.GPIO as GPIO
import time
import subprocess

def main():
    GPIO.setwarnings(False)
    GPIO.setmode(GPIO.BOARD)
    GPIO.setup(11, GPIO.IN)         #Read output from PIR motion sensor
    GPIO.setup(3, GPIO.OUT)         #LED output pin
    tvstatus = 1                    #tv starts on
    count = 0
    while True:
        i=GPIO.input(11)
        #Uncomment this code if tv works to power off. My tests show samsung works
        #insignia does not
        # if i==0:                 #When output from motion sensor is LOW
        #     print "No intruders",i
        #     if tvstatus==1 and count==200:  #tv stays on for about 30 sec. adjust count to change
        #                                     #adjust count to 200 for 30 second timeout. Higher for more
        #         turn_off()                  #for insignia tv this does not work. Need to manually turn off.
        #                                     #This will work with samsung tv with anynet+
        #         tvstatus=0
        #     count=count+1
        #     time.sleep(0.1)
        i==1:               #When output from motion sensor is HIGH
            print "Intruder detected",i
            if tvstatus==0:
                turn_on()
                tvstatus=1
            count=0
            time.sleep(0.1)

def turn_on():
    subprocess.call("sh /usr/local/sbin/SmartMirror/piPower/power.sh on", shell=True)

def turn_off():
    subprocess.call("sh /usr/local/sbin/SmartMirror/piPower/power.sh off", shell=True)

if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        io.cleanup()
<|endoftext|>


<commit_before>from ops import *
dtSet = dataset('data/')

batch = dtSet.next_batch(5)

for i in range(5):
    img = toImage(batch[0][i: i+1])
    img.save('data/%s.png'%i)<commit_msg>made it flexible<commit_after>from ops import *
dtSet = dataset('data/')

num = 20
batch = dtSet.next_batch(num)

for i in range(num):
    img = toImage(batch[0][i: i+1])
    img.save('results/%s.png'%i)<|endoftext|>


<commit_before>'''
Created on Jan 27, 2011

@author: peyman kazemian
'''
from headerspace.headerspace.hs import *
from headerspace.headerspace.tf import *
from headerspace.config_parser.openflow_parser import get_uniq_port_id

# What is a p_node?
# A hash apparently:
#  hdr -> foo
#  port -> foo
#  visits -> foo
def print_p_node(p_node):
    print "-----"
    print p_node["hdr"]
    print p_node["port"]
    print p_node["visits"]
    print "-----"

def find_reachability(NTF, TTF, in_port, out_ports, input_pkt):
    paths = []
    propagation = []

    p_node = {}
    p_node["hdr"] = input_pkt
    p_node["port"] = in_port
    p_node["visits"] = []
    p_node["hs_history"] = []
    propagation.append(p_node)

    while len(propagation)>0:
        #get the next node in propagation graph and apply it to NTF and TTF
        print "Propagation has length: %d"%len(propagation)
        tmp_propagate = []
        for p_node in propagation:
            next_hp = NTF.T(p_node["hdr"],p_node["port"])
            for (next_h,next_ps) in next_hp:
                for next_p in next_ps:
                    if next_p in out_ports:
                        reached = {}
                        reached["hdr"] = next_h
                        reached["port"] = next_p
                        reached["visits"] = list(p_node["visits"])
                        reached["visits"].append(p_node["port"])
                        reached["hs_history"] = list(p_node["hs_history"])
                        paths.append(reached)
                    else:
                        linked = TTF.T(next_h,next_p)
                        for (linked_h,linked_ports) in linked:
                            for linked_p in linked_ports:
                                new_p_node = {}
                                new_p_node["hdr"] = linked_h
                                new_p_node["port"] = linked_p
                                new_p_node["visits"] = list(p_node["visits"])
                                new_p_node["visits"].append(p_node["port"])
                                new_p_node["visits"].append(next_p)
                                new_p_node["hs_history"] = list(p_node["hs_history"])
                                new_p_node["hs_history"].append(p_node["hdr"])
                                if linked_p in out_ports:
                                    paths.append(new_p_node)
                                elif linked_p in new_p_node["visits"]:
                                    print "WARNING: detected a loop - branch aborted: \nHeaderSpace: %s\n Visited Ports: %s\nLast Port %d "%(\
                                        new_p_node["hdr"],new_p_node["visits"],new_p_node["port"])
                                else:
                                    tmp_propagate.append(new_p_node)
        propagation = tmp_propagate

    return paths

def get_all_x(NTF):
  all_x = byte_array_get_all_x(NTF.length)
  test_pkt = headerspace(NTF.length)
  test_pkt.add_hs(all_x)

def detect_loop(NTF, TTF, ports, reverse_map, test_packet = None):
    loops = []
    for port in ports:
        print "port %d is being checked"%port
        propagation = []

        # put all-x test packet in propagation graph
        test_pkt = test_packet
        if test_pkt == None:
          test_pkt = get_all_x(NTF)

        p_node = {}
        p_node["hdr"] = test_pkt
        p_node["port"] = port
        p_node["visits"] = []
        p_node["hs_history"] = []

        propagation.append(p_node)
        while len(propagation)>0:
            #get the next node in propagation graph and apply it to NTF and TTF
            print "Propagation has length: %d"%len(propagation)
            tmp_propag = []
            for p_node in propagation:
                # hp is "header port"
                next_hp = NTF.T(p_node["hdr"],p_node["port"])
                for (next_h,next_ps) in next_hp:
                    for next_p in next_ps:
                        linked = TTF.T(next_h,next_p)
                        for (linked_h,linked_ports) in linked:
                            for linked_p in linked_ports:
                                new_p_node = {}
                                new_p_node["hdr"] = linked_h
                                new_p_node["port"] = linked_p
                                new_p_node["visits"] = list(p_node["visits"])
                                new_p_node["visits"].append(p_node["port"])
                                #new_p_node["visits"].append(next_p)
                                new_p_node["hs_history"] = list(p_node["hs_history"])
                                new_p_node["hs_history"].append(p_node["hdr"])
                                #print new_p_node
                                if len(new_p_node["visits"]) > 0 and new_p_node["visits"][0] == linked_p:
                                    loops.append(new_p_node)
                                    print "loop detected"
                                elif linked_p in new_p_node["visits"]:
#                                    if (linked_p not in ports):
#                                        print "WARNING: detected a loop whose port is not in checked ports - branch aborted:"
#                                        print_loops([new_p_node],reverse_map)
#                                        print "END OF WARNING"
                                    pass
                                else:
                                    tmp_propag.append(new_p_node)
            propagation = tmp_propag
    return loops

# Omega defines the externally visible behavior of the network. Defined as a table:
#   (header space, edge_port) -> [(header_space, final_location),(header_space, final_location)...]
def compute_omega(NTF, TTF, edge_links, reverse_map={}, test_packet=None):
  omega = {}

  # TODO: need to model host end of link, or does switch end suffice?
  edge_ports = map(lambda access_link: get_uniq_port_id(access_link.switch, access_link.switch_port), edge_links)

  for start_port in edge_ports:
    print "port %d is being checked" % start_port
    propagation = []

    # put all-x test packet in propagation graph
    test_pkt = test_packet
    if test_pkt == None:
      test_pkt = get_all_x(NTF)

    p_node = {}
    p_node["hdr"] = test_pkt
    p_node["port"] = start_port
    p_node["visits"] = []
    p_node["hs_history"] = []

    propagation.append(p_node)
    while len(propagation) > 0: # TODO: is this stopping condition correct?!!!
      # get the next node in propagation graph and apply it to NTF and TTF
      print "Propagation has length: %d" % len(propagation)
      tmp_propag = []
      for p_node in propagation:
        # hp is "header port"
        next_hp = NTF.T(p_node["hdr"],p_node["port"])
        for (next_h,next_ps) in next_hp:
          for next_p in next_ps:
            # TODO: do I necessarily want to invoke the topology transfer function right off the bat?
            # TODO: how are packet drops represented?
            #       I believe that is represented as a null header space... Do I need to
            #       worry about that? Or do NTF/TTF take care of that for me?
            linked = TTF.T(next_h,next_p)
            for (linked_h,linked_ports) in linked:
              for linked_p in linked_ports:
                new_p_node = {}
                new_p_node["hdr"] = linked_h
                new_p_node["port"] = linked_p
                new_p_node["visits"] = list(p_node["visits"])
                new_p_node["visits"].append(p_node["port"])
                #new_p_node["visits"].append(next_p)
                new_p_node["hs_history"] = list(p_node["hs_history"])
                new_p_node["hs_history"].append(p_node["hdr"])
                #print new_p_node

                # TODO: This definitely doesn't properly handle dropped packets...
                if next_p in edge_ports:
                  # We've reached our final destination!
                  # ASSUMPTION: no edge port will send it back out into the network...

                  # use the inverse T trick to get original headerspace which led us here
                  original_headers = find_loop_original_header(NTF,TTF,new_p_node)
                  for original_header in original_headers:
                    key = (original_header, start_port)
                    if not key in omega:
                      # TODO: python default value for hash?
                      omega[key] = []
                    omega[key].append((next_h, next_p))

                if linked_p in new_p_node["visits"]:
                  print "WARNING: detected a loop - branch aborted: \nHeaderSpace: %s\n Visited Ports: %s\nLast Port %d "%(\
                         new_p_node["hdr"],new_p_node["visits"],new_p_node["port"])
                else:
                  tmp_propag.append(new_p_node)
      propagation = tmp_propag
  return omega

def print_reachability(paths, reverse_map):
    for p_node in paths:
        str = ""
        for port in p_node["visits"]:
            if str == "":
                str = reverse_map["%d"%port]
            else:
                str = "%s ---> %s"%(str,reverse_map["%d"%port])
        str = "%s ---> %s"%(str,reverse_map["%d"%p_node["port"]])
        print "Path: %s"%str
        print "HS Received: %s"%p_node["hdr"]
        print "----------------------------------------------"

def print_loops(loops, reverse_map):
    for p_node in loops:
        print "----------------------------------------------"
        str = ""
        for port in p_node["visits"]:
            if str == "":
                str = reverse_map["%d"%port]
            else:
                str = "%s ---> %s"%(str,reverse_map["%d"%port])
        str = "%s ---> %s"%(str,reverse_map["%d"%p_node["port"]])
        print "Path: %s"%str
        rl_id =  "applied rules: "
        for (n,r,s) in p_node["hdr"].applied_rule_ids:
            rl_id = rl_id + " -> %s"%r
        print rl_id
        for i in range(len(p_node["hs_history"])):
            print "*** %d) AT PORT: %s\nHS: %s\n"%(i,reverse_map["%d"%p_node["visits"][i]],p_node["hs_history"][i])
        print "*** %d) AT PORT: %s\nHS: %s\n"%(i+1,reverse_map["%d"%p_node["port"]],p_node["hdr"])
        print "----------------------------------------------"

def loop_path_to_str(p_node, reverse_map):
    str = ""
    for port in p_node["visits"]:
        if str == "":
            str = reverse_map["%d"%port]
        else:
            str = "%s ---> %s"%(str,reverse_map["%d"%port])
    str = "%s ---> %s"%(str,reverse_map["%d"%p_node["port"]])
    return str

def trace_hs_back(applied_rule_list,hs,last_port):

    port = last_port
    hs_list = [hs]
    hs.applied_rule_ids = []
    for (tf,rule_id,in_port) in reversed(applied_rule_list):
        tmp = []
        for next_hs in hs_list:
            hp = tf.T_inv_rule(rule_id,next_hs,port)
            for (h,p_list) in hp:
                if in_port in p_list:
                    tmp.append(h)
        if tmp == []:
            break
        hs_list = tmp
        port = in_port
    return hs_list

def find_loop_original_header(NTF,TTF,propagation_node):
    applied_rule_ids = list(propagation_node["hdr"].applied_rule_ids)
    hs_list = trace_hs_back(applied_rule_ids,propagation_node["hdr"],propagation_node["port"])
    return hs_list

#def is_loop_infinite(NTF, TTF, propagation_node):
#    p_node = propagation_node
#    header_cpy = p_node["hdr"].copy()
#    while len(p_node['visits']) > 0:
#        cur_port = p_node['visits'].pop()
#
#         transfer back on the link
#        linked = TTF.T_inv(p_node["hdr"],p_node["port"])
#
#         transfer back on node
#        prev_hp = NTF.T_inv(linked[0][0],linked[0][1])
#
#        prev_hs = headerspace(NTF.length)
#        for (h,p) in prev_hp:
#            if p == cur_port:
#                prev_hs.add_hs(h)
#
#        p_node["hdr"] = prev_hs
#        p_node["port"] = cur_port

    #print "hret = %s and"%propagation_node["hdr"]
    #print "hret_inv = %s"%(p_node["hdr"])
#    if propagation_node["hdr"].is_subset_of(p_node["hdr"]):
#        return (True,p_node["hdr"])
#    else:
#        insect = propagation_node["hdr"].copy_intersect(p_node["hdr"])
#        if insect.count() == 0:
#            return (False,[])
#        else:
#            new_cpy = propagation_node.copy()
#            new_cpy["hdr"] = insect
#            return is_loop_infinite(NTF, TTF, new_cpy)

<commit_msg>hsa.applications: fixing alleged bug in HSA code?<commit_after>'''
Created on Jan 27, 2011

@author: peyman kazemian
'''
from headerspace.headerspace.hs import *
from headerspace.headerspace.tf import *
from headerspace.config_parser.openflow_parser import get_uniq_port_id

# What is a p_node?
# A hash apparently:
#  hdr -> foo
#  port -> foo
#  visits -> foo
def print_p_node(p_node):
    print "-----"
    print p_node["hdr"]
    print p_node["port"]
    print p_node["visits"]
    print "-----"

def find_reachability(NTF, TTF, in_port, out_ports, input_pkt):
    paths = []
    propagation = []

    p_node = {}
    p_node["hdr"] = input_pkt
    p_node["port"] = in_port
    p_node["visits"] = []
    p_node["hs_history"] = []
    propagation.append(p_node)

    while len(propagation)>0:
        #get the next node in propagation graph and apply it to NTF and TTF
        print "Propagation has length: %d"%len(propagation)
        tmp_propagate = []
        for p_node in propagation:
            next_hp = NTF.T(p_node["hdr"],p_node["port"])
            for (next_h,next_ps) in next_hp:
                for next_p in next_ps:
                    if next_p in out_ports:
                        reached = {}
                        reached["hdr"] = next_h
                        reached["port"] = next_p
                        reached["visits"] = list(p_node["visits"])
                        reached["visits"].append(p_node["port"])
                        reached["hs_history"] = list(p_node["hs_history"])
                        paths.append(reached)
                    else:
                        linked = TTF.T(next_h,next_p)
                        for (linked_h,linked_ports) in linked:
                            for linked_p in linked_ports:
                                new_p_node = {}
                                new_p_node["hdr"] = linked_h
                                new_p_node["port"] = linked_p
                                new_p_node["visits"] = list(p_node["visits"])
                                new_p_node["visits"].append(p_node["port"])
                                new_p_node["visits"].append(next_p)
                                new_p_node["hs_history"] = list(p_node["hs_history"])
                                new_p_node["hs_history"].append(p_node["hdr"])
                                if linked_p in out_ports:
                                    paths.append(new_p_node)
                                elif linked_p in new_p_node["visits"]:
                                    print "WARNING: detected a loop - branch aborted: \nHeaderSpace: %s\n Visited Ports: %s\nLast Port %d "%(\
                                        new_p_node["hdr"],new_p_node["visits"],new_p_node["port"])
                                else:
                                    tmp_propagate.append(new_p_node)
        propagation = tmp_propagate

    return paths

def get_all_x(NTF):
  all_x = byte_array_get_all_x(NTF.length)
  test_pkt = headerspace(NTF.length)
  test_pkt.add_hs(all_x)
  return test_pkt

def detect_loop(NTF, TTF, ports, reverse_map, test_packet = None):
    loops = []
    for port in ports:
        print "port %d is being checked"%port
        propagation = []

        # put all-x test packet in propagation graph
        test_pkt = test_packet
        if test_pkt == None:
          test_pkt = get_all_x(NTF)

        p_node = {}
        p_node["hdr"] = test_pkt
        p_node["port"] = port
        p_node["visits"] = []
        p_node["hs_history"] = []

        propagation.append(p_node)
        while len(propagation)>0:
            #get the next node in propagation graph and apply it to NTF and TTF
            print "Propagation has length: %d"%len(propagation)
            tmp_propag = []
            for p_node in propagation:
                # hp is "header port"
                next_hp = NTF.T(p_node["hdr"],p_node["port"])
                for (next_h,next_ps) in next_hp:
                    for next_p in next_ps:
                        linked = TTF.T(next_h,next_p)
                        for (linked_h,linked_ports) in linked:
                            for linked_p in linked_ports:
                                new_p_node = {}
                                new_p_node["hdr"] = linked_h
                                new_p_node["port"] = linked_p
                                new_p_node["visits"] = list(p_node["visits"])
                                new_p_node["visits"].append(p_node["port"])
                                #new_p_node["visits"].append(next_p)
                                new_p_node["hs_history"] = list(p_node["hs_history"])
                                new_p_node["hs_history"].append(p_node["hdr"])
                                #print new_p_node
                                if len(new_p_node["visits"]) > 0 and new_p_node["visits"][0] == linked_p:
                                    loops.append(new_p_node)
                                    print "loop detected"
                                elif linked_p in new_p_node["visits"]:
#                                    if (linked_p not in ports):
#                                        print "WARNING: detected a loop whose port is not in checked ports - branch aborted:"
#                                        print_loops([new_p_node],reverse_map)
#                                        print "END OF WARNING"
                                    pass
                                else:
                                    tmp_propag.append(new_p_node)
            propagation = tmp_propag
    return loops

# Omega defines the externally visible behavior of the network. Defined as a table:
#   (header space, edge_port) -> [(header_space, final_location),(header_space, final_location)...]
def compute_omega(NTF, TTF, edge_links, reverse_map={}, test_packet=None):
  omega = {}

  # TODO: need to model host end of link, or does switch end suffice?
  edge_ports = map(lambda access_link: get_uniq_port_id(access_link.switch, access_link.switch_port), edge_links)

  for start_port in edge_ports:
    print "port %d is being checked" % start_port
    propagation = []

    # put all-x test packet in propagation graph
    test_pkt = test_packet
    if test_pkt == None:
      test_pkt = get_all_x(NTF)

    p_node = {}
    p_node["hdr"] = test_pkt
    p_node["port"] = start_port
    p_node["visits"] = []
    p_node["hs_history"] = []

    propagation.append(p_node)
    while len(propagation) > 0: # TODO: is this stopping condition correct?!!!
      # get the next node in propagation graph and apply it to NTF and TTF
      print "Propagation has length: %d" % len(propagation)
      tmp_propag = []
      for p_node in propagation:
        # hp is "header port"
        next_hp = NTF.T(p_node["hdr"],p_node["port"])
        for (next_h,next_ps) in next_hp:
          for next_p in next_ps:
            # TODO: do I necessarily want to invoke the topology transfer function right off the bat?
            # TODO: how are packet drops represented?
            #       I believe that is represented as a null header space... Do I need to
            #       worry about that? Or do NTF/TTF take care of that for me?
            linked = TTF.T(next_h,next_p)
            for (linked_h,linked_ports) in linked:
              for linked_p in linked_ports:
                new_p_node = {}
                new_p_node["hdr"] = linked_h
                new_p_node["port"] = linked_p
                new_p_node["visits"] = list(p_node["visits"])
                new_p_node["visits"].append(p_node["port"])
                #new_p_node["visits"].append(next_p)
                new_p_node["hs_history"] = list(p_node["hs_history"])
                new_p_node["hs_history"].append(p_node["hdr"])
                #print new_p_node

                # TODO: This definitely doesn't properly handle dropped packets...
                if next_p in edge_ports:
                  # We've reached our final destination!
                  # ASSUMPTION: no edge port will send it back out into the network...

                  # use the inverse T trick to get original headerspace which led us here
                  original_headers = find_loop_original_header(NTF,TTF,new_p_node)
                  for original_header in original_headers:
                    key = (original_header, start_port)
                    if not key in omega:
                      # TODO: python default value for hash?
                      omega[key] = []
                    omega[key].append((next_h, next_p))

                if linked_p in new_p_node["visits"]:
                  print "WARNING: detected a loop - branch aborted: \nHeaderSpace: %s\n Visited Ports: %s\nLast Port %d "%(\
                         new_p_node["hdr"],new_p_node["visits"],new_p_node["port"])
                else:
                  tmp_propag.append(new_p_node)
      propagation = tmp_propag
  return omega

def print_reachability(paths, reverse_map):
    for p_node in paths:
        str = ""
        for port in p_node["visits"]:
            if str == "":
                str = reverse_map["%d"%port]
            else:
                str = "%s ---> %s"%(str,reverse_map["%d"%port])
        str = "%s ---> %s"%(str,reverse_map["%d"%p_node["port"]])
        print "Path: %s"%str
        print "HS Received: %s"%p_node["hdr"]
        print "----------------------------------------------"

def print_loops(loops, reverse_map):
    for p_node in loops:
        print "----------------------------------------------"
        str = ""
        for port in p_node["visits"]:
            if str == "":
                str = reverse_map["%d"%port]
            else:
                str = "%s ---> %s"%(str,reverse_map["%d"%port])
        str = "%s ---> %s"%(str,reverse_map["%d"%p_node["port"]])
        print "Path: %s"%str
        rl_id =  "applied rules: "
        for (n,r,s) in p_node["hdr"].applied_rule_ids:
            rl_id = rl_id + " -> %s"%r
        print rl_id
        for i in range(len(p_node["hs_history"])):
            print "*** %d) AT PORT: %s\nHS: %s\n"%(i,reverse_map["%d"%p_node["visits"][i]],p_node["hs_history"][i])
        print "*** %d) AT PORT: %s\nHS: %s\n"%(i+1,reverse_map["%d"%p_node["port"]],p_node["hdr"])
        print "----------------------------------------------"

def loop_path_to_str(p_node, reverse_map):
    str = ""
    for port in p_node["visits"]:
        if str == "":
            str = reverse_map["%d"%port]
        else:
            str = "%s ---> %s"%(str,reverse_map["%d"%port])
    str = "%s ---> %s"%(str,reverse_map["%d"%p_node["port"]])
    return str

def trace_hs_back(applied_rule_list,hs,last_port):

    port = last_port
    hs_list = [hs]
    hs.applied_rule_ids = []
    for (tf,rule_id,in_port) in reversed(applied_rule_list):
        tmp = []
        for next_hs in hs_list:
            hp = tf.T_inv_rule(rule_id,next_hs,port)
            for (h,p_list) in hp:
                if in_port in p_list:
                    tmp.append(h)
        if tmp == []:
            break
        hs_list = tmp
        port = in_port
    return hs_list

def find_loop_original_header(NTF,TTF,propagation_node):
    applied_rule_ids = list(propagation_node["hdr"].applied_rule_ids)
    hs_list = trace_hs_back(applied_rule_ids,propagation_node["hdr"],propagation_node["port"])
    return hs_list

#def is_loop_infinite(NTF, TTF, propagation_node):
#    p_node = propagation_node
#    header_cpy = p_node["hdr"].copy()
#    while len(p_node['visits']) > 0:
#        cur_port = p_node['visits'].pop()
#
#         transfer back on the link
#        linked = TTF.T_inv(p_node["hdr"],p_node["port"])
#
#         transfer back on node
#        prev_hp = NTF.T_inv(linked[0][0],linked[0][1])
#
#        prev_hs = headerspace(NTF.length)
#        for (h,p) in prev_hp:
#            if p == cur_port:
#                prev_hs.add_hs(h)
#
#        p_node["hdr"] = prev_hs
#        p_node["port"] = cur_port

    #print "hret = %s and"%propagation_node["hdr"]
    #print "hret_inv = %s"%(p_node["hdr"])
#    if propagation_node["hdr"].is_subset_of(p_node["hdr"]):
#        return (True,p_node["hdr"])
#    else:
#        insect = propagation_node["hdr"].copy_intersect(p_node["hdr"])
#        if insect.count() == 0:
#            return (False,[])
#        else:
#            new_cpy = propagation_node.copy()
#            new_cpy["hdr"] = insect
#            return is_loop_infinite(NTF, TTF, new_cpy)

<|endoftext|>


<commit_before>""" Utilities to work with LaTeX files. """

import re
import requests
import unicodedata
import sys

def read_latex_file(latex_file, encoding="utf-8"):
    with open(latex_file, 'r', encoding=encoding) as data:
        latex_source = data.read()
    return latex_source

def write_latex_file(latex_file, latex_source):
    with open(latex_file, 'w') as open_file:
        open_file.write(latex_source)

def get_relevant_warnings(log_file):
    """Extract relevant warnings from a LaTeX log file."""
    overfull_lines = re.findall(r"Overfull \\hbox .*", log_file)
    undefined_references = re.findall(r"LaTeX Warning: Citation `.*?' on page .*", log_file)
    return overfull_lines + undefined_references

def remove_accented_characters(string):
    """Change accented characters to their ASCII equivalents."""
    return unicodedata.normalize('NFD', string).encode('ascii', 'ignore').decode('utf-8')

def open_webpage(address, exit_on_error=True):
    """Return the request server response for a webpage."""
    try:
        server_response = requests.get(address, timeout=10)
        server_response.raise_for_status()
    except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as e:
        if exit_on_error:
            sys.exit(e)
        else:
            return f"failed ({e})"
    else:
        return server_response
<commit_msg>Change to checking for latin-1 encoding automatically<commit_after>""" Utilities to work with LaTeX files. """

import re
import requests
import unicodedata
import sys

def read_latex_file(latex_file):
    try:
        with open(latex_file, 'r', encoding="utf-8") as data:
            latex_source = data.read()
    except UnicodeDecodeError:
        with open(latex_file, 'r', encoding="latin-1") as data:
            latex_source = data.read()
    return latex_source

def write_latex_file(latex_file, latex_source):
    with open(latex_file, 'w') as open_file:
        open_file.write(latex_source)

def get_relevant_warnings(log_file):
    """Extract relevant warnings from a LaTeX log file."""
    overfull_lines = re.findall(r"Overfull \\hbox .*", log_file)
    undefined_references = re.findall(r"LaTeX Warning: Citation `.*?' on page .*", log_file)
    return overfull_lines + undefined_references

def remove_accented_characters(string):
    """Change accented characters to their ASCII equivalents."""
    return unicodedata.normalize('NFD', string).encode('ascii', 'ignore').decode('utf-8')

def open_webpage(address, exit_on_error=True):
    """Return the request server response for a webpage."""
    try:
        server_response = requests.get(address, timeout=10)
        server_response.raise_for_status()
    except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as e:
        if exit_on_error:
            sys.exit(e)
        else:
            return f"failed ({e})"
    else:
        return server_response
<|endoftext|>


<commit_before>
import importlib.util
import inspect
import logging
import os
import types
from importlib import import_module
from pprint import pformat
from pydoc import locate

import tornado.log

import biothings.web.settings.default
from biothings import get_version
from biothings.web.handlers import BaseAPIHandler, BaseESRequestHandler
from biothings.web.options import OptionSets

try:
    from raven.contrib.tornado import AsyncSentryClient
except ImportError:
    __SENTRY_INSTALLED__ = False
else:
    __SENTRY_INSTALLED__ = True

class BiothingConfigError(Exception):
    pass

class BiothingWebSettings():
    '''
    A container for the settings that configure the web API.

    * Environment variables can override settings of the same names.
    * Default values are defined in biothings.web.settings.default.

    '''

    def __init__(self, config=None, **kwargs):
        '''
        :param config: a module that configures this biothing
            or its fully qualified name,
            or its module file path.
        '''
        self._default = biothings.web.settings.default
        self._user = self.load_module(config, self._default)
        self.logger.info("Biothings API %s", get_version())
        self.logger.info("%s", self._user)  # log file location

        # process keyword setting override
        for key, value in kwargs.items():
            setattr(self._user, key, value)

        # process environment variable override of named settings
        for name in os.environ:
            if hasattr(self, name) and isinstance(getattr(self, name), str):
                self.logger.info("Env %s = %s", name, os.environ[name])
                setattr(self._user, name, os.environ[name])

        # for metadata dev details
        if os.path.isdir(os.path.join(self.APP_GIT_REPOSITORY, '.git')):
            self._git_repo_path = self.APP_GIT_REPOSITORY
        else:
            self._git_repo_path = None

        self.optionsets = OptionSets()
        self.handlers = {}
        self.validate()

    @staticmethod
    def load_module(config, default=None):
        """
        Ensure config is a module.
        If config does not evaluate,
        Return default if it's provided.
        """
        if isinstance(config, types.ModuleType):
            return config
        elif isinstance(config, str) and config.endswith('.py'):
            spec = importlib.util.spec_from_file_location("config", config)
            config = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(config)
            return config
        elif isinstance(config, str) and config:
            return import_module(config)
        elif not config and default:
            return default
        raise BiothingConfigError()

    def __getattr__(self, name):
        if hasattr(self._user, name):
            return getattr(self._user, name)
        elif hasattr(self._default, name):
            return getattr(self._default, name)
        else:  # not provided and no default
            raise AttributeError()

    @property
    def logger(self):
        return logging.getLogger('biothings.web.settings')

    @staticmethod
    def load_class(kls):
        """
        Ensure config is a module.
        If config does not evaluate,
        Return default if it's provided.
        """
        if inspect.isclass(kls):
            return kls
        if isinstance(kls, str):
            return locate(kls)
        raise BiothingConfigError()

    def generate_app_settings(self, debug=False):
        """
        Generates settings for tornado.web.Application. This result and the
        method below can define a tornado application to start a web server.
        """
        settings = {
            'biothings': self,
            'debug': bool(debug)
        }
        supported_keywords = (
            'default_handler_class', 'default_handler_args',
            'log_function', 'compress_response', 'cookie_secret',
            'login_url', 'static_path', 'static_url_prefix')

        for setting in supported_keywords:
            if hasattr(self, setting.upper()):
                settings[setting] = getattr(self, setting.upper())

        if __SENTRY_INSTALLED__ and self.SENTRY_CLIENT_KEY:
            # Setup error monitoring with Sentry. More on:
            # https://docs.sentry.io/clients/python/integrations/#tornado
            settings['sentry_client'] = AsyncSentryClient(self.SENTRY_CLIENT_KEY)

        return settings

    def generate_app_handlers(self, addons=None):
        '''
        Generates the tornado.web.Application `(regex, handler_class, options) tuples
        <http://www.tornadoweb.org/en/stable/web.html#application-configuration>`_.
        '''
        handlers = {}
        addons = addons or []
        for rule in self.APP_LIST + addons:
            pattern = rule[0]
            handler = self.load_class(rule[1])
            setting = rule[2] if len(rule) == 3 else {}
            assert handler, rule[1]
            if issubclass(handler, BaseAPIHandler):
                handler_name = handler.name
                handler_options = handler.kwargs
                setting_attr = '_'.join((handler_name, 'kwargs')).upper()
                setting_options = getattr(self, setting_attr, {})
                self.optionsets.add(handler_name, handler_options)
                self.optionsets.add(handler_name, setting_options)
                self.optionsets.groups[handler_name] = handler.kwarg_groups
                self.optionsets.methods[handler_name] = handler.kwarg_methods
            if '{typ}' in pattern:
                if not issubclass(handler, BaseESRequestHandler):
                    raise BiothingConfigError()
                for biothing_type in self.BIOTHING_TYPES:
                    _pattern = pattern.format(
                        pre=self.API_PREFIX,
                        ver=self.API_VERSION,
                        typ=biothing_type).replace('//', '/')
                    _setting = dict(setting)
                    _setting['biothing_type'] = biothing_type
                    handlers[_pattern] = (_pattern, handler, _setting)
            elif '{pre}' in pattern or '{ver}' in pattern:
                pattern = pattern.format(
                    pre=self.API_PREFIX,
                    ver=self.API_VERSION).replace('//', '/')
                handlers[pattern] = (pattern, handler, setting)
            else:  # no pattern translation
                handlers[pattern] = (pattern, handler, setting)

        self.handlers = handlers
        handlers = list(handlers.values())
        self.logger.info('API Handlers:\n%s', pformat(handlers, width=200))
        return handlers

    def get_git_repo_path(self):
        '''
        Return the path of the codebase if the specified folder in settings exists or `None`.
        '''
        return self._git_repo_path

    def configure_logger(self, logger):
        '''
        Configure a logger's formatter to use the format defined in this web setting.
        '''
        try:
            if self.LOGGING_FORMAT and logger.hasHandlers():
                for handler in logger.handlers:
                    if isinstance(handler.formatter, tornado.log.LogFormatter):
                        handler.formatter._fmt = self.LOGGING_FORMAT
        except Exception:
            self.logger.exception('Error configuring logger %s.', logger)

    def validate(self):
        '''
        Validate the settings defined for this web server.
        '''
        assert self.API_VERSION or self.API_PREFIX
        assert isinstance(self.LIST_SIZE_CAP, int)
        assert isinstance(self.ACCESS_CONTROL_ALLOW_METHODS, str)
        assert isinstance(self.ACCESS_CONTROL_ALLOW_HEADERS, str)
<commit_msg>Add template path support<commit_after>
import importlib.util
import inspect
import logging
import os
import types
from importlib import import_module
from pprint import pformat
from pydoc import locate

import tornado.log

import biothings.web.settings.default
from biothings import get_version
from biothings.web.handlers import BaseAPIHandler, BaseESRequestHandler
from biothings.web.options import OptionSets

try:
    from raven.contrib.tornado import AsyncSentryClient
except ImportError:
    __SENTRY_INSTALLED__ = False
else:
    __SENTRY_INSTALLED__ = True

class BiothingConfigError(Exception):
    pass

class BiothingWebSettings():
    '''
    A container for the settings that configure the web API.

    * Environment variables can override settings of the same names.
    * Default values are defined in biothings.web.settings.default.

    '''

    def __init__(self, config=None, **kwargs):
        '''
        :param config: a module that configures this biothing
            or its fully qualified name,
            or its module file path.
        '''
        self._default = biothings.web.settings.default
        self._user = self.load_module(config, self._default)
        self.logger.info("Biothings API %s", get_version())
        self.logger.info("%s", self._user)  # log file location

        # process keyword setting override
        for key, value in kwargs.items():
            setattr(self._user, key, value)

        # process environment variable override of named settings
        for name in os.environ:
            if hasattr(self, name) and isinstance(getattr(self, name), str):
                self.logger.info("Env %s = %s", name, os.environ[name])
                setattr(self._user, name, os.environ[name])

        # for metadata dev details
        if os.path.isdir(os.path.join(self.APP_GIT_REPOSITORY, '.git')):
            self._git_repo_path = self.APP_GIT_REPOSITORY
        else:
            self._git_repo_path = None

        self.optionsets = OptionSets()
        self.handlers = {}
        self.validate()

    @staticmethod
    def load_module(config, default=None):
        """
        Ensure config is a module.
        If config does not evaluate,
        Return default if it's provided.
        """
        if isinstance(config, types.ModuleType):
            return config
        elif isinstance(config, str) and config.endswith('.py'):
            spec = importlib.util.spec_from_file_location("config", config)
            config = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(config)
            return config
        elif isinstance(config, str) and config:
            return import_module(config)
        elif not config and default:
            return default
        raise BiothingConfigError()

    def __getattr__(self, name):
        if hasattr(self._user, name):
            return getattr(self._user, name)
        elif hasattr(self._default, name):
            return getattr(self._default, name)
        else:  # not provided and no default
            raise AttributeError()

    @property
    def logger(self):
        return logging.getLogger('biothings.web.settings')

    @staticmethod
    def load_class(kls):
        """
        Ensure config is a module.
        If config does not evaluate,
        Return default if it's provided.
        """
        if inspect.isclass(kls):
            return kls
        if isinstance(kls, str):
            return locate(kls)
        raise BiothingConfigError()

    def generate_app_settings(self, debug=False):
        """
        Generates settings for tornado.web.Application. This result and the
        method below can define a tornado application to start a web server.
        """
        settings = {
            'biothings': self,
            'debug': bool(debug)
        }
        supported_keywords = (
            'default_handler_class', 'default_handler_args', 'template_path',
            'log_function', 'compress_response', 'cookie_secret',
            'login_url', 'static_path', 'static_url_prefix')

        for setting in supported_keywords:
            if hasattr(self, setting.upper()):
                settings[setting] = getattr(self, setting.upper())

        if __SENTRY_INSTALLED__ and self.SENTRY_CLIENT_KEY:
            # Setup error monitoring with Sentry. More on:
            # https://docs.sentry.io/clients/python/integrations/#tornado
            settings['sentry_client'] = AsyncSentryClient(self.SENTRY_CLIENT_KEY)

        return settings

    def generate_app_handlers(self, addons=None):
        '''
        Generates the tornado.web.Application `(regex, handler_class, options) tuples
        <http://www.tornadoweb.org/en/stable/web.html#application-configuration>`_.
        '''
        handlers = {}
        addons = addons or []
        for rule in self.APP_LIST + addons:
            pattern = rule[0]
            handler = self.load_class(rule[1])
            setting = rule[2] if len(rule) == 3 else {}
            assert handler, rule[1]
            if issubclass(handler, BaseAPIHandler):
                handler_name = handler.name
                handler_options = handler.kwargs
                setting_attr = '_'.join((handler_name, 'kwargs')).upper()
                setting_options = getattr(self, setting_attr, {})
                self.optionsets.add(handler_name, handler_options)
                self.optionsets.add(handler_name, setting_options)
                self.optionsets.groups[handler_name] = handler.kwarg_groups
                self.optionsets.methods[handler_name] = handler.kwarg_methods
            if '{typ}' in pattern:
                if not issubclass(handler, BaseESRequestHandler):
                    raise BiothingConfigError()
                for biothing_type in self.BIOTHING_TYPES:
                    _pattern = pattern.format(
                        pre=self.API_PREFIX,
                        ver=self.API_VERSION,
                        typ=biothing_type).replace('//', '/')
                    _setting = dict(setting)
                    _setting['biothing_type'] = biothing_type
                    handlers[_pattern] = (_pattern, handler, _setting)
            elif '{pre}' in pattern or '{ver}' in pattern:
                pattern = pattern.format(
                    pre=self.API_PREFIX,
                    ver=self.API_VERSION).replace('//', '/')
                handlers[pattern] = (pattern, handler, setting)
            else:  # no pattern translation
                handlers[pattern] = (pattern, handler, setting)

        self.handlers = handlers
        handlers = list(handlers.values())
        self.logger.info('API Handlers:\n%s', pformat(handlers, width=200))
        return handlers

    def get_git_repo_path(self):
        '''
        Return the path of the codebase if the specified folder in settings exists or `None`.
        '''
        return self._git_repo_path

    def configure_logger(self, logger):
        '''
        Configure a logger's formatter to use the format defined in this web setting.
        '''
        try:
            if self.LOGGING_FORMAT and logger.hasHandlers():
                for handler in logger.handlers:
                    if isinstance(handler.formatter, tornado.log.LogFormatter):
                        handler.formatter._fmt = self.LOGGING_FORMAT
        except Exception:
            self.logger.exception('Error configuring logger %s.', logger)

    def validate(self):
        '''
        Validate the settings defined for this web server.
        '''
        assert self.API_VERSION or self.API_PREFIX
        assert isinstance(self.LIST_SIZE_CAP, int)
        assert isinstance(self.ACCESS_CONTROL_ALLOW_METHODS, str)
        assert isinstance(self.ACCESS_CONTROL_ALLOW_HEADERS, str)
<|endoftext|>