<commit_before>#!/usr/bin/env python2.7
from Queue import Empty
from collections import Counter
from datetime import timedelta
from multiprocessing import Pool
import multiprocessing.pool
import multiprocessing
import sys
import argparse
import importlib
import os
import time
import pymongo
import pyprind
import yaml

from nupic.frameworks.opf.modelfactory import ModelFactory
from nupic.algorithms import anomaly_likelihood

global CACHE_MODELS

def getEngineDir():
    return os.path.dirname(os.path.realpath(__file__))

try:
    path = os.path.join(os.path.dirname(getEngineDir()), 'connection.yaml')
    with open(path, 'r') as f:
        conf = yaml.load(f)
        mongo_uri = conf['mongo_uri']
        mongo_database = conf['mongo_database']
        mongo_collection = conf['mongo_collection']
        MODEL_PARAMS_DIR = conf['MODEL_PARAMS_DIR']
        MODEL_CACHE_DIR = conf['MODEL_CACHE_DIR']
        SWARM_CONFIGS_DIR = conf['SWARM_CONFIGS_DIR']
        max_vehicles = conf['max_vehicles']
except:
    raise Exception('No connection.yaml with mongo_uri defined! please make one with a mongo_uri variable')

DESCRIPTION = """
Makes and runs a NuPIC model for an intersection in Adelaide and
determines its anomaly score based on historical traffic flow
"""

def setupFolders():
    for i in [MODEL_CACHE_DIR, MODEL_PARAMS_DIR, SWARM_CONFIGS_DIR]:
        directory = os.path.join(getEngineDir(), i)
        if not os.path.isdir(directory):
            print "Creating directory:", directory
            os.makedirs(directory)
            open(os.path.join(directory, '__init__.py'), 'wa').close()

def getModelDir(intersection):
    return os.path.join(getEngineDir(), MODEL_CACHE_DIR, intersection)

client = pymongo.MongoClient(mongo_uri, w=0)
readings_collection = client[mongo_database][mongo_collection]
locations_collection = client[mongo_database]['locations']

def get_most_used_sensors(intersection):
    records = readings_collection.find({'site_no': intersection})
    counter = Counter()
    for i in records:
        for s, c in i['readings'].items():
            if c < max_vehicles:
                counter[s] += c
    return counter

def get_sensor_encoder(name):
    return {
        'fieldname': name,
        'name': name,
        'clipInput': True,
        'minval': 0.0,
        'maxval': max_vehicles,
        'n': 50,
        'w': 21,
        'type': 'ScalarEncoder'
    }

def get_time_encoders():
    return [{
        'fieldname': u'timestamp',
        'name': u'timestamp_timeOfDay',
        'timeOfDay': (21, 9.5),
        'type': 'DateEncoder'
    }, {
        'fieldname': 'timestamp',
        'name': 'timestamp_dayOfWeek',
        'type': 'DateEncoder',
        'dayOfWeek': (21, 1)
    }]

def createModel(intersection):
    modelDir = getModelDir(intersection)
    if CACHE_MODELS and os.path.isdir(modelDir):
        # Read in the cached model
        print "Loading cached model for {} from {}...".format(intersection, modelDir)
        return ModelFactory.loadFromCheckpoint(modelDir)
    else:
        start = time.time()
        # redo the modelParams to use the actual sensor names
        modelParams = getModelParamsFromName('3001')

        sensor_counts = get_most_used_sensors(intersection)
        # try:
        #     pField = getSwarmConfig(intersection)['inferenceArgs']['predictedField']
        # except:
        #     print "Determining most used sensor for ", intersection
        try:
            counts = sensor_counts.most_common(1)
            if counts[0][1] == 0:
                return None
            else:
                pField = counts[0][0]
        except:
            return None
        print "Using", pField, "as predictedField for", intersection
        doc = readings_collection.find_one({'site_no': intersection})

        for k in doc['readings']:
            # don't model unused sensors
            # could run into errors when the sensor
            # was damaged for more than the sample period though
            if sensor_counts[k] == 0 or k != pField:
                continue
            modelParams['modelParams']['sensorParams']['encoders'][k] = get_sensor_encoder(k)

        for i in get_time_encoders():
            modelParams['modelParams']['sensorParams']['encoders'][i['name']] = i

        model = ModelFactory.create(modelParams)
        model.enableInference({'predictedField': pField})
        print "Creating model for {}, in {}s".format(intersection, time.time() - start)
        return model

def create_single_sensor_model(sensor, intersection):
    start = time.time()
    model_params = getModelParamsFromName('3001')
    model_params['modelParams']['sensorParams']['encoders'][sensor] = get_sensor_encoder(sensor)
    for i in get_time_encoders():
        model_params['modelParams']['sensorParams']['encoders'][i['name']] = i
    model = ModelFactory.create(model_params)
    model.enableInference({'predictedField': sensor})
  #  print "Creating model for {}:{} in {}s on pid {}".format(intersection, sensor, time.time() - start, os.getpid())
    return model

def setup_location_sensors(intersection):
    if not intersection:
        query = {}
    else:
        query = {'intersection_number': {'$in': intersection.split(',')}}
    for i in locations_collection.find(query):
        counts = get_most_used_sensors(i['intersection_number']).most_common()
        if len(counts) == 0:
            continue
        locations_collection.update_one({'_id': i['_id']},
                             {'$set': {'sensors':
                                           [i[0] for i in counts if i[1] != 0]
                                  }})

def getMax():
    readings = readings_collection.find()
    print "Max vehicle count:", max([max(
        filter(lambda x: x < max_vehicles, i.values())) for i in readings])

def getModelParamsFromName(intersection):
    """
    Given an intersection name, assumes a matching model params python module exists within
    the model_params directory and attempts to import it.
    :param intersection: intersection name, used to guess the model params module name.
    :return: OPF Model params dictionary
    """
    importName = "%s.model_params_%s" % (MODEL_PARAMS_DIR, intersection)
    # print "Importing model params from %s" % importName
    try:
        importedModelParams = importlib.import_module(importName).MODEL_PARAMS
    except ImportError:
       raise sys.exit("No model params exist for '%s'. Run swarm first!" % intersection)
    importedModelParams['modelParams']['sensorParams']['encoders'].clear()
    return importedModelParams

def get_encoders(model):
    return set([i.name for i in model._getSensorRegion().getSelf().encoder.getEncoderList()])

class Worker(multiprocessing.Process):
    def __init__(self, sensor, intersection):
        super(Worker, self).__init__()
        self.queue_in = multiprocessing.Queue()
        self.queue_out = multiprocessing.Queue()
        self.done = False
        self.sensor = sensor
        self.intersection = intersection

    def run(self):
        anomaly_likelihood_helper = anomaly_likelihood.AnomalyLikelihood(50, 50)
        model = create_single_sensor_model(self.sensor, self.intersection)
        while not self.done:
            try:
                val = self.queue_in.get(True, 1)
            except Empty:
                continue

            result = model.run(val)
            prediction = result.inferences["multiStepBestPredictions"][1]
            anomaly_score = result.inferences["anomalyScore"]
            if val[self.sensor] is None:
                likelihood = None
            else:
                likelihood = anomaly_likelihood_helper.anomalyProbability(
                    val[self.sensor], anomaly_score, val['timestamp'])
            self.queue_out.put((self.sensor, prediction, anomaly_score, likelihood))
        # could probably serialize the model here

    def finish(self):
        self.done = True

def process_readings(readings, intersection, write_anomaly, progress=True, multi_model=False, smoothing=True):
    counter = 0
    total = readings.count(True)

    if multi_model:

        loc = locations_collection.find_one({'intersection_number': intersection})
        models = {}
        for sensor in loc['sensors']:
            models[sensor] = Worker(sensor, intersection)
            models[sensor].start()
    else:
        model = createModel(intersection)
        anomaly_likelihood_helper = anomaly_likelihood.AnomalyLikelihood(1000, 200)
        if model is None:
            print "No model could be made for intersection", intersection
            return
        pfield = model.getInferenceArgs()['predictedField']
        encoders = get_encoders(model)
    if progress:
        progBar = pyprind.ProgBar(total, width=50)
    previous = None
    for i in readings:
        counter += 1
        if progress:
            progBar.update()
        timestamp = i['datetime']

        if multi_model:
            predictions, anomalies = {}, {}
            for sensor, proc in models.iteritems():
                vc = i['readings'][sensor]
                if vc > max_vehicles:
                    vc = None
                elif smoothing and previous:
                    vc = (vc + previous['readings'][sensor])/2.
                fields = {"timestamp": timestamp, sensor: vc}
                proc.queue_in.put(fields)
            for sensor, proc in models.iteritems():
                result = proc.queue_out.get()
                # (self.sensor, prediction, anomaly_score, likelihood)
                anomalies[result[0]] = {'score': result[2], 'likelihood': result[3]}
                predictions[result[0]] = result[1]
        else:
            fields = {"timestamp": timestamp}
            for p, j in enumerate(i['readings'].items()):
                if j[0] not in encoders:
                    continue
                vc = j[1]
                if vc > max_vehicles:
                    vc = None
                fields[j[0]] = vc
            result = model.run(fields)
            prediction = result.inferences["multiStepBestPredictions"][1]
            anomaly_score = result.inferences["anomalyScore"]
            predictions = {pfield: prediction}
            likelihood = anomaly_likelihood_helper.anomalyProbability(
                i['readings'][pfield], anomaly_score, timestamp)
            anomalies = {pfield: {'score': anomaly_score, 'likelihood': likelihood}}
        if write_anomaly:
            write_anomaly_out(i, anomalies, predictions)
        if smoothing:
            previous = dict(i)
    if multi_model:
        for proc in models.values():
            proc.terminate()
    else:
        save_model(model, intersection)
    if progress:
        print
    print "Read", counter, "lines"

def write_anomaly_out(doc, anomalies, predictions):
    readings_collection.update_one({"_id": doc["_id"]},
                          {"$set": {"anomalies": anomalies}})
    next_doc = readings_collection.find_one({'site_no': doc['site_no'],
                                   'datetime': doc['datetime'] + timedelta(minutes=5)})
    if next_doc is not None:
        readings_collection.update_one({'_id': next_doc['_id']},
                              {"$set": {"predictions": predictions}})

def save_model(model, site_no):
    if not CACHE_MODELS:
        return
    if model is None:
        print "Not saving model for", site_no
        return
    start = time.time()
    out_dir = getModelDir(site_no)
    model.save(out_dir)
    print "Caching model to {} in {}s".format(out_dir, time.time() - start)

def run_single_intersection(args):
    intersection, write_anomaly, incomplete, show_progress, multi_model, smooth = args[0], args[1], args[2], args[3],\
                                                                                args[4], args[5]
    start_time = time.time()

    query = {'site_no': intersection}
    if incomplete:
        query['anomaly'] = {'$exists': False}
    readings = readings_collection.find(query, {'anomalies': False, 'predictions': False}, no_cursor_timeout=True).\
        sort('datetime', pymongo.ASCENDING)
    if readings.count(True) == 0:
        print "No readings for intersection {}".format(intersection)
        return
    process_readings(readings, intersection, write_anomaly, show_progress, multi_model)

    print("Intersection %s complete: --- %s seconds ---" % (intersection, time.time() - start_time))

def run_all_intersections(write_anomaly, incomplete, intersections, multi_model, smooth):
    print "Running all on", os.getpid()
    start_time = time.time()

    if incomplete:
        key = '_id'
        query = [
            {'$match': {'anomaly': {'$exists': False}}},
            {'$group': {'_id': '$site_no'}}
        ]
        if intersections != '':
            query[0]['$match']['site_no'] = {'$in': intersections.split(',')}
        locations = list(readings_collection.aggregate(query))

    else:
        key = 'intersection_number'
        if intersections != '':
            query = {key: {'$in': intersections.split(',')}}
        else:
            query = {key:  {'$regex': '3\d\d\d'}}
        locations = list(locations_collection.find(query))
    gen = [(str(l[key]), write_anomaly, incomplete, False, multi_model, smooth) for l in locations]
    pool = Pool(8, maxtasksperchild=1)
    pool.map(run_single_intersection, gen)
    print("TOTAL TIME: --- %s seconds ---" % (time.time() - start_time))

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=DESCRIPTION)
    parser.add_argument('--write-anomaly', help="Write the anomaly score back into the document", action='store_true')
    parser.add_argument('--all', help="Run all readings through model", action="store_true")
    parser.add_argument('--intersection', type=str, help="Name of the intersection", default='')
    parser.add_argument('--incomplete', help="Analyse those intersections not done yet", action='store_true')
    parser.add_argument('--popular', help="Show the most popular sensor for an intersection", action='store_true')
    parser.add_argument('--cache-models', help="Cache models", action='store_true')
    parser.add_argument('--setup-sensors', help='store used sensors in locations. Use --intersection to specify '
                                                'many otherwise it will do all of them', action='store_true')
    parser.add_argument('--multi-model', help="Use a model per sensor", action='store_true')
    parser.add_argument('--smooth', help="Smooth the readings values using a mean filter with"
                                        "window size=2", action='store_true')
    args = parser.parse_args()
    if args.setup_sensors:
        setup_location_sensors(args.intersection)
        sys.exit()
    CACHE_MODELS = args.cache_models
    setupFolders()
    if args.all:
        run_all_intersections(args.write_anomaly, args.incomplete, args.intersection, args.multi_model, args)
    elif args.popular:
        print "Lane usage for ", args.intersection, "is:  "
        for i, j in get_most_used_sensors(args.intersection).most_common():
            print '\t', i, j
    else:
        if args.intersection == '':
            parser.error("Please specify an intersection")
        run_single_intersection((args.intersection, args.write_anomaly, args.incomplete, True, args.multi_model, args.smooth))
<commit_msg>stopped recording anomaly scores for none values too<commit_after>#!/usr/bin/env python2.7
from Queue import Empty
from collections import Counter
from datetime import timedelta
from multiprocessing import Pool
import multiprocessing.pool
import multiprocessing
import sys
import argparse
import importlib
import os
import time
import pymongo
import pyprind
import yaml

from nupic.frameworks.opf.modelfactory import ModelFactory
from nupic.algorithms import anomaly_likelihood

global CACHE_MODELS

def getEngineDir():
    return os.path.dirname(os.path.realpath(__file__))

try:
    path = os.path.join(os.path.dirname(getEngineDir()), 'connection.yaml')
    with open(path, 'r') as f:
        conf = yaml.load(f)
        mongo_uri = conf['mongo_uri']
        mongo_database = conf['mongo_database']
        mongo_collection = conf['mongo_collection']
        MODEL_PARAMS_DIR = conf['MODEL_PARAMS_DIR']
        MODEL_CACHE_DIR = conf['MODEL_CACHE_DIR']
        SWARM_CONFIGS_DIR = conf['SWARM_CONFIGS_DIR']
        max_vehicles = conf['max_vehicles']
except:
    raise Exception('No connection.yaml with mongo_uri defined! please make one with a mongo_uri variable')

DESCRIPTION = """
Makes and runs a NuPIC model for an intersection in Adelaide and
determines its anomaly score based on historical traffic flow
"""

def setupFolders():
    for i in [MODEL_CACHE_DIR, MODEL_PARAMS_DIR, SWARM_CONFIGS_DIR]:
        directory = os.path.join(getEngineDir(), i)
        if not os.path.isdir(directory):
            print "Creating directory:", directory
            os.makedirs(directory)
            open(os.path.join(directory, '__init__.py'), 'wa').close()

def getModelDir(intersection):
    return os.path.join(getEngineDir(), MODEL_CACHE_DIR, intersection)

client = pymongo.MongoClient(mongo_uri, w=0)
readings_collection = client[mongo_database][mongo_collection]
locations_collection = client[mongo_database]['locations']

def get_most_used_sensors(intersection):
    records = readings_collection.find({'site_no': intersection})
    counter = Counter()
    for i in records:
        for s, c in i['readings'].items():
            if c < max_vehicles:
                counter[s] += c
    return counter

def get_sensor_encoder(name):
    return {
        'fieldname': name,
        'name': name,
        'clipInput': True,
        'minval': 0.0,
        'maxval': max_vehicles,
        'n': 50,
        'w': 21,
        'type': 'ScalarEncoder'
    }

def get_time_encoders():
    return [{
        'fieldname': u'timestamp',
        'name': u'timestamp_timeOfDay',
        'timeOfDay': (21, 9.5),
        'type': 'DateEncoder'
    }, {
        'fieldname': 'timestamp',
        'name': 'timestamp_dayOfWeek',
        'type': 'DateEncoder',
        'dayOfWeek': (21, 1)
    }]

def createModel(intersection):
    modelDir = getModelDir(intersection)
    if CACHE_MODELS and os.path.isdir(modelDir):
        # Read in the cached model
        print "Loading cached model for {} from {}...".format(intersection, modelDir)
        return ModelFactory.loadFromCheckpoint(modelDir)
    else:
        start = time.time()
        # redo the modelParams to use the actual sensor names
        modelParams = getModelParamsFromName('3001')

        sensor_counts = get_most_used_sensors(intersection)
        # try:
        #     pField = getSwarmConfig(intersection)['inferenceArgs']['predictedField']
        # except:
        #     print "Determining most used sensor for ", intersection
        try:
            counts = sensor_counts.most_common(1)
            if counts[0][1] == 0:
                return None
            else:
                pField = counts[0][0]
        except:
            return None
        print "Using", pField, "as predictedField for", intersection
        doc = readings_collection.find_one({'site_no': intersection})

        for k in doc['readings']:
            # don't model unused sensors
            # could run into errors when the sensor
            # was damaged for more than the sample period though
            if sensor_counts[k] == 0 or k != pField:
                continue
            modelParams['modelParams']['sensorParams']['encoders'][k] = get_sensor_encoder(k)

        for i in get_time_encoders():
            modelParams['modelParams']['sensorParams']['encoders'][i['name']] = i

        model = ModelFactory.create(modelParams)
        model.enableInference({'predictedField': pField})
        print "Creating model for {}, in {}s".format(intersection, time.time() - start)
        return model

def create_single_sensor_model(sensor, intersection):
    start = time.time()
    model_params = getModelParamsFromName('3001')
    model_params['modelParams']['sensorParams']['encoders'][sensor] = get_sensor_encoder(sensor)
    for i in get_time_encoders():
        model_params['modelParams']['sensorParams']['encoders'][i['name']] = i
    model = ModelFactory.create(model_params)
    model.enableInference({'predictedField': sensor})
  #  print "Creating model for {}:{} in {}s on pid {}".format(intersection, sensor, time.time() - start, os.getpid())
    return model

def setup_location_sensors(intersection):
    if not intersection:
        query = {}
    else:
        query = {'intersection_number': {'$in': intersection.split(',')}}
    for i in locations_collection.find(query):
        counts = get_most_used_sensors(i['intersection_number']).most_common()
        if len(counts) == 0:
            continue
        locations_collection.update_one({'_id': i['_id']},
                             {'$set': {'sensors':
                                           [i[0] for i in counts if i[1] != 0]
                                  }})

def getMax():
    readings = readings_collection.find()
    print "Max vehicle count:", max([max(
        filter(lambda x: x < max_vehicles, i.values())) for i in readings])

def getModelParamsFromName(intersection):
    """
    Given an intersection name, assumes a matching model params python module exists within
    the model_params directory and attempts to import it.
    :param intersection: intersection name, used to guess the model params module name.
    :return: OPF Model params dictionary
    """
    importName = "%s.model_params_%s" % (MODEL_PARAMS_DIR, intersection)
    # print "Importing model params from %s" % importName
    try:
        importedModelParams = importlib.import_module(importName).MODEL_PARAMS
    except ImportError:
       raise sys.exit("No model params exist for '%s'. Run swarm first!" % intersection)
    importedModelParams['modelParams']['sensorParams']['encoders'].clear()
    return importedModelParams

def get_encoders(model):
    return set([i.name for i in model._getSensorRegion().getSelf().encoder.getEncoderList()])

class Worker(multiprocessing.Process):
    def __init__(self, sensor, intersection):
        super(Worker, self).__init__()
        self.queue_in = multiprocessing.Queue()
        self.queue_out = multiprocessing.Queue()
        self.done = False
        self.sensor = sensor
        self.intersection = intersection

    def run(self):
        anomaly_likelihood_helper = anomaly_likelihood.AnomalyLikelihood(50, 50)
        model = create_single_sensor_model(self.sensor, self.intersection)
        while not self.done:
            try:
                val = self.queue_in.get(True, 1)
            except Empty:
                continue

            result = model.run(val)
            prediction = result.inferences["multiStepBestPredictions"][1]

            if val[self.sensor] is None:
                anomaly_score = None
                likelihood = None
            else:
                anomaly_score = result.inferences["anomalyScore"]
                likelihood = anomaly_likelihood_helper.anomalyProbability(
                    val[self.sensor], anomaly_score, val['timestamp'])
            self.queue_out.put((self.sensor, prediction, anomaly_score, likelihood))
        # could probably serialize the model here

    def finish(self):
        self.done = True

def process_readings(readings, intersection, write_anomaly, progress=True, multi_model=False, smoothing=True):
    counter = 0
    total = readings.count(True)

    if multi_model:

        loc = locations_collection.find_one({'intersection_number': intersection})
        models = {}
        for sensor in loc['sensors']:
            models[sensor] = Worker(sensor, intersection)
            models[sensor].start()
    else:
        model = createModel(intersection)
        anomaly_likelihood_helper = anomaly_likelihood.AnomalyLikelihood(1000, 200)
        if model is None:
            print "No model could be made for intersection", intersection
            return
        pfield = model.getInferenceArgs()['predictedField']
        encoders = get_encoders(model)
    if progress:
        progBar = pyprind.ProgBar(total, width=50)
    previous = None
    for i in readings:
        counter += 1
        if progress:
            progBar.update()
        timestamp = i['datetime']

        if multi_model:
            predictions, anomalies = {}, {}
            for sensor, proc in models.iteritems():
                vc = i['readings'][sensor]
                if vc > max_vehicles:
                    vc = None
                elif smoothing and previous:
                    vc = (vc + previous['readings'][sensor])/2.
                fields = {"timestamp": timestamp, sensor: vc}
                proc.queue_in.put(fields)
            for sensor, proc in models.iteritems():
                result = proc.queue_out.get()
                # (self.sensor, prediction, anomaly_score, likelihood)
                anomalies[result[0]] = {'score': result[2], 'likelihood': result[3]}
                predictions[result[0]] = result[1]
        else:
            fields = {"timestamp": timestamp}
            for p, j in enumerate(i['readings'].items()):
                if j[0] not in encoders:
                    continue
                vc = j[1]
                if vc > max_vehicles:
                    vc = None
                fields[j[0]] = vc
            result = model.run(fields)
            prediction = result.inferences["multiStepBestPredictions"][1]
            anomaly_score = result.inferences["anomalyScore"]
            predictions = {pfield: prediction}
            likelihood = anomaly_likelihood_helper.anomalyProbability(
                i['readings'][pfield], anomaly_score, timestamp)
            anomalies = {pfield: {'score': anomaly_score, 'likelihood': likelihood}}
        if write_anomaly:
            write_anomaly_out(i, anomalies, predictions)
        if smoothing:
            previous = dict(i)
    if multi_model:
        for proc in models.values():
            proc.terminate()
    else:
        save_model(model, intersection)
    if progress:
        print
    print "Read", counter, "lines"

def write_anomaly_out(doc, anomalies, predictions):
    readings_collection.update_one({"_id": doc["_id"]},
                          {"$set": {"anomalies": anomalies}})
    next_doc = readings_collection.find_one({'site_no': doc['site_no'],
                                   'datetime': doc['datetime'] + timedelta(minutes=5)})
    if next_doc is not None:
        readings_collection.update_one({'_id': next_doc['_id']},
                              {"$set": {"predictions": predictions}})

def save_model(model, site_no):
    if not CACHE_MODELS:
        return
    if model is None:
        print "Not saving model for", site_no
        return
    start = time.time()
    out_dir = getModelDir(site_no)
    model.save(out_dir)
    print "Caching model to {} in {}s".format(out_dir, time.time() - start)

def run_single_intersection(args):
    intersection, write_anomaly, incomplete, show_progress, multi_model, smooth = args[0], args[1], args[2], args[3],\
                                                                                args[4], args[5]
    start_time = time.time()

    query = {'site_no': intersection}
    if incomplete:
        query['anomaly'] = {'$exists': False}
    readings = readings_collection.find(query, {'anomalies': False, 'predictions': False}, no_cursor_timeout=True).\
        sort('datetime', pymongo.ASCENDING)
    if readings.count(True) == 0:
        print "No readings for intersection {}".format(intersection)
        return
    process_readings(readings, intersection, write_anomaly, show_progress, multi_model)

    print("Intersection %s complete: --- %s seconds ---" % (intersection, time.time() - start_time))

def run_all_intersections(write_anomaly, incomplete, intersections, multi_model, smooth):
    print "Running all on", os.getpid()
    start_time = time.time()

    if incomplete:
        key = '_id'
        query = [
            {'$match': {'anomaly': {'$exists': False}}},
            {'$group': {'_id': '$site_no'}}
        ]
        if intersections != '':
            query[0]['$match']['site_no'] = {'$in': intersections.split(',')}
        locations = list(readings_collection.aggregate(query))

    else:
        key = 'intersection_number'
        if intersections != '':
            query = {key: {'$in': intersections.split(',')}}
        else:
            query = {key:  {'$regex': '3\d\d\d'}}
        locations = list(locations_collection.find(query))
    gen = [(str(l[key]), write_anomaly, incomplete, False, multi_model, smooth) for l in locations]
    pool = Pool(8, maxtasksperchild=1)
    pool.map(run_single_intersection, gen)
    print("TOTAL TIME: --- %s seconds ---" % (time.time() - start_time))

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=DESCRIPTION)
    parser.add_argument('--write-anomaly', help="Write the anomaly score back into the document", action='store_true')
    parser.add_argument('--all', help="Run all readings through model", action="store_true")
    parser.add_argument('--intersection', type=str, help="Name of the intersection", default='')
    parser.add_argument('--incomplete', help="Analyse those intersections not done yet", action='store_true')
    parser.add_argument('--popular', help="Show the most popular sensor for an intersection", action='store_true')
    parser.add_argument('--cache-models', help="Cache models", action='store_true')
    parser.add_argument('--setup-sensors', help='store used sensors in locations. Use --intersection to specify '
                                                'many otherwise it will do all of them', action='store_true')
    parser.add_argument('--multi-model', help="Use a model per sensor", action='store_true')
    parser.add_argument('--smooth', help="Smooth the readings values using a mean filter with"
                                        "window size=2", action='store_true')
    args = parser.parse_args()
    if args.setup_sensors:
        setup_location_sensors(args.intersection)
        sys.exit()
    CACHE_MODELS = args.cache_models
    setupFolders()
    if args.all:
        run_all_intersections(args.write_anomaly, args.incomplete, args.intersection, args.multi_model, args)
    elif args.popular:
        print "Lane usage for ", args.intersection, "is:  "
        for i, j in get_most_used_sensors(args.intersection).most_common():
            print '\t', i, j
    else:
        if args.intersection == '':
            parser.error("Please specify an intersection")
        run_single_intersection((args.intersection, args.write_anomaly, args.incomplete, True, args.multi_model, args.smooth))
<|endoftext|>


<commit_before># Copyright (C) 2014-2015 Andrey Antukh <niwi@niwi.be>
# Copyright (C) 2014-2015 Jesús Espino <jespinog@gmail.com>
# Copyright (C) 2014-2015 David Barragán <bameda@dbarragan.com>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from django.apps import apps
from taiga.base.api import serializers
from taiga.base.fields import TagsField
from taiga.base.fields import PgArrayField
from taiga.base.neighbors import NeighborsSerializerMixin
from taiga.base.utils import json

from taiga.mdrender.service import render as mdrender
from taiga.projects.validators import ProjectExistsValidator
from taiga.projects.validators import UserStoryStatusExistsValidator
from taiga.projects.userstories.validators import UserStoryExistsValidator
from taiga.projects.notifications.validators import WatchersValidator
from taiga.projects.serializers import BasicUserStoryStatusSerializer
from taiga.projects.notifications.mixins import EditableWatchedResourceModelSerializer
from taiga.projects.votes.mixins.serializers import VoteResourceSerializerMixin

from taiga.users.serializers import UserBasicInfoSerializer

from . import models

class RolePointsField(serializers.WritableField):
    def to_native(self, obj):
        return {str(o.role.id): o.points.id for o in obj.all()}

    def from_native(self, obj):
        if isinstance(obj, dict):
            return obj
        return json.loads(obj)

class MilestoneUserStorySerializer(serializers.ModelSerializer):
    total_points = serializers.SerializerMethodField("get_total_points")

    class Meta:
        model = models.UserStory
        depth = 0
        fields = ("id", "ref", "subject", "is_closed", "is_blocked", "total_points")

    def get_total_points(self, obj):
        return obj.get_total_points()

class UserStorySerializer(WatchersValidator, VoteResourceSerializerMixin, EditableWatchedResourceModelSerializer, serializers.ModelSerializer):
    tags = TagsField(default=[], required=False)
    external_reference = PgArrayField(required=False)
    points = RolePointsField(source="role_points", required=False)
    total_points = serializers.SerializerMethodField("get_total_points")
    comment = serializers.SerializerMethodField("get_comment")
    milestone_slug = serializers.SerializerMethodField("get_milestone_slug")
    milestone_name = serializers.SerializerMethodField("get_milestone_name")
    origin_issue = serializers.SerializerMethodField("get_origin_issue")
    blocked_note_html = serializers.SerializerMethodField("get_blocked_note_html")
    description_html = serializers.SerializerMethodField("get_description_html")
    status_extra_info = BasicUserStoryStatusSerializer(source="status", required=False, read_only=True)
    assigned_to_extra_info = UserBasicInfoSerializer(source="assigned_to", required=False, read_only=True)
    owner_extra_info = UserBasicInfoSerializer(source="owner", required=False, read_only=True)

    class Meta:
        model = models.UserStory
        depth = 0
        read_only_fields = ('created_date', 'modified_date')

    def get_total_points(self, obj):
        return obj.get_total_points()

    def get_comment(self, obj):
        # NOTE: This method and field is necessary to historical comments work
        return ""

    def get_milestone_slug(self, obj):
        if obj.milestone:
            return obj.milestone.slug
        else:
            return None

    def get_milestone_name(self, obj):
        if obj.milestone:
            return obj.milestone.name
        else:
            return None

    def get_origin_issue(self, obj):
        if obj.generated_from_issue:
            return {
                "id": obj.generated_from_issue.id,
                "ref": obj.generated_from_issue.ref,
                "subject": obj.generated_from_issue.subject,
            }
        return None

    def get_blocked_note_html(self, obj):
        return mdrender(obj.project, obj.blocked_note)

    def get_description_html(self, obj):
        return mdrender(obj.project, obj.description)

class UserStoryListSerializer(UserStorySerializer):
    class Meta:
        model = models.UserStory
        depth = 0
        read_only_fields = ('created_date', 'modified_date')
        exclude=("description", "description_html")

class UserStoryNeighborsSerializer(NeighborsSerializerMixin, UserStorySerializer):
    def serialize_neighbor(self, neighbor):
        return NeighborUserStorySerializer(neighbor).data

class NeighborUserStorySerializer(serializers.ModelSerializer):
    class Meta:
        model = models.UserStory
        fields = ("id", "ref", "subject")
        depth = 0

class UserStoriesBulkSerializer(ProjectExistsValidator, UserStoryStatusExistsValidator, serializers.Serializer):
    project_id = serializers.IntegerField()
    status_id = serializers.IntegerField(required=False)
    bulk_stories = serializers.CharField()

## Order bulk serializers

class _UserStoryOrderBulkSerializer(UserStoryExistsValidator, serializers.Serializer):
    us_id = serializers.IntegerField()
    order = serializers.IntegerField()

class UpdateUserStoriesOrderBulkSerializer(ProjectExistsValidator, UserStoryStatusExistsValidator, serializers.Serializer):
    project_id = serializers.IntegerField()
    bulk_stories = _UserStoryOrderBulkSerializer(many=True)
<commit_msg>fix issue #3630: I cannot open a story from the taskboard when I click on the story title.<commit_after># Copyright (C) 2014-2015 Andrey Antukh <niwi@niwi.be>
# Copyright (C) 2014-2015 Jesús Espino <jespinog@gmail.com>
# Copyright (C) 2014-2015 David Barragán <bameda@dbarragan.com>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from django.apps import apps
from taiga.base.api import serializers
from taiga.base.fields import TagsField
from taiga.base.fields import PgArrayField
from taiga.base.neighbors import NeighborsSerializerMixin
from taiga.base.utils import json

from taiga.mdrender.service import render as mdrender
from taiga.projects.validators import ProjectExistsValidator
from taiga.projects.validators import UserStoryStatusExistsValidator
from taiga.projects.userstories.validators import UserStoryExistsValidator
from taiga.projects.notifications.validators import WatchersValidator
from taiga.projects.serializers import BasicUserStoryStatusSerializer
from taiga.projects.notifications.mixins import EditableWatchedResourceModelSerializer
from taiga.projects.votes.mixins.serializers import VoteResourceSerializerMixin

from taiga.users.serializers import UserBasicInfoSerializer

from . import models

class RolePointsField(serializers.WritableField):
    def to_native(self, obj):
        return {str(o.role.id): o.points.id for o in obj.all()}

    def from_native(self, obj):
        if isinstance(obj, dict):
            return obj
        return json.loads(obj)

class MilestoneUserStorySerializer(serializers.ModelSerializer):
    total_points = serializers.SerializerMethodField("get_total_points")

    class Meta:
        model = models.UserStory
        depth = 0
        fields = ("id", "ref", "subject", "milestone", "is_closed", "is_blocked", "total_points")

    def get_total_points(self, obj):
        return obj.get_total_points()

class UserStorySerializer(WatchersValidator, VoteResourceSerializerMixin, EditableWatchedResourceModelSerializer,
                          serializers.ModelSerializer):
    tags = TagsField(default=[], required=False)
    external_reference = PgArrayField(required=False)
    points = RolePointsField(source="role_points", required=False)
    total_points = serializers.SerializerMethodField("get_total_points")
    comment = serializers.SerializerMethodField("get_comment")
    milestone_slug = serializers.SerializerMethodField("get_milestone_slug")
    milestone_name = serializers.SerializerMethodField("get_milestone_name")
    origin_issue = serializers.SerializerMethodField("get_origin_issue")
    blocked_note_html = serializers.SerializerMethodField("get_blocked_note_html")
    description_html = serializers.SerializerMethodField("get_description_html")
    status_extra_info = BasicUserStoryStatusSerializer(source="status", required=False, read_only=True)
    assigned_to_extra_info = UserBasicInfoSerializer(source="assigned_to", required=False, read_only=True)
    owner_extra_info = UserBasicInfoSerializer(source="owner", required=False, read_only=True)

    class Meta:
        model = models.UserStory
        depth = 0
        read_only_fields = ('created_date', 'modified_date')

    def get_total_points(self, obj):
        return obj.get_total_points()

    def get_comment(self, obj):
        # NOTE: This method and field is necessary to historical comments work
        return ""

    def get_milestone_slug(self, obj):
        if obj.milestone:
            return obj.milestone.slug
        else:
            return None

    def get_milestone_name(self, obj):
        if obj.milestone:
            return obj.milestone.name
        else:
            return None

    def get_origin_issue(self, obj):
        if obj.generated_from_issue:
            return {
                "id": obj.generated_from_issue.id,
                "ref": obj.generated_from_issue.ref,
                "subject": obj.generated_from_issue.subject,
            }
        return None

    def get_blocked_note_html(self, obj):
        return mdrender(obj.project, obj.blocked_note)

    def get_description_html(self, obj):
        return mdrender(obj.project, obj.description)

class UserStoryListSerializer(UserStorySerializer):
    class Meta:
        model = models.UserStory
        depth = 0
        read_only_fields = ('created_date', 'modified_date')
        exclude=("description", "description_html")

class UserStoryNeighborsSerializer(NeighborsSerializerMixin, UserStorySerializer):
    def serialize_neighbor(self, neighbor):
        return NeighborUserStorySerializer(neighbor).data

class NeighborUserStorySerializer(serializers.ModelSerializer):
    class Meta:
        model = models.UserStory
        fields = ("id", "ref", "subject")
        depth = 0

class UserStoriesBulkSerializer(ProjectExistsValidator, UserStoryStatusExistsValidator, serializers.Serializer):
    project_id = serializers.IntegerField()
    status_id = serializers.IntegerField(required=False)
    bulk_stories = serializers.CharField()

## Order bulk serializers

class _UserStoryOrderBulkSerializer(UserStoryExistsValidator, serializers.Serializer):
    us_id = serializers.IntegerField()
    order = serializers.IntegerField()

class UpdateUserStoriesOrderBulkSerializer(ProjectExistsValidator, UserStoryStatusExistsValidator, serializers.Serializer):
    project_id = serializers.IntegerField()
    bulk_stories = _UserStoryOrderBulkSerializer(many=True)
<|endoftext|>


<commit_before># Copyright 2018 Red Hat, Inc.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from tempest.api.image import base
from tempest.lib.common.utils import data_utils
from tempest.lib import decorators

class BasicOperationsImagesAdminTest(base.BaseV2ImageAdminTest):

    @decorators.related_bug('1420008')
    @decorators.idempotent_id('646a6eaa-135f-4493-a0af-12583021224e')
    def test_create_image_owner_param(self):
        # NOTE: Create image with owner different from tenant owner by
        # using "owner" parameter requires an admin privileges.
        random_id = data_utils.rand_uuid_hex()
        image = self.admin_client.create_image(
            container_format='bare', disk_format='raw', owner=random_id)
        self.addCleanup(self.admin_client.delete_image, image['id'])
        image_info = self.admin_client.show_image(image['id'])
        self.assertEqual(random_id, image_info['owner'])
<commit_msg>Update image using 'owner' parameter<commit_after># Copyright 2018 Red Hat, Inc.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from tempest.api.image import base
from tempest.lib.common.utils import data_utils
from tempest.lib import decorators

class BasicOperationsImagesAdminTest(base.BaseV2ImageAdminTest):

    @decorators.related_bug('1420008')
    @decorators.idempotent_id('646a6eaa-135f-4493-a0af-12583021224e')
    def test_create_image_owner_param(self):
        # NOTE: Create image with owner different from tenant owner by
        # using "owner" parameter requires an admin privileges.
        random_id = data_utils.rand_uuid_hex()
        image = self.admin_client.create_image(
            container_format='bare', disk_format='raw', owner=random_id)
        self.addCleanup(self.admin_client.delete_image, image['id'])
        image_info = self.admin_client.show_image(image['id'])
        self.assertEqual(random_id, image_info['owner'])

    @decorators.related_bug('1420008')
    @decorators.idempotent_id('525ba546-10ef-4aad-bba1-1858095ce553')
    def test_update_image_owner_param(self):
        random_id_1 = data_utils.rand_uuid_hex()
        image = self.admin_client.create_image(
            container_format='bare', disk_format='raw', owner=random_id_1)
        self.addCleanup(self.admin_client.delete_image, image['id'])
        created_image_info = self.admin_client.show_image(image['id'])

        random_id_2 = data_utils.rand_uuid_hex()
        self.admin_client.update_image(
            image['id'], [dict(replace="/owner", value=random_id_2)])
        updated_image_info = self.admin_client.show_image(image['id'])

        self.assertEqual(random_id_2, updated_image_info['owner'])
        self.assertNotEqual(created_image_info['owner'],
                            updated_image_info['owner'])
<|endoftext|>


<commit_before>Author: Funjando

from ggame import App, RectangleAsset, ImageAsset, SoundAsset
from ggame import LineStyle, Color, Sprite, Sound

myapp = App()
myapp.run()
<commit_msg>Updated from Brython Server: 3/1/2016 2:26:07 PM<commit_after>"""
Author: Funjando
"""

from ggame import App, RectangleAsset, ImageAsset, SoundAsset
from ggame import LineStyle, Color, Sprite, Sound

SCREEN_WIDTH = 640
SCREEN_HEIGHT = 480

green = Color(0x00ff00, 1)
black = Color(0, 1)
noline = LineStyle(0, black)
bg_asset = RectangleAsset(SCREEN_WIDTH, SCREEN_HEIGHT, noline, green)
bg = Sprite(bg_asset, (0,0))

myapp = App(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.run()
<|endoftext|>


<commit_before>import sublime
import sublime_plugin

from .lib.artlebedev import RemoteTypograf

__author__ = 'Anton Vakhrushev'
__email__ = 'anwinged@ya.ru'

PLUGIN_NAME = 'Typograph'

DEFAULT_SETTINGS = {
    'type': 'html',
    'breaks': False,
    'paragraphs': False,
    'max_no_break': 3,
}

class TypographSelectionCommand(sublime_plugin.TextCommand):
    """
    Process selections of html text
    """

    def run(self, edit):

        settings = self.get_settings()

        typograf = RemoteTypograf()
        typograf.htmlEntities()
        typograf.br(settings['breaks'])
        typograf.p(settings['paragraphs'])
        typograf.nobr(settings['max_no_break'])

        for region in self.view.sel():
            processed = typograf.processText(self.view.substr(region))
            self.view.replace(edit, region, processed)

    def get_settings(self):
        """
        Load plugin settings
        """
        settings = sublime.load_settings('{}.sublime-settings'.format(PLUGIN_NAME))
        project_settings = self.view.settings().get(PLUGIN_NAME, {})
        local_settings = DEFAULT_SETTINGS.copy()

        for key in DEFAULT_SETTINGS:
            local_settings[key] = settings.get(key)

        for key in project_settings:
            if key in DEFAULT_SETTINGS:
                local_settings[key] = project_settings[key]
            else:
                print('{}: invalid key "{}" in project settings'.format(PLUGIN_NAME, key))

        return local_settings
<commit_msg>Stripe processed text<commit_after>import sublime
import sublime_plugin

from .lib.artlebedev import RemoteTypograf

__author__ = 'Anton Vakhrushev'
__email__ = 'anwinged@ya.ru'

PLUGIN_NAME = 'Typograph'

DEFAULT_SETTINGS = {
    'type': 'html',
    'breaks': False,
    'paragraphs': False,
    'max_no_break': 3,
}

class TypographSelectionCommand(sublime_plugin.TextCommand):
    """
    Process selections of html text
    """

    def run(self, edit):

        settings = self.get_settings()

        typograf = RemoteTypograf()
        typograf.htmlEntities()
        typograf.br(settings['breaks'])
        typograf.p(settings['paragraphs'])
        typograf.nobr(settings['max_no_break'])

        for region in self.view.sel():
            processed = typograf.processText(self.view.substr(region))
            self.view.replace(edit, region, processed.strip())

    def get_settings(self):
        """
        Load plugin settings
        """
        settings = sublime.load_settings('{}.sublime-settings'.format(PLUGIN_NAME))
        project_settings = self.view.settings().get(PLUGIN_NAME, {})
        local_settings = DEFAULT_SETTINGS.copy()

        for key in DEFAULT_SETTINGS:
            local_settings[key] = settings.get(key)

        for key in project_settings:
            if key in DEFAULT_SETTINGS:
                local_settings[key] = project_settings[key]
            else:
                print('{}: invalid key "{}" in project settings'.format(PLUGIN_NAME, key))

        return local_settings
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from pythinclient.client import BasicThinClient

if __name__ == "__main__":
  client = BasicThinClient(65000)
  running = True
  while running:
    line = input("> ")
    if line == "": continue
    # process the line with the client
    try:
      # attempt to connect, send, receive, and close interactions with the server
      response = client.send_receive(line)
    except Exception as ex:
      response = "Could not make a connection to the server\n"
      response += "reason: %s" % ex
    print(response)<commit_msg>Changed the client example to follow suit with the async client example, an EOF (ctrl-d) or keyboard interrupt (ctrl-c) will cleanly exit<commit_after>#!/usr/bin/env python3

from pythinclient.client import BasicThinClient

if __name__ == "__main__":
  client = BasicThinClient(65000)
  running = True
  while running:
    try:
      line = input("> ")
    except (KeyboardInterrupt, EOFError):
      break
    if line == "": continue
    # process the line with the client
    try:
      # attempt to connect, send, receive, and close interactions with the server
      response = client.send_receive(line)
    except Exception as ex:
      response = "Could not make a connection to the server\n"
      response += "reason: %s" % ex
    print(response)<|endoftext|>


<commit_before>"""Tests for decorators."""
import pytest
import numpy as np
import functools
from tensornetwork.backends.abstract_backend import AbstractBackend
from tensornetwork.backends import backend_factory
from tensornetwork import backends
import tensornetwork

def jittest_init(backend):
  backend_obj = backends.backend_factory.get_backend(backend)
  def fun(x, A, y):
    return backend_obj.multiply(x, backend_obj.multiply(A, y))
  x = backend_obj.randn((4,))
  y = backend_obj.randn((4,))
  A = backend_obj.randn((4, 4))
  return (x, y, A, fun)

def test_jit(backend):
  x, y, A, fun = jittest_init(backend)
  fun_jit = tensornetwork.jit(fun, backend=backend)
  res1 = fun(x, A, y)
  res2 = fun_jit(x, A, y)
  np.testing.assert_allclose(res1, res2)

def test_jit_ampersand(backend):
  x, y, A, fun = jittest_init(backend)
  @functools.partial(tensornetwork.jit, static_argnums=(3,), backend=backend)
  def fun_jit(x, A, y, dummy):
    _ = dummy
    return fun(x, A, y)
  res1 = fun(x, A, y)
  res2 = fun_jit(x, A, y, 2)
  np.testing.assert_allclose(res1, res2)

def test_jit_args(backend):
  x, y, A, fun = jittest_init(backend)
  fun_jit = tensornetwork.jit(fun, backend=backend)
  res1 = fun(x, A, y)
  res2 = fun_jit(x, A, y)
  res3 = fun_jit(x, y=y, A=A)
  np.testing.assert_allclose(res1, res2)
  np.testing.assert_allclose(res1, res3)

def test_jit_backend_argnum_is_string(backend):
  x, y, A, fun = jittest_init(backend)

  @functools.partial(tensornetwork.jit, backend_argnum=3)
  def fun_jit(x, A, y, the_backend):
    _ = the_backend
    return fun(x, A, y)
  res1 = fun(x, A, y)
  res2 = fun_jit(x, A, y, backend)
  np.testing.assert_allclose(res1, res2)

def test_jit_backend_argnum_is_obj(backend):
  x, y, A, fun = jittest_init(backend)

  @functools.partial(tensornetwork.jit, backend_argnum=3)
  def fun_jit(x, A, y, the_backend):
    _ = the_backend
    return fun(x, A, y)
  res1 = fun(x, A, y)
  backend_obj = backends.backend_factory.get_backend(backend)
  res2 = fun_jit(x, A, y, backend_obj)
  np.testing.assert_allclose(res1, res2)

def test_jit_backend_argnum_invalid(backend):
  x, y, A, fun = jittest_init(backend)

  with pytest.raises(ValueError):
    @functools.partial(tensornetwork.jit, backend_argnum=3, backend=backend)
    def fun_jit(x, A, y, the_backend):
      _ = the_backend
      return fun(x, A, y)
    _ = fun_jit(x, A, y, 99)

def test_jit_backend_and_backend_obj_raises_error(backend):
  x, y, A, fun = jittest_init(backend)

  with pytest.raises(ValueError):
    @functools.partial(tensornetwork.jit, backend_argnum=3, backend=backend)
    def fun_jit(x, A, y, the_backend):
      _ = the_backend
      return fun(x, A, y)
    _ = fun_jit(x, A, y, backend)
<commit_msg>Jit test seeds (#730)<commit_after>"""Tests for decorators."""
import pytest
import numpy as np
import functools
from tensornetwork.backends.abstract_backend import AbstractBackend
from tensornetwork.backends import backend_factory
from tensornetwork import backends
import tensornetwork

def jittest_init(backend):
  """
  Helper to initialize data for the other Jit tests.
  """
  backend_obj = backends.backend_factory.get_backend(backend)
  def fun(x, A, y):
    return backend_obj.multiply(x, backend_obj.multiply(A, y))
  x = backend_obj.randn((4,), seed=11)
  y = backend_obj.randn((4,), seed=11)
  A = backend_obj.randn((4, 4), seed=11)
  return (x, y, A, fun)

def test_jit(backend):
  """
  Tests that tn.jit gives the right answer.
  """
  x, y, A, fun = jittest_init(backend)
  fun_jit = tensornetwork.jit(fun, backend=backend)
  res1 = fun(x, A, y)
  res2 = fun_jit(x, A, y)
  np.testing.assert_allclose(res1, res2)

def test_jit_ampersand(backend):
  """
  Tests that tn.jit gives the right answer when used as a decorator.
  """
  x, y, A, fun = jittest_init(backend)
  @functools.partial(tensornetwork.jit, static_argnums=(3,), backend=backend)
  def fun_jit(x, A, y, dummy):
    _ = dummy
    return fun(x, A, y)
  res1 = fun(x, A, y)
  res2 = fun_jit(x, A, y, 2)
  np.testing.assert_allclose(res1, res2)

def test_jit_args(backend):
  """
  Tests that tn.jit gives the right answer when given extra arguments.
  """
  x, y, A, fun = jittest_init(backend)
  fun_jit = tensornetwork.jit(fun, backend=backend)
  res1 = fun(x, A, y)
  res2 = fun_jit(x, A, y)
  res3 = fun_jit(x, y=y, A=A)
  np.testing.assert_allclose(res1, res2)
  np.testing.assert_allclose(res1, res3)

def test_jit_backend_argnum_is_string(backend):
  """
  Tests that tn.jit gives the right answer when the backend is supplied
  via backend_argnum as a string.
  """
  x, y, A, fun = jittest_init(backend)

  @functools.partial(tensornetwork.jit, backend_argnum=3)
  def fun_jit(x, A, y, the_backend):
    _ = the_backend
    return fun(x, A, y)
  res1 = fun(x, A, y)
  res2 = fun_jit(x, A, y, backend)
  np.testing.assert_allclose(res1, res2)

def test_jit_backend_argnum_is_obj(backend):
  """
  Tests that tn.jit gives the right answer when the backend is supplied
  via backend_argnum as a backend object.
  """
  x, y, A, fun = jittest_init(backend)

  @functools.partial(tensornetwork.jit, backend_argnum=3)
  def fun_jit(x, A, y, the_backend):
    _ = the_backend
    return fun(x, A, y)
  res1 = fun(x, A, y)
  backend_obj = backends.backend_factory.get_backend(backend)
  res2 = fun_jit(x, A, y, backend_obj)
  np.testing.assert_allclose(res1, res2)

def test_jit_backend_argnum_invalid(backend):
  """
  Tests that tn.jit raises ValueError when backend_argnum points to something
  other than a backend.
  """
  x, y, A, fun = jittest_init(backend)

  with pytest.raises(ValueError):
    @functools.partial(tensornetwork.jit, backend_argnum=3)
    def fun_jit(x, A, y, the_backend):
      _ = the_backend
      return fun(x, A, y)
    _ = fun_jit(x, A, y, 99)

def test_jit_backend_and_backend_obj_raises_error(backend):
  """
  Tests that tn.jit raises ValueError when backend_argnum and backend
  are both specified.
  """
  x, y, A, fun = jittest_init(backend)

  with pytest.raises(ValueError):
    @functools.partial(tensornetwork.jit, backend_argnum=3, backend=backend)
    def fun_jit(x, A, y, the_backend):
      _ = the_backend
      return fun(x, A, y)
    _ = fun_jit(x, A, y, backend)
<|endoftext|>


<commit_before>state = {
    'Facility': 'LCLS',
    'LCLS/DataSource': 'exp=XCS/xcstut13:run=15'
}

def onEvent(evt):
    analysis.event.printKeys(evt)
    analysis.event.printNativeKeys(evt)
    analysis.beamline.printPulseEnergy(evt['pulseEnergies'])
    analysis.beamline.printPhotonEnergy(evt['photonEnergies'])
    analysis.event.printProcessingRate(evt)

<commit_msg>Add missing imports<commit_after>import analysis.event
import analysis.beamline

state = {
    'Facility': 'LCLS',
    'LCLS/DataSource': 'exp=XCS/xcstut13:run=15'
}

def onEvent(evt):
    analysis.event.printKeys(evt)
    analysis.event.printNativeKeys(evt)
    analysis.beamline.printPulseEnergy(evt['pulseEnergies'])
    analysis.beamline.printPhotonEnergy(evt['photonEnergies'])
    analysis.event.printProcessingRate(evt)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# IMPORT
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

import ctypes

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# STRUCT CLASS
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

class type_test_struct(ctypes.Structure):

	_fields_ = [
		('el_char', ctypes.c_char),
		('el_int8t', ctypes.c_int8),
		('el_int16t', ctypes.c_int16),
		('el_float', ctypes.c_float),
		('el_double', ctypes.c_double),
		('el_int8t_4', ctypes.c_int8 * 4),
		('el_int8t_2x3', ctypes.c_int8 * 2 * 3)
		]

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# CALLER CLASS
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

class demo_routine_caller_class():

	def __init__(self):

		# Generate reference on routine in DLL
		self._call_demo_routine_ = ctypes.windll.LoadLibrary('demo_dll.dll').demo_routine

		# Define parameter datatypes
		self._call_demo_routine_.argtypes = [
			ctypes.c_char_p,
			ctypes.c_int,
			ctypes.POINTER(type_test_struct)
			]

		# Define return datatype
		self._call_demo_routine_.restype = ctypes.c_void_p

	def call(self, param_char_p, param_int, param_type_test):

		try:

			# Call routine in DLL
			self._call_demo_routine_(
				param_char_p,
				param_int,
				ctypes.byref(param_type_test)
				)

			return True

		except:

			return False

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# MAKE A CALL
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

if __name__ == '__main__':

	# Initialize caller
	demo_routine_caller = demo_routine_caller_class()

	# Initialize some test parameters
	sample_param_char_p = 7
	sample_param_int = 42
	sample_type_test = type_test_struct()
	sample_type_test.el_char = 5
	sample_type_test.el_int8t = 17
	sample_type_test.el_int16t = 25874
	sample_type_test.el_float = 2.4
	sample_type_test.el_double = 8.598e31
	el_int8t_4 = [1, 2, 3, 4]
	sample_type_test.el_int8t_4 = (ctypes.c_int8 * len(el_int8t_4))(*el_int8t_4)
	el_int8t_2x3 = [[1, 2], [3, 4], [5, 6]]
	sample_type_test.el_int8t_2x3 = (ctypes.c_int8 * len(el_int8t_2x3[0]) * len(el_int8t_2x3))(*(tuple(i) for i in el_int8t_2x3))

	# Call
	demo_routine_caller.call(sample_param_char_p, sample_param_int, sample_type_test)

<commit_msg>demo call works under wine<commit_after># -*- coding: utf-8 -*-

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# IMPORT
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

import ctypes

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# STRUCT CLASS
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

class type_test_struct(ctypes.Structure):

	_fields_ = [
		('el_char', ctypes.c_char),
		('el_int8t', ctypes.c_int8),
		('el_int16t', ctypes.c_int16),
		('el_float', ctypes.c_float),
		('el_double', ctypes.c_double),
		('el_int8t_4', ctypes.c_int8 * 4),
		('el_int8t_2x3', ctypes.c_int8 * 2 * 3)
		]

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# CALLER CLASS
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

class demo_routine_caller_class():

	def __init__(self):

		# Generate reference on routine in DLL
		self._call_demo_routine_ = ctypes.windll.LoadLibrary('demo_dll.dll').demo_routine

		# Define parameter datatypes
		self._call_demo_routine_.argtypes = [
			ctypes.POINTER(ctypes.c_char),
			ctypes.c_int,
			ctypes.POINTER(type_test_struct)
			]

		# Define return datatype
		self._call_demo_routine_.restype = ctypes.c_void_p

	def call(self, param_char, param_int, param_type_test):

		# Call routine in DLL
		self._call_demo_routine_(
			ctypes.byref(param_char),
			param_int,
			ctypes.byref(param_type_test)
			)

		print('Called!')
		return True

# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# MAKE A CALL
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

if __name__ == '__main__':

	# Initialize some test parameters
	sample_param_char = ctypes.c_char(7)
	sample_param_int = ctypes.c_int(42)
	sample_type_test = type_test_struct()
	sample_type_test.el_char = ctypes.c_char(5)
	sample_type_test.el_int8t = ctypes.c_int8(17)
	sample_type_test.el_int16t = ctypes.c_int16(25874)
	sample_type_test.el_float = ctypes.c_float(2.4)
	sample_type_test.el_double = ctypes.c_double(8.598e31)
	el_int8t_4 = [1, 2, 3, 4]
	sample_type_test.el_int8t_4 = (ctypes.c_int8 * len(el_int8t_4))(*el_int8t_4)
	el_int8t_2x3 = [[1, 2], [3, 4], [5, 6]]
	sample_type_test.el_int8t_2x3 = (ctypes.c_int8 * len(el_int8t_2x3[0]) * len(el_int8t_2x3))(*(tuple(i) for i in el_int8t_2x3))

	# Initialize caller
	demo_routine_caller = demo_routine_caller_class()

	# Call
	demo_routine_caller.call(sample_param_char, sample_param_int, sample_type_test)

<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-
#!/usr/bin/env python
"""
This is a simple script for tests esky.pl website functionality
by Selenium/Webdriver Python module.
"""
from datetime import date
from time import sleep
from random import sample

from dateutil.relativedelta import relativedelta
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.common.by import By
from selenium.webdriver.support.select import Select

MONTHS_PL = {1 : u"styczeń",
             2 : u"luty",
             3 : u"marzec",
             4 : u"kwiecień",
             5 : u"maj",
             6 : u"czerwiec",
             7 : u"lipiec",
             8 : u"sierpień",
             9 : u"wrzesień",
             10 : u"październik",
             11 : u"listopad",
             12 : u"grudzień"
            }

def click_date_on_calendar(calendar_div, dt_date):
    """
    Function for click date on pop up calendar.
    Params:
    # calendar_div - html div contain pop up calendar
    # dt_date - datetime instance of date
    """
    calendar_year = calendar_div.find_element_by_css_selector("span.ui-datepicker-year").text
    calendar_month = calendar_div.find_element_by_css_selector("span.ui-datepicker-month").text
    while not (calendar_month.lower() == MONTHS_PL[dt_date.month]\
               and int(calendar_year) == dt_date.year):
        calendar_div.find_element_by_xpath("./div/a[@data-handler='next']").click()
        sleep(1)
        calendar_year = calendar_div.find_element_by_css_selector("span.ui-datepicker-year").text
        calendar_month = calendar_div.find_element_by_css_selector("span.ui-datepicker-month").text
    for link in  calendar_div.find_elements_by_xpath('./table/tbody/tr/td/a'):
        if int(link.text) == dt_date.day:
            link.click()
            break

def fill_flights_form(flights_form, departure, arrival, dep_date):
    """
    Function for fill up flights form.
    With departure, arrival and departure date (dep_date) as params.
    Return date is equal to departure date plus 3 months.
    """
    departure_input = flights_form.find_element_by_id("departureRoundtrip0")
    departure_input.clear()
    departure_input.send_keys(departure)
    arrival_input = flights_form.find_element_by_id("arrivalRoundtrip0")
    arrival_input.clear()
    arrival_input.send_keys(arrival)
    dep_date_input = flights_form.find_element_by_id("departureDateRoundtrip0")
    dep_date_input.click()
    sleep(1)
    click_date_on_calendar(flights_form.find_element_by_xpath("//*[@id='ui-datepicker-div']"),
                           dep_date
                           )

    return_date_input = flights_form.find_element_by_id("departureDateRoundtrip1")
    return_date_input.click()
    click_date_on_calendar(flights_form.find_element_by_xpath("//*[@id='ui-datepicker-div']"),
                           dep_date + relativedelta(months=+3)
                           )

def click_filter(filters_div, filter_grp_id, filter_id):
    """
    Function for select/deselect chosen filter by filter_id:
    Params:
    # filters_div - holder of all filters as selenium/webdriver object
    # filter_grp_id - id of group of filter contain choosen one
    """
    filter_container = filters_div.find_element_by_xpath("./div/div[@data-dropdown-content-id='%s']" % filter_grp_id)
    filter = filters_div.find_element_by_id(filter_id)
    if not filter_container.is_displayed():
        filters_div.find_element_by_xpath("./div/a[@data-content-id='%s']" % filter_grp_id).click()
    filter.click()

def get_flight_details(esky_webdriver, flight_details_link):
    """
    Function return details from single flight as a dictionary.

    """
#     flight_details_link.click()
#     esky_webdriver.find_element_by_xpath("//div[@class='custom-rwd-dialog-wrapper open']")
    fl_details_dict = {'all_single_flights': [],
                       'all_changes' : []}

    return fl_details_dict

def fill_payments_form(payments_form,
                       first_name,
                       last_name,
                       gender,
                       phone_number,
                       email_address,
#                        birthday_date
                       ):
    """
    Function for fill up payments form.
    Params:
    # first_name,
    # last_name,
    # gender - value "mrs" means female and "mr" means male,
    # phone_number,
    # email_address.
    """
#      birthday_date
#     """
    _genders = ["mrs", "mr"]
    if not gender in _genders:
        raise Exception('Bad gender value %s - please choose from %s' % (gender, _genders))
    payments_form.find_element_by_id("bookFlight_paxes_1_name").send_keys(first_name)
    payments_form.find_element_by_id("bookFlight_paxes_1_surname").send_keys(last_name)
    gender_select = Select(payments_form.find_element_by_id("bookFlight_paxes_1_title"))
    gender_select.select_by_value(gender)
    payments_form.find_element_by_id("bookFlight_contactDetails_phoneNumber_phoneNumber").send_keys(phone_number)
    payments_form.find_element_by_id("bookFlight_contactDetails_email").send_keys(email_address)
    payments_form.find_element_by_id("bookFlight_statute").click()
#     payments_form.find_element_by_id(

def main():
    """
    Main program for realization of esky.pl task
    with guidelines from eskypl_guidelines_in_polish.txt.
    """
    # 1.       Otworzenie strony
    main_url = "http://www.esky.pl/"
    esky_wd = webdriver.Firefox()
    esky_wd.get(main_url)
    print "Task #1: opening website: %s" % main_url
    # 2.       Pobranie i wyświetlenie w konsoli wartości ciastka esky_TCSI
    cookie_name = "esky_TCSI"
    print "Task #2: display value of cookie %s: %s" % (
        cookie_name, esky_wd.get_cookie(cookie_name)["value"]
    )
    # 3.       Wypełnienie danych formularza wyszukiwania
    #(parametryzowane destynacje oraz daty wylotu/powrotu wyliczane zawsze 3mce do przodu)
    flights_form = esky_wd.find_element_by_css_selector("form.flights-qsf")
    fill_flights_form(flights_form,
                      departure="Hamburg",
                      arrival="Katowice",
                      dep_date=date.today()+relativedelta(days=+10)
                      )
    flights_form.submit()
    print "Task #3: fill up flights form from %s" % main_url
    # 4.       Wykonanie prostego filtrowania na wynikach wyszukiwania (dowolna kombinacja)
    wait = WebDriverWait(esky_wd, 10)
    filters_div = wait.until(
        expected_conditions.visibility_of_element_located((By.ID, "filters"))
    )
    filters_div.find_element_by_xpath("./div/span[@class='filters-text']").click() #show filters
    print "Show filters"
    click_filter(filters_div,
                 filter_grp_id = 'filterConnections',
                 filter_id = "filterConnections_opt_1")
    click_filter(filters_div,
                 filter_grp_id = 'filterDepartureTime0',
                 filter_id = "filterDepartureTime1_opt_12_18")
    click_filter(filters_div,
                 filter_grp_id = 'filterDepartureTime0',
                 filter_id = "filterDepartureTime0_opt_12_18")
    click_filter(filters_div,
                 filter_grp_id = 'filterDepartureTime0',
                 filter_id = "filterDepartureTime0_opt_12_18")
    print "Task #4: using filters on results website"
    # 5.       Pobieranie kilku istotnych elementów pojedynczego lotu
    #TODO: 20160425-04 Task #5 take data from single flight
    sleep(1)
    available_flights_div = wait.until(
        expected_conditions.visibility_of_element_located((By.ID, "available-flights"))
    )
    available_flights = available_flights_div.find_elements_by_tag_name("form")
    for flight_details_link in available_flights[0].find_elements_by_link_text("szczegóły"):
        print get_flight_details(esky_wd, flight_details_link)
    print "Task #5: get single flight details"
#     raise Exception("TESTING")
    # 6.       Przejście na kolejny ekran płatności i wypełnienie formularza dowolnymi wartościami
    # spełniającymi reguły walidacji
    #TODO: 20160425-02 fix reserve flights issue - remove random?
    sample(available_flights, 1)[0].find_element_by_xpath("./div[1]/a").click()
    payment_website_title = u"eSky: Hamburg - Katowice"
    wait.until(
        expected_conditions.title_is(payment_website_title)
    )
    assert esky_wd.title == payment_website_title
    payments_form = esky_wd.find_element_by_css_selector("form.booking-form")
    #TODO: 20160425-01 fix birthday_date issue
    fill_payments_form(payments_form,
                       first_name="John",
                       last_name="Doe",
                       gender="mr",
                       phone_number=789456123,
                       email_address="john.doe@hotmail.com",
#                        birthday_date=date.today()+relativedelta(years=-20)
                       )
    payments_form.submit()
    print "Task #6 fill up payments form"
    # 7.       Wykonanie kilku asercji, biorąc pod uwagę dane uzyskane na formularzu wyszukiwania,
    # wynikach i ekranie płatności.
    #TODO: 20160425-03 Task #7 prepare more asserts

#     esky_wd.close()

if __name__ == "__main__":
    print "START"
    main()
    print "STOP"<commit_msg>extend get_flight_details for get flights info wrapper<commit_after>#-*- coding: utf-8 -*-
#!/usr/bin/env python
"""
This is a simple script for tests esky.pl website functionality
by Selenium/Webdriver Python module.
"""
from datetime import date
from time import sleep
from random import sample

from dateutil.relativedelta import relativedelta
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.common.by import By
from selenium.webdriver.support.select import Select
from selenium.common.exceptions import WebDriverException

MONTHS_PL = {1 : u"styczeń",
             2 : u"luty",
             3 : u"marzec",
             4 : u"kwiecień",
             5 : u"maj",
             6 : u"czerwiec",
             7 : u"lipiec",
             8 : u"sierpień",
             9 : u"wrzesień",
             10 : u"październik",
             11 : u"listopad",
             12 : u"grudzień"
            }

def click_date_on_calendar(calendar_div, dt_date):
    """
    Function for click date on pop up calendar.
    Params:
    # calendar_div - html div contain pop up calendar
    # dt_date - datetime instance of date
    """
    calendar_year = calendar_div.find_element_by_css_selector("span.ui-datepicker-year").text
    calendar_month = calendar_div.find_element_by_css_selector("span.ui-datepicker-month").text
    while not (calendar_month.lower() == MONTHS_PL[dt_date.month]\
               and int(calendar_year) == dt_date.year):
        calendar_div.find_element_by_xpath("./div/a[@data-handler='next']").click()
        sleep(1)
        calendar_year = calendar_div.find_element_by_css_selector("span.ui-datepicker-year").text
        calendar_month = calendar_div.find_element_by_css_selector("span.ui-datepicker-month").text
    for link in  calendar_div.find_elements_by_xpath('./table/tbody/tr/td/a'):
        if int(link.text) == dt_date.day:
            link.click()
            break

def fill_flights_form(flights_form, departure, arrival, dep_date):
    """
    Function for fill up flights form.
    With departure, arrival and departure date (dep_date) as params.
    Return date is equal to departure date plus 3 months.
    """
    departure_input = flights_form.find_element_by_id("departureRoundtrip0")
    departure_input.clear()
    departure_input.send_keys(departure)
    arrival_input = flights_form.find_element_by_id("arrivalRoundtrip0")
    arrival_input.clear()
    arrival_input.send_keys(arrival)
    dep_date_input = flights_form.find_element_by_id("departureDateRoundtrip0")
    dep_date_input.click()
    sleep(1)
    click_date_on_calendar(flights_form.find_element_by_xpath("//*[@id='ui-datepicker-div']"),
                           dep_date
                           )

    return_date_input = flights_form.find_element_by_id("departureDateRoundtrip1")
    return_date_input.click()
    click_date_on_calendar(flights_form.find_element_by_xpath("//*[@id='ui-datepicker-div']"),
                           dep_date + relativedelta(months=+3)
                           )

def click_filter(filters_div, filter_grp_id, filter_id):
    """
    Function for select/deselect chosen filter by filter_id:
    Params:
    # filters_div - holder of all filters as selenium/webdriver object
    # filter_grp_id - id of group of filter contain choosen one
    """
    filter_container = filters_div.find_element_by_xpath("./div/div[@data-dropdown-content-id='%s']" % filter_grp_id)
    filter = filters_div.find_element_by_id(filter_id)
    if not filter_container.is_displayed():
        filters_div.find_element_by_xpath("./div/a[@data-content-id='%s']" % filter_grp_id).click()
    filter.click()

def get_flight_details(esky_webdriver, flight_details_link):
    """
    Function return details from single flight as a dictionary.

    """
    try:
        flight_details_link.click()
    except WebDriverException:
        pass #TODO: 20160426-01 some problem with click on ahref inside label

    open_info_wrapper = esky_webdriver.find_element_by_xpath("//div[@class='custom-rwd-dialog-wrapper open']")
    fl_details_dict = {'all_single_flights': [],
                       'all_changes' : []}
    open_info_wrapper.find_element_by_xpath(".//i[@class='qa-dialog-closse']").click()
    return fl_details_dict

def fill_payments_form(payments_form,
                       first_name,
                       last_name,
                       gender,
                       phone_number,
                       email_address,
#                        birthday_date
                       ):
    """
    Function for fill up payments form.
    Params:
    # first_name,
    # last_name,
    # gender - value "mrs" means female and "mr" means male,
    # phone_number,
    # email_address.
    """
#      birthday_date
#     """
    _genders = ["mrs", "mr"]
    if not gender in _genders:
        raise Exception('Bad gender value %s - please choose from %s' % (gender, _genders))
    payments_form.find_element_by_id("bookFlight_paxes_1_name").send_keys(first_name)
    payments_form.find_element_by_id("bookFlight_paxes_1_surname").send_keys(last_name)
    gender_select = Select(payments_form.find_element_by_id("bookFlight_paxes_1_title"))
    gender_select.select_by_value(gender)
    payments_form.find_element_by_id("bookFlight_contactDetails_phoneNumber_phoneNumber").send_keys(phone_number)
    payments_form.find_element_by_id("bookFlight_contactDetails_email").send_keys(email_address)
    payments_form.find_element_by_id("bookFlight_statute").click()
#     payments_form.find_element_by_id(

def main():
    """
    Main program for realization of esky.pl task
    with guidelines from eskypl_guidelines_in_polish.txt.
    """
    # 1.       Otworzenie strony
    main_url = "http://www.esky.pl/"
    esky_wd = webdriver.Firefox()
    esky_wd.get(main_url)
    print "Task #1: opening website: %s" % main_url
    # 2.       Pobranie i wyświetlenie w konsoli wartości ciastka esky_TCSI
    cookie_name = "esky_TCSI"
    print "Task #2: display value of cookie %s: %s" % (
        cookie_name, esky_wd.get_cookie(cookie_name)["value"]
    )
    # 3.       Wypełnienie danych formularza wyszukiwania
    #(parametryzowane destynacje oraz daty wylotu/powrotu wyliczane zawsze 3mce do przodu)
    flights_form = esky_wd.find_element_by_css_selector("form.flights-qsf")
    fill_flights_form(flights_form,
                      departure="Hamburg",
                      arrival="Katowice",
                      dep_date=date.today()+relativedelta(days=+10)
                      )
    flights_form.submit()
    print "Task #3: fill up flights form from %s" % main_url
    # 4.       Wykonanie prostego filtrowania na wynikach wyszukiwania (dowolna kombinacja)
    wait = WebDriverWait(esky_wd, 10)
    filters_div = wait.until(
        expected_conditions.visibility_of_element_located((By.ID, "filters"))
    )
    filters_div.find_element_by_xpath("./div/span[@class='filters-text']").click() #show filters
    print "Show filters"
    click_filter(filters_div,
                 filter_grp_id = 'filterConnections',
                 filter_id = "filterConnections_opt_1")
    click_filter(filters_div,
                 filter_grp_id = 'filterDepartureTime0',
                 filter_id = "filterDepartureTime1_opt_12_18")
    click_filter(filters_div,
                 filter_grp_id = 'filterDepartureTime0',
                 filter_id = "filterDepartureTime0_opt_12_18")
    click_filter(filters_div,
                 filter_grp_id = 'filterDepartureTime0',
                 filter_id = "filterDepartureTime0_opt_12_18")
    print "Task #4: using filters on results website"
    # 5.       Pobieranie kilku istotnych elementów pojedynczego lotu
    #TODO: 20160425-04 Task #5 take data from single flight
    sleep(1)
    available_flights_div = wait.until(
        expected_conditions.visibility_of_element_located((By.ID, "available-flights"))
    )
    available_flights = available_flights_div.find_elements_by_tag_name("form")
    for flight_details_link in available_flights[0].find_elements_by_link_text("szczegóły"):
        print get_flight_details(esky_wd, flight_details_link)
    print "Task #5: get single flight details"
#     raise Exception("TESTING")
    # 6.       Przejście na kolejny ekran płatności i wypełnienie formularza dowolnymi wartościami
    # spełniającymi reguły walidacji
    #TODO: 20160425-02 fix reserve flights issue - remove random?
    sample(available_flights, 1)[0].find_element_by_xpath("./div[1]/a").click()
    payment_website_title = u"eSky: Hamburg - Katowice"
    wait.until(
        expected_conditions.title_is(payment_website_title)
    )
    assert esky_wd.title == payment_website_title
    payments_form = esky_wd.find_element_by_css_selector("form.booking-form")
    #TODO: 20160425-01 fix birthday_date issue
    fill_payments_form(payments_form,
                       first_name="John",
                       last_name="Doe",
                       gender="mr",
                       phone_number=789456123,
                       email_address="john.doe@hotmail.com",
#                        birthday_date=date.today()+relativedelta(years=-20)
                       )
    payments_form.submit()
    print "Task #6 fill up payments form"
    # 7.       Wykonanie kilku asercji, biorąc pod uwagę dane uzyskane na formularzu wyszukiwania,
    # wynikach i ekranie płatności.
    #TODO: 20160425-03 Task #7 prepare more asserts

#     esky_wd.close()

if __name__ == "__main__":
    print "START"
    main()
    print "STOP"<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright (c) 2019-2020 The PIVX developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

from test_framework.test_framework import PivxTestFramework
from test_framework.util import (
    assert_equal,
)
import random
from time import sleep

class BlockHashCacheTest(PivxTestFramework):

    def set_test_params(self):
        self.num_nodes = 1
        self.setup_clean_chain = True

    def log_title(self):
        title = "*** Starting %s ***" % self.__class__.__name__
        underline = "-" * len(title)
        description = "Tests the block-hashes rolling cache."
        self.log.info("\n\n%s\n%s\n%s\n", title, underline, description)

    def run_test(self):
        self.log_title()
        self.node = self.nodes[0]
        CACHE_SIZE = 200
        # Start with the genesis block
        vBlocks = [self.node.getbestblockhash()]

        # Mine blocks and save the hashes
        self.log.info("Mining %d blocks..." % (CACHE_SIZE - 1))
        for i in range(CACHE_SIZE-1):
            vBlocks.append(self.node.generate(1)[0])

        # Check the cache
        cache = self.node.getcachedblockhashes()
        assert_equal(vBlocks, cache)
        self.log.info("First %d blocks check out" % (CACHE_SIZE - 1))

        # Mine a block and check the cache
        self.log.info("Mining another block...")
        vBlocks.append(self.node.generate(1)[0])
        height = self.node.getblockcount()
        assert_equal(height, CACHE_SIZE)
        assert_equal(len(vBlocks), height + 1)      # genesis included
        cache = self.node.getcachedblockhashes()
        assert_equal(vBlocks[CACHE_SIZE], cache[0])
        self.log.info("Block %d correctly cached (overwriting genesis hash)" % CACHE_SIZE)

        # Mine a random number of blocks between 1 and CACHE_SIZE-1
        x = random.randint(1, CACHE_SIZE)
        self.log.info("Mining %d more blocks..." % x)
        for i in range(x):
            vBlocks.append(self.node.generate(1)[0])

        # Check that the cache is (partially) overwritten
        cache = self.node.getcachedblockhashes()
        for i in range(1, x+1):
            assert_equal(vBlocks[CACHE_SIZE + i], cache[i])
        # ...but not completely
        for i in range(x+1, CACHE_SIZE):
            assert_equal(vBlocks[i], cache[i])
        self.log.info("Cached correctly")

        # Disconnect one block
        self.log.info("Disconnecting one block...")
        height = self.node.getblockcount()
        old_hash = self.node.getbestblockhash()
        self.node.invalidateblock(old_hash)
        assert_equal(self.node.getblockcount(), height - 1)
        vBlocks.pop()
        assert_equal(len(vBlocks), height)  # genesis included

        # Check that the cache cycles back
        cache = self.node.getcachedblockhashes()
        assert_equal(vBlocks[height - CACHE_SIZE], cache[height % CACHE_SIZE])
        self.log.info("Disconnecting %d, block at height %d cycles back in" % (
            height, height - CACHE_SIZE))

        # Reconsider the block, mine another on top, and check the cache again
        self.log.info("Reconsidering and mining one more block...")
        self.node.reconsiderblock(old_hash)
        assert_equal(self.node.getblockcount(), height)
        vBlocks.append(old_hash)
        vBlocks.append(self.node.generate(1)[0])
        assert_equal(self.node.getblockcount(), height + 1)
        assert_equal(len(vBlocks), height + 2)  # genesis included
        assert_equal(vBlocks[height], old_hash)
        cache = self.node.getcachedblockhashes()
        assert_equal(vBlocks[height], cache[height % CACHE_SIZE])
        assert_equal(vBlocks[height + 1], cache[(height + 1) % CACHE_SIZE])
        self.log.info("Cached correctly")

        # Restart the node and check that the cache is correctly initialized
        self.log.info("Restarting the node...")
        self.stop_nodes()
        sleep(5)
        self.start_nodes()
        assert_equal(self.node.getcachedblockhashes(), cache)
        self.log.info("All good.")

if __name__ == '__main__':
    BlockHashCacheTest().main()<commit_msg>[Tests][BUG] Fix random range in feature_blockhashcache<commit_after>#!/usr/bin/env python3
# Copyright (c) 2019-2020 The PIVX developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

from test_framework.test_framework import PivxTestFramework
from test_framework.util import (
    assert_equal,
)
import random
from time import sleep

class BlockHashCacheTest(PivxTestFramework):

    def set_test_params(self):
        self.num_nodes = 1
        self.setup_clean_chain = True

    def log_title(self):
        title = "*** Starting %s ***" % self.__class__.__name__
        underline = "-" * len(title)
        description = "Tests the block-hashes rolling cache."
        self.log.info("\n\n%s\n%s\n%s\n", title, underline, description)

    def run_test(self):
        self.log_title()
        self.node = self.nodes[0]
        CACHE_SIZE = 200
        # Start with the genesis block
        vBlocks = [self.node.getbestblockhash()]

        # Mine blocks and save the hashes
        self.log.info("Mining %d blocks..." % (CACHE_SIZE - 1))
        for i in range(CACHE_SIZE-1):
            vBlocks.append(self.node.generate(1)[0])

        # Check the cache
        cache = self.node.getcachedblockhashes()
        assert_equal(vBlocks, cache)
        self.log.info("First %d blocks check out" % (CACHE_SIZE - 1))

        # Mine a block and check the cache
        self.log.info("Mining another block...")
        vBlocks.append(self.node.generate(1)[0])
        height = self.node.getblockcount()
        assert_equal(height, CACHE_SIZE)
        assert_equal(len(vBlocks), height + 1)      # genesis included
        cache = self.node.getcachedblockhashes()
        assert_equal(vBlocks[CACHE_SIZE], cache[0])
        self.log.info("Block %d correctly cached (overwriting genesis hash)" % CACHE_SIZE)

        # Mine a random number of blocks between 1 and CACHE_SIZE-1
        x = random.randint(1, CACHE_SIZE - 1)
        self.log.info("Mining %d more blocks..." % x)
        for i in range(x):
            vBlocks.append(self.node.generate(1)[0])

        # Check that the cache is (partially) overwritten
        cache = self.node.getcachedblockhashes()
        for i in range(1, x+1):
            assert_equal(vBlocks[CACHE_SIZE + i], cache[i])
        # ...but not completely
        for i in range(x+1, CACHE_SIZE):
            assert_equal(vBlocks[i], cache[i])
        self.log.info("Cached correctly")

        # Disconnect one block
        self.log.info("Disconnecting one block...")
        height = self.node.getblockcount()
        old_hash = self.node.getbestblockhash()
        self.node.invalidateblock(old_hash)
        assert_equal(self.node.getblockcount(), height - 1)
        vBlocks.pop()
        assert_equal(len(vBlocks), height)  # genesis included

        # Check that the cache cycles back
        cache = self.node.getcachedblockhashes()
        assert_equal(vBlocks[height - CACHE_SIZE], cache[height % CACHE_SIZE])
        self.log.info("Disconnecting %d, block at height %d cycles back in" % (
            height, height - CACHE_SIZE))

        # Reconsider the block, mine another on top, and check the cache again
        self.log.info("Reconsidering and mining one more block...")
        self.node.reconsiderblock(old_hash)
        assert_equal(self.node.getblockcount(), height)
        vBlocks.append(old_hash)
        vBlocks.append(self.node.generate(1)[0])
        assert_equal(self.node.getblockcount(), height + 1)
        assert_equal(len(vBlocks), height + 2)  # genesis included
        assert_equal(vBlocks[height], old_hash)
        cache = self.node.getcachedblockhashes()
        assert_equal(vBlocks[height], cache[height % CACHE_SIZE])
        assert_equal(vBlocks[height + 1], cache[(height + 1) % CACHE_SIZE])
        self.log.info("Cached correctly")

        # Restart the node and check that the cache is correctly initialized
        self.log.info("Restarting the node...")
        self.stop_nodes()
        sleep(5)
        self.start_nodes()
        assert_equal(self.node.getcachedblockhashes(), cache)
        self.log.info("All good.")

if __name__ == '__main__':
    BlockHashCacheTest().main()<|endoftext|>


<commit_before>import json
import os.path

import memcache
from flask import Flask, abort

from common import file, page, console

rss = None
menu_list = None
restful_switch = False
page_name_list = list()
template_config = None
cache_switch = False
cache_list = list()
cache_page = dict()
app = Flask(__name__)

console.log("info", "Loading configuration")
system_config = json.loads(file.read_file("./config/system.json"))
system_config["API_Password"] = None

page_list = json.loads(file.read_file("./config/page.json"))

if os.path.exists("./config/menu.json"):
    menu_list = json.loads(file.read_file("./config/menu.json"))

if "Restful_API" in system_config:
    if system_config["Restful_API"]:
        restful_switch = True

for item in page_list:
    page_name_list.append(item["name"])

if os.path.exists("./document/rss.xml"):
    rss = file.read_file("document/rss.xml")

if os.path.exists("./templates/{0}/config.json".format(system_config["Theme"])):
    template_config = json.loads(file.read_file("./templates/{0}/config.json".format(system_config["Theme"])))
console.log("Success", "load the configuration file successfully")

# Subscribe
@app.route("/rss")
@app.route("/feed")
def load_rss():
    return rss, 200, {'Content-Type': 'text/xml; charset=utf-8'}

@app.route("/")
@app.route("/<file_name>")
@app.route("/<file_name>/")
@app.route('/<file_name>/p/<int:page_index>')
@app.route('/<file_name>x/p/<int:page_index>/')
def route(file_name = "index", page_index = 1):
    result = None
    get_from_cache = False

    page_index_url = ""
    if file_name == "index":
        page_index_url = "/p/{0}".format(str(page_index))
    page_url = "/{0}{1}".format(file_name, page_index_url)

    if page_url in cache_list:
        console.log("info", "Trying to get cache: {0}".format(page_url))
        get_from_cache = True
        result=cache_page[page_url]

    if result is None:
        console.log("info", "Trying to build: {0}".format(page_url))
        if restful_switch:
            result = page.build_restful_result(file_name, system_config, page_list, page_name_list, menu_list)
        else:
            if file_name == "index":
                result, row = page.build_index(page_index, system_config, page_list, menu_list, False, template_config)
            if os.path.exists("document/{0}.md".format(file_name)):
                result = page.build_page(file_name, system_config, page_list, page_name_list, menu_list, False,
                                         template_config)

    if result is not None:
        if not get_from_cache:
            console.log("info", "Writing to cache: {0}".format(page_url))
            cache_list.append(page_url)
            cache_page[page_url]=result
        console.log("Success", "Get success: {0}".format(page_url))
        return result
    console.log("Error", "Can not build: {0}".format(page_url))
    abort(404)
<commit_msg>修改缓存逻辑<commit_after>import json
import os.path

from flask import Flask, abort

from common import file, page, console

rss = None
menu_list = None
page_name_list = list()
cache_page = dict()
template_config = None
cache_switch = False
app = Flask(__name__)

console.log("info", "Loading configuration")

system_config = json.loads(file.read_file("./config/system.json"))

page_list = json.loads(file.read_file("./config/page.json"))

if os.path.exists("./config/menu.json"):
    menu_list = json.loads(file.read_file("./config/menu.json"))

for item in page_list:
    page_name_list.append(item["name"])

if os.path.exists("./document/rss.xml"):
    rss = file.read_file("document/rss.xml")

if os.path.exists("./templates/{0}/config.json".format(system_config["Theme"])):
    template_config = json.loads(file.read_file("./templates/{0}/config.json".format(system_config["Theme"])))

system_config["API_Password"] = None

console.log("Success", "load the configuration file successfully")

# Subscribe
@app.route("/rss")
@app.route("/feed")
def load_rss():
    if rss is None:
        abort(400)
    return rss, 200, {'Content-Type': 'text/xml; charset=utf-8'}

@app.route("/")
@app.route("/<file_name>")
@app.route("/<file_name>/")
@app.route('/<file_name>/p/<int:page_index>')
@app.route('/<file_name>/p/<int:page_index>/')
def route(file_name = "index", page_index = 1):
    result = None

    page_index_url = ""
    if file_name == "index":
        page_index_url = "/p/{0}".format(str(page_index))
    page_url = "/{0}{1}".format(file_name, page_index_url)

    if page_url in cache_page:
        console.log("info", "Get cache Success: {0}".format(page_url))
        return cache_page[page_url]

    if result is None:
        console.log("info", "Trying to build: {0}".format(page_url))
        if system_config["Restful_API"]:
            result = page.build_restful_result(file_name, system_config, page_list, page_name_list, menu_list)
        else:
            if file_name == "index":
                result, row = page.build_index(page_index, system_config, page_list, menu_list, False, template_config)
            if os.path.exists("document/{0}.md".format(file_name)):
                result = page.build_page(file_name, system_config, page_list, page_name_list, menu_list, False,
                                         template_config)

    if result is not None:
        console.log("info", "Writing to cache: {0}".format(page_url))
        cache_page[page_url]=result
        console.log("Success", "Get success: {0}".format(page_url))

        return result

    console.log("Error", "Can not build: {0}".format(page_url))
    abort(404)
<|endoftext|>