<commit_before># Copyright 2019 The SQLFlow Authors. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
# Disable Tensorflow INFO and WARNING logs
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

import sys, json
import tensorflow as tf
import functools
import sys
import numpy as np
import copy
try:
    import sqlflow_models
except:
    pass

from sqlflow_submitter.db import connect_with_data_source, db_generator, parseMaxComputeDSN

TF_VERSION_2 = True  # TODO(shendiaomo): Remove after we fully upgrade to TF2.0
# Disable Tensorflow INFO and WARNING
try:
    if tf.version.VERSION > '1':
        import logging
        tf.get_logger().setLevel(logging.ERROR)
    else:
        raise ImportError
except:
    tf.logging.set_verbosity(tf.logging.ERROR)
    TF_VERSION_2 = False

# FLAGS only available for TF 1.x to support PAI TF training
if not TF_VERSION_2:
    tf.app.flags.DEFINE_string("tables", "", "tables info")
    FLAGS = tf.app.flags.FLAGS

def get_dtype(type_str):
    if type_str == "float32":
        return tf.float32
    elif type_str == "int64":
        return tf.int64
    else:
        raise TypeError("not supported dtype: %s" % type_str)

def parse_sparse_feature(features, label, feature_column_names, feature_metas):
    features_dict = dict()
    for idx, col in enumerate(features):
        name = feature_column_names[idx]
        if feature_metas[name]["is_sparse"]:
            i, v, s = col
            features_dict[name] = tf.SparseTensor(indices=i, values=v, dense_shape=s)
        else:
            features_dict[name] = col
    return features_dict, label

if TF_VERSION_2:
    class PrintStatusHook(tf.estimator.LoggingTensorHook):
        def __init__(self, prefix="", every_n_iter=None, every_n_secs=None,
                at_end=False, formatter=None):
            super().__init__([], every_n_iter=every_n_iter, every_n_secs=every_n_secs,
                at_end=at_end, formatter=formatter)
            self.prefix = prefix

        def before_run(self, run_context):
            self._should_trigger = self._timer.should_trigger_for_step(self._iter_count)
            loss_vars = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.LOSSES)
            step_vars = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.GLOBAL_STEP)
            fetch = {"loss": loss_vars[0], "step": step_vars[0]}
            if self._should_trigger:
                return tf.estimator.SessionRunArgs(fetch)
            else:
                return None

        def _log_tensors(self, tensor_values):
            elapsed_secs, _ = self._timer.update_last_triggered_step(self._iter_count)
            stats = []
            for k in tensor_values.keys():
                stats.append("%s = %s" % (k, tensor_values[k]))
            if self.prefix == "eval":
                print("============Evaluation=============")
            print("%s: %s" % (self.prefix, ", ".join(stats)))
            if self.prefix == "eval":
                print("============Evaluation End=============")

def train(is_keras_model,
          datasource,
          estimator,
          select,
          validate_select,
          feature_columns,
          feature_column_names,
          feature_metas={},
          label_meta={},
          model_params={},
          save="",
          batch_size=1,
          epochs=1,
          verbose=0,
          train_max_steps=None,
          eval_start_delay_secs=0,
          eval_throttle_secs=0,
          save_checkpoints_steps=100,
          log_every_n_iter=10,
          is_pai=False,
          pai_table=""):
    if is_keras_model:
        if verbose == 1:
            # show keras training progress
            tf.get_logger().setLevel(logging.INFO)
        elif verbose >= 2:
            tf.get_logger().setLevel(logging.DEBUG)
    else:
        if verbose >= 2:
            if TF_VERSION_2:
                tf.get_logger().setLevel(logging.INFO)
            else:
                tf.logging.set_verbosity(tf.logging.INFO)
    if not is_pai:
        conn = connect_with_data_source(datasource)
    model_params.update(feature_columns)

    def input_fn(datasetStr):
        feature_types = []
        for name in feature_column_names:
            # NOTE: vector columns like 23,21,3,2,0,0 should use shape None
            if feature_metas[name]["is_sparse"]:
                feature_types.append((tf.int64, tf.int32, tf.int64))
            else:
                feature_types.append(get_dtype(feature_metas[name]["dtype"]))

        gen = db_generator(conn.driver, conn, datasetStr, feature_column_names, label_meta["feature_name"], feature_metas)
        dataset = tf.data.Dataset.from_generator(gen, (tuple(feature_types), eval("tf.%s" % label_meta["dtype"])))
        ds_mapper = functools.partial(parse_sparse_feature, feature_column_names=feature_column_names, feature_metas=feature_metas)
        return dataset.map(ds_mapper)

    def pai_maxcompute_input_fn(datasetStr):
        table_parts = pai_table.split(".")
        if len(table_parts) == 2:
            database, table_name = table_parts
        elif len(table_parts) == 1:
            table_name = pai_table
            driver, dsn = datasource.split("://")
            database = parseMaxComputeDSN(dsn)[-1]
        else:
            raise ValueError("error database.table format: %s" % pai_table)

        tables = ["odps://%s/tables/%s" % (database, table_name)]
        record_defaults = []
        for name in feature_column_names:
            dtype = get_dtype(feature_metas[name]["dtype"])
            record_defaults.append(tf.constant(0, dtype=dtype, shape=feature_metas[name]["shape"]))
        record_defaults.append(
            tf.constant(0, get_dtype(label_meta["dtype"]), shape=label_meta["shape"]))

        selected_cols = copy.copy(feature_column_names)
        selected_cols.append(label_meta["feature_name"])
        dataset = tf.data.TableRecordDataset(tables,
                                     record_defaults=record_defaults,
                                     selected_cols=",".join(selected_cols))
        def tensor_to_dict(*args):
            num_features = len(feature_column_names)
            label = args[num_features]
            features_dict = dict()
            for idx in range(num_features):
                name = feature_column_names[idx]
                features_dict[name] = tf.reshape(args[idx], [-1])
            return features_dict, label

        return dataset.map(tensor_to_dict)

    def train_input_fn(batch_size):
        if is_pai:
            dataset = pai_maxcompute_input_fn(select)
        else:
            dataset = input_fn(select)
        # FIXME(typhoonzero): find a way to cache to local file and avoid cache lockfile already exists issue.
        dataset = dataset.shuffle(1000).batch(batch_size).cache()
        if not is_keras_model:
            dataset = dataset.repeat(epochs if epochs else 1)
        return dataset

    def validate_input_fn(batch_size):
        if is_pai:
            dataset = pai_maxcompute_input_fn(validate_select)
        else:
            dataset = input_fn(validate_select)
        return dataset.batch(batch_size).cache()

    if is_keras_model:
        if not issubclass(estimator, tf.keras.Model):
            # functional model need field_metas parameter
            model_params["field_metas"] = feature_metas
        classifier = estimator(**model_params)
        classifier_pkg = sys.modules[estimator.__module__]

        classifier.compile(optimizer=classifier_pkg.optimizer(),
            loss=classifier_pkg.loss,
            metrics=["accuracy"])
        if hasattr(classifier, 'sqlflow_train_loop'):
            classifier.sqlflow_train_loop(train_input_fn(batch_size))
        else:
            ds = train_input_fn(batch_size)
            if label_meta["feature_name"] != "" and validate_select != "":
                history = classifier.fit(ds,
                    epochs=epochs if epochs else classifier.default_training_epochs(),
                    validation_data=validate_input_fn(batch_size),
                    verbose=verbose)
            else:
                history = classifier.fit(ds,
                    epochs=epochs if epochs else classifier.default_training_epochs(),
                    verbose=verbose)
            train_keys = []
            val_keys = []
            for k in history.history.keys():
                if k.startswith("val_"):
                    val_keys.append(k)
                else:
                    train_keys.append(k)
            print("====== Result for training set: ======")
            for k in train_keys:
                print("%s: %s" % (k, history.history[k][-1]))
            print("====== Result for validation set: ======")
            for k in val_keys:
                print("%s: %s" % (k, history.history[k][-1]))
        classifier.save_weights(save, save_format="h5")
    else:
        model_params["model_dir"] = save
        model_params["config"] = tf.estimator.RunConfig(save_checkpoints_steps=save_checkpoints_steps)
        classifier = estimator(**model_params)

        if validate_select == "":
            classifier.train(input_fn=lambda:train_input_fn(batch_size))
        else:
            # TODO(typhoonzero): able to config metrics by calling tf.estimators.add_metrics()
            train_hooks = []
            if verbose == 1 and TF_VERSION_2:
                train_hooks = [PrintStatusHook("train", every_n_iter=log_every_n_iter)]
            train_spec = tf.estimator.TrainSpec(input_fn=lambda:train_input_fn(batch_size), max_steps=train_max_steps, hooks=train_hooks)
            eval_hooks = []
            if verbose == 1 and TF_VERSION_2:
                eval_hooks = [PrintStatusHook("eval", every_n_iter=log_every_n_iter)]
            eval_spec = tf.estimator.EvalSpec(input_fn=lambda:validate_input_fn(batch_size), hooks=eval_hooks, start_delay_secs=eval_start_delay_secs, throttle_secs=eval_throttle_secs)
            result = tf.estimator.train_and_evaluate(classifier, train_spec, eval_spec)
            print(result[0])

    print("Done training")

<commit_msg>support custom metrics for keras models (#1467)<commit_after># Copyright 2019 The SQLFlow Authors. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
# Disable Tensorflow INFO and WARNING logs
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

import sys, json
import tensorflow as tf
import functools
import sys
import numpy as np
import copy
try:
    import sqlflow_models
except:
    pass

from sqlflow_submitter.db import connect_with_data_source, db_generator, parseMaxComputeDSN

TF_VERSION_2 = True  # TODO(shendiaomo): Remove after we fully upgrade to TF2.0
# Disable Tensorflow INFO and WARNING
try:
    if tf.version.VERSION > '1':
        import logging
        tf.get_logger().setLevel(logging.ERROR)
    else:
        raise ImportError
except:
    tf.logging.set_verbosity(tf.logging.ERROR)
    TF_VERSION_2 = False

# FLAGS only available for TF 1.x to support PAI TF training
if not TF_VERSION_2:
    tf.app.flags.DEFINE_string("tables", "", "tables info")
    FLAGS = tf.app.flags.FLAGS

def get_dtype(type_str):
    if type_str == "float32":
        return tf.float32
    elif type_str == "int64":
        return tf.int64
    else:
        raise TypeError("not supported dtype: %s" % type_str)

def parse_sparse_feature(features, label, feature_column_names, feature_metas):
    features_dict = dict()
    for idx, col in enumerate(features):
        name = feature_column_names[idx]
        if feature_metas[name]["is_sparse"]:
            i, v, s = col
            features_dict[name] = tf.SparseTensor(indices=i, values=v, dense_shape=s)
        else:
            features_dict[name] = col
    return features_dict, label

if TF_VERSION_2:
    class PrintStatusHook(tf.estimator.LoggingTensorHook):
        def __init__(self, prefix="", every_n_iter=None, every_n_secs=None,
                at_end=False, formatter=None):
            super().__init__([], every_n_iter=every_n_iter, every_n_secs=every_n_secs,
                at_end=at_end, formatter=formatter)
            self.prefix = prefix

        def before_run(self, run_context):
            self._should_trigger = self._timer.should_trigger_for_step(self._iter_count)
            loss_vars = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.LOSSES)
            step_vars = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.GLOBAL_STEP)
            fetch = {"loss": loss_vars[0], "step": step_vars[0]}
            if self._should_trigger:
                return tf.estimator.SessionRunArgs(fetch)
            else:
                return None

        def _log_tensors(self, tensor_values):
            elapsed_secs, _ = self._timer.update_last_triggered_step(self._iter_count)
            stats = []
            for k in tensor_values.keys():
                stats.append("%s = %s" % (k, tensor_values[k]))
            if self.prefix == "eval":
                print("============Evaluation=============")
            print("%s: %s" % (self.prefix, ", ".join(stats)))
            if self.prefix == "eval":
                print("============Evaluation End=============")

def train(is_keras_model,
          datasource,
          estimator,
          select,
          validate_select,
          feature_columns,
          feature_column_names,
          feature_metas={},
          label_meta={},
          model_params={},
          save="",
          batch_size=1,
          epochs=1,
          verbose=0,
          train_max_steps=None,
          eval_start_delay_secs=0,
          eval_throttle_secs=0,
          save_checkpoints_steps=100,
          log_every_n_iter=10,
          is_pai=False,
          pai_table=""):
    if is_keras_model:
        if verbose == 1:
            # show keras training progress
            tf.get_logger().setLevel(logging.INFO)
        elif verbose >= 2:
            tf.get_logger().setLevel(logging.DEBUG)
    else:
        if verbose >= 2:
            if TF_VERSION_2:
                tf.get_logger().setLevel(logging.INFO)
            else:
                tf.logging.set_verbosity(tf.logging.INFO)
    if not is_pai:
        conn = connect_with_data_source(datasource)
    model_params.update(feature_columns)

    def input_fn(datasetStr):
        feature_types = []
        for name in feature_column_names:
            # NOTE: vector columns like 23,21,3,2,0,0 should use shape None
            if feature_metas[name]["is_sparse"]:
                feature_types.append((tf.int64, tf.int32, tf.int64))
            else:
                feature_types.append(get_dtype(feature_metas[name]["dtype"]))

        gen = db_generator(conn.driver, conn, datasetStr, feature_column_names, label_meta["feature_name"], feature_metas)
        dataset = tf.data.Dataset.from_generator(gen, (tuple(feature_types), eval("tf.%s" % label_meta["dtype"])))
        ds_mapper = functools.partial(parse_sparse_feature, feature_column_names=feature_column_names, feature_metas=feature_metas)
        return dataset.map(ds_mapper)

    def pai_maxcompute_input_fn(datasetStr):
        table_parts = pai_table.split(".")
        if len(table_parts) == 2:
            database, table_name = table_parts
        elif len(table_parts) == 1:
            table_name = pai_table
            driver, dsn = datasource.split("://")
            database = parseMaxComputeDSN(dsn)[-1]
        else:
            raise ValueError("error database.table format: %s" % pai_table)

        tables = ["odps://%s/tables/%s" % (database, table_name)]
        record_defaults = []
        for name in feature_column_names:
            dtype = get_dtype(feature_metas[name]["dtype"])
            record_defaults.append(tf.constant(0, dtype=dtype, shape=feature_metas[name]["shape"]))
        record_defaults.append(
            tf.constant(0, get_dtype(label_meta["dtype"]), shape=label_meta["shape"]))

        selected_cols = copy.copy(feature_column_names)
        selected_cols.append(label_meta["feature_name"])
        dataset = tf.data.TableRecordDataset(tables,
                                     record_defaults=record_defaults,
                                     selected_cols=",".join(selected_cols))
        def tensor_to_dict(*args):
            num_features = len(feature_column_names)
            label = args[num_features]
            features_dict = dict()
            for idx in range(num_features):
                name = feature_column_names[idx]
                features_dict[name] = tf.reshape(args[idx], [-1])
            return features_dict, label

        return dataset.map(tensor_to_dict)

    def train_input_fn(batch_size):
        if is_pai:
            dataset = pai_maxcompute_input_fn(select)
        else:
            dataset = input_fn(select)
        # FIXME(typhoonzero): find a way to cache to local file and avoid cache lockfile already exists issue.
        dataset = dataset.shuffle(1000).batch(batch_size).cache()
        if not is_keras_model:
            dataset = dataset.repeat(epochs if epochs else 1)
        return dataset

    def validate_input_fn(batch_size):
        if is_pai:
            dataset = pai_maxcompute_input_fn(validate_select)
        else:
            dataset = input_fn(validate_select)
        return dataset.batch(batch_size).cache()

    if is_keras_model:
        if not issubclass(estimator, tf.keras.Model):
            # functional model need field_metas parameter
            model_params["field_metas"] = feature_metas
        classifier = estimator(**model_params)
        classifier_pkg = sys.modules[estimator.__module__]
        if hasattr(classifier_pkg, "eval_metrics_fn"):
            metrics_functions = classifier_pkg.eval_metrics_fn()
            metrics = []
            for key, func in metrics_functions.items():
                func.__name__ = key
                metrics.append(func)
        else:
            metrics = ["accuracy"]

        classifier.compile(optimizer=classifier_pkg.optimizer(),
            loss=classifier_pkg.loss,
            metrics=metrics)
        if hasattr(classifier, 'sqlflow_train_loop'):
            classifier.sqlflow_train_loop(train_input_fn(batch_size))
        else:
            ds = train_input_fn(batch_size)
            if label_meta["feature_name"] != "" and validate_select != "":
                history = classifier.fit(ds,
                    epochs=epochs if epochs else classifier.default_training_epochs(),
                    validation_data=validate_input_fn(batch_size),
                    verbose=verbose)
            else:
                history = classifier.fit(ds,
                    epochs=epochs if epochs else classifier.default_training_epochs(),
                    verbose=verbose)
            train_keys = []
            val_keys = []
            for k in history.history.keys():
                if k.startswith("val_"):
                    val_keys.append(k)
                else:
                    train_keys.append(k)
            print("====== Result for training set: ======")
            for k in train_keys:
                print("%s: %s" % (k, history.history[k][-1]))
            print("====== Result for validation set: ======")
            for k in val_keys:
                print("%s: %s" % (k, history.history[k][-1]))
        classifier.save_weights(save, save_format="h5")
    else:
        model_params["model_dir"] = save
        model_params["config"] = tf.estimator.RunConfig(save_checkpoints_steps=save_checkpoints_steps)
        classifier = estimator(**model_params)

        if validate_select == "":
            classifier.train(input_fn=lambda:train_input_fn(batch_size))
        else:
            # TODO(typhoonzero): able to config metrics by calling tf.estimators.add_metrics()
            train_hooks = []
            if verbose == 1 and TF_VERSION_2:
                train_hooks = [PrintStatusHook("train", every_n_iter=log_every_n_iter)]
            train_spec = tf.estimator.TrainSpec(input_fn=lambda:train_input_fn(batch_size), max_steps=train_max_steps, hooks=train_hooks)
            eval_hooks = []
            if verbose == 1 and TF_VERSION_2:
                eval_hooks = [PrintStatusHook("eval", every_n_iter=log_every_n_iter)]
            eval_spec = tf.estimator.EvalSpec(input_fn=lambda:validate_input_fn(batch_size), hooks=eval_hooks, start_delay_secs=eval_start_delay_secs, throttle_secs=eval_throttle_secs)
            result = tf.estimator.train_and_evaluate(classifier, train_spec, eval_spec)
            print(result[0])

    print("Done training")

<|endoftext|>


<commit_before>from app import db
from sqlalchemy.dialects.postgresql import ARRAY
from datetime import datetime

class Listing(db.Model):
    __tablename__ = 'listings'

    id = db.Column(db.Integer, primary_key=True)
    url = db.Column(db.String())
    company = db.Column(db.String(32))
    title = db.Column(db.String(128))
    status = db.Column(db.String(10))
    scraped_dt = db.Column(db.DateTime())
    last_seen = db.Column(db.DateTime())

    def seen_now(self):
        """Function for updating an entry on successive sightings"""
        self.last_seen = datetime.now()

    def __init__(self, url, company, title):
        self.url = url
        self.company = company
        self.title = title
        self.status = "NEW"
        self.scraped_dt = datetime.now()
        self.last_seen = datetime.now()

    def __repr__(self):
        return '{} : {}'.format(self.company, self.title)

class Company(db.Model):
    __tablename__ = 'companies'

    id = db.Column(db.Integer, primary_key=True)
    interest = db.Column(db.Boolean())

    careers_url = db.Column(db.String())
    last_checked = db.Column(db.DateTime())

    keywords = db.Column(ARRAY(db.String(32)))
    locations = db.Column(ARRAY(db.String(32)))
    departments = db.Column(ARRAY(db.String(16)))

    def __init__(self, careers_url, keywords, locations, departments):
        self.interest = True
        self.careers_url = careers_url
        self.last_checked = datetime.now()
        self.keywords = keywords
        self.locations = locations
        self.departments = departments

    def __repr__(self):
        return '{} ({})'.format(self.company, self.interest)
<commit_msg>included name field for company<commit_after>from app import db
from sqlalchemy.dialects.postgresql import ARRAY
from datetime import datetime

class Listing(db.Model):
    __tablename__ = 'listings'

    id = db.Column(db.Integer, primary_key=True)
    url = db.Column(db.String())
    company = db.Column(db.String(32))
    title = db.Column(db.String(128))
    status = db.Column(db.String(10))
    scraped_dt = db.Column(db.DateTime())
    last_seen = db.Column(db.DateTime())

    def seen_now(self):
        """Function for updating an entry on successive sightings"""
        self.last_seen = datetime.now()

    def __init__(self, url, company, title):
        self.url = url
        self.company = company
        self.title = title
        self.status = "NEW"
        self.scraped_dt = datetime.now()
        self.last_seen = datetime.now()

    def __repr__(self):
        return '{} : {}'.format(self.company, self.title)

class Company(db.Model):
    __tablename__ = 'companies'

    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(24))
    interest = db.Column(db.Boolean())

    careers_url = db.Column(db.String())
    last_checked = db.Column(db.DateTime())

    keywords = db.Column(ARRAY(db.String(32)))
    locations = db.Column(ARRAY(db.String(32)))
    departments = db.Column(ARRAY(db.String(16)))

    def __init__(self, name, careers_url, keywords, locations, departments):
        self.name = name
        self.interest = True
        self.careers_url = careers_url
        self.last_checked = datetime.now()
        self.keywords = keywords
        self.locations = locations
        self.departments = departments

    def __repr__(self):
        return '{} ({})'.format(self.company, self.interest)
<|endoftext|>


<commit_before>"""
This module handles all the phishing related operations for
Wifiphisher.py
"""

import os
from constants import *
from shutil import copyfile

import ConfigParser
from jinja2 import Environment, FileSystemLoader

def config_section_map(config_file, section):
    """
    Map the values of a config file to a dictionary.
    """

    config = ConfigParser.ConfigParser()
    config.read(config_file)
    dict1 = {}

    if section not in config.sections():
        return dict1

    options = config.options(section)
    for option in options:
        try:
            dict1[option] = config.get(section, option)
        except:
            dict1[option] = None
    return dict1

class InvalidTemplate(Exception):
    """ Exception class to raise in case of a invalid template """

    def __init__(self):
        Exception.__init__(self, "The given template is either invalid or " +
                           "not available locally!")

class PhishingTemplate(object):
    """ This class represents phishing templates """

    def __init__(self, name):
        """
        Construct object.

        :param self: A PhishingTemplate object
        :type self: PhishingScenario
        :return: None
        :rtype: None
        .. todo:: Maybe add a category field
        """

        # setup all the variables

        config_path = os.path.join(PHISHING_PAGES_DIR, name, 'config.ini')
        info = config_section_map(config_path, 'info')

        self._name = name
        self._display_name = info['name']
        self._description = info['description']
        self._payload = False
        if 'payloadpath' in info:
            self._payload = info['payloadpath']

        self._path = PHISHING_PAGES_DIR + self._name.lower() + "/"

        self._context = config_section_map(config_path, 'context')
        self._env = Environment(loader=FileSystemLoader(self._path))
        self._extra_files = []

    def merge_context(self, context):
        """
            Merge dict context with current one
            In case of confict always keep current values
        """
        context.update(self._context)
        self._context = context

    def get_display_name(self):
        """
        Return the display name of the template.

        :param self: A PhishingTemplate object
        :type self: PhishingTemplate
        :return: the display name of the template
        :rtype: str
        """

        return self._display_name

    def get_payload_path(self):
        """
        Return the payload path of the template.

        :param self: A PhishingTemplate object
        :type self: PhishingTemplate
        :return: The path of the template
        :rtype: bool
        """

        return self._payload

    def has_payload(self):
        """
        Return whether the template has a payload.

        :param self: A PhishingTemplate object
        :type self: PhishingTemplate
        :return: boolean if it needs payload
        :rtype: bool
        """

        if self._payload:
            return True
        return False

    def get_description(self):
        """
        Return the description of the template.

        :param self: A PhishingTemplate object
        :type self: PhishingTemplate
        :return: the description of the template
        :rtype: str
        """

        return self._description

    def get_path(self):
        """
        Return the path of the template files.

        :param self: A PhishingTemplate object
        :type self: PhishingTemplate
        :return: the path of template files
        :rtype: str
        """

        return self._path

    def use_file(self, path):
        """
        Copies a file in the filesystem to the path
        of the template files.

        :param self: A PhishingTemplate object
        :type self: PhishingTemplate
        :param path: path of the file that is to be copied
        :type self: str
        :return: the path of the file under the template files
        :rtype: str
        """

        if os.path.isfile(path):
            filename = os.path.basename(path)
            copyfile(path, self.get_path() + filename)
            self._extra_files.append(self.get_path() + filename)
            return filename

    def remove_extra_files(self):
        """
        Removes extra used files (if any)

        :param self: A PhishingTemplate object
        :type self: PhishingTemplate
        :return: None
        :rtype: None
        """

        for f in self._extra_files:
            if os.path.isfile(f):
                os.remove(f)

    def render(self, path):
        t = self._env.get_template(path)
        return t.render(self._context)

    def __str__(self):
        """
        Return a string representation of the template.

        :param self: A PhishingTemplate object
        :type self: PhishingTemplate
        :return: the name followed by the description of the template
        :rtype: str
        """

        return (self._display_name + "\n\t" + self._description + "\n")

class TemplateManager(object):
    """ This class handles all the template management operations """

    def __init__(self):
        """
        Construct object.

        :param self: A TemplateManager object
        :type self: TemplateManager
        :return: None
        :rtype: None
        """

        # setup the templates
        self._template_directory = PHISHING_PAGES_DIR

        page_dirs = os.listdir(PHISHING_PAGES_DIR)

        self._templates = {}

        for page in page_dirs:
            if os.path.isdir(page):
                self._templates[page] = PhishingTemplate(page)

        # add all the user templates to the database
        self.add_user_templates()

    def get_templates(self):
        """
        Return all the available templates.

        :param self: A TemplateManager object
        :type self: TemplateManager
        :return: all the available templates
        :rtype: dict
        """

        return self._templates

    def find_user_templates(self):
        """
        Return all the user's templates available.

        :param self: A TemplateManager object
        :type self: TemplateManager
        :return: all the local templates available
        :rtype: list
        .. todo:: check to make sure directory contains HTML files
        """

        # a list to store file names in
        local_templates = []

        # loop through the directory content
        for name in os.listdir(self._template_directory):
            # check to see if it is a directory and not in the database
            if (os.path.isdir(os.path.join(self._template_directory, name)) and
                    name not in self._templates):
                # add it to the list
                local_templates.append(name)

        return local_templates

    def add_user_templates(self):
        """
        Add all the user templates to the database.

        :param self: A TemplateManager object
        :type: self: TemplateManager
        :return: None
        :rtype: None
        """

        # get all the user's templates
        user_templates = self.find_user_templates()

        # loop through the templates
        for template in user_templates:
            # create a template object and add it to the database
            local_template = PhishingTemplate(template)
            self._templates[template] = local_template
<commit_msg>Added sanity check.<commit_after>"""
This module handles all the phishing related operations for
Wifiphisher.py
"""

import os
from constants import *
from shutil import copyfile

import ConfigParser
from jinja2 import Environment, FileSystemLoader

def config_section_map(config_file, section):
    """
    Map the values of a config file to a dictionary.
    """

    config = ConfigParser.ConfigParser()
    config.read(config_file)
    dict1 = {}

    if section not in config.sections():
        return dict1

    options = config.options(section)
    for option in options:
        try:
            dict1[option] = config.get(section, option)
        except:
            dict1[option] = None
    return dict1

class InvalidTemplate(Exception):
    """ Exception class to raise in case of a invalid template """

    def __init__(self):
        Exception.__init__(self, "The given template is either invalid or " +
                           "not available locally!")

class PhishingTemplate(object):
    """ This class represents phishing templates """

    def __init__(self, name):
        """
        Construct object.

        :param self: A PhishingTemplate object
        :type self: PhishingScenario
        :return: None
        :rtype: None
        .. todo:: Maybe add a category field
        """

        # setup all the variables

        config_path = os.path.join(PHISHING_PAGES_DIR, name, 'config.ini')
        info = config_section_map(config_path, 'info')

        self._name = name
        self._display_name = info['name']
        self._description = info['description']
        self._payload = False
        if 'payloadpath' in info:
            self._payload = info['payloadpath']

        self._path = PHISHING_PAGES_DIR + self._name.lower() + "/"

        self._context = config_section_map(config_path, 'context')
        self._env = Environment(loader=FileSystemLoader(self._path))
        self._extra_files = []

    def merge_context(self, context):
        """
            Merge dict context with current one
            In case of confict always keep current values
        """
        context.update(self._context)
        self._context = context

    def get_display_name(self):
        """
        Return the display name of the template.

        :param self: A PhishingTemplate object
        :type self: PhishingTemplate
        :return: the display name of the template
        :rtype: str
        """

        return self._display_name

    def get_payload_path(self):
        """
        Return the payload path of the template.

        :param self: A PhishingTemplate object
        :type self: PhishingTemplate
        :return: The path of the template
        :rtype: bool
        """

        return self._payload

    def has_payload(self):
        """
        Return whether the template has a payload.

        :param self: A PhishingTemplate object
        :type self: PhishingTemplate
        :return: boolean if it needs payload
        :rtype: bool
        """

        if self._payload:
            return True
        return False

    def get_description(self):
        """
        Return the description of the template.

        :param self: A PhishingTemplate object
        :type self: PhishingTemplate
        :return: the description of the template
        :rtype: str
        """

        return self._description

    def get_path(self):
        """
        Return the path of the template files.

        :param self: A PhishingTemplate object
        :type self: PhishingTemplate
        :return: the path of template files
        :rtype: str
        """

        return self._path

    def use_file(self, path):
        """
        Copies a file in the filesystem to the path
        of the template files.

        :param self: A PhishingTemplate object
        :type self: PhishingTemplate
        :param path: path of the file that is to be copied
        :type self: str
        :return: the path of the file under the template files
        :rtype: str
        """

        if path is not None and os.path.isfile(path):
            filename = os.path.basename(path)
            copyfile(path, self.get_path() + filename)
            self._extra_files.append(self.get_path() + filename)
            return filename

    def remove_extra_files(self):
        """
        Removes extra used files (if any)

        :param self: A PhishingTemplate object
        :type self: PhishingTemplate
        :return: None
        :rtype: None
        """

        for f in self._extra_files:
            if os.path.isfile(f):
                os.remove(f)

    def render(self, path):
        t = self._env.get_template(path)
        return t.render(self._context)

    def __str__(self):
        """
        Return a string representation of the template.

        :param self: A PhishingTemplate object
        :type self: PhishingTemplate
        :return: the name followed by the description of the template
        :rtype: str
        """

        return (self._display_name + "\n\t" + self._description + "\n")

class TemplateManager(object):
    """ This class handles all the template management operations """

    def __init__(self):
        """
        Construct object.

        :param self: A TemplateManager object
        :type self: TemplateManager
        :return: None
        :rtype: None
        """

        # setup the templates
        self._template_directory = PHISHING_PAGES_DIR

        page_dirs = os.listdir(PHISHING_PAGES_DIR)

        self._templates = {}

        for page in page_dirs:
            if os.path.isdir(page):
                self._templates[page] = PhishingTemplate(page)

        # add all the user templates to the database
        self.add_user_templates()

    def get_templates(self):
        """
        Return all the available templates.

        :param self: A TemplateManager object
        :type self: TemplateManager
        :return: all the available templates
        :rtype: dict
        """

        return self._templates

    def find_user_templates(self):
        """
        Return all the user's templates available.

        :param self: A TemplateManager object
        :type self: TemplateManager
        :return: all the local templates available
        :rtype: list
        .. todo:: check to make sure directory contains HTML files
        """

        # a list to store file names in
        local_templates = []

        # loop through the directory content
        for name in os.listdir(self._template_directory):
            # check to see if it is a directory and not in the database
            if (os.path.isdir(os.path.join(self._template_directory, name)) and
                    name not in self._templates):
                # add it to the list
                local_templates.append(name)

        return local_templates

    def add_user_templates(self):
        """
        Add all the user templates to the database.

        :param self: A TemplateManager object
        :type: self: TemplateManager
        :return: None
        :rtype: None
        """

        # get all the user's templates
        user_templates = self.find_user_templates()

        # loop through the templates
        for template in user_templates:
            # create a template object and add it to the database
            local_template = PhishingTemplate(template)
            self._templates[template] = local_template
<|endoftext|>


<commit_before>#!/usr/bin/env python -u
from __future__ import absolute_import, division, print_function, unicode_literals

import json, os, sys, traceback
from collections import defaultdict
from gratipay import wireup
from decimal import Decimal as D
from pprint import pprint
from gratipay.models.exchange_route import ExchangeRoute

db = wireup.db(wireup.env())
dirname = os.path.join(os.path.dirname(__file__), 'balanced', 'transactions')

class UnknownExchange(Exception):
    def __str__(self):
        return "\n\n{}".format(self.args[0])

class UnknownCustomer(Exception): pass
class UnknownRoute(Exception): pass
class AmbiguousCustomer(Exception): pass
class AmbiguousRoute(Exception): pass

class CountsOff(Exception):
    def __str__(self):
        msg, counts = self.args
        out = [msg]
        cur = ''
        for name, val in sorted(counts.__dict__.items()):
            if name[0] != cur:
                cur = name[0]
                out.append('')
            out.append("{:<24} {:>5}".format(name, val))
        return '\n'.join(out)

class Counts(object):
    def __init__(self):
        self.customer_known = 0
        self.customer_unknown = 0

        self.exchange_in_transaction = 0
        self.exchange_triangulated = 0
        self.exchange_unknown = 0

        self.route_in_exchange = 0
        self.route_ambiguous = 0
        self.route_matched_via_card = 0
        self.route_matched_via_customer = 0
        self.route_ambiguous_customer = 0
        self.route_created = 0
        self.route_unknown = 0

        self.z_linked_already = 0
        self.z_linking_now = 0

        self.z_n = 0

    def check(self):
        n = sum(( self.z_linking_now
                , self.z_linked_already
                , self.customer_unknown
                , self.exchange_unknown
                , self.route_ambiguous
                , self.route_ambiguous_customer
                , self.route_unknown
                 ))
        if self.z_n != n:
            raise CountsOff("Total N is out from {}.".format(n), counts)

        ncustomers = sum((self.customer_known,))
        nexchanges = sum(( self.exchange_in_transaction
                         , self.exchange_triangulated
                         , self.exchange_unknown
                          ))
        if ncustomers != nexchanges:
            raise CountsOff( "Customer count ({}) doesn't match exchange count ({})."
                             .format(ncustomers, nexchanges)
                           , counts
                            )

        nexchanges = sum(( self.exchange_in_transaction
                         , self.exchange_triangulated
                          ))
        nroutes = sum(( self.route_in_exchange
                      , self.route_ambiguous
                      , self.route_matched_via_card
                      , self.route_matched_via_customer
                      , self.route_ambiguous_customer
                      , self.route_created
                      , self.route_unknown
                       ))
        if nexchanges != nroutes:
            raise CountsOff( "Exchange count ({}) doesn't match route count ({})."
                             .format(nexchanges, nroutes)
                           , counts
                            )

def log(msg, w=0):
    print("{msg:<{w}}".format(msg=msg, w=w), end=' | ')

def get_usernames(cur, transaction, customers, counts):

    # First strategy: check known customers.
    usernames = customers.get(transaction['links']['customer'])
    if usernames:
        counts.customer_known += 1
        log('known customer')
        log('participants: {}'.format(len(usernames)))
        return usernames

    # Second strategy: blah
    counts.customer_unknown += 1
    raise UnknownCustomer()

def get_exchange_id(cur, transaction, counts, usernames):

    # First strategy: use the one in the transaction!
    exchange_id = transaction['meta'].get('exchange_id')
    if exchange_id:
        counts.exchange_in_transaction += 1
        log("exchange_id in transaction", 26)
        return exchange_id

    # Second strategy: triangulate.
    params = dict( usernames = usernames
                 , amount    = D(transaction['amount']) / 100
                 , ts        = transaction['created_at']
                  )
    mogrified = cur.mogrify("""\
        SELECT id FROM exchanges
         WHERE participant = ANY(%(usernames)s)
           AND amount + fee = %(amount)s
           AND ((   (%(ts)s::timestamptz - "timestamp") < interval '0'
                AND (%(ts)s::timestamptz - "timestamp") > interval '-15m'
                ) OR (
                    (%(ts)s::timestamptz - "timestamp") > interval '0'
                AND (%(ts)s::timestamptz - "timestamp") < interval '15m'
                ))
    """, params)
    exchange_id = cur.one(mogrified)
    if exchange_id:
        counts.exchange_triangulated += 1
        log("triangulated an exchange", 26)
        return exchange_id

    counts.exchange_unknown += 1
    raise UnknownExchange(mogrified)

def resolve_route(cur, transaction, counts, routes, usernames):
    log('routes: {routes:<12}'.format(routes=', '.join(str(r.id) for r in routes)))

    nroutes = len(routes)
    if nroutes == 0:    return None
    elif nroutes == 1:  return routes[0].id
    assert nroutes == 2
    # Pick the route for the participant that was active at the time of the transaction.

    nusernames = len(usernames)
    assert nusernames in (1, 2), (usernames, routes)
    if nusernames == 1:
        counts.route_ambiguous += 1
        raise AmbiguousRoute()
    else:
        # See if we can assume it's a case of absorption.
        user_ids = [r.participant for r in routes]
        i2n = dict(cur.all( "SELECT id, username FROM participants WHERE id = ANY(%s)"
                          , (user_ids,)
                           ))
        assert sorted(i2n.values()) == sorted(usernames)
        u2r = {i2n[r.participant]: r.id for r in routes}

        keys = u2r.keys()
        absorption = cur.one("""\
            SELECT *
              FROM absorptions
             WHERE (archived_as=%(one)s AND absorbed_by=%(two)s)
                OR (archived_as=%(two)s AND absorbed_by=%(one)s)
        """, dict(one=keys[0], two=keys[1]))
        if absorption is None:
            counts.route_ambiguous += 1
            raise AmbiguousRoute()
        if transaction['created_at'] < str(absorption.timestamp).replace('+00:00', 'Z'):
            key = absorption.archived_as
        else:
            assert transaction['created_at'] > str(absorption.timestamp).replace('+00:00', 'Z')
            key = absorption.absorbed_by
        return u2r[key]

def get_route_id(cur, transaction, counts, usernames, exchange_id):

    # First strategy: match on exchange id.
    route_id = cur.one("SELECT route FROM exchanges WHERE id=%s", (exchange_id,))
    if route_id:
        counts.route_in_exchange += 1
        log("exchange has a route", 22)
        return route_id

    # Second strategy: match on known cards.
    if not route_id:
        routes = cur.all( "SELECT * FROM exchange_routes "
                          "WHERE network='balanced-cc' AND address='/cards/'||%s"
                        , (transaction['links']['source'],)
                         )
        route_id = resolve_route(cur, transaction, counts, routes, usernames)
    if route_id:
        counts.route_matched_via_card += 1
        log("card matches {}".format(route_id), 22)
        return route_id

    # Third strategy: match on usernames.
    if not route_id:
        routes = cur.all("""\
            SELECT *
              FROM exchange_routes
             WHERE network='balanced-cc' and participant in (
                    SELECT id
                      FROM participants
                     WHERE username=ANY(%s)
                   )
        """, (usernames,))
        route_id = resolve_route(cur, transaction, counts, routes, usernames)
    if route_id:
        counts.route_matched_via_customer += 1
        log("customer matches {}".format(route_id), 22)
        return route_id

    # Fourth strategy: make a route!
    if not route_id:
        if len(usernames) > 1:
            counts.route_ambiguous_customer += 1
            raise AmbiguousCustomer()
        username = usernames[0]
        participant = cur.one("SELECT participants.*::participants FROM participants "
                              "WHERE username=%s", (username,))
        route = ExchangeRoute.insert( participant
                                    , 'balanced-cc'
                                    , '/cards/'+transaction['links']['source']
                                    , cursor=cur
                                     )
        route_id = route.id
    if route_id:
        counts.route_created += 1
        log("created a route", 22)
        return route_id

    counts.route_unknown += 1
    raise UnknownRoute()

def link_exchange_to_transaction(cur, transaction, customers, counts):
    print("{created_at} | {description:>24} | ".format(**transaction), end='')
    counts.z_n += 1

    usernames = get_usernames(cur, transaction, customers, counts)
    exchange_id = get_exchange_id(cur, transaction, counts, usernames)
    status = transaction['status']
    route_id = get_route_id(cur, transaction, counts, usernames, exchange_id)
    ref = transaction['id']

    existing = cur.one("SELECT * FROM exchanges WHERE id=%s", (exchange_id,), Exception)
    if existing.route:
        assert existing.route == route_id, exchange_id
        assert existing.status == status, exchange_id
        counts.z_linked_already += 1
        print("Already linked: {} & {}.".format(transaction['id'], exchange_id))
    else:
        assert existing.status in (None, status), exchange_id  # don't want to mutate status
        counts.z_linking_now += 1
        print("Linking {} to {}.".format(transaction['id'], exchange_id))
        cur.run( "UPDATE exchanges SET status=%s, route=%s, ref=%s WHERE id=%s"
               , (status, route_id, ref, exchange_id)
                )

    counts.check()

def link_participant_to_customer(cur, transaction, customers):
    participant_id = transaction['meta'].get('participant_id')
    if participant_id:
        customer_id = transaction['links']['customer']
        if customer_id not in customers:
            import pdb; pdb.set_trace()  # not hit! :-(
            balanced_customer_href = '/customers/' + customer_id
            cur.run( 'UPDATE participants SET balanced_customer_href=%s WHERE id=%s'
                   , (balanced_customer_href, participant_id)
                    )
            print('Linked customer {} to participant {}.'
                  .format(balanced_customer_href, participant_id))

def log_exc(transaction):
    print("Exception! Transaction:")
    pprint(transaction)
    print('-'*78)
    traceback.print_exc(file=sys.stdout)
    print('='*78)

def walk(cur, visit):
    for filename in reversed(sorted(os.listdir(dirname))):
        data = json.load(open(os.path.join(dirname, filename)))

        for key in data:
            if key in ('links', 'meta'): continue
            transactions = reversed(data[key])

            if key == 'card_holds':
                continue  # We don't track these.

            if key != 'debits':
                continue  # Let's start with credit card charges.

            for transaction in transactions:
                if transaction['links']['source'].startswith('BA'):
                    continue  # Filter out escrow shuffles (we didn't do any other ACH debits).
                try:
                    visit(cur, transaction)
                except KeyboardInterrupt:
                    raise
                except CountsOff:
                    log_exc(transaction)
                    raise
                except:
                    log_exc(transaction)
    print()

def get_customers():
    customers = cur.all("SELECT balanced_customer_href, username FROM participants "
                        "WHERE balanced_customer_href is not null")
    out = defaultdict(list)
    for balanced_customer_href, username in customers:
        customer_id = balanced_customer_href[len('/customers/'):]
        out[customer_id].append(username)
    return out

with db.get_cursor() as cur:
    try:
        customers = get_customers()
        counts = Counts()
        walk(cur, lambda *a: link_exchange_to_transaction(*a, counts=counts, customers=customers))
    finally:
        print('Rolling back ...')
        cur.connection.rollback()
<commit_msg>Set exchanges.ref for already linked<commit_after>#!/usr/bin/env python -u
from __future__ import absolute_import, division, print_function, unicode_literals

import json, os, sys, traceback
from collections import defaultdict
from gratipay import wireup
from decimal import Decimal as D
from pprint import pprint
from gratipay.models.exchange_route import ExchangeRoute

db = wireup.db(wireup.env())
dirname = os.path.join(os.path.dirname(__file__), 'balanced', 'transactions')

class UnknownExchange(Exception):
    def __str__(self):
        return "\n\n{}".format(self.args[0])

class UnknownCustomer(Exception): pass
class UnknownRoute(Exception): pass
class AmbiguousCustomer(Exception): pass
class AmbiguousRoute(Exception): pass

class CountsOff(Exception):
    def __str__(self):
        msg, counts = self.args
        out = [msg]
        cur = ''
        for name, val in sorted(counts.__dict__.items()):
            if name[0] != cur:
                cur = name[0]
                out.append('')
            out.append("{:<24} {:>5}".format(name, val))
        return '\n'.join(out)

class Counts(object):
    def __init__(self):
        self.customer_known = 0
        self.customer_unknown = 0

        self.exchange_in_transaction = 0
        self.exchange_triangulated = 0
        self.exchange_unknown = 0

        self.route_in_exchange = 0
        self.route_ambiguous = 0
        self.route_matched_via_card = 0
        self.route_matched_via_customer = 0
        self.route_ambiguous_customer = 0
        self.route_created = 0
        self.route_unknown = 0

        self.z_linked_already = 0
        self.z_linking_now = 0

        self.z_n = 0

    def check(self):
        n = sum(( self.z_linking_now
                , self.z_linked_already
                , self.customer_unknown
                , self.exchange_unknown
                , self.route_ambiguous
                , self.route_ambiguous_customer
                , self.route_unknown
                 ))
        if self.z_n != n:
            raise CountsOff("Total N is out from {}.".format(n), counts)

        ncustomers = sum((self.customer_known,))
        nexchanges = sum(( self.exchange_in_transaction
                         , self.exchange_triangulated
                         , self.exchange_unknown
                          ))
        if ncustomers != nexchanges:
            raise CountsOff( "Customer count ({}) doesn't match exchange count ({})."
                             .format(ncustomers, nexchanges)
                           , counts
                            )

        nexchanges = sum(( self.exchange_in_transaction
                         , self.exchange_triangulated
                          ))
        nroutes = sum(( self.route_in_exchange
                      , self.route_ambiguous
                      , self.route_matched_via_card
                      , self.route_matched_via_customer
                      , self.route_ambiguous_customer
                      , self.route_created
                      , self.route_unknown
                       ))
        if nexchanges != nroutes:
            raise CountsOff( "Exchange count ({}) doesn't match route count ({})."
                             .format(nexchanges, nroutes)
                           , counts
                            )

def log(msg, w=0):
    print("{msg:<{w}}".format(msg=msg, w=w), end=' | ')

def get_usernames(cur, transaction, customers, counts):

    # First strategy: check known customers.
    usernames = customers.get(transaction['links']['customer'])
    if usernames:
        counts.customer_known += 1
        log('known customer')
        log('participants: {}'.format(len(usernames)))
        return usernames

    # Second strategy: blah
    counts.customer_unknown += 1
    raise UnknownCustomer()

def get_exchange_id(cur, transaction, counts, usernames):

    # First strategy: use the one in the transaction!
    exchange_id = transaction['meta'].get('exchange_id')
    if exchange_id:
        counts.exchange_in_transaction += 1
        log("exchange_id in transaction", 26)
        return exchange_id

    # Second strategy: triangulate.
    params = dict( usernames = usernames
                 , amount    = D(transaction['amount']) / 100
                 , ts        = transaction['created_at']
                  )
    mogrified = cur.mogrify("""\
        SELECT id FROM exchanges
         WHERE participant = ANY(%(usernames)s)
           AND amount + fee = %(amount)s
           AND ((   (%(ts)s::timestamptz - "timestamp") < interval '0'
                AND (%(ts)s::timestamptz - "timestamp") > interval '-15m'
                ) OR (
                    (%(ts)s::timestamptz - "timestamp") > interval '0'
                AND (%(ts)s::timestamptz - "timestamp") < interval '15m'
                ))
    """, params)
    exchange_id = cur.one(mogrified)
    if exchange_id:
        counts.exchange_triangulated += 1
        log("triangulated an exchange", 26)
        return exchange_id

    counts.exchange_unknown += 1
    raise UnknownExchange(mogrified)

def resolve_route(cur, transaction, counts, routes, usernames):
    log('routes: {routes:<12}'.format(routes=', '.join(str(r.id) for r in routes)))

    nroutes = len(routes)
    if nroutes == 0:    return None
    elif nroutes == 1:  return routes[0].id
    assert nroutes == 2
    # Pick the route for the participant that was active at the time of the transaction.

    nusernames = len(usernames)
    assert nusernames in (1, 2), (usernames, routes)
    if nusernames == 1:
        counts.route_ambiguous += 1
        raise AmbiguousRoute()
    else:
        # See if we can assume it's a case of absorption.
        user_ids = [r.participant for r in routes]
        i2n = dict(cur.all( "SELECT id, username FROM participants WHERE id = ANY(%s)"
                          , (user_ids,)
                           ))
        assert sorted(i2n.values()) == sorted(usernames)
        u2r = {i2n[r.participant]: r.id for r in routes}

        keys = u2r.keys()
        absorption = cur.one("""\
            SELECT *
              FROM absorptions
             WHERE (archived_as=%(one)s AND absorbed_by=%(two)s)
                OR (archived_as=%(two)s AND absorbed_by=%(one)s)
        """, dict(one=keys[0], two=keys[1]))
        if absorption is None:
            counts.route_ambiguous += 1
            raise AmbiguousRoute()
        if transaction['created_at'] < str(absorption.timestamp).replace('+00:00', 'Z'):
            key = absorption.archived_as
        else:
            assert transaction['created_at'] > str(absorption.timestamp).replace('+00:00', 'Z')
            key = absorption.absorbed_by
        return u2r[key]

def get_route_id(cur, transaction, counts, usernames, exchange_id):

    # First strategy: match on exchange id.
    route_id = cur.one("SELECT route FROM exchanges WHERE id=%s", (exchange_id,))
    if route_id:
        counts.route_in_exchange += 1
        log("exchange has a route", 22)
        return route_id

    # Second strategy: match on known cards.
    if not route_id:
        routes = cur.all( "SELECT * FROM exchange_routes "
                          "WHERE network='balanced-cc' AND address='/cards/'||%s"
                        , (transaction['links']['source'],)
                         )
        route_id = resolve_route(cur, transaction, counts, routes, usernames)
    if route_id:
        counts.route_matched_via_card += 1
        log("card matches {}".format(route_id), 22)
        return route_id

    # Third strategy: match on usernames.
    if not route_id:
        routes = cur.all("""\
            SELECT *
              FROM exchange_routes
             WHERE network='balanced-cc' and participant in (
                    SELECT id
                      FROM participants
                     WHERE username=ANY(%s)
                   )
        """, (usernames,))
        route_id = resolve_route(cur, transaction, counts, routes, usernames)
    if route_id:
        counts.route_matched_via_customer += 1
        log("customer matches {}".format(route_id), 22)
        return route_id

    # Fourth strategy: make a route!
    if not route_id:
        if len(usernames) > 1:
            counts.route_ambiguous_customer += 1
            raise AmbiguousCustomer()
        username = usernames[0]
        participant = cur.one("SELECT participants.*::participants FROM participants "
                              "WHERE username=%s", (username,))
        route = ExchangeRoute.insert( participant
                                    , 'balanced-cc'
                                    , '/cards/'+transaction['links']['source']
                                    , cursor=cur
                                     )
        route_id = route.id
    if route_id:
        counts.route_created += 1
        log("created a route", 22)
        return route_id

    counts.route_unknown += 1
    raise UnknownRoute()

def link_exchange_to_transaction(cur, transaction, customers, counts):
    print("{created_at} | {description:>24} | ".format(**transaction), end='')
    counts.z_n += 1

    usernames = get_usernames(cur, transaction, customers, counts)
    exchange_id = get_exchange_id(cur, transaction, counts, usernames)
    status = transaction['status']
    route_id = get_route_id(cur, transaction, counts, usernames, exchange_id)
    ref = transaction['id']

    existing = cur.one("SELECT * FROM exchanges WHERE id=%s", (exchange_id,), Exception)
    if existing.route:
        assert existing.route == route_id, exchange_id
        assert existing.status == status, exchange_id
        counts.z_linked_already += 1
        print("Already linked: {} & {}.".format(transaction['id'], exchange_id))
        cur.run( "UPDATE exchanges SET ref=%s WHERE id=%s", (ref, exchange_id))
    else:
        assert existing.status in (None, status), exchange_id  # don't want to mutate status
        counts.z_linking_now += 1
        print("Linking {} to {}.".format(transaction['id'], exchange_id))
        cur.run( "UPDATE exchanges SET status=%s, route=%s, ref=%s WHERE id=%s"
               , (status, route_id, ref, exchange_id)
                )

    counts.check()

def link_participant_to_customer(cur, transaction, customers):
    participant_id = transaction['meta'].get('participant_id')
    if participant_id:
        customer_id = transaction['links']['customer']
        if customer_id not in customers:
            import pdb; pdb.set_trace()  # not hit! :-(
            balanced_customer_href = '/customers/' + customer_id
            cur.run( 'UPDATE participants SET balanced_customer_href=%s WHERE id=%s'
                   , (balanced_customer_href, participant_id)
                    )
            print('Linked customer {} to participant {}.'
                  .format(balanced_customer_href, participant_id))

def log_exc(transaction):
    print("Exception! Transaction:")
    pprint(transaction)
    print('-'*78)
    traceback.print_exc(file=sys.stdout)
    print('='*78)

def walk(cur, visit):
    for filename in reversed(sorted(os.listdir(dirname))):
        data = json.load(open(os.path.join(dirname, filename)))

        for key in data:
            if key in ('links', 'meta'): continue
            transactions = reversed(data[key])

            if key == 'card_holds':
                continue  # We don't track these.

            if key != 'debits':
                continue  # Let's start with credit card charges.

            for transaction in transactions:
                if transaction['links']['source'].startswith('BA'):
                    continue  # Filter out escrow shuffles (we didn't do any other ACH debits).
                try:
                    visit(cur, transaction)
                except KeyboardInterrupt:
                    raise
                except CountsOff:
                    log_exc(transaction)
                    raise
                except:
                    log_exc(transaction)
    print()

def get_customers():
    customers = cur.all("SELECT balanced_customer_href, username FROM participants "
                        "WHERE balanced_customer_href is not null")
    out = defaultdict(list)
    for balanced_customer_href, username in customers:
        customer_id = balanced_customer_href[len('/customers/'):]
        out[customer_id].append(username)
    return out

with db.get_cursor() as cur:
    try:
        customers = get_customers()
        counts = Counts()
        walk(cur, lambda *a: link_exchange_to_transaction(*a, counts=counts, customers=customers))
    finally:
        print('Rolling back ...')
        cur.connection.rollback()
<|endoftext|>


<commit_before>from nose.plugins.attrib import attr
from openquake.commonlib.tests.calculators import CalculatorTestCase
from openquake.qa_tests_data.classical import (
    case_1, case_2, case_3, case_4, case_5, case_6, case_7, case_8, case_9,
    case_10, case_11, case_12, case_13, case_14, case_15, case_16, case_17,
    case_19)

class ClassicalTestCase(CalculatorTestCase):
    @attr('qa', 'hazard', 'classical')
    def test_case_1(self):
        out = self.run_calc(case_1.__file__, 'job.ini', exports='csv')
        self.assertEqualFiles(
            'expected/hazard_curve-smltp_b1-gsimltp_b1.csv',
            out['hazard_curve-smltp_b1-gsimltp_b1.csv'])

    @attr('qa', 'hazard', 'classical')
    def test_case_2(self):
        out = self.run_calc(case_2.__file__, 'job.ini', exports='csv')
        self.assertEqualFiles(
            'expected/hazard_curve-smltp_b1-gsimltp_b1.csv',
            out['hazard_curve-smltp_b1-gsimltp_b1.csv'])

    @attr('qa', 'hazard', 'classical')
    def test_case_3(self):
        out = self.run_calc(case_3.__file__, 'job.ini', exports='csv')
        self.assertEqualFiles(
            'expected/hazard_curve-smltp_b1-gsimltp_b1.csv',
            out['hazard_curve-smltp_b1-gsimltp_b1.csv'])

    @attr('qa', 'hazard', 'classical', 'slow')
    def test_case_4(self):
        out = self.run_calc(case_4.__file__, 'job.ini', exports='csv')
        self.assertEqualFiles(
            'expected/hazard_curve-smltp_b1-gsimltp_b1.csv',
            out['hazard_curve-smltp_b1-gsimltp_b1.csv'])

    @attr('qa', 'hazard', 'classical', 'slow')
    def test_case_5(self):
        out = self.run_calc(case_5.__file__, 'job.ini', exports='csv')
        self.assertEqualFiles(
            'expected/hazard_curve-smltp_b1-gsimltp_b1.csv',
            out['hazard_curve-smltp_b1-gsimltp_b1.csv'])

    @attr('qa', 'hazard', 'classical', 'slow')
    def test_case_6(self):
        out = self.run_calc(case_6.__file__, 'job.ini', exports='csv')
        self.assertEqualFiles(
            'expected/hazard_curve-smltp_b1-gsimltp_b1.csv',
            out['hazard_curve-smltp_b1-gsimltp_b1.csv'])

    @attr('qa', 'hazard', 'classical', 'slow')
    def test_case_7(self):
        expected = [
            'hazard_curve-smltp_b1-gsimltp_b1.csv',
            'hazard_curve-smltp_b2-gsimltp_b1.csv',
            'hazard_curve-mean.csv',
        ]
        out = self.run_calc(case_7.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_8(self):
        expected = [
            'hazard_curve-smltp_b1_b2-gsimltp_b1.csv',
            'hazard_curve-smltp_b1_b3-gsimltp_b1.csv',
            'hazard_curve-smltp_b1_b4-gsimltp_b1.csv',
        ]
        out = self.run_calc(case_8.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_9(self):
        expected = [
            'hazard_curve-smltp_b1_b2-gsimltp_b1.csv',
            'hazard_curve-smltp_b1_b3-gsimltp_b1.csv',
        ]
        out = self.run_calc(case_9.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_10(self):
        expected = [
            'hazard_curve-smltp_b1_b2-gsimltp_b1.csv',
            'hazard_curve-smltp_b1_b3-gsimltp_b1.csv',
        ]
        out = self.run_calc(case_10.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_11(self):
        expected = [
            'hazard_curve-smltp_b1_b2-gsimltp_b1.csv',
            'hazard_curve-smltp_b1_b3-gsimltp_b1.csv',
            'hazard_curve-smltp_b1_b4-gsimltp_b1.csv',
            'hazard_curve-mean.csv',
        ]
        out = self.run_calc(case_11.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_12(self):
        expected = [
            'hazard_curve-smltp_b1-gsimltp_b1_b2.csv',
        ]
        out = self.run_calc(case_12.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_13(self):
        expected = [
            'hazard_curve-smltp_aFault_aPriori_D2.1-gsimltp_BooreAtkinson2008.csv',
            'hazard_curve-smltp_aFault_aPriori_D2.1-gsimltp_ChiouYoungs2008.csv',
            'hazard_curve-smltp_bFault_stitched_D2.1_Char-gsimltp_BooreAtkinson2008.csv',
            'hazard_curve-smltp_bFault_stitched_D2.1_Char-gsimltp_ChiouYoungs2008.csv',
            'hazard_curve-mean.csv',
            'hazard_map-smltp_aFault_aPriori_D2.1-gsimltp_BooreAtkinson2008.csv',
            'hazard_map-smltp_aFault_aPriori_D2.1-gsimltp_ChiouYoungs2008.csv',
            'hazard_map-smltp_bFault_stitched_D2.1_Char-gsimltp_BooreAtkinson2008.csv',
            'hazard_map-smltp_bFault_stitched_D2.1_Char-gsimltp_ChiouYoungs2008.csv',
            'hazard_map-mean.csv',
        ]
        out = self.run_calc(case_13.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_14(self):
        expected = [
            'hazard_curve-smltp_simple_fault-gsimltp_AbrahamsonSilva2008.csv',
            'hazard_curve-smltp_simple_fault-gsimltp_CampbellBozorgnia2008.csv',
        ]
        out = self.run_calc(case_14.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_15(self):  # full enumeration
        expected = '''\
hazard_curve-mean.csv
hazard_curve-smltp_SM1-gsimltp_BA2008_C2003.csv
hazard_curve-smltp_SM1-gsimltp_BA2008_T2002.csv
hazard_curve-smltp_SM1-gsimltp_CB2008_C2003.csv
hazard_curve-smltp_SM1-gsimltp_CB2008_T2002.csv
hazard_curve-smltp_SM2_a3b1-gsimltp_BA2008_*.csv
hazard_curve-smltp_SM2_a3b1-gsimltp_CB2008_*.csv
hazard_curve-smltp_SM2_a3pt2b0pt8-gsimltp_BA2008_*.csv
hazard_curve-smltp_SM2_a3pt2b0pt8-gsimltp_CB2008_*.csv
hazard_uhs-mean.csv
hazard_uhs-smltp_SM1-gsimltp_BA2008_C2003.csv
hazard_uhs-smltp_SM1-gsimltp_BA2008_T2002.csv
hazard_uhs-smltp_SM1-gsimltp_CB2008_C2003.csv
hazard_uhs-smltp_SM1-gsimltp_CB2008_T2002.csv
hazard_uhs-smltp_SM2_a3b1-gsimltp_BA2008_*.csv
hazard_uhs-smltp_SM2_a3b1-gsimltp_CB2008_*.csv
hazard_uhs-smltp_SM2_a3pt2b0pt8-gsimltp_BA2008_*.csv
hazard_uhs-smltp_SM2_a3pt2b0pt8-gsimltp_CB2008_*.csv'''.split()
        out = self.run_calc(case_15.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_16(self):   # sampling
        expected = [
            'hazard_curve-mean.csv',
            'quantile_curve-0.1.csv',
            'quantile_curve-0.9.csv',
        ]
        out = self.run_calc(case_16.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_17(self):  # oversampling
        expected = [
            'hazard_curve-smltp_b1-gsimltp_b1-ltr_0.csv',
            'hazard_curve-smltp_b2-gsimltp_b1-ltr_1.csv',
        ]
        out = self.run_calc(case_17.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_19(self):
        expected = [
            'hazard_curve-mean.csv',
            'hazard_curve-smltp_b1-gsimltp_*_*_*_*_b51_*_*.csv',
            'hazard_curve-smltp_b1-gsimltp_*_*_*_*_b52_*_*.csv',
            'hazard_curve-smltp_b1-gsimltp_*_*_*_*_b53_*_*.csv',
            'hazard_curve-smltp_b1-gsimltp_*_*_*_*_b54_*_*.csv',
        ]
        out = self.run_calc(case_19.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname],
                                  ignore_last_digits=1)
<commit_msg>Reduced the precision on case_19<commit_after>from nose.plugins.attrib import attr
from openquake.commonlib.tests.calculators import CalculatorTestCase
from openquake.qa_tests_data.classical import (
    case_1, case_2, case_3, case_4, case_5, case_6, case_7, case_8, case_9,
    case_10, case_11, case_12, case_13, case_14, case_15, case_16, case_17,
    case_19)

class ClassicalTestCase(CalculatorTestCase):
    @attr('qa', 'hazard', 'classical')
    def test_case_1(self):
        out = self.run_calc(case_1.__file__, 'job.ini', exports='csv')
        self.assertEqualFiles(
            'expected/hazard_curve-smltp_b1-gsimltp_b1.csv',
            out['hazard_curve-smltp_b1-gsimltp_b1.csv'])

    @attr('qa', 'hazard', 'classical')
    def test_case_2(self):
        out = self.run_calc(case_2.__file__, 'job.ini', exports='csv')
        self.assertEqualFiles(
            'expected/hazard_curve-smltp_b1-gsimltp_b1.csv',
            out['hazard_curve-smltp_b1-gsimltp_b1.csv'])

    @attr('qa', 'hazard', 'classical')
    def test_case_3(self):
        out = self.run_calc(case_3.__file__, 'job.ini', exports='csv')
        self.assertEqualFiles(
            'expected/hazard_curve-smltp_b1-gsimltp_b1.csv',
            out['hazard_curve-smltp_b1-gsimltp_b1.csv'])

    @attr('qa', 'hazard', 'classical', 'slow')
    def test_case_4(self):
        out = self.run_calc(case_4.__file__, 'job.ini', exports='csv')
        self.assertEqualFiles(
            'expected/hazard_curve-smltp_b1-gsimltp_b1.csv',
            out['hazard_curve-smltp_b1-gsimltp_b1.csv'])

    @attr('qa', 'hazard', 'classical', 'slow')
    def test_case_5(self):
        out = self.run_calc(case_5.__file__, 'job.ini', exports='csv')
        self.assertEqualFiles(
            'expected/hazard_curve-smltp_b1-gsimltp_b1.csv',
            out['hazard_curve-smltp_b1-gsimltp_b1.csv'])

    @attr('qa', 'hazard', 'classical', 'slow')
    def test_case_6(self):
        out = self.run_calc(case_6.__file__, 'job.ini', exports='csv')
        self.assertEqualFiles(
            'expected/hazard_curve-smltp_b1-gsimltp_b1.csv',
            out['hazard_curve-smltp_b1-gsimltp_b1.csv'])

    @attr('qa', 'hazard', 'classical', 'slow')
    def test_case_7(self):
        expected = [
            'hazard_curve-smltp_b1-gsimltp_b1.csv',
            'hazard_curve-smltp_b2-gsimltp_b1.csv',
            'hazard_curve-mean.csv',
        ]
        out = self.run_calc(case_7.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_8(self):
        expected = [
            'hazard_curve-smltp_b1_b2-gsimltp_b1.csv',
            'hazard_curve-smltp_b1_b3-gsimltp_b1.csv',
            'hazard_curve-smltp_b1_b4-gsimltp_b1.csv',
        ]
        out = self.run_calc(case_8.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_9(self):
        expected = [
            'hazard_curve-smltp_b1_b2-gsimltp_b1.csv',
            'hazard_curve-smltp_b1_b3-gsimltp_b1.csv',
        ]
        out = self.run_calc(case_9.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_10(self):
        expected = [
            'hazard_curve-smltp_b1_b2-gsimltp_b1.csv',
            'hazard_curve-smltp_b1_b3-gsimltp_b1.csv',
        ]
        out = self.run_calc(case_10.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_11(self):
        expected = [
            'hazard_curve-smltp_b1_b2-gsimltp_b1.csv',
            'hazard_curve-smltp_b1_b3-gsimltp_b1.csv',
            'hazard_curve-smltp_b1_b4-gsimltp_b1.csv',
            'hazard_curve-mean.csv',
        ]
        out = self.run_calc(case_11.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_12(self):
        expected = [
            'hazard_curve-smltp_b1-gsimltp_b1_b2.csv',
        ]
        out = self.run_calc(case_12.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_13(self):
        expected = [
            'hazard_curve-smltp_aFault_aPriori_D2.1-gsimltp_BooreAtkinson2008.csv',
            'hazard_curve-smltp_aFault_aPriori_D2.1-gsimltp_ChiouYoungs2008.csv',
            'hazard_curve-smltp_bFault_stitched_D2.1_Char-gsimltp_BooreAtkinson2008.csv',
            'hazard_curve-smltp_bFault_stitched_D2.1_Char-gsimltp_ChiouYoungs2008.csv',
            'hazard_curve-mean.csv',
            'hazard_map-smltp_aFault_aPriori_D2.1-gsimltp_BooreAtkinson2008.csv',
            'hazard_map-smltp_aFault_aPriori_D2.1-gsimltp_ChiouYoungs2008.csv',
            'hazard_map-smltp_bFault_stitched_D2.1_Char-gsimltp_BooreAtkinson2008.csv',
            'hazard_map-smltp_bFault_stitched_D2.1_Char-gsimltp_ChiouYoungs2008.csv',
            'hazard_map-mean.csv',
        ]
        out = self.run_calc(case_13.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_14(self):
        expected = [
            'hazard_curve-smltp_simple_fault-gsimltp_AbrahamsonSilva2008.csv',
            'hazard_curve-smltp_simple_fault-gsimltp_CampbellBozorgnia2008.csv',
        ]
        out = self.run_calc(case_14.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_15(self):  # full enumeration
        expected = '''\
hazard_curve-mean.csv
hazard_curve-smltp_SM1-gsimltp_BA2008_C2003.csv
hazard_curve-smltp_SM1-gsimltp_BA2008_T2002.csv
hazard_curve-smltp_SM1-gsimltp_CB2008_C2003.csv
hazard_curve-smltp_SM1-gsimltp_CB2008_T2002.csv
hazard_curve-smltp_SM2_a3b1-gsimltp_BA2008_*.csv
hazard_curve-smltp_SM2_a3b1-gsimltp_CB2008_*.csv
hazard_curve-smltp_SM2_a3pt2b0pt8-gsimltp_BA2008_*.csv
hazard_curve-smltp_SM2_a3pt2b0pt8-gsimltp_CB2008_*.csv
hazard_uhs-mean.csv
hazard_uhs-smltp_SM1-gsimltp_BA2008_C2003.csv
hazard_uhs-smltp_SM1-gsimltp_BA2008_T2002.csv
hazard_uhs-smltp_SM1-gsimltp_CB2008_C2003.csv
hazard_uhs-smltp_SM1-gsimltp_CB2008_T2002.csv
hazard_uhs-smltp_SM2_a3b1-gsimltp_BA2008_*.csv
hazard_uhs-smltp_SM2_a3b1-gsimltp_CB2008_*.csv
hazard_uhs-smltp_SM2_a3pt2b0pt8-gsimltp_BA2008_*.csv
hazard_uhs-smltp_SM2_a3pt2b0pt8-gsimltp_CB2008_*.csv'''.split()
        out = self.run_calc(case_15.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_16(self):   # sampling
        expected = [
            'hazard_curve-mean.csv',
            'quantile_curve-0.1.csv',
            'quantile_curve-0.9.csv',
        ]
        out = self.run_calc(case_16.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_17(self):  # oversampling
        expected = [
            'hazard_curve-smltp_b1-gsimltp_b1-ltr_0.csv',
            'hazard_curve-smltp_b2-gsimltp_b1-ltr_1.csv',
        ]
        out = self.run_calc(case_17.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname])

    @attr('qa', 'hazard', 'classical')
    def test_case_19(self):
        expected = [
            'hazard_curve-mean.csv',
            'hazard_curve-smltp_b1-gsimltp_*_*_*_*_b51_*_*.csv',
            'hazard_curve-smltp_b1-gsimltp_*_*_*_*_b52_*_*.csv',
            'hazard_curve-smltp_b1-gsimltp_*_*_*_*_b53_*_*.csv',
            'hazard_curve-smltp_b1-gsimltp_*_*_*_*_b54_*_*.csv',
        ]
        out = self.run_calc(case_19.__file__, 'job.ini', exports='csv')
        for fname in expected:
            self.assertEqualFiles('expected/%s' % fname, out[fname],
                                  ignore_last_digits=2)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""models.py

Udacity conference server-side Python App Engine data & ProtoRPC models

$Id: models.py,v 1.1 2014/05/24 22:01:10 wesc Exp $

created/forked from conferences.py by wesc on 2014 may 24

"""

__author__ = 'wesc+api@google.com (Wesley Chun)'

import httplib
import endpoints
from protorpc import messages
from google.appengine.ext import ndb

class Session(ndb.Model):
    """Session Object - represents a specific session of a Conference"""
    conferenceKey   = ndb.StringProperty(repeated=True)
    sessionName     = ndb.StringProperty()
    highlights      = ndb.StringProperty()
    speaker         = ndb.StringProperty()
    duration        = ndb.IntegerProperty()
    typeOfSession   = ndb.StringProperty()
    date            = ndb.DateProperty()
    startTime       = ndb.StringProperty()

class SessionForm(messages.Message):
    """SessionForm -- create a Session"""
    sessionName     = messages.StringField(1)
    highlights      = messages.StringField(2)
    speaker         = messages.StringField(3)
    duration        = messages.IntegerField(4)
    typeOfSession   = messages.StringField(5)
    date            = messages.StringField(6)
    startTime       = messages.StringField(7)
    websafeKey      = messages.StringField(8)

class SessionForms(messages.Message):
    """SessionForms -- multiple Session outbound form message"""
    items = messages.MessageField(SessionForm, 1, repeated=True)

class ConflictException(endpoints.ServiceException):
    """ConflictException -- exception mapped to HTTP 409 response"""
    http_status = httplib.CONFLICT

class Profile(ndb.Model):
    """Profile -- User profile object"""
    displayName = ndb.StringProperty()
    mainEmail = ndb.StringProperty()
    teeShirtSize = ndb.StringProperty(default='NOT_SPECIFIED')
    conferenceKeysToAttend = ndb.StringProperty(repeated=True)

class ProfileMiniForm(messages.Message):
    """ProfileMiniForm -- update Profile form message"""
    displayName = messages.StringField(1)
    teeShirtSize = messages.EnumField('TeeShirtSize', 2)

class ProfileForm(messages.Message):
    """ProfileForm -- Profile outbound form message"""
    displayName = messages.StringField(1)
    mainEmail = messages.StringField(2)
    teeShirtSize = messages.EnumField('TeeShirtSize', 3)
    conferenceKeysToAttend = messages.StringField(4, repeated=True)

class StringMessage(messages.Message):
    """StringMessage-- outbound (single) string message"""
    data = messages.StringField(1, required=True)

class BooleanMessage(messages.Message):
    """BooleanMessage-- outbound Boolean value message"""
    data = messages.BooleanField(1)

class Conference(ndb.Model):
    """Conference -- Conference object"""
    name            = ndb.StringProperty(required=True)
    description     = ndb.StringProperty()
    organizerUserId = ndb.StringProperty()
    topics          = ndb.StringProperty(repeated=True)
    city            = ndb.StringProperty()
    startDate       = ndb.DateProperty()
    month           = ndb.IntegerProperty() # TODO: do we need for indexing like Java?
    endDate         = ndb.DateProperty()
    maxAttendees    = ndb.IntegerProperty()
    seatsAvailable  = ndb.IntegerProperty()

class ConferenceForm(messages.Message):
    """ConferenceForm -- Conference outbound form message"""
    name            = messages.StringField(1)
    description     = messages.StringField(2)
    organizerUserId = messages.StringField(3)
    topics          = messages.StringField(4, repeated=True)
    city            = messages.StringField(5)
    startDate       = messages.StringField(6) #DateTimeField()
    month           = messages.IntegerField(7)
    maxAttendees    = messages.IntegerField(8)
    seatsAvailable  = messages.IntegerField(9)
    endDate         = messages.StringField(10) #DateTimeField()
    websafeKey      = messages.StringField(11)
    organizerDisplayName = messages.StringField(12)

class ConferenceForms(messages.Message):
    """ConferenceForms -- multiple Conference outbound form message"""
    items = messages.MessageField(ConferenceForm, 1, repeated=True)

class TeeShirtSize(messages.Enum):
    """TeeShirtSize -- t-shirt size enumeration value"""
    NOT_SPECIFIED = 1
    XS_M = 2
    XS_W = 3
    S_M = 4
    S_W = 5
    M_M = 6
    M_W = 7
    L_M = 8
    L_W = 9
    XL_M = 10
    XL_W = 11
    XXL_M = 12
    XXL_W = 13
    XXXL_M = 14
    XXXL_W = 15

class ConferenceQueryForm(messages.Message):
    """ConferenceQueryForm -- Conference query inbound form message"""
    field = messages.StringField(1)
    operator = messages.StringField(2)
    value = messages.StringField(3)

class ConferenceQueryForms(messages.Message):
    """ConferenceQueryForms -- multiple ConferenceQueryForm inbound form message"""
    filters = messages.MessageField(ConferenceQueryForm, 1, repeated=True)

<commit_msg>FIX: modify conferenceKey field<commit_after>#!/usr/bin/env python

"""models.py

Udacity conference server-side Python App Engine data & ProtoRPC models

$Id: models.py,v 1.1 2014/05/24 22:01:10 wesc Exp $

created/forked from conferences.py by wesc on 2014 may 24

"""

__author__ = 'wesc+api@google.com (Wesley Chun)'

import httplib
import endpoints
from protorpc import messages
from google.appengine.ext import ndb

class Session(ndb.Model):
    """Session Object - represents a specific session of a Conference"""
    conferenceKey   = ndb.StringProperty()
    sessionName     = ndb.StringProperty()
    highlights      = ndb.StringProperty()
    speaker         = ndb.StringProperty()
    duration        = ndb.IntegerProperty()
    typeOfSession   = ndb.StringProperty()
    date            = ndb.DateProperty()
    startTime       = ndb.StringProperty()

class SessionForm(messages.Message):
    """SessionForm -- create a Session"""
    sessionName     = messages.StringField(1)
    highlights      = messages.StringField(2)
    speaker         = messages.StringField(3)
    duration        = messages.IntegerField(4)
    typeOfSession   = messages.StringField(5)
    date            = messages.StringField(6)
    startTime       = messages.StringField(7)
    conferenceKey   = messages.StringField(8)

class SessionForms(messages.Message):
    """SessionForms -- multiple Session outbound form message"""
    items = messages.MessageField(SessionForm, 1, repeated=True)

class ConflictException(endpoints.ServiceException):
    """ConflictException -- exception mapped to HTTP 409 response"""
    http_status = httplib.CONFLICT

class Profile(ndb.Model):
    """Profile -- User profile object"""
    displayName = ndb.StringProperty()
    mainEmail = ndb.StringProperty()
    teeShirtSize = ndb.StringProperty(default='NOT_SPECIFIED')
    conferenceKeysToAttend = ndb.StringProperty(repeated=True)

class ProfileMiniForm(messages.Message):
    """ProfileMiniForm -- update Profile form message"""
    displayName = messages.StringField(1)
    teeShirtSize = messages.EnumField('TeeShirtSize', 2)

class ProfileForm(messages.Message):
    """ProfileForm -- Profile outbound form message"""
    displayName = messages.StringField(1)
    mainEmail = messages.StringField(2)
    teeShirtSize = messages.EnumField('TeeShirtSize', 3)
    conferenceKeysToAttend = messages.StringField(4, repeated=True)

class StringMessage(messages.Message):
    """StringMessage-- outbound (single) string message"""
    data = messages.StringField(1, required=True)

class BooleanMessage(messages.Message):
    """BooleanMessage-- outbound Boolean value message"""
    data = messages.BooleanField(1)

class Conference(ndb.Model):
    """Conference -- Conference object"""
    name            = ndb.StringProperty(required=True)
    description     = ndb.StringProperty()
    organizerUserId = ndb.StringProperty()
    topics          = ndb.StringProperty(repeated=True)
    city            = ndb.StringProperty()
    startDate       = ndb.DateProperty()
    month           = ndb.IntegerProperty() # TODO: do we need for indexing like Java?
    endDate         = ndb.DateProperty()
    maxAttendees    = ndb.IntegerProperty()
    seatsAvailable  = ndb.IntegerProperty()

class ConferenceForm(messages.Message):
    """ConferenceForm -- Conference outbound form message"""
    name            = messages.StringField(1)
    description     = messages.StringField(2)
    organizerUserId = messages.StringField(3)
    topics          = messages.StringField(4, repeated=True)
    city            = messages.StringField(5)
    startDate       = messages.StringField(6) #DateTimeField()
    month           = messages.IntegerField(7)
    maxAttendees    = messages.IntegerField(8)
    seatsAvailable  = messages.IntegerField(9)
    endDate         = messages.StringField(10) #DateTimeField()
    websafeKey      = messages.StringField(11)
    organizerDisplayName = messages.StringField(12)

class ConferenceForms(messages.Message):
    """ConferenceForms -- multiple Conference outbound form message"""
    items = messages.MessageField(ConferenceForm, 1, repeated=True)

class TeeShirtSize(messages.Enum):
    """TeeShirtSize -- t-shirt size enumeration value"""
    NOT_SPECIFIED = 1
    XS_M = 2
    XS_W = 3
    S_M = 4
    S_W = 5
    M_M = 6
    M_W = 7
    L_M = 8
    L_W = 9
    XL_M = 10
    XL_W = 11
    XXL_M = 12
    XXL_W = 13
    XXXL_M = 14
    XXXL_W = 15

class ConferenceQueryForm(messages.Message):
    """ConferenceQueryForm -- Conference query inbound form message"""
    field = messages.StringField(1)
    operator = messages.StringField(2)
    value = messages.StringField(3)

class ConferenceQueryForms(messages.Message):
    """ConferenceQueryForms -- multiple ConferenceQueryForm inbound form message"""
    filters = messages.MessageField(ConferenceQueryForm, 1, repeated=True)

<|endoftext|>


<commit_before>''' Workbench Server Version '''
__version_info__ = (0, 3, 2)
__version__      = '.'.join(map(str, __version_info__))
<commit_msg>server version bump<commit_after>''' Workbench Server Version '''
__version_info__ = (0, 3, 3)
__version__      = '.'.join(map(str, __version_info__))
<|endoftext|>