<commit_before>
#!/usr/bin/env python
# encoding: utf-8
"""
tempodb/client.py

Copyright (c) 2012 TempoDB, Inc. All rights reserved.
"""

from dateutil import parser
import requests
import simplejson
import urllib
import urllib2

API_HOST = 'api.tempo-db.com'
API_PORT = 80
API_VERSION = 'v1'

class Database(object):

    def __init__(self, key, secret):
        self.key = key
        self.secret = secret

class Series(object):

    def __init__(self, i, key, attributes={}, tags=[]):
        self.id = i
        self.key = key
        self.attributes = attributes
        self.tags = tags

class DataPoint(object):

    def __init__(self, ts, value):
        self.ts = ts
        self.value = value

    def __str__(self):
        return "t: %s, v: %s" % (self.ts, self.value)

class DataSet(object):

    def __init__(self, series, start, end, data=[]):
        self.series = series
        self.start = start
        self.end = end
        self.data = data

class Client(object):

    def __init__(self, key, secret, host=API_HOST, port=API_PORT, secure=True):
        self.key = key
        self.secret = secret
        self.host = host
        self.port = port
        self.secure = secure

    def create_database(self, name=""):
        params = {
            'name': name,
        }
        json = self.request('/database/', method='POST', params=params)
        key = json.get('id', '')
        secret = json.get('password', '')
        database = Database(key, secret)
        return database

    def get_series(self, ids=[], keys=[], tags=[], attributes={}):
        params = {}
        if ids:
            params['id'] = ids
        if keys:
            params['key'] = keys
        if tags:
            params['tag'] = tags
        if attributes:
            params['attr'] = attributes

        json = self.request('/series/', method='GET', params=params)
        series = []
        for s in json:
            i = s.get('id', '')
            key = s.get('key', '')
            attr = s.get('attributes', {})
            tags = s.get('tags', [])
            series.append(Series(i, key, attr, tags))
        return series

    def update_series(self, series):
        json = self.request('/series/id/%s/' % (series.id,), method='PUT', params=series.__dict__)
        i = json.get('id', '')
        key = json.get('key', '')
        attr = json.get('attributes', {})
        tags = json.get('tags', [])
        return Series(i, key, attr, tags)

    def read(self, start, end, interval="", function="", ids=[], keys=[], tags=[], attributes={}):
        params = {
            'start': start.isoformat(),
            'end': end.isoformat()
        }

        if ids:
            params['id'] = ids
        if keys:
            params['key'] = keys
        if interval:
            params['interval'] = interval
        if function:
            params['function'] = function
        if tags:
            params['tag'] = tags
        if attributes:
            params['attr'] = attributes

        url = '/data/'
        json = self.request(url, method='GET', params=params)

        datasets = []
        for j in json:
            id = j.get('series', {}).get('id', '')
            key = j.get('series', {}).get('key', '')
            attributes = j.get('series', {}).get('attributes', {})
            tags = j.get('series', {}).get('tags', [])
            series = Series(id, key, attributes=attributes, tags=tags)

            start_date = parser.parse(j.get('start', ''))
            end_date = parser.parse(j.get('end', ''))

            data = [DataPoint(parser.parse(dp.get('t', '')), dp.get('v', None)) for dp in j.get("data", [])]
            datasets.append(DataSet(series, start_date, end_date, data))
        return datasets

    def read_id(self, series_id, start, end, interval="", function=""):
        series_type = 'id'
        series_val = series_id
        return self._read(series_type, series_val, start, end, interval, function)

    def read_key(self, series_key, start, end, interval="", function=""):
        series_type = 'key'
        series_val = series_key
        return self._read(series_type, series_val, start, end, interval, function)

    def _read(self, series_type, series_val, start, end, interval="", function=""):
        params = {
            'start': start.isoformat(),
            'end': end.isoformat(),
        }

        # add rollup interval and function if supplied
        if interval:
            params['interval'] = interval
        if function:
            params['function'] = function

        url = '/series/%s/%s/data/' % (series_type, series_val)
        json = self.request(url, method='GET', params=params)

        #we got an error
        if 'error' in json:
            return json

        id = json.get('series', {}).get('id', '')
        key = json.get('series', {}).get('key', '')
        attributes = json.get('series', {}).get('attributes', {})
        tags = json.get('series', {}).get('tags', [])
        series = Series(id, key, attributes=attributes, tags=tags)

        start_date = parser.parse(json.get('start', ''))
        end_date = parser.parse(json.get('end', ''))

        data = [DataPoint(parser.parse(dp.get('t', '')), dp.get('v', None)) for dp in json.get("data", [])]
        return DataSet(series, start_date, end_date, data)

    def write_id(self, series_id, data):
        series_type = 'id'
        series_val = series_id
        return self.write(series_type, series_val, data)

    def write_key(self, series_key, data):
        series_type = 'key'
        series_val = series_key
        return self.write(series_type, series_val, data)

    def write(self, series_type, series_val, data):
        url = '/series/%s/%s/data/' % (series_type, series_val)
        json = self.request(url, method='POST', params=data)
        return json

    def write_bulk(self, data):
        json = self.request('/data/', method='POST', params=data)
        return json

    def request(self, target, method='GET', params={}):
        assert method in ['GET', 'POST', 'PUT'], "Only 'GET' and 'POST' are allowed for method."

        if method == 'POST':
            base = self.build_full_url(target)
            response = requests.post(base, data=simplejson.dumps(params), auth=(self.key, self.secret))
        elif method == 'PUT':
            base = self.build_full_url(target)
            response = requests.put(base, data=simplejson.dumps(params), auth=(self.key, self.secret))
        else:
            base = self.build_full_url(target, params)
            response = requests.get(base, auth=(self.key, self.secret))

        if response.status_code == 200:
            if response.text:
                json = simplejson.loads(response.text)
            else:
                json = ''
            #try:
            #    json = simplejson.loads(response.text)
            #except simplejson.decoder.JSONDecodeError, err:
            #    json = dict(error="JSON Parse Error (%s):\n%s" % (err, response.text))
        else:
            json = dict(error=response.text)
        return json

    def build_full_url(self, target, params={}):
        port = "" if self.port == 80 else ":%d" % self.port
        protocol = "https://" if self.secure else "http://"
        base_full_url = "%s%s%s" % (protocol, self.host, port)
        return base_full_url + self.build_url(target, params)

    def build_url(self, url, params={}):
        target_path = urllib2.quote(url)

        if params:
            return "/%s%s?%s" % (API_VERSION, target_path, self._urlencode(params))
        else:
            return "/%s%s" % (API_VERSION, target_path)

    def _urlencode(self, params):
        p = []
        for key, value in params.iteritems():
            if isinstance(value, (list, tuple)):
                for v in value:
                    p.append((key, v))
            elif isinstance(value, dict):
                for k, v in value.items():
                    p.append(('%s[%s]' % (key, k), v))
            else:
                p.append((key, value))
        return urllib.urlencode(p).encode("UTF-8")

class TempoDBApiException(Exception):
    pass
<commit_msg>Add a static method to DataSet to create from json<commit_after>
#!/usr/bin/env python
# encoding: utf-8
"""
tempodb/client.py

Copyright (c) 2012 TempoDB, Inc. All rights reserved.
"""

from dateutil import parser
import requests
import simplejson
import urllib
import urllib2

API_HOST = 'api.tempo-db.com'
API_PORT = 80
API_VERSION = 'v1'

class Database(object):

    def __init__(self, key, secret):
        self.key = key
        self.secret = secret

class Series(object):

    def __init__(self, i, key, attributes={}, tags=[]):
        self.id = i
        self.key = key
        self.attributes = attributes
        self.tags = tags

class DataPoint(object):

    def __init__(self, ts, value):
        self.ts = ts
        self.value = value

    def __str__(self):
        return "t: %s, v: %s" % (self.ts, self.value)

class DataSet(object):

    def __init__(self, series, start, end, data=[]):
        self.series = series
        self.start = start
        self.end = end
        self.data = data

    @staticmethod
    def from_json(json):
        id = json.get('series', {}).get('id', '')
        key = json.get('series', {}).get('key', '')
        attributes = json.get('series', {}).get('attributes', {})
        tags = json.get('series', {}).get('tags', [])
        series = Series(id, key, attributes=attributes, tags=tags)

        start_date = parser.parse(json.get('start', ''))
        end_date = parser.parse(json.get('end', ''))

        data = [DataPoint(parser.parse(dp.get('t', '')), dp.get('v', None)) for dp in json.get("data", [])]
        return DataSet(series, start_date, end_date, data)

class Client(object):

    def __init__(self, key, secret, host=API_HOST, port=API_PORT, secure=True):
        self.key = key
        self.secret = secret
        self.host = host
        self.port = port
        self.secure = secure

    def create_database(self, name=""):
        params = {
            'name': name,
        }
        json = self.request('/database/', method='POST', params=params)
        key = json.get('id', '')
        secret = json.get('password', '')
        database = Database(key, secret)
        return database

    def get_series(self, ids=[], keys=[], tags=[], attributes={}):
        params = {}
        if ids:
            params['id'] = ids
        if keys:
            params['key'] = keys
        if tags:
            params['tag'] = tags
        if attributes:
            params['attr'] = attributes

        json = self.request('/series/', method='GET', params=params)
        series = []
        for s in json:
            i = s.get('id', '')
            key = s.get('key', '')
            attr = s.get('attributes', {})
            tags = s.get('tags', [])
            series.append(Series(i, key, attr, tags))
        return series

    def update_series(self, series):
        json = self.request('/series/id/%s/' % (series.id,), method='PUT', params=series.__dict__)
        i = json.get('id', '')
        key = json.get('key', '')
        attr = json.get('attributes', {})
        tags = json.get('tags', [])
        return Series(i, key, attr, tags)

    def read(self, start, end, interval="", function="", ids=[], keys=[], tags=[], attributes={}):
        params = {
            'start': start.isoformat(),
            'end': end.isoformat()
        }

        if ids:
            params['id'] = ids
        if keys:
            params['key'] = keys
        if interval:
            params['interval'] = interval
        if function:
            params['function'] = function
        if tags:
            params['tag'] = tags
        if attributes:
            params['attr'] = attributes

        url = '/data/'
        json = self.request(url, method='GET', params=params)
        return [DataSet.from_json(j) for j in json]

    def read_id(self, series_id, start, end, interval="", function=""):
        series_type = 'id'
        series_val = series_id
        return self._read(series_type, series_val, start, end, interval, function)

    def read_key(self, series_key, start, end, interval="", function=""):
        series_type = 'key'
        series_val = series_key
        return self._read(series_type, series_val, start, end, interval, function)

    def _read(self, series_type, series_val, start, end, interval="", function=""):
        params = {
            'start': start.isoformat(),
            'end': end.isoformat(),
        }

        # add rollup interval and function if supplied
        if interval:
            params['interval'] = interval
        if function:
            params['function'] = function

        url = '/series/%s/%s/data/' % (series_type, series_val)
        json = self.request(url, method='GET', params=params)

        #we got an error
        if 'error' in json:
            return json
        return DataSet.from_json(json)

    def write_id(self, series_id, data):
        series_type = 'id'
        series_val = series_id
        return self.write(series_type, series_val, data)

    def write_key(self, series_key, data):
        series_type = 'key'
        series_val = series_key
        return self.write(series_type, series_val, data)

    def write(self, series_type, series_val, data):
        url = '/series/%s/%s/data/' % (series_type, series_val)
        json = self.request(url, method='POST', params=data)
        return json

    def write_bulk(self, data):
        json = self.request('/data/', method='POST', params=data)
        return json

    def request(self, target, method='GET', params={}):
        assert method in ['GET', 'POST', 'PUT'], "Only 'GET' and 'POST' are allowed for method."

        if method == 'POST':
            base = self.build_full_url(target)
            response = requests.post(base, data=simplejson.dumps(params), auth=(self.key, self.secret))
        elif method == 'PUT':
            base = self.build_full_url(target)
            response = requests.put(base, data=simplejson.dumps(params), auth=(self.key, self.secret))
        else:
            base = self.build_full_url(target, params)
            response = requests.get(base, auth=(self.key, self.secret))

        if response.status_code == 200:
            if response.text:
                json = simplejson.loads(response.text)
            else:
                json = ''
            #try:
            #    json = simplejson.loads(response.text)
            #except simplejson.decoder.JSONDecodeError, err:
            #    json = dict(error="JSON Parse Error (%s):\n%s" % (err, response.text))
        else:
            json = dict(error=response.text)
        return json

    def build_full_url(self, target, params={}):
        port = "" if self.port == 80 else ":%d" % self.port
        protocol = "https://" if self.secure else "http://"
        base_full_url = "%s%s%s" % (protocol, self.host, port)
        return base_full_url + self.build_url(target, params)

    def build_url(self, url, params={}):
        target_path = urllib2.quote(url)

        if params:
            return "/%s%s?%s" % (API_VERSION, target_path, self._urlencode(params))
        else:
            return "/%s%s" % (API_VERSION, target_path)

    def _urlencode(self, params):
        p = []
        for key, value in params.iteritems():
            if isinstance(value, (list, tuple)):
                for v in value:
                    p.append((key, v))
            elif isinstance(value, dict):
                for k, v in value.items():
                    p.append(('%s[%s]' % (key, k), v))
            else:
                p.append((key, value))
        return urllib.urlencode(p).encode("UTF-8")

class TempoDBApiException(Exception):
    pass
<|endoftext|>


<commit_before>#
# Copyright 2018 Analytics Zoo Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from pyspark.ml.param.shared import *
from pyspark.ml.wrapper import JavaModel, JavaEstimator, JavaTransformer
from bigdl.optim.optimizer import SGD
from bigdl.util.common import *
from zoo.feature.common import *

if sys.version >= '3':
    long = int
    unicode = str

class HasBatchSize(Params):
    """
    Mixin for param batchSize: batch size.
    """

    # a placeholder to make it appear in the generated doc
    batchSize = Param(Params._dummy(), "batchSize", "batchSize (>= 0).")

    def __init__(self):
        super(HasBatchSize, self).__init__()
        #: param for batch size.
        self.batchSize = Param(self, "batchSize", "batchSize")
        self._setDefault(batchSize=1)

    def setBatchSize(self, val):
        """
        Sets the value of :py:attr:`batchSize`.
        """
        self._paramMap[self.batchSize] = val
        return self

    def getBatchSize(self):
        """
        Gets the value of batchSize or its default value.
        """
        return self.getOrDefault(self.batchSize)

class HasSamplePreprocessing:
    """
    Mixin for param samplePreprocessing
    """
    samplePreprocessing = None

    def __init__(self):
        super(HasSamplePreprocessing, self).__init__()

    def setSamplePreprocessing(self, val):
        """
        Sets samplePreprocessing
        """
        pythonBigDL_method_name = "setSamplePreprocessing"
        callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val)
        self.samplePreprocessing = val
        return self

    def getSamplePreprocessing(self):
        return self.samplePreprocessing

class HasOptimMethod:

    optimMethod = SGD()

    def __init__(self):
        super(HasOptimMethod, self).__init__()

    def setOptimMethod(self, val):
        """
        Sets optimization method. E.g. SGD, Adam, LBFGS etc. from bigdl.optim.optimizer.
        default: SGD()
        """
        pythonBigDL_method_name = "setOptimMethod"
        callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val)
        self.optimMethod = val
        return self

    def getOptimMethod(self):
        """
        Gets the optimization method
        """
        return self.optimMethod

class NNEstimator(JavaEstimator, HasFeaturesCol, HasLabelCol, HasPredictionCol, HasBatchSize,
                  HasOptimMethod, HasSamplePreprocessing, JavaValue):
    """
    NNEstimator extends org.apache.spark.ml.Estimator and supports training a BigDL model with
    Spark DataFrame data. It can be integrated into a standard Spark ML Pipeline to enable
    users for combined usage with Spark MLlib.

    NNEstimator supports different feature and label data type through operation defined in
    Preprocessing. We provide pre-defined Preprocessing for popular data types like Array
    or Vector in package zoo.feature, while user can also develop customized Preprocess
    which extends from feature.common.Preprocessing. During fit, NNEstimator
    will extract feature and label data from input DataFrame and use the Preprocessing to prepare
    data for the model.
    Using the Preprocessing allows NNEstimator to cache only the raw data and decrease the
    memory consumption during feature conversion and training.

    More concrete examples are available in package com.intel.analytics.zoo.examples.nnframes
    """

    def __init__(self, model, criterion,
                 feature_preprocessing=SeqToTensor(),
                 label_preprocessing=SeqToTensor(),
                 jvalue=None, bigdl_type="float"):
        """
        Construct a NNEstimator with BigDL model, criterion and Preprocessing for feature and label
        data.
        :param model: BigDL Model to be trained.
        :param criterion: BigDL criterion.
        :param feature_preprocessing: The param converts the data in feature column to a
               Tensor or to a Sample directly. It expects a List of Int as the size of the
               converted Tensor, or a Preprocessing[F, Tensor[T]]

               If a List of Int is set as feature_preprocessing, it can only handle the case that
               feature column contains the following data types:
               Float, Double, Int, Array[Float], Array[Double], Array[Int] and MLlib Vector. The
               feature data are converted to Tensors with the specified sizes before
               sending to the model. Internally, a SeqToTensor is generated according to the
               size, and used as the feature_preprocessing.

               Alternatively, user can set feature_preprocessing as Preprocessing[F, Tensor[T]]
               that transforms the feature data to a Tensor[T]. Some pre-defined Preprocessing are
               provided in package zoo.feature. Multiple Preprocessing can be combined as a
               ChainedPreprocessing.

               The feature_preprocessing will also be copied to the generated NNModel and applied
               to feature column during transform.
        :param label_preprocessing: similar to feature_preprocessing, but applies to Label data.
        :param jvalue: Java object create by Py4j
        :param bigdl_type: optional parameter. data type of model, "float"(default) or "double".
        """
        super(NNEstimator, self).__init__()
        if type(feature_preprocessing) is list:
            assert(all(isinstance(x, int) for x in feature_preprocessing))
            feature_preprocessing = SeqToTensor(feature_preprocessing)

        if type(label_preprocessing) is list:
            assert(all(isinstance(x, int) for x in label_preprocessing))
            label_preprocessing = SeqToTensor(label_preprocessing)

        sample_preprocessing = FeatureLabelPreprocessing(feature_preprocessing, label_preprocessing)

        self.value = jvalue if jvalue else callBigDlFunc(
            bigdl_type, self.jvm_class_constructor(), model, criterion, sample_preprocessing)
        self.samplePreprocessing = sample_preprocessing
        self.bigdl_type = bigdl_type
        self._java_obj = self.value

        self.maxEpoch = Param(self, "maxEpoch", "number of max Epoch")
        self.learningRate = Param(self, "learningRate", "learning rate")
        self._setDefault(maxEpoch=50, learningRate=1e-3, batchSize=1)

        self.train_summary = None
        self.validation_config = None
        self.validation_summary = None

    def setSamplePreprocessing(self, val):
        """
        Sets the value of sample_preprocessing
        :param val: a Preprocesing[(Feature, Option(Label), Sample]
        """
        super(NNEstimator, self).setSamplePreprocessing(val)
        return self

    def setMaxEpoch(self, val):
        """
        Sets the value of :py:attr:`maxEpoch`.
        """
        self._paramMap[self.maxEpoch] = val
        return self

    def getMaxEpoch(self):
        """
        Gets the value of maxEpoch or its default value.
        """
        return self.getOrDefault(self.maxEpoch)

    def setLearningRate(self, val):
        """
        Sets the value of :py:attr:`learningRate`.
        """
        self._paramMap[self.learningRate] = val
        return self

    def getLearningRate(self):
        """
        Gets the value of learningRate or its default value.
        """
        return self.getOrDefault(self.learningRate)

    def setTrainSummary(self, val):
        """
        Statistics (LearningRate, Loss, Throughput, Parameters) collected during training for the
        training data, which can be used for visualization via Tensorboard.
        Use setTrainSummary to enable train logger. Then the log will be saved to
        logDir/appName/train as specified by the parameters of TrainSummary.
        Default: Not enabled

        :param summary: a TrainSummary object
        """
        pythonBigDL_method_name = "setTrainSummary"
        callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val)
        self.train_summary = val
        return self

    def getTrainSummary(self):
        """
        Gets the train summary
        """
        return self.train_summary

    def setValidationSummary(self, val):
        """
        Statistics (LearningRate, Loss, Throughput, Parameters) collected during training for the
        validation data if validation data is set, which can be used for visualization via
        Tensorboard. Use setValidationSummary to enable validation logger. Then the log will be
        saved to logDir/appName/ as specified by the parameters of validationSummary.
        Default: None
        """
        pythonBigDL_method_name = "setValidationSummary"
        callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val)
        self.validation_summary = val
        return self

    def getValidationSummary(self):
        """
        Gets the Validation summary
        """
        return self.validation_summary

    def setValidation(self, trigger, val_df, val_method, batch_size):
        """
        Set a validate evaluation during training

        :param trigger: validation interval
        :param val_df: validation dataset
        :param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss"
        :param batch_size: validation batch size
        """
        pythonBigDL_method_name = "setValidation"
        callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value,
                      trigger, val_df, val_method, batch_size)
        self.validation_config = [trigger, val_df, val_method, batch_size]
        return self

    def getValidation(self):
        """
        Gets the validate configuration. If validation config has been set, getValidation will
        return a List of [ValidationTrigger, Validation data, Array[ValidationMethod[T]],
        batchsize]
        """
        return self.validation_config

    def clearGradientClippingParams(self):
        """
        Clear clipping params, in this case, clipping will not be applied.
        In order to take effect, it needs to be called before fit.
        """
        callBigDlFunc(self.bigdl_type, "nnEstimatorClearGradientClippingParams",
                      self.value)
        return self

    def setConstantGradientClipping(self, min, max):
        """
        Set constant gradient clipping during the training process.
        In order to take effect, it needs to be called before fit.

        # Arguments
        min: The minimum value to clip by. Float.
        max: The maximum value to clip by. Float.
        """
        callBigDlFunc(self.bigdl_type, "nnEstimatorSetConstantGradientClipping",
                      self.value,
                      float(min),
                      float(max))
        return self

    def setGradientClippingByL2Norm(self, clip_norm):
        """
        Clip gradient to a maximum L2-Norm during the training process.
        In order to take effect, it needs to be called before fit.

        # Arguments
        clip_norm: Gradient L2-Norm threshold. Float.
        """
        callBigDlFunc(self.bigdl_type, "nnEstimatorSetGradientClippingByL2Norm",
                      self.value,
                      float(clip_norm))
        return self

    def _create_model(self, java_model):
        nnModel = NNModel.of(java_model,
                             ChainedPreprocessing([ToTuple(), self.getSamplePreprocessing()]),
                             self.bigdl_type)
        nnModel.setFeaturesCol(self.getFeaturesCol()) \
            .setPredictionCol(self.getPredictionCol()) \
            .setBatchSize(self.getBatchSize())
        return nnModel

class NNModel(JavaTransformer, HasFeaturesCol, HasPredictionCol, HasBatchSize,
              HasSamplePreprocessing, JavaValue):
    """
    NNModel extends Spark ML Transformer and supports BigDL model with Spark DataFrame.

    NNModel supports different feature data type through Preprocessing. Some common
    Preprocessing have been defined in com.intel.analytics.zoo.feature.

    After transform, the prediction column contains the output of the model as Array[T], where
    T (Double or Float) is decided by the model type.
    """
    def __init__(self, model, feature_preprocessing=SeqToTensor(), jvalue=None, bigdl_type="float"):
        """
        create a NNModel with a BigDL model
        :param model: trained BigDL model to use in prediction.
        :param feature_preprocessing: The param converts the data in feature column to a
                                      Tensor. It expects a List of Int as
                                      the size of the converted Tensor, or a
                                      Preprocessing[F, Tensor[T]]
        :param jvalue: Java object create by Py4j
        :param bigdl_type: optional parameter. data type of model, "float"(default) or "double".
        """
        super(NNModel, self).__init__()
        if type(feature_preprocessing) is list:
            assert(all(isinstance(x, int) for x in feature_preprocessing))
            feature_preprocessing = SeqToTensor(feature_preprocessing)

        sample_preprocessing = ChainedPreprocessing([feature_preprocessing, TensorToSample()])

        self.value = jvalue if jvalue else callBigDlFunc(
            bigdl_type, self.jvm_class_constructor(), model, sample_preprocessing)
        self.samplePreprocessing = sample_preprocessing
        self._java_obj = self.value
        self.bigdl_type = bigdl_type

    @classmethod
    def of(self, jvalue, sample_preprocessing=None, bigdl_type="float"):
        model = NNModel(model=None, jvalue=jvalue, bigdl_type=bigdl_type)\
            .setSamplePreprocessing(sample_preprocessing)
        return model

    def setSamplePreprocessing(self, val):
        """
        Sets the value of sample_preprocessing
        :param val: a Preprocesing[Feature, Sample]
        """
        super(NNModel, self).setSamplePreprocessing(val)
        return self

class NNClassifier(NNEstimator):
    """
    NNClassifier is a specialized NNEstimator that simplifies the data format for
    classification tasks. It only supports label column of DoubleType, and the fitted
    NNClassifierModel will have the prediction column of DoubleType.
    """
    def __init__(self, model, criterion, feature_preprocessing=SeqToTensor(),
                 jvalue=None, bigdl_type="float"):
        """
        :param model: BigDL module to be optimized
        :param criterion: BigDL criterion method
        :param feature_preprocessing: The param converts the data in feature column to a
                                      Tensor. It expects a List of Int as
                                      the size of the converted Tensor, or a
                                      Preprocessing[F, Tensor[T]]
        :param bigdl_type(optional): Data type of BigDL model, "float"(default) or "double".
        """
        super(NNClassifier, self).__init__(
            model, criterion, feature_preprocessing, ScalarToTensor(), jvalue, bigdl_type)

    def setSamplePreprocessing(self, val):
        """
        Sets the value of sample_preprocessing
        :param val: a Preprocesing[(Feature, Option(Label), Sample]
        """
        super(NNClassifier, self).setSamplePreprocessing(val)
        return self

    def _create_model(self, java_model):
        classifierModel = NNClassifierModel.of(java_model, ChainedPreprocessing(
            [ToTuple(), self.getSamplePreprocessing()]), self.bigdl_type)
        classifierModel.setFeaturesCol(self.getFeaturesCol()) \
            .setPredictionCol(self.getPredictionCol()) \
            .setBatchSize(self.getBatchSize())
        return classifierModel

class NNClassifierModel(NNModel):
    """
    NNClassifierModel is a specialized [[NNModel]] for classification tasks. The prediction
    column will have the datatype of Double.
    """
    def __init__(self,  model, feature_preprocessing=SeqToTensor(), jvalue=None,
                 bigdl_type="float"):
        """
        :param model: trained BigDL model to use in prediction.
        :param feature_preprocessing: The param converts the data in feature column to a
                                      Tensor. It expects a List of Int as
                                      the size of the converted Tensor, or a
                                      Preprocessing[F, Tensor[T]]
        :param jvalue: Java object create by Py4j
        :param bigdl_type(optional): Data type of BigDL model, "float"(default) or "double".
        """
        super(NNClassifierModel, self).__init__(model, feature_preprocessing, jvalue, bigdl_type)

    @classmethod
    def of(self, jvalue, sample_preprocessing=None, bigdl_type="float"):
        model = NNClassifierModel(model=None, jvalue=jvalue, bigdl_type=bigdl_type) \
            .setSamplePreprocessing(sample_preprocessing)
        return model

    def setSamplePreprocessing(self, val):
        """
        Sets the value of sample_preprocessing
        :param val: a Preprocesing[Feature, Sample]
        """
        super(NNClassifierModel, self).setSamplePreprocessing(val)
        return self
<commit_msg>fix trailing space (#194)<commit_after>#
# Copyright 2018 Analytics Zoo Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from pyspark.ml.param.shared import *
from pyspark.ml.wrapper import JavaModel, JavaEstimator, JavaTransformer
from bigdl.optim.optimizer import SGD
from bigdl.util.common import *
from zoo.feature.common import *

if sys.version >= '3':
    long = int
    unicode = str

class HasBatchSize(Params):
    """
    Mixin for param batchSize: batch size.
    """

    # a placeholder to make it appear in the generated doc
    batchSize = Param(Params._dummy(), "batchSize", "batchSize (>= 0).")

    def __init__(self):
        super(HasBatchSize, self).__init__()
        #: param for batch size.
        self.batchSize = Param(self, "batchSize", "batchSize")
        self._setDefault(batchSize=1)

    def setBatchSize(self, val):
        """
        Sets the value of :py:attr:`batchSize`.
        """
        self._paramMap[self.batchSize] = val
        return self

    def getBatchSize(self):
        """
        Gets the value of batchSize or its default value.
        """
        return self.getOrDefault(self.batchSize)

class HasSamplePreprocessing:
    """
    Mixin for param samplePreprocessing
    """
    samplePreprocessing = None

    def __init__(self):
        super(HasSamplePreprocessing, self).__init__()

    def setSamplePreprocessing(self, val):
        """
        Sets samplePreprocessing
        """
        pythonBigDL_method_name = "setSamplePreprocessing"
        callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val)
        self.samplePreprocessing = val
        return self

    def getSamplePreprocessing(self):
        return self.samplePreprocessing

class HasOptimMethod:

    optimMethod = SGD()

    def __init__(self):
        super(HasOptimMethod, self).__init__()

    def setOptimMethod(self, val):
        """
        Sets optimization method. E.g. SGD, Adam, LBFGS etc. from bigdl.optim.optimizer.
        default: SGD()
        """
        pythonBigDL_method_name = "setOptimMethod"
        callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val)
        self.optimMethod = val
        return self

    def getOptimMethod(self):
        """
        Gets the optimization method
        """
        return self.optimMethod

class NNEstimator(JavaEstimator, HasFeaturesCol, HasLabelCol, HasPredictionCol, HasBatchSize,
                  HasOptimMethod, HasSamplePreprocessing, JavaValue):
    """
    NNEstimator extends org.apache.spark.ml.Estimator and supports training a BigDL model with
    Spark DataFrame data. It can be integrated into a standard Spark ML Pipeline to enable
    users for combined usage with Spark MLlib.

    NNEstimator supports different feature and label data type through operation defined in
    Preprocessing. We provide pre-defined Preprocessing for popular data types like Array
    or Vector in package zoo.feature, while user can also develop customized Preprocess
    which extends from feature.common.Preprocessing. During fit, NNEstimator
    will extract feature and label data from input DataFrame and use the Preprocessing to prepare
    data for the model.
    Using the Preprocessing allows NNEstimator to cache only the raw data and decrease the
    memory consumption during feature conversion and training.

    More concrete examples are available in package com.intel.analytics.zoo.examples.nnframes
    """

    def __init__(self, model, criterion,
                 feature_preprocessing=SeqToTensor(),
                 label_preprocessing=SeqToTensor(),
                 jvalue=None, bigdl_type="float"):
        """
        Construct a NNEstimator with BigDL model, criterion and Preprocessing for feature and label
        data.
        :param model: BigDL Model to be trained.
        :param criterion: BigDL criterion.
        :param feature_preprocessing: The param converts the data in feature column to a
               Tensor or to a Sample directly. It expects a List of Int as the size of the
               converted Tensor, or a Preprocessing[F, Tensor[T]]

               If a List of Int is set as feature_preprocessing, it can only handle the case that
               feature column contains the following data types:
               Float, Double, Int, Array[Float], Array[Double], Array[Int] and MLlib Vector. The
               feature data are converted to Tensors with the specified sizes before
               sending to the model. Internally, a SeqToTensor is generated according to the
               size, and used as the feature_preprocessing.

               Alternatively, user can set feature_preprocessing as Preprocessing[F, Tensor[T]]
               that transforms the feature data to a Tensor[T]. Some pre-defined Preprocessing are
               provided in package zoo.feature. Multiple Preprocessing can be combined as a
               ChainedPreprocessing.

               The feature_preprocessing will also be copied to the generated NNModel and applied
               to feature column during transform.
        :param label_preprocessing: similar to feature_preprocessing, but applies to Label data.
        :param jvalue: Java object create by Py4j
        :param bigdl_type: optional parameter. data type of model, "float"(default) or "double".
        """
        super(NNEstimator, self).__init__()
        if type(feature_preprocessing) is list:
            assert(all(isinstance(x, int) for x in feature_preprocessing))
            feature_preprocessing = SeqToTensor(feature_preprocessing)

        if type(label_preprocessing) is list:
            assert(all(isinstance(x, int) for x in label_preprocessing))
            label_preprocessing = SeqToTensor(label_preprocessing)

        sample_preprocessing = FeatureLabelPreprocessing(feature_preprocessing, label_preprocessing)

        self.value = jvalue if jvalue else callBigDlFunc(
            bigdl_type, self.jvm_class_constructor(), model, criterion, sample_preprocessing)
        self.samplePreprocessing = sample_preprocessing
        self.bigdl_type = bigdl_type
        self._java_obj = self.value

        self.maxEpoch = Param(self, "maxEpoch", "number of max Epoch")
        self.learningRate = Param(self, "learningRate", "learning rate")
        self._setDefault(maxEpoch=50, learningRate=1e-3, batchSize=1)

        self.train_summary = None
        self.validation_config = None
        self.validation_summary = None

    def setSamplePreprocessing(self, val):
        """
        Sets the value of sample_preprocessing
        :param val: a Preprocesing[(Feature, Option(Label), Sample]
        """
        super(NNEstimator, self).setSamplePreprocessing(val)
        return self

    def setMaxEpoch(self, val):
        """
        Sets the value of :py:attr:`maxEpoch`.
        """
        self._paramMap[self.maxEpoch] = val
        return self

    def getMaxEpoch(self):
        """
        Gets the value of maxEpoch or its default value.
        """
        return self.getOrDefault(self.maxEpoch)

    def setLearningRate(self, val):
        """
        Sets the value of :py:attr:`learningRate`.
        """
        self._paramMap[self.learningRate] = val
        return self

    def getLearningRate(self):
        """
        Gets the value of learningRate or its default value.
        """
        return self.getOrDefault(self.learningRate)

    def setTrainSummary(self, val):
        """
        Statistics (LearningRate, Loss, Throughput, Parameters) collected during training for the
        training data, which can be used for visualization via Tensorboard.
        Use setTrainSummary to enable train logger. Then the log will be saved to
        logDir/appName/train as specified by the parameters of TrainSummary.
        Default: Not enabled

        :param summary: a TrainSummary object
        """
        pythonBigDL_method_name = "setTrainSummary"
        callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val)
        self.train_summary = val
        return self

    def getTrainSummary(self):
        """
        Gets the train summary
        """
        return self.train_summary

    def setValidationSummary(self, val):
        """
        Statistics (LearningRate, Loss, Throughput, Parameters) collected during training for the
        validation data if validation data is set, which can be used for visualization via
        Tensorboard. Use setValidationSummary to enable validation logger. Then the log will be
        saved to logDir/appName/ as specified by the parameters of validationSummary.
        Default: None
        """
        pythonBigDL_method_name = "setValidationSummary"
        callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val)
        self.validation_summary = val
        return self

    def getValidationSummary(self):
        """
        Gets the Validation summary
        """
        return self.validation_summary

    def setValidation(self, trigger, val_df, val_method, batch_size):
        """
        Set a validate evaluation during training

        :param trigger: validation interval
        :param val_df: validation dataset
        :param val_method: the ValidationMethod to use,e.g. "Top1Accuracy", "Top5Accuracy", "Loss"
        :param batch_size: validation batch size
        """
        pythonBigDL_method_name = "setValidation"
        callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value,
                      trigger, val_df, val_method, batch_size)
        self.validation_config = [trigger, val_df, val_method, batch_size]
        return self

    def getValidation(self):
        """
        Gets the validate configuration. If validation config has been set, getValidation will
        return a List of [ValidationTrigger, Validation data, Array[ValidationMethod[T]],
        batchsize]
        """
        return self.validation_config

    def clearGradientClippingParams(self):
        """
        Clear clipping params, in this case, clipping will not be applied.
        In order to take effect, it needs to be called before fit.
        """
        callBigDlFunc(self.bigdl_type, "nnEstimatorClearGradientClippingParams",
                      self.value)
        return self

    def setConstantGradientClipping(self, min, max):
        """
        Set constant gradient clipping during the training process.
        In order to take effect, it needs to be called before fit.

        # Arguments
        min: The minimum value to clip by. Float.
        max: The maximum value to clip by. Float.
        """
        callBigDlFunc(self.bigdl_type, "nnEstimatorSetConstantGradientClipping",
                      self.value,
                      float(min),
                      float(max))
        return self

    def setGradientClippingByL2Norm(self, clip_norm):
        """
        Clip gradient to a maximum L2-Norm during the training process.
        In order to take effect, it needs to be called before fit.

        # Arguments
        clip_norm: Gradient L2-Norm threshold. Float.
        """
        callBigDlFunc(self.bigdl_type, "nnEstimatorSetGradientClippingByL2Norm",
                      self.value,
                      float(clip_norm))
        return self

    def _create_model(self, java_model):
        nnModel = NNModel.of(java_model,
                             ChainedPreprocessing([ToTuple(), self.getSamplePreprocessing()]),
                             self.bigdl_type)
        nnModel.setFeaturesCol(self.getFeaturesCol()) \
            .setPredictionCol(self.getPredictionCol()) \
            .setBatchSize(self.getBatchSize())
        return nnModel

class NNModel(JavaTransformer, HasFeaturesCol, HasPredictionCol, HasBatchSize,
              HasSamplePreprocessing, JavaValue):
    """
    NNModel extends Spark ML Transformer and supports BigDL model with Spark DataFrame.

    NNModel supports different feature data type through Preprocessing. Some common
    Preprocessing have been defined in com.intel.analytics.zoo.feature.

    After transform, the prediction column contains the output of the model as Array[T], where
    T (Double or Float) is decided by the model type.
    """
    def __init__(self, model, feature_preprocessing=SeqToTensor(), jvalue=None, bigdl_type="float"):
        """
        create a NNModel with a BigDL model
        :param model: trained BigDL model to use in prediction.
        :param feature_preprocessing: The param converts the data in feature column to a
                                      Tensor. It expects a List of Int as
                                      the size of the converted Tensor, or a
                                      Preprocessing[F, Tensor[T]]
        :param jvalue: Java object create by Py4j
        :param bigdl_type: optional parameter. data type of model, "float"(default) or "double".
        """
        super(NNModel, self).__init__()
        if type(feature_preprocessing) is list:
            assert(all(isinstance(x, int) for x in feature_preprocessing))
            feature_preprocessing = SeqToTensor(feature_preprocessing)

        sample_preprocessing = ChainedPreprocessing([feature_preprocessing, TensorToSample()])

        self.value = jvalue if jvalue else callBigDlFunc(
            bigdl_type, self.jvm_class_constructor(), model, sample_preprocessing)
        self.samplePreprocessing = sample_preprocessing
        self._java_obj = self.value
        self.bigdl_type = bigdl_type

    @classmethod
    def of(self, jvalue, sample_preprocessing=None, bigdl_type="float"):
        model = NNModel(model=None, jvalue=jvalue, bigdl_type=bigdl_type)\
            .setSamplePreprocessing(sample_preprocessing)
        return model

    def setSamplePreprocessing(self, val):
        """
        Sets the value of sample_preprocessing
        :param val: a Preprocesing[Feature, Sample]
        """
        super(NNModel, self).setSamplePreprocessing(val)
        return self

class NNClassifier(NNEstimator):
    """
    NNClassifier is a specialized NNEstimator that simplifies the data format for
    classification tasks. It only supports label column of DoubleType, and the fitted
    NNClassifierModel will have the prediction column of DoubleType.
    """
    def __init__(self, model, criterion, feature_preprocessing=SeqToTensor(),
                 jvalue=None, bigdl_type="float"):
        """
        :param model: BigDL module to be optimized
        :param criterion: BigDL criterion method
        :param feature_preprocessing: The param converts the data in feature column to a
                                      Tensor. It expects a List of Int as
                                      the size of the converted Tensor, or a
                                      Preprocessing[F, Tensor[T]]
        :param bigdl_type(optional): Data type of BigDL model, "float"(default) or "double".
        """
        super(NNClassifier, self).__init__(
            model, criterion, feature_preprocessing, ScalarToTensor(), jvalue, bigdl_type)

    def setSamplePreprocessing(self, val):
        """
        Sets the value of sample_preprocessing
        :param val: a Preprocesing[(Feature, Option(Label), Sample]
        """
        super(NNClassifier, self).setSamplePreprocessing(val)
        return self

    def _create_model(self, java_model):
        classifierModel = NNClassifierModel.of(java_model, ChainedPreprocessing(
            [ToTuple(), self.getSamplePreprocessing()]), self.bigdl_type)
        classifierModel.setFeaturesCol(self.getFeaturesCol()) \
            .setPredictionCol(self.getPredictionCol()) \
            .setBatchSize(self.getBatchSize())
        return classifierModel

class NNClassifierModel(NNModel):
    """
    NNClassifierModel is a specialized [[NNModel]] for classification tasks. The prediction
    column will have the datatype of Double.
    """
    def __init__(self,  model, feature_preprocessing=SeqToTensor(), jvalue=None,
                 bigdl_type="float"):
        """
        :param model: trained BigDL model to use in prediction.
        :param feature_preprocessing: The param converts the data in feature column to a
                                      Tensor. It expects a List of Int as
                                      the size of the converted Tensor, or a
                                      Preprocessing[F, Tensor[T]]
        :param jvalue: Java object create by Py4j
        :param bigdl_type(optional): Data type of BigDL model, "float"(default) or "double".
        """
        super(NNClassifierModel, self).__init__(model, feature_preprocessing, jvalue, bigdl_type)

    @classmethod
    def of(self, jvalue, sample_preprocessing=None, bigdl_type="float"):
        model = NNClassifierModel(model=None, jvalue=jvalue, bigdl_type=bigdl_type) \
            .setSamplePreprocessing(sample_preprocessing)
        return model

    def setSamplePreprocessing(self, val):
        """
        Sets the value of sample_preprocessing
        :param val: a Preprocesing[Feature, Sample]
        """
        super(NNClassifierModel, self).setSamplePreprocessing(val)
        return self
<|endoftext|>


<commit_before>from django import template
from django.template.defaultfilters import stringfilter
from django.utils.safestring import mark_safe
import re

register = template.Library()

@register.filter(is_safe=True, needs_autoescape=False)
@stringfilter
def url_target_blank(text):
	return mark_safe(re.sub("]+)(?<!target=)>",'', text))

# url_target_blank.is_safe = True
# url_target_blank = register.filter(url_target_blank)<commit_msg>add a simple template print debug tag<commit_after>from django import template
from django.template.defaultfilters import stringfilter
from django.utils.safestring import mark_safe
import pprint
import re

register = template.Library()

@register.filter(is_safe=True, needs_autoescape=False)
@stringfilter
def url_target_blank(text):
	return mark_safe(re.sub("]+)(?<!target=)>",'', text))

def template_print(value):
	print '>>>>>>>>>>>TEMPLATE_DEBUG_PRINT<<<<<<<<<<<<<<'
	pprint.pprint(value)
	print '>>>>>>>>>>>TEMPLATE_DEBUG_PRINT<<<<<<<<<<<<<<'
	return u''

register.simple_tag(template_print)

# url_target_blank.is_safe = True
# url_target_blank = register.filter(url_target_blank)<|endoftext|>


<commit_before>#!python3

###############################################################################
# Copyright (c) 2013 Evgeny Proydakov <lord.tiran@gmail.com>
###############################################################################
# -*- coding: utf-8 -*-
###############################################################################

import sys
import utils
import re

# compate type
COMPARE_FULL   = 'FULL'
COMPARE_TRACE  = 'TRACE'
COMPARE_RESULT = 'RESULT'

def cmp(a, b):
    return (a > b) - (a < b)

def _read_result_data(name):
    file_data_temp = open(name, 'r', encoding = 'utf-8').readlines()
    utils.cut_slash(file_data_temp)

    file_data = []

    push = False
    for line in file_data_temp:
        if (push):
            line = re.sub(r'(\.+)Медиана:\s+(\S+)', r'\1(Медиана:)\2', line)
            file_data.append(line)
        if (line.find('BExpCalcCounter') != -1):
            push = True

    return file_data

def _read_trace_data(name):
    file_data_temp = open(name, 'r', encoding = 'utf-8').readlines()
    utils.cut_slash(file_data_temp)

    file_data = []

    push = False
    for line in file_data_temp:
        if (push):
            line = re.sub(r'SES\s+(\S+)\s(\S+)\s(\S+)', r'SES \1', line)
            line = re.sub(r'DPS_TM\s(\S+)(\s+)(\S+)(\s+)(\S+)', r'DPS_TM', line)
            line = re.sub(r'DPS_ME\s(\S+)(\s+)(\S+)(\s+)(\S+)', r'DPS_ME', line)
            line = re.sub(r'DPS_MM\s(\S+)', r'DPS_MM', line)
            file_data.append(line)

        if (line.find('$Changes') != -1):
            push = True

    return file_data

def full(file1, file2):
    file1_data = open(file1, 'r', encoding = 'utf-8').readlines()
    file2_data = open(file2, 'r', encoding = 'utf-8').readlines()

    utils.cut_slash(file1_data)
    utils.cut_slash(file2_data)

    if cmp(file1_data, file2_data) == 0:
        return True

    return False

def result(file1, file2):
    file1_data = _read_result_data(file1)
    file2_data = _read_result_data(file2)

    if cmp(file1_data, file2_data) == 0:
        return True

    return False

def trace(file1, file2):
    file1_data = _read_trace_data(file1)
    file2_data = _read_trace_data(file2)

    if cmp(file1_data, file2_data) == 0:
        return True

    return False

def files(file1, file2, type):
    if   type == COMPARE_FULL:
        return full(file1, file2)
    elif type == COMPARE_TRACE:
        return trace(file1, file2)
    elif type == COMPARE_RESULT:
        return result(file1, file2)

    return False

if __name__ == '__main__':

    utils.enc_print (str(files('game5.trc', 'game5_etalon.trc', COMPARE_TRACE)))
<commit_msg> - медиана не учавствует в сравнении<commit_after>#!python3

###############################################################################
# Copyright (c) 2013 Evgeny Proydakov <lord.tiran@gmail.com>
###############################################################################
# -*- coding: utf-8 -*-
###############################################################################

import sys
import utils
import re

# compate type
COMPARE_FULL   = 'FULL'
COMPARE_TRACE  = 'TRACE'
COMPARE_RESULT = 'RESULT'

def cmp(a, b):
    return (a > b) - (a < b)

def _read_result_data(name):
    file_data_temp = open(name, 'r', encoding = 'utf-8').readlines()
    utils.cut_slash(file_data_temp)

    file_data = []

    push = False
    for line in file_data_temp:
        if (push):
            line = re.sub(r'(\.+)Медиана:(\s+)(\S+)', r'\1(Медиана:)\2', line)
            file_data.append(line)
        if (line.find('BExpCalcCounter') != -1):
            push = True

    return file_data

def _read_trace_data(name):
    file_data_temp = open(name, 'r', encoding = 'utf-8').readlines()
    utils.cut_slash(file_data_temp)

    file_data = []

    push = False
    for line in file_data_temp:
        if (push):
            line = re.sub(r'SES\s+(\S+)\s(\S+)\s(\S+)', r'SES \1', line)
            line = re.sub(r'DPS_TM\s(\S+)(\s+)(\S+)(\s+)(\S+)', r'DPS_TM', line)
            line = re.sub(r'DPS_ME\s(\S+)(\s+)(\S+)(\s+)(\S+)', r'DPS_ME', line)
            line = re.sub(r'DPS_MM\s(\S+)', r'DPS_MM', line)
            file_data.append(line)

        if (line.find('$Changes') != -1):
            push = True

    return file_data

def full(file1, file2):
    file1_data = open(file1, 'r', encoding = 'utf-8').readlines()
    file2_data = open(file2, 'r', encoding = 'utf-8').readlines()

    utils.cut_slash(file1_data)
    utils.cut_slash(file2_data)

    if cmp(file1_data, file2_data) == 0:
        return True

    return False

def result(file1, file2):
    file1_data = _read_result_data(file1)
    file2_data = _read_result_data(file2)

    if cmp(file1_data, file2_data) == 0:
        return True

    return False

def trace(file1, file2):
    file1_data = _read_trace_data(file1)
    file2_data = _read_trace_data(file2)

    if cmp(file1_data, file2_data) == 0:
        return True

    return False

def files(file1, file2, type):
    if   type == COMPARE_FULL:
        return full(file1, file2)
    elif type == COMPARE_TRACE:
        return trace(file1, file2)
    elif type == COMPARE_RESULT:
        return result(file1, file2)

    return False

if __name__ == '__main__':

    utils.enc_print (str(files('game5.trc', 'game5_etalon.trc', COMPARE_TRACE)))
<|endoftext|>


<commit_before>"""
Definition of network object.
"""

import numpy as num
import matplotlib.pyplot as plt
import copy
import itertools
import logging
from kreveik.classes import *
import kreveik.probes as probes
from kreveik import *

class TopologicalNetwork(ProbeableObj):
    """
    This object is a stripped down network, designated to be a core
    object for all network-like objects, like sub-graphs and motifs.
    """
    def __init__ (self,adjacency_matrix):
        ProbeableObj.__init__(self)
        self.adjacency = num.array(adjacency_matrix,dtype=bool)
        self.code = str(len(self.adjacency))+"-"+str(reduce(lambda x,y : 2*x+y,
                                                              self.adjacency.flatten()*1))
        self.n_nodes = len(self.adjacency)

    def indegree(self):
        return self.adjacency.sum(axis=0)

    def outdegree(self):
        return self.adjacency.sum(axis=1)

    def plot(self):
        """Opens a window, draws the graph into the window.
           Requires Tk, and of course a windowing system.
        """
        import Tkinter as tk
        import math
        window= tk.Tk()
        canvas_size = 400
        drawing = tk.Canvas(window, height=canvas_size, width=canvas_size, background="white")
        n_nodes = self.n_nodes
        radius = 150
        node_radius = 10

        drawing.create_text(200,10,text = "Network:"+str(id(self)))

        list_of_coordinates = [(radius*math.sin(2*math.pi*n/n_nodes)+canvas_size/2,radius*math.cos(2*math.pi*n/n_nodes)+canvas_size/2) for n in range(n_nodes)]

        for linksto,node in enumerate(self.adjacency):
            for linksfrom,link in enumerate(node):

                if linksto == linksfrom and link==True:
                    angle = math.atan2(list_of_coordinates[linksto][1]-200,
                                      list_of_coordinates[linksto][0]-200)

                    drawing.create_line(list_of_coordinates[linksto][0]+node_radius*math.cos(angle),
                                        list_of_coordinates[linksto][1]+node_radius*math.sin(angle),
                                        list_of_coordinates[linksto][0]+node_radius*2*(math.cos(angle+20)),
                                        list_of_coordinates[linksto][1]+node_radius*2*math.sin(angle+20),
                                        list_of_coordinates[linksto][0]+node_radius*4*(math.cos(angle)),
                                        list_of_coordinates[linksto][1]+node_radius*4*math.sin(angle),
                                        list_of_coordinates[linksto][0]+node_radius*2*math.cos(angle-20),
                                        list_of_coordinates[linksto][1]+node_radius*2*(math.sin(angle-20)),
                                        list_of_coordinates[linksto][0]+node_radius*math.cos(angle),
                                        list_of_coordinates[linksto][1]+node_radius*math.sin(angle),
                                        smooth=True,joinstyle="round",fill="black",width=2,arrow="last"
                                        )

                elif link == True:
                    angle = math.atan2(list_of_coordinates[linksto][1]-list_of_coordinates[linksfrom][1],
                                   list_of_coordinates[linksto][0]-list_of_coordinates[linksfrom][0])

                    drawing.create_line(list_of_coordinates[linksfrom][0]+node_radius*math.cos(angle),
                                        list_of_coordinates[linksfrom][1]+node_radius*math.sin(angle),
                                        list_of_coordinates[linksto][0]-node_radius*math.cos(angle),
                                        list_of_coordinates[linksto][1]-node_radius*math.sin(angle),
                                        fill="black",width=2,arrow="last")

        for node_ctr,(x,y) in enumerate(list_of_coordinates):
            if self.state == []:
                node_color = "white"
                text_color = "black"
            else:
                if self.state[-1][node_ctr] == True:
                    node_color = "black"
                    text_color = "white"
                else:
                    node_color = "white"
                    text_color = "black"

            drawing.create_oval(x-node_radius,y-node_radius,x+node_radius,y+node_radius,width=2,fill=node_color)
            drawing.create_text(x,y,text =  str(node_ctr),fill = text_color, font="Arial")

        drawing.pack()
        window.mainloop()

    def laplacian(self):
        """
        Returns the graph laplacian of the network
        """
        symmetric = self.adjacency+self.adjacency.T-num.diag(self.adjacency.diagonal())
        degrees = num.diag(symmetric.sum(axis=0))
        laplacian = degrees-symmetric
        return laplacian

    def directed_laplacian(self):
        """
        Returns the laplacian of the network. It differs from laplacian function by using
        the original adjacency matrix, not the symmetricised version of it.
        """
        original = self.adjacency-num.diag(self.adjacency.diagonal())
        degrees = num.diag(original.sum(axis=0)+original.sum(axis=1))
        laplacian = degrees-original
        return laplacian

    def indegree_laplacian(self):
        """
        Returns the laplacian composed of in-degrees of the nodes
        """
        original = self.adjacency-num.diag(self.adjacency.diagonal())
        degrees = num.diag(original.sum(axis=1))
        laplacian = degrees-original
        return laplacian

    def outdegree_laplacian(self):
        """
        Returns the laplacian composed of out-degrees of the nodes
        """
        original = self.adjacency-num.diag(self.adjacency.diagonal())
        degrees = num.diag(original.sum(axis=0))
        laplacian = degrees-original
        return laplacian

    def is_connected(self):
        """
        Returns True if the graph is connected, False if not.
        uses the algorithm explained in
        http://keithbriggs.info/documents/graph-eigenvalues.pdf
        """
        symmetric = self.adjacency+self.adjacency.T-num.diag(
                                 self.adjacency.diagonal())
        if (0 in symmetric.sum(axis=0) or 0 in symmetric.sum(axis=1)):
            return False
        degrees = num.diagflat(symmetric.sum(axis=0))
        laplacian = degrees-symmetric
        determinant = num.linalg.det(laplacian +num.ones((len(laplacian),len(laplacian) )))
        return not(determinant == 0)

    def copy(self):
        """
        Returns a copy of the Topological Network object.
        """
        return copy.deepcopy(self)

    def save(self,filename):
        """
        Saves the Network as an object to a file specified.
        """
        import pickle
        try:
            filehandler = open(filename+".net", 'w')
            pickle.dump(self,filehandler)
        except pickle.PickleError:
            logging.error("The object failed to be pickled.")

class Motif(TopologicalNetwork):
    """
    Motif is a
    """
    def __init__(self, adjacency_matrix):
        TopologicalNetwork.__init__(self, adjacency_matrix)
        self.degree = len(adjacency_matrix)

    def __eq__(self,other):
        permutation_list = itertools.permutations(range(self.degree),self.degree)
        for permutation in permutation_list:

            if num.sum(self.indegree()) != num.sum(other.indegree()):
                return False

            newarray = num.zeros((len(self.adjacency),len(self.adjacency)),dtype=bool)
            #newarray[[node_init,node_end]] = newarray[[node_end,node_init]]
            #newarray[:,[node_init,node_end]] = newarray[:,[node_end,node_init]]
            for rowctr,row in enumerate(self.adjacency):
                for colctr,col in enumerate(row):
                    if col == True:
                        newarray[permutation[rowctr]][permutation[colctr]]= True

            if num.all(newarray == other.adjacency):
                return True

        return False

class Network(TopologicalNetwork,Element):
    '''
    Network Class

    Input Arguments
        adjacency_matrix
        mask
        state_vec
    '''
    def __init__ (self,adjacency_matrix,mask,function,state_vec=None):
        Element.__init__(self)
        TopologicalNetwork.__init__(self,adjacency_matrix)
        self.n_nodes= num.size(adjacency_matrix,0)
        self.mask=mask
        if state_vec == None:
            state_vec= (num.random.random((1,self.n_nodes))< 0.5)
        self.state=num.array(state_vec)
        self.function = function

    def __str__(self):
        return str(id(self))

    def info(self):
        '''
        Prints out an identification of the Network.
        Prints:
            Id
            Mothers
            Children
            Orbits
            Score
            Adjacency matrix
            sTate
            masK
        '''
        print "This network is : "+str(id(self))+"."
        print "Nodes: "+str(self.n_nodes)
        print "Score: "+str(self.score)
        print "Its children are: "
        for child in self.children:
            print "   "+str(child)
        print "It has the following adjacency matrix: "
        print self.adjacency
        print "The following are the masks for each node: "
        for (num,node) in enumerate(self.mask):
            print str(num)+" th node : "+str(node)
        print "The following are the states with respect to time "
        for (t,state) in enumerate(self.state):
            print "t= "+str(t)+" : "+str(node)
        print "The scorer is : "
        print self.scorer

    def __getitem__(self, index):
        """
        nth item of a network object is the state that it is in, in the nth
        iteration
        """
        if index > len(self.state):
            raise IndexError
        return self.state[index]

    def __contains__(self, state):
        """
        Returns a boolean according to whether a network includes the state
        """
        item = num.array(state*True)
        return item in self.state

    def __call__ (self,state):
        """
        When a  network is called as a function, it sets the initial condition
        as the given vector, finds the equilibrium of that state.
        """
        self.set_state(state)
        self.search_equilibrium(2**self.n_nodes,state,orbit_extraction=False,def_advance=1)

    def advance(self,times,start_from=None,*args):
        '''
        Advances the state in the phase space a given number of times.
        If a starter state is given, the initial condition is taken as the given state.
        If not, the last state is used instead.
        Input Arguments
            times -> the number of iterations to be taken.
            starter_state -> the initial state to be used
        '''

        if start_from != None:
            self.set_set(start_from)

        for counter in xrange(times):
            newstate = self.function(self,self.state[-1])
            self.state = num.append(self.state,[newstate],axis=0)

        self.populate_probes(probes.advance)

    def set_state(self,state):
        """
        Flushes the state of the system, and sets the new state as the given one
        """

        if type(state) == int:
            state = [int(strings)==True for strings in list(num.binary_repr(
                                                (state),width=self.n_nodes))]
        state_bool = [i == True for i in state]
        state = [list(state_bool)]
        self.state = num.array(state)

    def plot_state(self,last=20):
        '''
        Plots the last 20 states as a black and white strips vertically.
        The vertical axis is time, whereas each strip is a single state.
        Input Arguments
            last -> the number of states that will be plotted
        '''
        # Take the state vector, convert the list of arrays into a 2d array, then show it as an image
        # Black and white.

        plt.imshow(self.state[-last:],cmap=plt.cm.binary,interpolation='nearest')
        plt.show()

    def plot_equilibria(self):
        """Creates a plot of the equilibria for all possible initial conditions
        in the phase space. Every point in the phase space corresponds to the
        length of the orbit that initial condition is attracted to.
        """
        rowsandcols = 2**(len(self.adjacency)/2)
        if self.n_nodes % 2 == 0:
            im_matrix = self.equilibria.reshape((rowsandcols,rowsandcols))

        if self.n_nodes % 2 == 1:
            im_matrix = self.equilibria.reshape((rowsandcols,rowsandcols*2))

        plt.imshow(im_matrix,cmap=plt.cm.gray,interpolation='nearest')

        plt.grid()
        plt.colorbar()
        plt.show()

    def search_equilibrium(self,chaos_limit,starter_state,orbit_extraction=False,def_advance=1):
        '''
        Searches for an equilibrium point, or a limit cycle.
        Returns the state vector, or the state vector list, if the equilibrium is a limit cycle.
        If no equilibrium is found, returns False.
        Input Arguments:
            starter_state -> the initial state vector that the state will evolve on.
            chaos_limit -> the degree that an orbit will be considered as chaotic.
                The calculation will stop when this point is reached.
            orbit_extraction -> True when every individual orbit is recorded with its degree.
        '''
        if not(hasattr(self,"equilibria")):
            self.equilibria = num.zeros(2**self.n_nodes)
        if not(hasattr(self,"orbits")):
            if orbit_extraction:
                self.orbits = num.array([None]*2**self.n_nodes)

        self.set_state(starter_state)
        starter_state = self.state[-1]

        for ctr in xrange(chaos_limit):

            self.advance(def_advance)
            row = num.all(self.state[-1] == self.state, axis=1)
            where = num.where(row==True)

            if len(where[0])> 1:
                frst_where = where[0][0]
                scnd_where = where[0][1]

                orbit_length = scnd_where-frst_where

                orbit = None
                location = reduce(lambda x,y : 2*x+y, starter_state)

                if orbit_extraction:
                    orbit = self.state[frst_where:scnd_where]
                    self.orbits[location] = orbit

                self.equilibria[location] = orbit_length

                self.populate_probes(probes.search_equilibrium)
                return (orbit_length,orbit)

    def populate_equilibria(self,orbit_extraction=False):
        '''
        Creates all possible initial conditions by listing all possible 2^n boolean states.
        Then runs populate_equilibrium for each of them.
            populate_equilibrium returns orbits and-or degrees of the orbits.
        Gives scores to each of the networks, depending on the degree of the orbit each initial condition
        rests on.
        Input Arguments:
            normalize -> normalizes the scores to the value given.
        '''

        self.equilibria = num.zeros(2**self.n_nodes)
        if orbit_extraction:
            self.orbits = num.array([None]*2**self.n_nodes)

        binspace = range(0,num.power(2,self.n_nodes))

        for state in binspace:
            self.search_equilibrium(2**self.n_nodes,state,orbit_extraction)
        self.populate_probes(probes.populate_equilibria)

<commit_msg>coded for a text based plotting<commit_after>"""
Definition of network object.
"""

import numpy as num
import matplotlib.pyplot as plt
import copy
import itertools
import logging
from kreveik.classes import *
import kreveik.probes as probes
from kreveik import *

class TopologicalNetwork(ProbeableObj):
    """
    This object is a stripped down network, designated to be a core
    object for all network-like objects, like sub-graphs and motifs.
    """
    def __init__ (self,adjacency_matrix):
        ProbeableObj.__init__(self)
        self.adjacency = num.array(adjacency_matrix,dtype=bool)
        self.code = str(len(self.adjacency))+"-"+str(reduce(lambda x,y : 2*x+y,
                                                              self.adjacency.flatten()*1))
        self.n_nodes = len(self.adjacency)

    def text_plot(self):
        for i in range(len(self.adjacency)):
            for j in range(len(self.adjacency)):
                if self.adjacency[i][j]==True:
                    print str(j)+"--->"+str(i)

    def indegree(self):
        return self.adjacency.sum(axis=0)

    def outdegree(self):
        return self.adjacency.sum(axis=1)

    def plot(self):
        """Opens a window, draws the graph into the window.
           Requires Tk, and of course a windowing system.
        """
        import Tkinter as tk
        import math
        window= tk.Tk()
        canvas_size = 400
        drawing = tk.Canvas(window, height=canvas_size, width=canvas_size, background="white")
        n_nodes = self.n_nodes
        radius = 150
        node_radius = 10

        drawing.create_text(200,10,text = "Network:"+str(id(self)))

        list_of_coordinates = [(radius*math.sin(2*math.pi*n/n_nodes)+canvas_size/2,radius*math.cos(2*math.pi*n/n_nodes)+canvas_size/2) for n in range(n_nodes)]

        for linksto,node in enumerate(self.adjacency):
            for linksfrom,link in enumerate(node):

                if linksto == linksfrom and link==True:
                    angle = math.atan2(list_of_coordinates[linksto][1]-200,
                                      list_of_coordinates[linksto][0]-200)

                    drawing.create_line(list_of_coordinates[linksto][0]+node_radius*math.cos(angle),
                                        list_of_coordinates[linksto][1]+node_radius*math.sin(angle),
                                        list_of_coordinates[linksto][0]+node_radius*2*(math.cos(angle+20)),
                                        list_of_coordinates[linksto][1]+node_radius*2*math.sin(angle+20),
                                        list_of_coordinates[linksto][0]+node_radius*4*(math.cos(angle)),
                                        list_of_coordinates[linksto][1]+node_radius*4*math.sin(angle),
                                        list_of_coordinates[linksto][0]+node_radius*2*math.cos(angle-20),
                                        list_of_coordinates[linksto][1]+node_radius*2*(math.sin(angle-20)),
                                        list_of_coordinates[linksto][0]+node_radius*math.cos(angle),
                                        list_of_coordinates[linksto][1]+node_radius*math.sin(angle),
                                        smooth=True,joinstyle="round",fill="black",width=2,arrow="last"
                                        )

                elif link == True:
                    angle = math.atan2(list_of_coordinates[linksto][1]-list_of_coordinates[linksfrom][1],
                                   list_of_coordinates[linksto][0]-list_of_coordinates[linksfrom][0])

                    drawing.create_line(list_of_coordinates[linksfrom][0]+node_radius*math.cos(angle),
                                        list_of_coordinates[linksfrom][1]+node_radius*math.sin(angle),
                                        list_of_coordinates[linksto][0]-node_radius*math.cos(angle),
                                        list_of_coordinates[linksto][1]-node_radius*math.sin(angle),
                                        fill="black",width=2,arrow="last")

        for node_ctr,(x,y) in enumerate(list_of_coordinates):
            if self.state = = []:
                node_color = "white"
                text_color = "black"
            else:
                if self.state[-1][node_ctr] == True:
                    node_color = "black"
                    text_color = "white"
                else:
                    node_color = "white"
                    text_color = "black"

            drawing.create_oval(x-node_radius,y-node_radius,x+node_radius,y+node_radius,width=2,fill=node_color)
            drawing.create_text(x,y,text =  str(node_ctr),fill = text_color, font="Arial")

        drawing.pack()
        window.mainloop()

    def laplacian(self):
        """
        Returns the graph laplacian of the network
        """
        symmetric = self.adjacency+self.adjacency.T-num.diag(self.adjacency.diagonal())
        degrees = num.diag(symmetric.sum(axis=0))
        laplacian = degrees-symmetric
        return laplacian

    def directed_laplacian(self):
        """
        Returns the laplacian of the network. It differs from laplacian function by using
        the original adjacency matrix, not the symmetricised version of it.
        """
        original = self.adjacency-num.diag(self.adjacency.diagonal())
        degrees = num.diag(original.sum(axis=0)+original.sum(axis=1))
        laplacian = degrees-original
        return laplacian

    def indegree_laplacian(self):
        """
        Returns the laplacian composed of in-degrees of the nodes
        """
        original = self.adjacency-num.diag(self.adjacency.diagonal())
        degrees = num.diag(original.sum(axis=1))
        laplacian = degrees-original
        return laplacian

    def outdegree_laplacian(self):
        """
        Returns the laplacian composed of out-degrees of the nodes
        """
        original = self.adjacency-num.diag(self.adjacency.diagonal())
        degrees = num.diag(original.sum(axis=0))
        laplacian = degrees-original
        return laplacian

    def is_connected(self):
        """
        Returns True if the graph is connected, False if not.
        uses the algorithm explained in
        http://keithbriggs.info/documents/graph-eigenvalues.pdf
        """
        symmetric = self.adjacency+self.adjacency.T-num.diag(
                                 self.adjacency.diagonal())
        if (0 in symmetric.sum(axis=0) or 0 in symmetric.sum(axis=1)):
            return False
        degrees = num.diagflat(symmetric.sum(axis=0))
        laplacian = degrees-symmetric
        determinant = num.linalg.det(laplacian +num.ones((len(laplacian),len(laplacian) )))
        return not(determinant == 0)

    def copy(self):
        """
        Returns a copy of the Topological Network object.
        """
        return copy.deepcopy(self)

    def save(self,filename):
        """
        Saves the Network as an object to a file specified.
        """
        import pickle
        try:
            filehandler = open(filename+".net", 'w')
            pickle.dump(self,filehandler)
        except pickle.PickleError:
            logging.error("The object failed to be pickled.")

class Motif(TopologicalNetwork):
    """
    Motif is a
    """
    def __init__(self, adjacency_matrix):
        TopologicalNetwork.__init__(self, adjacency_matrix)
        self.degree = len(adjacency_matrix)

    def __eq__(self,other):
        permutation_list = itertools.permutations(range(self.degree),self.degree)
        for permutation in permutation_list:

            if num.sum(self.indegree()) != num.sum(other.indegree()):
                return False

            newarray = num.zeros((len(self.adjacency),len(self.adjacency)),dtype=bool)
            #newarray[[node_init,node_end]] = newarray[[node_end,node_init]]
            #newarray[:,[node_init,node_end]] = newarray[:,[node_end,node_init]]
            for rowctr,row in enumerate(self.adjacency):
                for colctr,col in enumerate(row):
                    if col == True:
                        newarray[permutation[rowctr]][permutation[colctr]]= True

            if num.all(newarray == other.adjacency):
                return True

        return False

class Network(TopologicalNetwork,Element):
    '''
    Network Class

    Input Arguments
        adjacency_matrix
        mask
        state_vec
    '''
    def __init__ (self,adjacency_matrix,mask,function,state_vec=None):
        Element.__init__(self)
        TopologicalNetwork.__init__(self,adjacency_matrix)
        self.n_nodes= num.size(adjacency_matrix,0)
        self.mask=mask
        if state_vec == None:
            state_vec= (num.random.random((1,self.n_nodes))< 0.5)
        self.state=num.array(state_vec)
        self.function = function

    def __str__(self):
        return str(id(self))

    def info(self):
        '''
        Prints out an identification of the Network.
        Prints:
            Id
            Mothers
            Children
            Orbits
            Score
            Adjacency matrix
            sTate
            masK
        '''
        print "This network is : "+str(id(self))+"."
        print "Nodes: "+str(self.n_nodes)
        print "Score: "+str(self.score)
        print "Its children are: "
        for child in self.children:
            print "   "+str(child)
        print "It has the following adjacency matrix: "
        print self.adjacency
        print "The following are the masks for each node: "
        for (num,node) in enumerate(self.mask):
            print str(num)+" th node : "+str(node)
        print "The following are the states with respect to time "
        for (t,state) in enumerate(self.state):
            print "t= "+str(t)+" : "+str(node)
        print "The scorer is : "
        print self.scorer

    def __getitem__(self, index):
        """
        nth item of a network object is the state that it is in, in the nth
        iteration
        """
        if index > len(self.state):
            raise IndexError
        return self.state[index]

    def __contains__(self, state):
        """
        Returns a boolean according to whether a network includes the state
        """
        item = num.array(state*True)
        return item in self.state

    def __call__ (self,state):
        """
        When a  network is called as a function, it sets the initial condition
        as the given vector, finds the equilibrium of that state.
        """
        self.set_state(state)
        self.search_equilibrium(2**self.n_nodes,state,orbit_extraction=False,def_advance=1)

    def advance(self,times,start_from=None,*args):
        '''
        Advances the state in the phase space a given number of times.
        If a starter state is given, the initial condition is taken as the given state.
        If not, the last state is used instead.
        Input Arguments
            times -> the number of iterations to be taken.
            starter_state -> the initial state to be used
        '''

        if start_from != None:
            self.set_set(start_from)

        for counter in xrange(times):
            newstate = self.function(self,self.state[-1])
            self.state = num.append(self.state,[newstate],axis=0)

        self.populate_probes(probes.advance)

    def set_state(self,state):
        """
        Flushes the state of the system, and sets the new state as the given one
        """

        if type(state) == int:
            state = [int(strings)==True for strings in list(num.binary_repr(
                                                (state),width=self.n_nodes))]
        state_bool = [i == True for i in state]
        state = [list(state_bool)]
        self.state = num.array(state)

    def plot_state(self,last=20):
        '''
        Plots the last 20 states as a black and white strips vertically.
        The vertical axis is time, whereas each strip is a single state.
        Input Arguments
            last -> the number of states that will be plotted
        '''
        # Take the state vector, convert the list of arrays into a 2d array, then show it as an image
        # Black and white.

        plt.imshow(self.state[-last:],cmap=plt.cm.binary,interpolation='nearest')
        plt.show()

    def plot_equilibria(self):
        """Creates a plot of the equilibria for all possible initial conditions
        in the phase space. Every point in the phase space corresponds to the
        length of the orbit that initial condition is attracted to.
        """
        rowsandcols = 2**(len(self.adjacency)/2)
        if self.n_nodes % 2 == 0:
            im_matrix = self.equilibria.reshape((rowsandcols,rowsandcols))

        if self.n_nodes % 2 == 1:
            im_matrix = self.equilibria.reshape((rowsandcols,rowsandcols*2))

        plt.imshow(im_matrix,cmap=plt.cm.gray,interpolation='nearest')

        plt.grid()
        plt.colorbar()
        plt.show()

    def search_equilibrium(self,chaos_limit,starter_state,orbit_extraction=False,def_advance=1):
        '''
        Searches for an equilibrium point, or a limit cycle.
        Returns the state vector, or the state vector list, if the equilibrium is a limit cycle.
        If no equilibrium is found, returns False.
        Input Arguments:
            starter_state -> the initial state vector that the state will evolve on.
            chaos_limit -> the degree that an orbit will be considered as chaotic.
                The calculation will stop when this point is reached.
            orbit_extraction -> True when every individual orbit is recorded with its degree.
        '''
        if not(hasattr(self,"equilibria")):
            self.equilibria = num.zeros(2**self.n_nodes)
        if not(hasattr(self,"orbits")):
            if orbit_extraction:
                self.orbits = num.array([None]*2**self.n_nodes)

        self.set_state(starter_state)
        starter_state = self.state[-1]

        for ctr in xrange(chaos_limit):

            self.advance(def_advance)
            row = num.all(self.state[-1] == self.state, axis=1)
            where = num.where(row==True)

            if len(where[0])> 1:
                frst_where = where[0][0]
                scnd_where = where[0][1]

                orbit_length = scnd_where-frst_where

                orbit = None
                location = reduce(lambda x,y : 2*x+y, starter_state)

                if orbit_extraction:
                    orbit = self.state[frst_where:scnd_where]
                    self.orbits[location] = orbit

                self.equilibria[location] = orbit_length

                self.populate_probes(probes.search_equilibrium)
                return (orbit_length,orbit)

    def populate_equilibria(self,orbit_extraction=False):
        '''
        Creates all possible initial conditions by listing all possible 2^n boolean states.
        Then runs populate_equilibrium for each of them.
            populate_equilibrium returns orbits and-or degrees of the orbits.
        Gives scores to each of the networks, depending on the degree of the orbit each initial condition
        rests on.
        Input Arguments:
            normalize -> normalizes the scores to the value given.
        '''

        self.equilibria = num.zeros(2**self.n_nodes)
        if orbit_extraction:
            self.orbits = num.array([None]*2**self.n_nodes)

        binspace = range(0,num.power(2,self.n_nodes))

        for state in binspace:
            self.search_equilibrium(2**self.n_nodes,state,orbit_extraction)
        self.populate_probes(probes.populate_equilibria)

<|endoftext|>


<commit_before>import tensorflow as tf
import hyperchamber as hc

from hypergan.losses.base_loss import BaseLoss

class AliLoss(BaseLoss):

    def required(self):
        return "reduce".split()

    def _create(self, d_real, d_fake):
        ops = self.ops
        config = self.config
        gan = self.gan

        pq = d_real
        pp = d_fake
        zeros = tf.zeros_like(d_fake)
        ones = tf.ones_like(d_fake)

        if config.type == 'original':
            d_loss = -tf.log(tf.nn.sigmoid(pq))-tf.log(1-tf.nn.sigmoid(pp))
            g_loss = -tf.log(1-tf.nn.sigmoid(pq))-tf.log(tf.nn.sigmoid(pp))
        elif config.type == 'least_squares':
            a,b,c = config.labels
            square = ops.lookup('square')
            d_loss = 0.5*square(d_real - b) + 0.5*square(d_fake - a)
            g_loss = 0.5*square(d_fake - c) + 0.5*square(d_real - a)
            #g_loss = 0.5*square(d_fake - c) - d_real

            #g_loss = 0.5*square(d_fake - c) + 0.5*(b-d_real)

        elif config.type == 'wasserstein':
            d_loss = -pq+pp
            g_loss = pq-pp
        elif config.type == 'label_smoothing':
            generator_target_probability = config.generator_target_probability or 0.8
            label_smooth = config.label_smooth or 0.2
            g_loss = self.sigmoid_kl_with_logits(d_fake, generator_target_probability) + \
                    tf.nn.sigmoid_cross_entropy_with_logits(logits=d_real, labels=zeros)
            d_loss = self.sigmoid_kl_with_logits(d_real, 1.-label_smooth) + \
                    tf.nn.sigmoid_cross_entropy_with_logits(logits=d_fake, labels=zeros)
        else:
            g_loss = tf.nn.sigmoid_cross_entropy_with_logits(logits=d_fake, labels=zeros) + \
                     tf.nn.sigmoid_cross_entropy_with_logits(logits=d_real, labels=ones)
            d_loss = tf.nn.sigmoid_cross_entropy_with_logits(logits=d_real, labels=zeros) + \
                     tf.nn.sigmoid_cross_entropy_with_logits(logits=d_fake, labels=ones)

        return [d_loss, g_loss]

<commit_msg>[feature] aligan config.type 'logistic'<commit_after>import tensorflow as tf
import hyperchamber as hc

from hypergan.losses.base_loss import BaseLoss

class AliLoss(BaseLoss):

    def required(self):
        return "reduce".split()

    def _create(self, d_real, d_fake):
        ops = self.ops
        config = self.config
        gan = self.gan

        pq = d_real
        pp = d_fake
        zeros = tf.zeros_like(d_fake)
        ones = tf.ones_like(d_fake)

        if config.type == 'original':
            d_loss = -tf.log(tf.nn.sigmoid(pq))-tf.log(1-tf.nn.sigmoid(pp))
            g_loss = -tf.log(1-tf.nn.sigmoid(pq))-tf.log(tf.nn.sigmoid(pp))
        elif config.type == 'least_squares':
            a,b,c = config.labels
            square = ops.lookup('square')
            d_loss = 0.5*square(d_real - b) + 0.5*square(d_fake - a)
            g_loss = 0.5*square(d_fake - c) + 0.5*square(d_real - a)
            #g_loss = 0.5*square(d_fake - c) - d_real

            #g_loss = 0.5*square(d_fake - c) + 0.5*(b-d_real)

        elif config.type == 'logistic':
            d_loss = tf.nn.softplus(-d_real) + tf.nn.softplus(d_fake)
            g_loss = tf.nn.softplus(-d_fake) + tf.nn.softplus(d_real)
        elif config.type == 'wasserstein':
            d_loss = -pq+pp
            g_loss = pq-pp
        elif config.type == 'label_smoothing':
            generator_target_probability = config.generator_target_probability or 0.8
            label_smooth = config.label_smooth or 0.2
            g_loss = self.sigmoid_kl_with_logits(d_fake, generator_target_probability) + \
                    tf.nn.sigmoid_cross_entropy_with_logits(logits=d_real, labels=zeros)
            d_loss = self.sigmoid_kl_with_logits(d_real, 1.-label_smooth) + \
                    tf.nn.sigmoid_cross_entropy_with_logits(logits=d_fake, labels=zeros)
        else:
            g_loss = tf.nn.sigmoid_cross_entropy_with_logits(logits=d_fake, labels=zeros) + \
                     tf.nn.sigmoid_cross_entropy_with_logits(logits=d_real, labels=ones)
            d_loss = tf.nn.sigmoid_cross_entropy_with_logits(logits=d_real, labels=zeros) + \
                     tf.nn.sigmoid_cross_entropy_with_logits(logits=d_fake, labels=ones)

        return [d_loss, g_loss]

<|endoftext|>


<commit_before>from craigslist import CraigslistHousing
import settings

class Scraper:
    def results(self):
        cl_h = CraigslistHousing(
                    site=settings.CRAIGSLIST_SITE,
                    category=settings.CRAIGSLIST_CATEGORY,
                    filters={
                        'min_ft2': settings.MIN_FT2,
                        'min_price': settings.MIN_PRICE,
                        'max_price': settings.MAX_PRICE,
                        'search_distance': 3.5,
                        'zip_code': '07307'
                    }
            )

        return cl_h.get_results(sort_by='newest', geotagged=True, limit=50)
<commit_msg>Fuzz search query<commit_after>from craigslist import CraigslistHousing
from random import randint
import settings

class Scraper:
    def results(self):
        cl_h = CraigslistHousing(
                    site=settings.CRAIGSLIST_SITE,
                    category=settings.CRAIGSLIST_CATEGORY,
                    filters={
                        'min_ft2': settings.MIN_FT2 + randint(-5, 5) * 10,
                        'min_price': settings.MIN_PRICE + randint(-10, 10) * 10,
                        'max_price': settings.MAX_PRICE + randint(-10, 10) * 10
                    }
            )

        return cl_h.get_results(sort_by='newest', geotagged=True, limit=50)
<|endoftext|>