<commit_before>import itertools

import mock
import rdflib, rdflib.term
import unittest2

from django.test.client import Client
from django.http import HttpResponseNotFound

from humfrey.desc import rdf_processors, views
from humfrey.utils import sparql, resource
from humfrey.tests.stubs import stub_reverse_crossdomain

class GraphTestMixin(object):
    def check_valid_terms(self, graph):
        for term in itertools.chain.from_iterable(graph):
            self.assertIsInstance(term, rdflib.term.Identifier)

@mock.patch('humfrey.linkeddata.uri.reverse_crossdomain', stub_reverse_crossdomain)
class RDFProcessorsTestCase(unittest2.TestCase, GraphTestMixin):
    _ALL = [
        rdf_processors.formats,
        rdf_processors.doc_meta,
    ]

    def testAll(self):
        for rdf_processor in self._ALL:
            endpoint = mock.Mock(spec=sparql.Endpoint)
            graph = rdflib.ConjunctiveGraph()
            doc_uri = rdflib.URIRef('http://example.com/doc/Foo')
            subject_uri = rdflib.URIRef('http://example.com/id/Foo')
            subject = resource.Resource(subject_uri, graph, endpoint)
            renderers = views.DocView().FORMATS.values()

            context = rdf_processor(graph, doc_uri, subject_uri, subject, endpoint, renderers)

            self.assertFalse(endpoint.query.called, "The rdf procesor should not be touching the endpoint (at the moment)")
            self.check_valid_terms(graph)
            self.assertIsInstance(context, (type(None), dict))

#@mock.patch('humfrey.linkeddata.uri.reverse_crossdomain', stub_reverse_crossdomain)
class DocViewTestCase(unittest2.TestCase, GraphTestMixin):
    _TEST_URI = 'http://data/example.com/id/Foo'
    _HTTP_HOST = 'data.example.org'

    def setUp(self):
        self.client = Client()

    @mock.patch('humfrey.desc.views.DocView.get_types')
    def testNoTypes(self, get_types):
        get_types.return_value = ()
        response = self.client.get('/doc/', {'uri': self._TEST_URI}, HTTP_HOST=self._HTTP_HOST)
        self.assertIsInstance(response, HttpResponseNotFound)

    @mock.patch('humfrey.desc.views.DocView.get_types')
    @mock.patch('humfrey.desc.views.DocView.endpoint')
    def testGraphValid(self, endpoint, get_types):
        get_types.return_value = (rdflib.URIRef('http://example.org/vocab/Thing'),)
        endpoint.query.return_value = rdflib.ConjunctiveGraph()
        response = self.client.get('/doc/', {'uri': self._TEST_URI, 'format': 'html'}, HTTP_HOST='data.example.org')
        self.assertIsInstance(response.context['subject_uri'], rdflib.URIRef)
        self.assertIsInstance(response.context['doc_uri'], rdflib.URIRef)
        self.check_valid_terms(response.context['graph'])

<commit_msg>Added test to make sure the SRJ serializer is creating valid JSON. Should highlight issue described by #7.<commit_after>import collections
import itertools

import mock
import rdflib, rdflib.term
import simplejson
import unittest2

from django.test.client import Client
from django.http import HttpResponseNotFound

from humfrey.desc import rdf_processors, views
from humfrey.utils import sparql, resource, namespaces
from humfrey.tests.stubs import stub_reverse_crossdomain

TEST_RESULTSET_RESULT = collections.namedtuple('Result', 'one two')
TEST_RESULTSET = sparql.ResultList(list(itertools.starmap(TEST_RESULTSET_RESULT, [
    (rdflib.URIRef('http://example.org/one'), rdflib.BNode()),
    (rdflib.Literal('hello'), rdflib.Literal('hello', lang='en')),
    (rdflib.Literal('foo"bar'), rdflib.Literal('foo\nbar')),
    (rdflib.Literal('foo bar'), rdflib.Literal('foo\tbar')),
    (rdflib.Literal(1), rdflib.Literal('2011-01-02T12:34:56Z', datatype=namespaces.NS.xsd.timeDate)),
    (None, None),
    (rdflib.URIRef('http://example.org/'), rdflib.URIRef('mailto:alice@example.org')),
    (rdflib.URIRef('urn:isbn:9781449306595'), rdflib.URIRef('tag:bob@example.org,2011:foo')),
])))
TEST_RESULTSET.fields = ('one', 'two')
TEST_RESULTSET.query = 'The query that was run'
TEST_RESULTSET.duration = 1

class GraphTestMixin(object):
    def check_valid_terms(self, graph):
        for term in itertools.chain.from_iterable(graph):
            self.assertIsInstance(term, rdflib.term.Identifier)

@mock.patch('humfrey.linkeddata.uri.reverse_crossdomain', stub_reverse_crossdomain)
class RDFProcessorsTestCase(unittest2.TestCase, GraphTestMixin):
    _ALL = [
        rdf_processors.formats,
        rdf_processors.doc_meta,
    ]

    def testAll(self):
        for rdf_processor in self._ALL:
            endpoint = mock.Mock(spec=sparql.Endpoint)
            graph = rdflib.ConjunctiveGraph()
            doc_uri = rdflib.URIRef('http://example.com/doc/Foo')
            subject_uri = rdflib.URIRef('http://example.com/id/Foo')
            subject = resource.Resource(subject_uri, graph, endpoint)
            renderers = views.DocView().FORMATS.values()

            context = rdf_processor(graph, doc_uri, subject_uri, subject, endpoint, renderers)

            self.assertFalse(endpoint.query.called, "The rdf procesor should not be touching the endpoint (at the moment)")
            self.check_valid_terms(graph)
            self.assertIsInstance(context, (type(None), dict))

#@mock.patch('humfrey.linkeddata.uri.reverse_crossdomain', stub_reverse_crossdomain)
class DocViewTestCase(unittest2.TestCase, GraphTestMixin):
    _TEST_URI = 'http://data/example.com/id/Foo'
    _HTTP_HOST = 'data.example.org'

    def setUp(self):
        self.client = Client()

    @mock.patch('humfrey.desc.views.DocView.get_types')
    def testNoTypes(self, get_types):
        get_types.return_value = ()
        response = self.client.get('/doc/', {'uri': self._TEST_URI}, HTTP_HOST=self._HTTP_HOST)
        self.assertIsInstance(response, HttpResponseNotFound)

    @mock.patch('humfrey.desc.views.DocView.get_types')
    @mock.patch('humfrey.desc.views.DocView.endpoint')
    def testGraphValid(self, endpoint, get_types):
        get_types.return_value = (rdflib.URIRef('http://example.org/vocab/Thing'),)
        endpoint.query.return_value = rdflib.ConjunctiveGraph()
        response = self.client.get('/doc/', {'uri': self._TEST_URI, 'format': 'html'}, HTTP_HOST='data.example.org')
        self.assertIsInstance(response.context['subject_uri'], rdflib.URIRef)
        self.assertIsInstance(response.context['doc_uri'], rdflib.URIRef)
        self.check_valid_terms(response.context['graph'])

    @mock.patch('humfrey.desc.views.redis.client.Redis')
    @mock.patch('humfrey.desc.views.SparqlView.endpoint')
    def testValidSparqlResultsJSON(self, endpoint, redis_client_class):
        endpoint.query.return_value = TEST_RESULTSET
        redis_client = redis_client_class.return_value
        redis_client.setnx.return_value = True
        redis_client.get.return_value = 0

        response = self.client.get('/sparql/', {'query': 'irrelevant'}, HTTP_HOST='data.example.org', HTTP_ACCEPT='application/sparql-results+json')
        endpoint.query.assert_called_once_with('irrelevant', common_prefixes=False)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response['Content-Type'], 'application/sparql-results+json')

        try:
            data = simplejson.loads(response.content)
        except Exception, e:
        	   raise AssertionError(e)

        self.assertEqual(data['head']['vars'], ['one', 'two'])
        self.assertEqual(len(data['results']), 8)

<|endoftext|>


<commit_before>"""
Persistent profile-storage module for the HXL Proxy

Started by David Megginson, February 2015
"""

import shelve
import base64
import hashlib
import random
import time
import copy

VERSION=1.0

from hxl_proxy import app

class Profile(object):

    def __init__(self, args):
        self.version = 1.0
        self.args = args

    def set_password(self, password):
        """Assign a new password to this profile (None to clear)."""
        if password:
            self.passhash = hashlib.md5(password).digest()
        else:
            self.passhash = None

    def check_password(self, password):
        """Check the password for this profile."""
        # if none is set, also succeeds
        if not hasattr(self, 'passhash') or self.passhash is None:
            return True
        return (self.passhash == hashlib.md5(password).digest())

def make_profile(args):
    return Profile(args)

def get_profile(key):
    """
    Get an existing profile.
    @param key the string key for the profile.
    @return the profile as an associative array.
    """
    key = str(key)
    dict = shelve.open(app.config['PROFILE_FILE'])
    try:
        if dict.has_key(key):
            return copy.copy(dict[key])
        else:
            return None
    finally:
        dict.close()

def update_profile(key, profile):
    """
    Update an existing profile.
    @param key the string key for the profile
    @param profile the profile as an associative array
    @return the key provided
    """
    print str(profile)
    dict = shelve.open(app.config['PROFILE_FILE'])
    try:
        dict[str(key)] = profile
        return key
    finally:
        dict.close()

def add_profile(profile):
    """
    Add a new profile.
    @param profile the profile as an associative array.
    @return the generated key for the new profile.
    """
    key = _gen_key()
    dict = shelve.open(app.config['PROFILE_FILE'])
    try:
        # check for collisions
        while dict.has_key(key):
            key = _gen_key()
        dict[str(key)] = _check_profile(profile)
        return key
    finally:
        dict.close()

def _gen_key():
    """
    Generate a pseudo-random, 6-character hash for use as a key.
    """
    hash = base64.urlsafe_b64encode(hashlib.md5(str(time.time() * random.random())).digest())
    return hash[:6]

# end
<commit_msg>Fix call to missing function.<commit_after>"""
Persistent profile-storage module for the HXL Proxy

Started by David Megginson, February 2015
"""

import shelve
import base64
import hashlib
import random
import time
import copy

VERSION=1.0

from hxl_proxy import app

class Profile(object):

    def __init__(self, args):
        self.version = 1.0
        self.args = args

    def set_password(self, password):
        """Assign a new password to this profile (None to clear)."""
        if password:
            self.passhash = hashlib.md5(password).digest()
        else:
            self.passhash = None

    def check_password(self, password):
        """Check the password for this profile."""
        # if none is set, also succeeds
        if not hasattr(self, 'passhash') or self.passhash is None:
            return True
        return (self.passhash == hashlib.md5(password).digest())

def make_profile(args):
    return Profile(args)

def get_profile(key):
    """
    Get an existing profile.
    @param key the string key for the profile.
    @return the profile as an associative array.
    """
    key = str(key)
    dict = shelve.open(app.config['PROFILE_FILE'])
    try:
        if dict.has_key(key):
            return copy.copy(dict[key])
        else:
            return None
    finally:
        dict.close()

def update_profile(key, profile):
    """
    Update an existing profile.
    @param key the string key for the profile
    @param profile the profile as an associative array
    @return the key provided
    """
    print str(profile)
    dict = shelve.open(app.config['PROFILE_FILE'])
    try:
        dict[str(key)] = profile
        return key
    finally:
        dict.close()

def add_profile(profile):
    """
    Add a new profile.
    @param profile the profile as an associative array.
    @return the generated key for the new profile.
    """
    key = _gen_key()
    dict = shelve.open(app.config['PROFILE_FILE'])
    try:
        # check for collisions
        while dict.has_key(key):
            key = _gen_key()
        dict[str(key)] = profile
        return key
    finally:
        dict.close()

def _gen_key():
    """
    Generate a pseudo-random, 6-character hash for use as a key.
    """
    hash = base64.urlsafe_b64encode(hashlib.md5(str(time.time() * random.random())).digest())
    return hash[:6]

# end
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
""" Hyperparameter optimization of DNN model
Uses hyperopt module to search for optimal DNN hyper parameters.

Note: 'output_size' can't be changed during hyperparameter search as data preprocessing depends on this parameter.
      If you need to search over this parameter, make sure to re-execute 'feature_engineering.load_data' in '_objective' function.

.. See https://github.com/PaulEmmanuelSotir/NYC_TaxiTripDuration
"""
from sklearn.utils import resample
import tensorflow as tf
import hyperopt as ho
import pandas as pd
import numpy as np
import argparse
import time
import math
import os

import utils
import nyc_dnn

# Full example of hyperparameter search space:
# HP_SPACE = {'epochs': 200,
#            'early_stopping': 20,
#            'lr': ho.hp.loguniform('lr', math.log(2e-5), math.log(2e-3)),
#            'opt': ho.hp.choice('opt', [tf.train.AdamOptimizer,
#                                        tf.train.RMSPropOptimizer,
#                                        tf.train.GradientDescentOptimizer]),
#            'depth': ho.hp.choice('depth', [4, 5, 6, 7, 8, 9]),
#            'batch_size': ho.hp.choice('batch_size', [512, 1024, 2048, 4096, 8192]),
#            'hidden_size': ho.hp.choice('hidden_size', [256, 512, 1024]),
#            'dropout_keep_prob': ho.hp.uniform('dropout_keep_prob', 0.75, 0.9),
#            'max_norm_threshold': ho.hp.uniform('max_norm_threshold', [0.8, 2.]),
#            'duration_std_margin': ho.hp.choice('duration_std_margin', [5, 6]),
#            'output_size': ho.hp.choice('output_size', [1, 128, 256, 512])}

# Hyperparameter optimization space and algorithm
MAX_EVALS = 35
SUB_TRAINSET_SIZE = 1.
ALLOW_GPU_MEM_GROWTH = True
OPT_ALGO = ho.tpe.suggest
HP_SPACE = {'epochs': 200,
            'early_stopping': 20,
            'lr': ho.hp.loguniform('lr', math.log(2e-5), math.log(2e-3)),
            'opt': {'algo': tf.train.AdamOptimizer},
            'depth': ho.hp.choice('depth', [4, 5]),
            'batch_size': ho.hp.choice('batch_size', [512, 1024, 2048, 4096, 8192]),
            'hidden_size': 512,
            'dropout_keep_prob': ho.hp.uniform('dropout_keep_prob', 0.75, 0.9),
            'max_norm_threshold': ho.hp.uniform('max_norm_threshold', 0.8, 2.),
            'duration_std_margin': 5,
            'output_size': 1}

def main():
    # Define working directories
    source_dir = os.path.dirname(os.path.abspath(__file__))
    save_dir = '/output/' if tf.flags.FLAGS.floyd_job else os.path.join(source_dir, 'models/hyperopt_models/')
    dataset_dir = '/input/' if tf.flags.FLAGS.floyd_job else os.path.join(source_dir, 'NYC_taxi_data_2016/')

    # Load and preprocess data
    features_len, (test_ids, testset), dataset, bucket_means = nyc_dnn.feature_engineering.load_data(dataset_dir,
                                                                                'train.csv',
                                                                                'test.csv',
                                                                                nyc_dnn.VALID_SIZE,
                                                                                HP_SPACE['output_size'],
                                                                                tf.flags.FLAGS.floyd_job)

    # Reduce trainset size (sample a subset of trainset)
    train_data, test_data, train_targets, test_targets, train_labels, test_labels = dataset
    train_data, train_targets, train_labels = resample(train_data, train_targets, train_labels,
                                                       replace=False, n_samples=int(SUB_TRAINSET_SIZE * len(train_data)))
    dataset = (train_data, test_data, train_targets, test_targets, train_labels, test_labels)

    # Define the objective function optimized by hyperopt
    eval_count = 0

    def _objective(hyperparameters):
        nonlocal eval_count
        eval_count += 1
        model_save_dir = save_dir + str(eval_count) + '/'
        print('\n\n' + '#' * 10 + ' TRYING HYPERPERPARAMETER SET #' + str(eval_count) + ' ' + '#' * 10)
        print(hyperparameters)
        # Reset default tensorflow graph
        tf.reset_default_graph()
        # Write hyperparameters to summary as text (once per session/model)
        hp_string = tf.placeholder(tf.string, name='hp')
        hp_summary_op = tf.summary.text('hyperparameters', hp_string, collections=['per_session'])
        with tf.Session(config=utils.tf_config(ALLOW_GPU_MEM_GROWTH)) as sess:
            summary_writer = tf.summary.FileWriter(model_save_dir, sess.graph)
            summary = sess.run(hp_summary_op, feed_dict={hp_string: 'Trial #' + str(eval_count) + ' hyperparameters:\n' + str(hyperparameters)})
            summary_writer.add_summary(summary, 0)
        # Get buckets from train targets
        train_labels, test_labels, bucket_means = nyc_dnn.bucketize(dataset[2], dataset[3], hyperparameters['output_size'])
        # Build model
        model = nyc_dnn.build_model(features_len, hyperparameters, bucket_means, summarize_parameters=False)
        # Train model
        test_rmse, predictions = nyc_dnn.train(model, dataset, train_labels, test_labels, hyperparameters, model_save_dir, testset)
        # Save predictions to csv file for Kaggle submission
        predictions = np.int32(np.round(np.exp(predictions))) - 1
        pd.DataFrame(np.column_stack([test_ids, predictions]), columns=['id', 'trip_duration']
                     ).to_csv(os.path.join(save_dir, str(eval_count), 'preds.csv'), index=False)
        return {'loss': test_rmse,  # TODO: put last batch average loss here?
                'true_loss': test_rmse,
                'status': ho.STATUS_OK,
                'eval_time': time.time()}

    # Run optimization algorithm
    trials = ho.Trials()
    best_parameters = ho.fmin(_objective,
                              space=HP_SPACE,
                              algo=OPT_ALGO,
                              max_evals=MAX_EVALS,
                              trials=trials)
    print('\n\n' + '#' * 20 + '  BEST HYPERPARAMETERS  ' + '#' * 20)
    print(best_parameters)  # TODO: translate ho.hp.choice hyperarameters from index to their actual value

if __name__ == '__main__':
    tf.flags.DEFINE_bool('floyd-job', False, 'Change working directories for training on Floyd.')
    tf.app.run()
<commit_msg>Modified hyperparameter_opt.py to include last modifications of nyc_dnn and feature_engineering<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
""" Hyperparameter optimization of DNN model
Uses hyperopt module to search for optimal DNN hyper parameters.

Note: 'output_size' can't be changed during hyperparameter search as data preprocessing depends on this parameter.
      If you need to search over this parameter, make sure to re-execute 'feature_engineering.load_data' in '_objective' function (can be expensive).

.. See https://github.com/PaulEmmanuelSotir/NYC_TaxiTripDuration
"""
from sklearn.utils import resample
import tensorflow as tf
import hyperopt as ho
import pandas as pd
import numpy as np
import argparse
import time
import math
import os

import utils
import nyc_dnn

# Full example of hyperparameter search space:
# HP_SPACE = {'epochs': 200,
#            'early_stopping': 20,
#            'lr': ho.hp.loguniform('lr', math.log(2e-5), math.log(2e-3)),
#            'momentum': ho.hp.uniform('momentum', 0.8, 0.95),
#            'depth': ho.hp.choice('depth', [4, 5, 6, 7, 8, 9]),
#            'hidden_size': ho.hp.choice('hidden_size', [256, 512, 1024]),
#            'batch_size': ho.hp.choice('batch_size', [256, 512, 1024, 2048]),
#            'dropout_keep_prob': ho.hp.uniform('dropout_keep_prob', 0.75, 0.9),
#            'l2_regularization': ho.hp.loguniform('l2_regularization', math.log(1e-6), math.log(1e-3)),
#            'output_size': ho.hp.choice('output_size', [1, 128, 256, 512])}

# Hyperparameter optimization space and algorithm
MAX_EVALS = 35
SUB_TRAINSET_SIZE = 1.
ALLOW_GPU_MEM_GROWTH = True
OPT_ALGO = ho.tpe.suggest
HP_SPACE = {'epochs': 200,
            'early_stopping': 20,
            'lr': ho.hp.loguniform('lr', math.log(2e-5), math.log(2e-3)),
            'momentum': ho.hp.uniform('momentum', 0.8, 0.95),
            'depth': 5,
            'hidden_size': 512,
            'batch_size': 1024,
            'dropout_keep_prob': ho.hp.uniform('dropout_keep_prob', 0.75, 0.9),
            'l2_regularization': ho.hp.loguniform('l2_regularization', math.log(1e-6), math.log(1e-3)),
            'output_size': 512}

def main(_=None):
    # Define working directories
    source_dir = os.path.dirname(os.path.abspath(__file__))
    save_dir = '/output/' if tf.flags.FLAGS.floyd_job else os.path.join(source_dir, 'models/hyperopt_models/')
    dataset_dir = '/input/' if tf.flags.FLAGS.floyd_job else os.path.join(source_dir, 'NYC_taxi_data_2016/')

    # Load and preprocess data
    features_len, (test_ids, testset), dataset, bucket_means = nyc_dnn.feature_engineering.load_data(dataset_dir,
                                                                                'train.csv',
                                                                                'test.csv',
                                                                                nyc_dnn.VALID_SIZE,
                                                                                HP_SPACE['output_size'],
                                                                                tf.flags.FLAGS.floyd_job)

    # Reduce trainset size (sample a subset of trainset)
    train_data, test_data, train_targets, test_targets, train_labels, test_labels = dataset
    train_data, train_targets, train_labels = resample(train_data, train_targets, train_labels,
                                                       replace=False, n_samples=int(SUB_TRAINSET_SIZE * len(train_data)))
    dataset = (train_data, test_data, train_targets, test_targets, train_labels, test_labels)

    # Define the objective function optimized by hyperopt
    eval_count = 0

    def _objective(hyperparameters):
        nonlocal eval_count
        eval_count += 1
        model_save_dir = save_dir + str(eval_count) + '/'
        print('\n\n' + '#' * 10 + ' TRYING HYPERPERPARAMETER SET #' + str(eval_count) + ' ' + '#' * 10)
        print(hyperparameters)
        # Reset default tensorflow graph
        tf.reset_default_graph()
        # Write hyperparameters to summary as text (once per session/model)
        hp_string = tf.placeholder(tf.string, name='hp')
        hp_summary_op = tf.summary.text('hyperparameters', hp_string, collections=['per_session'])
        with tf.Session(config=utils.tf_config(ALLOW_GPU_MEM_GROWTH)) as sess:
            summary_writer = tf.summary.FileWriter(model_save_dir, sess.graph)
            summary = sess.run(hp_summary_op, feed_dict={hp_string: 'Trial #' + str(eval_count) + ' hyperparameters:\n' + str(hyperparameters)})
            summary_writer.add_summary(summary, 0)
        # Build model
        model = nyc_dnn.build_graph(features_len, hyperparameters, bucket_means, summarize=False)
        # Train model
        test_rmse, predictions = nyc_dnn.train(model, dataset, hyperparameters, model_save_dir, testset)
        # Save predictions to csv file for Kaggle submission
        predictions = np.int32(np.round(np.exp(predictions))) - 1
        pd.DataFrame(np.column_stack([test_ids, predictions]), columns=['id', 'trip_duration']
                     ).to_csv(os.path.join(save_dir, str(eval_count), 'preds.csv'), index=False)
        return {'loss': test_rmse,  # TODO: put last batch average loss here?
                'true_loss': test_rmse,
                'status': ho.STATUS_OK,
                'eval_time': time.time()}

    # Run optimization algorithm
    trials = ho.Trials()
    best_parameters = ho.fmin(_objective,
                              space=HP_SPACE,
                              algo=OPT_ALGO,
                              max_evals=MAX_EVALS,
                              trials=trials)
    print('\n\n' + '#' * 20 + '  BEST HYPERPARAMETERS  ' + '#' * 20)
    print(best_parameters)  # TODO: translate ho.hp.choice hyperarameters from index to their actual value

if __name__ == '__main__':
    tf.logging.set_verbosity(tf.logging.INFO)  # Set log level to debug
    tf.flags.DEFINE_bool('floyd-job', False, 'Change working directories for training on Floyd.')
    tf.app.run()
<|endoftext|>


<commit_before>"""
Handles hyphenation of words or sentences
"""

from .language_patterns import LanguagePatterns

class KnuthLiang(object):
    """
    This class handles hyphenation
    """

    def __init__(self, code, limit_left=2, limit_right=3):
        self.language_patterns = LanguagePatterns(code)
        self.limit_left = limit_left
        self.limit_right = limit_right

    def hyphenate_word(self, word):
        "Hyphenates a word"
        word = '.' + word + '.'
        word_length = len(word)
        found_patterns = dict()
        for left_pos in range(word_length + 1):
            for right_pos in range(self.limit_left, word_length-left_pos+1):
                word_concat = word[left_pos:right_pos]
                if word_concat not in self.language_patterns:
                    continue

                pattern = self.language_patterns[word_concat]
                # TODO: Find simplification to this step
                digit_cnt = 1
                for i, char in enumerate(pattern):
                    if not char.isdigit():
                        continue

                    digit_pos = left_pos + i - digit_cnt
                    if (digit_pos not in found_patterns or
                            found_patterns[digit_pos] < int(char)):
                        found_patterns[digit_pos] = int(char)

                    digit_cnt += 1

        hyphens = (h for h in found_patterns.keys() if found_patterns[h] & 1)
        for i, hyphen in enumerate(hyphens):
            index = i + hyphen + 1
            word = word[:index] + '-' + word[index:]

        return word[1:-1]
<commit_msg>less bloat in the algorithm<commit_after>"""
Handles hyphenation of words or sentences
"""

from .language_patterns import LanguagePatterns
from string import digits

class KnuthLiang(object):
    """
    This class handles hyphenation
    """

    def __init__(self, code, limit_left=2, limit_right=3):
        self.language_patterns = LanguagePatterns(code)
        self.limit_left = limit_left
        self.limit_right = limit_right

    def hyphenate_word(self, word):
        "Hyphenates a word"
        word = '.' + word + '.'
        word_len = len(word)
        found_patterns = dict()

        # we find all the patterns that match our word
        patterns = ((self.language_patterns[word[l:r]], l)
                    for l in range(word_len) for r in range(word_len-l)
                    if word[l:r] in self.language_patterns)
        for pattern, left_position in patterns:
            # double enumeration to save i (index in pattern)
            #   and j (index in digit tuple)
            patts = [(i, p) for i, p in enumerate(pattern) if p in digits]
            for j, (i, char) in enumerate(patts):
                digit_pos = left_position + i - j - 1
                if (digit_pos not in found_patterns or
                        found_patterns[digit_pos] < int(char)):
                    found_patterns[digit_pos] = int(char)

        # we find all the odd-numbered digits in the pattern and hyphenate
        hyphens = (h for h in found_patterns.keys() if found_patterns[h] & 1)
        for i, hyphen in enumerate(hyphens):
            index = i + hyphen + 1
            word = word[:index] + '-' + word[index:]

        return word[1:-1]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import itertools

from . import ACTION_DEPENDENCY_FLAG
from .base import (BasePolicy, AttachedPreparationMixin, ForwardActionGeneratorMixin, AbstractActionGenerator,
                   ReverseActionGeneratorMixin)
from .utils import is_initial

class SimpleCreateGenerator(ForwardActionGeneratorMixin, AbstractActionGenerator):
    def generate_item_actions(self, map_name, c_map, container_name, c_config, instances, flags, *args, **kwargs):
        for client_name, client, client_config in self._policy.get_clients(c_config, c_map):
            existing_containers = self._policy.container_names[client_name]
            images = self._policy.images[client_name]
            for a in c_config.attaches:
                a_name = self._policy.cname(map_name, a)
                if a_name not in existing_containers:
                    a_kwargs = self._policy.get_attached_create_kwargs(c_map, c_config, client_config, a_name, a)
                    images.ensure_image(a_kwargs['image'])
                    client.create_container(**a_kwargs)
                    existing_containers.add(a_name)
            for ci in instances:
                ci_name = self._policy.cname(map_name, container_name, ci)
                if ci_name not in existing_containers:
                    c_kwargs = self._policy.get_create_kwargs(c_map, c_config, client_config, ci_name, container_name,
                                                              kwargs=kwargs)
                    images.ensure_image(c_kwargs['image'])
                    yield client_name, client.create_container(**c_kwargs)
                    existing_containers.add(ci_name)

class SimpleCreateMixin(object):
    def create_actions(self, map_name, container, instances=None, **kwargs):
        """
        Generates actions for creating any configured container that does not already exist, including all of its
        dependencies.

        :param map_name: Container map name.
        :type map_name: unicode
        :param container: Container configuration name.
        :type container: unicode
        :param instances: Instance names. Optional, if ``None`` the configured instances or one default instance is
          created.
        :type instances: list[unicode]
        :param kwargs: Additional keyword args for the create action.
        :return: Return values of created main containers.
        :rtype: list[(unicode, dict)]
        """
        return SimpleCreateGenerator(self).get_actions(map_name, container, instances=instances, **kwargs)

class SimpleStartGenerator(AttachedPreparationMixin, ForwardActionGeneratorMixin, AbstractActionGenerator):
    def generate_item_actions(self, map_name, c_map, container_name, c_config, instances, flags, *args, **kwargs):
        for client_name, client, client_config in self._policy.get_clients(c_config, c_map):
            images = self._policy.images[client_name]
            for a in c_config.attaches:
                a_name = self._policy.cname(map_name, a)
                a_status = client.inspect_container(a_name)['State']
                if a_status['ExitCode'] != 0 or is_initial(a_status):
                    a_kwargs = self._policy.get_attached_start_kwargs(c_map, c_config, client_config, a_name, a)
                    client.start(**a_kwargs)
                    self.prepare_container(images, client, c_map, c_config, client_config, a, a_name)
            for instance in instances:
                ci_name = self._policy.cname(map_name, container_name, instance)
                ci_status = client.inspect_container(ci_name)['State']
                if not ci_status['Running']:
                    c_kwargs = self._policy.get_start_kwargs(c_map, c_config, client_config, ci_name, instance,
                                                             kwargs=kwargs)
                    client.start(**c_kwargs)

class SimpleStartMixin(object):
    def start_actions(self, map_name, container, instances=None, **kwargs):
        """
        Generates actions for starting any configured container that is not running, including all of its dependencies.

        :param map_name: Container map name.
        :type map_name: unicode
        :param container: Container configuration name.
        :type container: unicode
        :param instances: Instance names. Optional, if ``None`` the configured instances or one default instance is
          started.
        :type instances: list[unicode]
        :param kwargs: Additional keyword args for the start action.
        """
        SimpleStartGenerator(self).get_actions(map_name, container, instances=instances, **kwargs)

class SimpleRestartMixin(object):
    def restart_actions(self, map_name, container, instances=None, **kwargs):
        """
        Generates actions for restarting a configured container. Does not consider dependencies.

        :param map_name: Container map name.
        :type map_name: unicode
        :param container: Container configuration name.
        :type container: unicode
        :param instances: Instance names. Optional, if ``None`` the configured instances or one default instance is
          restarted.
        :type instances: list[unicode]
        :param kwargs: Additional keyword args for the restart action.
        """
        c_map = self._maps[map_name]
        c_config = c_map.get_existing(container)
        c_instances = instances or c_config.instances or [None]
        for client_name, client, client_config in self._policy.get_clients(c_config, c_map):
            existing_containers = self._policy.container_names[client_name]
            for instance in c_instances:
                ci_name = self.cname(map_name, container, instance)
                ci_status = client.inspect_container(ci_name)['State'] if ci_name in existing_containers else None
                if ci_status and ci_status['Running']:
                    c_kwargs = self._policy.get_restart_kwargs(c_map, c_config, client_config, ci_name, instance,
                                                               kwargs=kwargs)
                    client.restart(**c_kwargs)

class SimpleStopGenerator(ReverseActionGeneratorMixin, AbstractActionGenerator):
    def __init__(self, policy, *args, **kwargs):
        super(SimpleStopGenerator, self).__init__(policy, *args, **kwargs)
        self._stop_dependent = policy.stop_dependent

    def generate_item_actions(self, map_name, c_map, container_name, c_config, instances, flags, *args, **kwargs):
        if self._stop_dependent or not flags & ACTION_DEPENDENCY_FLAG:
            for client_name, client, client_config in self._policy.get_clients(c_config, c_map):
                existing_containers = self._policy.container_names[client_name]
                for instance in instances:
                    ci_name = self._policy.cname(map_name, container_name, instance)
                    ci_status = client.inspect_container(ci_name)['State'] if ci_name in existing_containers else None
                    if ci_status and ci_status['Running']:
                        c_kwargs = self._policy.get_stop_kwargs(c_map, c_config, client_config, ci_name, instance,
                                                                kwargs=kwargs)
                        client.stop(**c_kwargs)

class SimpleStopMixin(object):
    stop_dependent = True

    def stop_actions(self, map_name, container, instances=None, **kwargs):
        """
        Generates actions for stopping any configured container that is running, including all of its
        dependents. Stopping dependent containers can be prevented by setting ``stop_dependent`` to ``False`` in
        subclasses.

        :param map_name: Container map name.
        :type map_name: unicode
        :param container: Container configuration name.
        :type container: unicode
        :param instances: Instance names. Optional, if ``None`` the configured instances or one default instance is
          stopped.
        :type instances: list[unicode]
        :param kwargs: Additional keyword args for the stop action.
        """
        SimpleStopGenerator(self).get_actions(map_name, container, instances=instances, **kwargs)

class SimpleRemoveGenerator(ReverseActionGeneratorMixin, AbstractActionGenerator):
    def __init__(self, policy, *args, **kwargs):
        super(SimpleRemoveGenerator, self).__init__(policy, *args, **kwargs)
        self._remove_dependent = policy.remove_dependent
        self._remove_persistent = policy.remove_persistent
        self._remove_attached = policy.remove_attached

    def generate_item_actions(self, map_name, c_map, container_name, c_config, instances, flags, *args, **kwargs):
        if (self._remove_dependent or not flags & ACTION_DEPENDENCY_FLAG) and (self._remove_persistent or not c_config.persistent):
            for client_name, client, client_config in self._policy.get_clients(c_config, c_map):
                existing_containers = self._policy.container_names[client_name]
                for instance in instances:
                    ci_name = self._policy.cname(map_name, container_name, instance)
                    if ci_name in existing_containers:
                        c_kwargs = self._policy.get_remove_kwargs(map_name, c_config, client_config, ci_name,
                                                                  kwargs=kwargs)
                        client.remove_container(**c_kwargs)
                        existing_containers.remove(ci_name)
                if self._remove_attached:
                    for a in c_config.attaches:
                        a_name = self._policy.cname(map_name, a)
                        if a_name in existing_containers:
                            a_kwargs = self._policy.get_remove_kwargs(c_map, c_config, client_config, a_name,
                                                                      kwargs=kwargs)
                            client.remove_container(**a_kwargs)
                            existing_containers.remove(a_name)

class SimpleRemoveMixin(object):
    remove_dependent = True
    remove_persistent = True
    remove_attached = False

    def remove_actions(self, map_name, container, instances=None, **kwargs):
        """
        Generates actions for removing any configured container that exists, including all of its
        dependents. Removing dependent containers can be prevented by setting ``remove_dependent`` to ``False`` in
        subclasses. Containers with the :attr:`~dockermap.map.config.ContainerConfiguration.persistent` property are
        also removed, but attached containers are not.
        This behavior can be changed by setting ``remove_persistent`` and ``remove_attached`` in subclasses.

        :param map_name: Container map name.
        :type map_name: unicode
        :param container: Container configuration name.
        :type container: unicode
        :param instances: Instance names. Optional, if ``None`` the configured instances or one default instance is
          removed.
        :type instances: list[unicode]
        :param kwargs: Additional keyword args for the remove action.
        """
        SimpleRemoveGenerator(self).get_actions(map_name, container, instances=instances, **kwargs)

class SimpleStartupMixin(object):
    def startup_actions(self, map_name, container, instances=None, **kwargs):
        """
        Generates actions for creating any configured container that does not already exist, and start any non-running
        containers. This also applies to all of its dependencies.

        :param map_name: Container map name.
        :type map_name: unicode
        :param container: Container configuration name.
        :type container: unicode
        :param instances: Instance names. Optional, if ``None`` the configured instances or one default instance is
          created and started.
        :type instances: list[unicode]
        :param kwargs: Has no effect in this implementation.
        :return: Return values of created main containers.
        :rtype: list[(unicode, dict)]
        """
        return itertools.chain(self.create_actions(map_name, container, instances) or (),
                               self.start_actions(map_name, container, instances) or ())

class SimpleShutdownMixin(object):
    def shutdown_actions(self, map_name, container, instances=None, **kwargs):
        """
        Generates actions for stopping any configured container that is running, and removing any container that exists.
        This also applies to all of its dependent container configurations.

        Stopping dependent containers can be prevented by setting ``stop_dependent`` to ``False`` in
        subclasses. Removing dependent containers can be prevented by setting ``remove_dependent`` to ``False`` in
        subclasses. Containers with the :attr:`~dockermap.map.config.ContainerConfiguration.persistent` property are also
        removed, but attached containers are not. This behavior can be changed by setting ``remove_persistent`` and
        ``remove_attached`` in subclasses.

        :param map_name: Container map name.
        :type map_name: unicode
        :param container: Container configuration name.
        :type container: unicode
        :param instances: Instance names. Optional, if ``None`` the configured instances or one default instance is
          stopped and removed.
        :type instances: list[unicode]
        :param kwargs: Has no effect in this implementation.
        """
        self.stop_actions(map_name, container, instances)
        self.remove_actions(map_name, container, instances)

class SimplePolicy(SimpleCreateMixin, SimpleStartMixin, SimpleRestartMixin, SimpleStopMixin, SimpleRemoveMixin,
                   SimpleStartupMixin, SimpleShutdownMixin, BasePolicy):
    pass
<commit_msg>Fixed restart policy.<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import itertools

from . import ACTION_DEPENDENCY_FLAG
from .base import (BasePolicy, AttachedPreparationMixin, ForwardActionGeneratorMixin, AbstractActionGenerator,
                   ReverseActionGeneratorMixin)
from .utils import is_initial

class SimpleCreateGenerator(ForwardActionGeneratorMixin, AbstractActionGenerator):
    def generate_item_actions(self, map_name, c_map, container_name, c_config, instances, flags, *args, **kwargs):
        for client_name, client, client_config in self._policy.get_clients(c_config, c_map):
            existing_containers = self._policy.container_names[client_name]
            images = self._policy.images[client_name]
            for a in c_config.attaches:
                a_name = self._policy.cname(map_name, a)
                if a_name not in existing_containers:
                    a_kwargs = self._policy.get_attached_create_kwargs(c_map, c_config, client_config, a_name, a)
                    images.ensure_image(a_kwargs['image'])
                    client.create_container(**a_kwargs)
                    existing_containers.add(a_name)
            for ci in instances:
                ci_name = self._policy.cname(map_name, container_name, ci)
                if ci_name not in existing_containers:
                    c_kwargs = self._policy.get_create_kwargs(c_map, c_config, client_config, ci_name, container_name,
                                                              kwargs=kwargs)
                    images.ensure_image(c_kwargs['image'])
                    yield client_name, client.create_container(**c_kwargs)
                    existing_containers.add(ci_name)

class SimpleCreateMixin(object):
    def create_actions(self, map_name, container, instances=None, **kwargs):
        """
        Generates actions for creating any configured container that does not already exist, including all of its
        dependencies.

        :param map_name: Container map name.
        :type map_name: unicode
        :param container: Container configuration name.
        :type container: unicode
        :param instances: Instance names. Optional, if ``None`` the configured instances or one default instance is
          created.
        :type instances: list[unicode]
        :param kwargs: Additional keyword args for the create action.
        :return: Return values of created main containers.
        :rtype: list[(unicode, dict)]
        """
        return SimpleCreateGenerator(self).get_actions(map_name, container, instances=instances, **kwargs)

class SimpleStartGenerator(AttachedPreparationMixin, ForwardActionGeneratorMixin, AbstractActionGenerator):
    def generate_item_actions(self, map_name, c_map, container_name, c_config, instances, flags, *args, **kwargs):
        for client_name, client, client_config in self._policy.get_clients(c_config, c_map):
            images = self._policy.images[client_name]
            for a in c_config.attaches:
                a_name = self._policy.cname(map_name, a)
                a_status = client.inspect_container(a_name)['State']
                if a_status['ExitCode'] != 0 or is_initial(a_status):
                    a_kwargs = self._policy.get_attached_start_kwargs(c_map, c_config, client_config, a_name, a)
                    client.start(**a_kwargs)
                    self.prepare_container(images, client, c_map, c_config, client_config, a, a_name)
            for instance in instances:
                ci_name = self._policy.cname(map_name, container_name, instance)
                ci_status = client.inspect_container(ci_name)['State']
                if not ci_status['Running']:
                    c_kwargs = self._policy.get_start_kwargs(c_map, c_config, client_config, ci_name, instance,
                                                             kwargs=kwargs)
                    client.start(**c_kwargs)

class SimpleStartMixin(object):
    def start_actions(self, map_name, container, instances=None, **kwargs):
        """
        Generates actions for starting any configured container that is not running, including all of its dependencies.

        :param map_name: Container map name.
        :type map_name: unicode
        :param container: Container configuration name.
        :type container: unicode
        :param instances: Instance names. Optional, if ``None`` the configured instances or one default instance is
          started.
        :type instances: list[unicode]
        :param kwargs: Additional keyword args for the start action.
        """
        SimpleStartGenerator(self).get_actions(map_name, container, instances=instances, **kwargs)

class SimpleRestartMixin(object):
    def restart_actions(self, map_name, container, instances=None, **kwargs):
        """
        Generates actions for restarting a configured container. Does not consider dependencies.

        :param map_name: Container map name.
        :type map_name: unicode
        :param container: Container configuration name.
        :type container: unicode
        :param instances: Instance names. Optional, if ``None`` the configured instances or one default instance is
          restarted.
        :type instances: list[unicode]
        :param kwargs: Additional keyword args for the restart action.
        """
        c_map = self._maps[map_name]
        c_config = c_map.get_existing(container)
        c_instances = instances or c_config.instances or [None]
        for client_name, client, client_config in self.get_clients(c_config, c_map):
            existing_containers = self.container_names[client_name]
            for instance in c_instances:
                ci_name = self.cname(map_name, container, instance)
                ci_status = client.inspect_container(ci_name)['State'] if ci_name in existing_containers else None
                if ci_status and ci_status['Running']:
                    c_kwargs = self.get_restart_kwargs(c_map, c_config, client_config, ci_name, instance,
                                                       kwargs=kwargs)
                    client.restart(**c_kwargs)

class SimpleStopGenerator(ReverseActionGeneratorMixin, AbstractActionGenerator):
    def __init__(self, policy, *args, **kwargs):
        super(SimpleStopGenerator, self).__init__(policy, *args, **kwargs)
        self._stop_dependent = policy.stop_dependent

    def generate_item_actions(self, map_name, c_map, container_name, c_config, instances, flags, *args, **kwargs):
        if self._stop_dependent or not flags & ACTION_DEPENDENCY_FLAG:
            for client_name, client, client_config in self._policy.get_clients(c_config, c_map):
                existing_containers = self._policy.container_names[client_name]
                for instance in instances:
                    ci_name = self._policy.cname(map_name, container_name, instance)
                    ci_status = client.inspect_container(ci_name)['State'] if ci_name in existing_containers else None
                    if ci_status and ci_status['Running']:
                        c_kwargs = self._policy.get_stop_kwargs(c_map, c_config, client_config, ci_name, instance,
                                                                kwargs=kwargs)
                        client.stop(**c_kwargs)

class SimpleStopMixin(object):
    stop_dependent = True

    def stop_actions(self, map_name, container, instances=None, **kwargs):
        """
        Generates actions for stopping any configured container that is running, including all of its
        dependents. Stopping dependent containers can be prevented by setting ``stop_dependent`` to ``False`` in
        subclasses.

        :param map_name: Container map name.
        :type map_name: unicode
        :param container: Container configuration name.
        :type container: unicode
        :param instances: Instance names. Optional, if ``None`` the configured instances or one default instance is
          stopped.
        :type instances: list[unicode]
        :param kwargs: Additional keyword args for the stop action.
        """
        SimpleStopGenerator(self).get_actions(map_name, container, instances=instances, **kwargs)

class SimpleRemoveGenerator(ReverseActionGeneratorMixin, AbstractActionGenerator):
    def __init__(self, policy, *args, **kwargs):
        super(SimpleRemoveGenerator, self).__init__(policy, *args, **kwargs)
        self._remove_dependent = policy.remove_dependent
        self._remove_persistent = policy.remove_persistent
        self._remove_attached = policy.remove_attached

    def generate_item_actions(self, map_name, c_map, container_name, c_config, instances, flags, *args, **kwargs):
        if (self._remove_dependent or not flags & ACTION_DEPENDENCY_FLAG) and (self._remove_persistent or not c_config.persistent):
            for client_name, client, client_config in self._policy.get_clients(c_config, c_map):
                existing_containers = self._policy.container_names[client_name]
                for instance in instances:
                    ci_name = self._policy.cname(map_name, container_name, instance)
                    if ci_name in existing_containers:
                        c_kwargs = self._policy.get_remove_kwargs(map_name, c_config, client_config, ci_name,
                                                                  kwargs=kwargs)
                        client.remove_container(**c_kwargs)
                        existing_containers.remove(ci_name)
                if self._remove_attached:
                    for a in c_config.attaches:
                        a_name = self._policy.cname(map_name, a)
                        if a_name in existing_containers:
                            a_kwargs = self._policy.get_remove_kwargs(c_map, c_config, client_config, a_name,
                                                                      kwargs=kwargs)
                            client.remove_container(**a_kwargs)
                            existing_containers.remove(a_name)

class SimpleRemoveMixin(object):
    remove_dependent = True
    remove_persistent = True
    remove_attached = False

    def remove_actions(self, map_name, container, instances=None, **kwargs):
        """
        Generates actions for removing any configured container that exists, including all of its
        dependents. Removing dependent containers can be prevented by setting ``remove_dependent`` to ``False`` in
        subclasses. Containers with the :attr:`~dockermap.map.config.ContainerConfiguration.persistent` property are
        also removed, but attached containers are not.
        This behavior can be changed by setting ``remove_persistent`` and ``remove_attached`` in subclasses.

        :param map_name: Container map name.
        :type map_name: unicode
        :param container: Container configuration name.
        :type container: unicode
        :param instances: Instance names. Optional, if ``None`` the configured instances or one default instance is
          removed.
        :type instances: list[unicode]
        :param kwargs: Additional keyword args for the remove action.
        """
        SimpleRemoveGenerator(self).get_actions(map_name, container, instances=instances, **kwargs)

class SimpleStartupMixin(object):
    def startup_actions(self, map_name, container, instances=None, **kwargs):
        """
        Generates actions for creating any configured container that does not already exist, and start any non-running
        containers. This also applies to all of its dependencies.

        :param map_name: Container map name.
        :type map_name: unicode
        :param container: Container configuration name.
        :type container: unicode
        :param instances: Instance names. Optional, if ``None`` the configured instances or one default instance is
          created and started.
        :type instances: list[unicode]
        :param kwargs: Has no effect in this implementation.
        :return: Return values of created main containers.
        :rtype: list[(unicode, dict)]
        """
        return itertools.chain(self.create_actions(map_name, container, instances) or (),
                               self.start_actions(map_name, container, instances) or ())

class SimpleShutdownMixin(object):
    def shutdown_actions(self, map_name, container, instances=None, **kwargs):
        """
        Generates actions for stopping any configured container that is running, and removing any container that exists.
        This also applies to all of its dependent container configurations.

        Stopping dependent containers can be prevented by setting ``stop_dependent`` to ``False`` in
        subclasses. Removing dependent containers can be prevented by setting ``remove_dependent`` to ``False`` in
        subclasses. Containers with the :attr:`~dockermap.map.config.ContainerConfiguration.persistent` property are also
        removed, but attached containers are not. This behavior can be changed by setting ``remove_persistent`` and
        ``remove_attached`` in subclasses.

        :param map_name: Container map name.
        :type map_name: unicode
        :param container: Container configuration name.
        :type container: unicode
        :param instances: Instance names. Optional, if ``None`` the configured instances or one default instance is
          stopped and removed.
        :type instances: list[unicode]
        :param kwargs: Has no effect in this implementation.
        """
        self.stop_actions(map_name, container, instances)
        self.remove_actions(map_name, container, instances)

class SimplePolicy(SimpleCreateMixin, SimpleStartMixin, SimpleRestartMixin, SimpleStopMixin, SimpleRemoveMixin,
                   SimpleStartupMixin, SimpleShutdownMixin, BasePolicy):
    pass
<|endoftext|>


<commit_before>from denim import *

## Configuration ######################

# Name of the project
env.project_name = 'Denim'

# Name of the application package
env.package_name = 'denim_demo'

# Override default source control management tool
# env.deploy_scm = 'hg' # Options 'git', 'hg'

# Override default web-server
# env.deploy_web_server = 'nginx' # Options 'apache', 'nginx'

## Deployment environments ############

@task(alias='prod')
def production():
    env.deploy_env = 'production'
    env.hosts = ['192.168.1.10', '192.168.1.12']

@task
def staging():
    env.deploy_env = 'staging'
    env.hosts = ['192.168.2.10', '192.168.2.12']

## Actions ############################

@task
def provision():
    defaults.standard_provision()

@task(alias='d')
def deploy(revision):
    defaults.standard_deploy(revision)
<commit_msg>Updated example with defaults<commit_after>from denim import *
from denim import defaults

## Configuration ######################

# Name of the project
env.project_name = 'Denim'

# Name of the application package
env.package_name = 'denim_demo'

# Override default source control management tool
# env.deploy_scm = 'hg' # Options 'git', 'hg'

# Override default web-server
# env.deploy_web_server = 'nginx' # Options 'apache', 'nginx'

## Deployment environments ############

@task(alias='prod')
def production():
    env.deploy_env = 'production'
    env.hosts = ['192.168.1.10', '192.168.1.12']

@task
def staging():
    env.deploy_env = 'staging'
    env.hosts = ['192.168.2.10', '192.168.2.12']

## Actions ############################

@task
def provision():
    defaults.standard_provision()

@task(alias='d')
def deploy(revision):
    defaults.standard_deploy(revision)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging

from dino import environ
from dino import utils
from dino.utils.decorators import timeit
from dino.db.manager import UserManager
from dino.rest.resources.base import BaseResource

from flask import request

logger = logging.getLogger(__name__)

__author__ = 'Oscar Eriksson <oscar.eriks@gmail.com>'

def fail(error_message):
    return {
        'status': 'FAIL',
        'message': error_message
    }

class SendResource(BaseResource):
    def __init__(self):
        super(SendResource, self).__init__()
        self.user_manager = UserManager(environ.env)
        self.request = request

    @timeit(logger, 'on_rest_send')
    def do_post(self):
        is_valid, msg, json = self.validate_json(self.request, silent=False)
        if not is_valid:
            logger.error('invalid json: %s' % msg)
            raise RuntimeError('invalid json')

        if json is None:
            raise RuntimeError('no json in request')
        if not isinstance(json, dict):
            raise RuntimeError('need a dict')

        logger.debug('POST request: %s' % str(json))

        if 'content' not in json:
            raise RuntimeError('no key [content] in json message')

        msg_content = json.get('content')
        if msg_content is None or len(msg_content.strip()) == 0:
            raise RuntimeError('content may not be blank')
        if not utils.is_base64(msg_content):
            raise RuntimeError('content in json message must be base64')

        user_id = json.get('user_id', '0')
        user_name = utils.b64d(json.get('user_name', 'admin'))
        object_type = json.get('object_type')
        target_id = json.get('target_id')
        target_name = json.get('target_name')

        data = utils.activity_for_message(user_id, user_name)
        data['target'] = {
            'objectType': object_type,
            'id': str(target_id),
            'displayName': target_name
        }
        data['object'] = {
            'content': msg_content
        }

        environ.env.internal_publisher.publish(data)
<commit_msg>default to /wio namespac<commit_after>#!/usr/bin/env python

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging

from dino import environ
from dino import utils
from dino.utils.decorators import timeit
from dino.db.manager import UserManager
from dino.rest.resources.base import BaseResource

from flask import request

logger = logging.getLogger(__name__)

__author__ = 'Oscar Eriksson <oscar.eriks@gmail.com>'

def fail(error_message):
    return {
        'status': 'FAIL',
        'message': error_message
    }

class SendResource(BaseResource):
    def __init__(self):
        super(SendResource, self).__init__()
        self.user_manager = UserManager(environ.env)
        self.request = request

    @timeit(logger, 'on_rest_send')
    def do_post(self):
        is_valid, msg, json = self.validate_json(self.request, silent=False)
        if not is_valid:
            logger.error('invalid json: %s' % msg)
            raise RuntimeError('invalid json')

        if json is None:
            raise RuntimeError('no json in request')
        if not isinstance(json, dict):
            raise RuntimeError('need a dict')

        logger.debug('POST request: %s' % str(json))

        if 'content' not in json:
            raise RuntimeError('no key [content] in json message')

        msg_content = json.get('content')
        if msg_content is None or len(msg_content.strip()) == 0:
            raise RuntimeError('content may not be blank')
        if not utils.is_base64(msg_content):
            raise RuntimeError('content in json message must be base64')

        user_id = json.get('user_id', '0')
        user_name = utils.b64d(json.get('user_name', 'admin'))
        object_type = json.get('object_type')
        target_id = json.get('target_id')
        namespace = json.get('namespace', '/wio')
        target_name = json.get('target_name')

        data = utils.activity_for_message(user_id, user_name)
        data['target'] = {
            'objectType': object_type,
            'id': str(target_id),
            'displayName': target_name,
            'url': namespace
        }
        data['object'] = {
            'content': msg_content
        }

        environ.env.internal_publisher.publish(data)
<|endoftext|>


<commit_before>import numpy as np
import nibabel as nib
from numpy.testing import assert_equal, run_module_suite
from dipy.data import get_fnames
from dipy.io.streamline import load_trk, save_trk
from dipy.tracking.streamline import Streamlines
from dipy.segment.metric import AveragePointwiseEuclideanMetric
import os
from dipy.io.image import load_nifti, save_nifti
from nibabel.tmpdirs import TemporaryDirectory
from dipy.stats.analysis import bundle_analysis
from dipy.testing import assert_true

def test_ba():

    with TemporaryDirectory() as dirpath:

        streams, hdr = nib.trackvis.read(get_fnames('fornix'))
        fornix = [s[0] for s in streams]

        f = Streamlines(fornix)

        mb = os.path.join(dirpath, "model_bundles")

        os.mkdir(mb)

        save_trk(os.path.join(mb, "temp.trk"),
                 f, affine=np.eye(4))

        rb = os.path.join(dirpath, "rec_bundles")
        os.mkdir(rb)

        save_trk(os.path.join(rb, "temp.trk"), f,
                 affine=np.eye(4))

        ob = os.path.join(dirpath, "org_bundles")
        os.mkdir(ob)

        save_trk(os.path.join(ob, "temp.trk"), f,
                 affine=np.eye(4))

        dt = os.path.join(dirpath, "dti_measures")
        os.mkdir(dt)

        fa = np.random.rand(255, 255, 255)

        save_nifti(os.path.join(dt, "fa.nii.gz"),
                   fa, affine=np.eye(4))

        out_dir = os.path.join(dirpath, "output")
        os.mkdir(out_dir)

        bundle_analysis(mb, rb, ob, dt, group="patient", subject="10001",
                        no_disks=100, out_dir=out_dir)

        assert_true(os.path.exists(os.path.join(out_dir, 'fa.h5')))

if __name__ == '__main__':

    run_module_suite()
<commit_msg>fixed a test<commit_after>import numpy as np
import nibabel as nib
from numpy.testing import assert_equal, run_module_suite
from dipy.data import get_fnames
from dipy.io.streamline import save_trk
from dipy.tracking.streamline import Streamlines
import os
import numpy.testing as npt
from dipy.utils.optpkg import optional_package
from dipy.io.image import save_nifti
from nibabel.tmpdirs import TemporaryDirectory
from dipy.stats.analysis import bundle_analysis
from dipy.testing import assert_true
_, have_pd, _ = optional_package("pandas")
_, have_smf, _ = optional_package("statsmodels")

@npt.dec.skipif(not have_pd or not have_smf)
def test_ba():

    with TemporaryDirectory() as dirpath:

        streams, hdr = nib.trackvis.read(get_fnames('fornix'))
        fornix = [s[0] for s in streams]

        f = Streamlines(fornix)

        mb = os.path.join(dirpath, "model_bundles")

        os.mkdir(mb)

        save_trk(os.path.join(mb, "temp.trk"),
                 f, affine=np.eye(4))

        rb = os.path.join(dirpath, "rec_bundles")
        os.mkdir(rb)

        save_trk(os.path.join(rb, "temp.trk"), f,
                 affine=np.eye(4))

        ob = os.path.join(dirpath, "org_bundles")
        os.mkdir(ob)

        save_trk(os.path.join(ob, "temp.trk"), f,
                 affine=np.eye(4))

        dt = os.path.join(dirpath, "dti_measures")
        os.mkdir(dt)

        fa = np.random.rand(255, 255, 255)

        save_nifti(os.path.join(dt, "fa.nii.gz"),
                   fa, affine=np.eye(4))

        out_dir = os.path.join(dirpath, "output")
        os.mkdir(out_dir)

        bundle_analysis(mb, rb, ob, dt, group="patient", subject="10001",
                        no_disks=100, out_dir=out_dir)

        assert_true(os.path.exists(os.path.join(out_dir, 'fa.h5')))

if __name__ == '__main__':

    run_module_suite()
<|endoftext|>


<commit_before>"""
Validators library

Complex validators
"""
from .basic import BaseValidator
from dirty_validators.basic import NotNone
from collections import OrderedDict
from dirty_models.models import BaseModel

class Chain(BaseValidator):

    """
    Use a chain of validators for one value
    """

    def __init__(self, validators=None, stop_on_fail=True, *args, **kwargs):
        super(Chain, self).__init__(*args, **kwargs)

        self.stop_on_fail = stop_on_fail
        self.validators = validators.copy() if validators is not None else []

    def _internal_is_valid(self, value, *args, **kwargs):
        result = True
        for validator in self.validators:
            if not validator.is_valid(value, *args, **kwargs):
                self.messages.update(validator.messages)
                result = False
                if self.stop_on_fail:
                    return False
        return result

class Some(BaseValidator):

    """
    Pass some validators for one value
    """

    def __init__(self, validators=None, *args, **kwargs):
        super(Some, self).__init__(*args, **kwargs)

        self.validators = validators.copy() if validators is not None else []

    def _internal_is_valid(self, value, *args, **kwargs):
        messages = {}
        for validator in self.validators:
            if validator.is_valid(value, *args, **kwargs):
                return True
            messages.update(validator.messages)

        self.messages.update(messages)
        return False

class ComplexValidator(BaseValidator):

    """
    Base for validator which inject context
    """

    def is_valid(self, value, *args, **kwargs):
        context = kwargs.get('context', [])
        context.append(value)
        kwargs['context'] = context

        result = super(ComplexValidator, self).is_valid(value, *args, **kwargs)

        context.pop()
        return result

    def import_messages(self, prefix, messages):
        base_messages = {}
        for key, message in messages.items():
            if isinstance(message, dict):
                self.messages[str(prefix) + "." + str(key)] = message
            else:
                base_messages[key] = message

        if len(base_messages):
            self.messages[prefix] = base_messages

class ListValidator(ComplexValidator):

    """
    Validate items on list
    """

    def __init__(self, validator, stop_on_fail=True, *args, **kwargs):
        super(ListValidator, self).__init__(*args, **kwargs)

        self.validator = validator
        self.stop_on_fail = stop_on_fail

class AllItems(ListValidator):

    """
    Validate all items on list
    """

    def _internal_is_valid(self, value, *args, **kwargs):
        result = True
        for item_index in range(len(value)):
            if not self.validator.is_valid(value[item_index], *args, **kwargs):
                self.import_messages(item_index, self.validator.messages)
                result = False
                if self.stop_on_fail:
                    return False
        return result

class SomeItems(ListValidator):

    """
    Validate some items on list
    """

    TOO_MANY_VALID_ITEMS = 'tooManyValidItems'
    TOO_FEW_VALID_ITEMS = 'tooFewValidItems'

    error_messages = {
        TOO_MANY_VALID_ITEMS: "Too many items pass validation",
        TOO_FEW_VALID_ITEMS: "Too few items pass validation"
    }

    def __init__(self, min=1, max=-1, *args, **kwargs):
        super(SomeItems, self).__init__(*args, **kwargs)
        assert min != -1 or max != -1, 'At least one of `min` or `max` must be specified.'
        assert max == -1 or min <= max, '`min` cannot be more than `max`.'
        self.min = min
        self.max = max

        self.message_values.update({"min": self.min, "max": self.max})

    def _internal_is_valid(self, value, *args, **kwargs):
        item_pass = 0
        for item_index in range(len(value)):
            if not self.validator.is_valid(value[item_index], *args, **kwargs):
                self.import_messages(item_index, self.validator.messages)
            else:
                item_pass += 1
                if self.stop_on_fail and max != -1 and item_pass > self.max:
                    self.error(self.TOO_MANY_VALID_ITEMS, value)
                    return False

        if max != -1 and item_pass > self.max:
            self.error(self.TOO_MANY_VALID_ITEMS, value)
            return False

        if min != -1 and item_pass < self.min:
            self.error(self.TOO_FEW_VALID_ITEMS, value)
            return False

        self.messages = {}
        return True

class ItemLimitedOccuerrences(BaseValidator):

    """
    Validate whether item in list are distincts
    """

    TOO_MANY_ITEM_OCCURRENCES = 'tooManyItemOccurrences'
    TOO_FEW_ITEM_OCCURRENCES = 'tooFewItemOccurrences'

    error_messages = {
        TOO_MANY_ITEM_OCCURRENCES: "Item '$value' is repeated to many times. Limit to $max_occ.",
        TOO_FEW_ITEM_OCCURRENCES: "Item '$value' is not enough repeated. Limit to $min_occ.",
    }

    def __init__(self, min_occ=0, max_occ=1, *args, **kwargs):
        super(ItemLimitedOccuerrences, self).__init__(*args, **kwargs)
        self.min_occ = min_occ
        self.max_occ = max_occ

        self.message_values['min_occ'] = self.min_occ
        self.message_values['max_occ'] = self.max_occ

    def _get_checking_value(self, value):
        """
        It must be override on descendant validators
        """
        return value

    def _internal_is_valid(self, value, *args, **kwargs):

        def add_occurrence(val, counter):
            if val not in counter:
                counter[val] = 0

            counter[val] += 1

        counter = {}

        for item in value:
            val = self._get_checking_value(item)
            add_occurrence(val, counter)

            if counter[val] > self.max_occ:
                self.error(self.TOO_MANY_ITEM_OCCURRENCES, val)
                return False

        for val, occ in counter.items():
            if occ < self.min_occ:
                self.error(self.TOO_FEW_ITEM_OCCURRENCES, val)
                return False

        return True

def get_field_value_from_context(field_name, context_list):
    """
    Helper to get field value from string path.
    String '<context>' is used to go up on context stack. It just
    can be used at the beginning of path: <context>.<context>.field_name_1
    """
    field_path = field_name.split('.')
    context_index = -1
    while field_path[0] == '<context>':
        context_index -= 1
        field_path.pop(0)

    try:
        field_value = context_list[context_index]

        while len(field_path):
            field = field_path.pop(0)
            if isinstance(field_value, (list, tuple)):
                if field.isdigit():
                    field = int(field)
                field_value = field_value[field]
            elif isinstance(field_value, dict):
                try:
                    field_value = field_value[field]
                except KeyError:
                    if field.isdigit():
                        field = int(field)
                        field_value = field_value[field]
                    else:
                        field_value = None

            else:
                field_value = getattr(field_value, field)

        return field_value
    except (IndexError, AttributeError, KeyError):
        return None

class IfField(BaseValidator):

    """
    Conditional validator. It run validators if a specific field value pass validations.
    """

    NEEDS_VALIDATE = 'needsValidate'

    error_messages = {
        NEEDS_VALIDATE: "Some validate error due to field '$field_name' has value '$field_value'.",
    }

    def __init__(self, validator, field_name, field_validator=None,
                 run_if_none=False, add_check_info=True, *args, **kwargs):
        super(IfField, self).__init__(*args, **kwargs)

        self.validator = validator
        self.field_name = field_name
        self.field_validator = field_validator
        self.run_if_none = run_if_none
        self.add_check_info = add_check_info

        self.message_values['field_name'] = field_name

    def _internal_is_valid(self, value, *args, **kwargs):
        field_value = get_field_value_from_context(self.field_name, kwargs.get('context', []))

        if (self.run_if_none or field_value is not None) and \
                (self.field_validator is None or self.field_validator.is_valid(field_value, *args, **kwargs)) and \
                not self.validator.is_valid(value, *args, **kwargs):
            self.messages.update(self.validator.messages)
            if self.add_check_info:
                self.error(self.NEEDS_VALIDATE, value, field_value=field_value)
            return False

        return True

class BaseSpec(ComplexValidator):

    """
    Base class to use spec
    """

    def __init__(self, spec=None, stop_on_fail=True, *args, **kwargs):
        super(BaseSpec, self).__init__(*args, **kwargs)

        if spec is not None:
            self.spec = spec.copy()

        try:
            self.spec
        except AttributeError:
            self.spec = OrderedDict()

        self.stop_on_fail = stop_on_fail

    def _internal_is_valid(self, value, *args, **kwargs):
        result = True
        for field_name, validator in self.spec.items():
            field_value = self.get_field_value(field_name, value, kwargs)

            if not validator.is_valid(field_value, *args, **kwargs):
                self.import_messages(field_name, validator.messages)
                result = False
                if self.stop_on_fail:
                    return False

        return result

class DictValidate(BaseSpec):

    INVALID_TYPE = 'notDict'

    error_messages = {
        INVALID_TYPE: "'$value' is not a dictionary",
    }

    def get_field_value(self, field_name, value, kwargs):
        return value.get(field_name, None)

    def _internal_is_valid(self, value, *args, **kwargs):
        if not isinstance(value, dict):
            self.error(self.INVALID_TYPE, value)
            return False
        return super(DictValidate, self)._internal_is_valid(value, *args, **kwargs)

class Required(Chain):

    REQUIRED = 'required'

    error_messages = {
        REQUIRED: "Value is required and can not be empty",
    }

    def __init__(self, empty_validator=None, *args, **kwargs):
        self.empty_validator = empty_validator or NotNone()
        super(Required, self).__init__(*args, **kwargs)

    def _internal_is_valid(self, value, *args, **kwargs):
        if not self.empty_validator.is_valid(value):
            self.error(self.REQUIRED, value)
            return False

        return super(Required, self)._internal_is_valid(value, *args, **kwargs)

class Optional(Chain):

    def __init__(self, empty_validator=None, *args, **kwargs):
        self.empty_validator = empty_validator or NotNone()
        super(Optional, self).__init__(*args, **kwargs)

    def _internal_is_valid(self, value, *args, **kwargs):
        if not self.empty_validator.is_valid(value):
            return True

        return Chain._internal_is_valid(self, value, *args, **kwargs)

class ModelValidateMetaclass(type):

    @classmethod
    def __prepare__(metacls, name, bases):  # No keywords in this case
        return OrderedDict()

    def __new__(cls, name, bases, classdict):

        result = super(ModelValidateMetaclass, cls).__new__(
            cls, name, bases, classdict)

        spec = OrderedDict([(field, validator) for field, validator in classdict.items()
                            if hasattr(validator, 'is_valid')])

        setattr(result, 'spec', spec)
        return result

class ModelValidate(BaseSpec, metaclass=ModelValidateMetaclass):

    __modelclass__ = BaseModel

    INVALID_MODEL = 'notModel'

    error_messages = {
        INVALID_MODEL: "'$value' is not an instance of $model",
    }

    def __init__(self, spec=None, *args, **kwargs):
        self.spec = self.spec.copy()
        if spec is not None:
            self.spec.update(spec)

        super(ModelValidate, self).__init__(*args, **kwargs)

    def get_field_value(self, field_name, value, kwargs):
        kwargs['is_modified'] = value.is_modified_field(field_name)
        return getattr(value, field_name)

    def _internal_is_valid(self, value, *args, **kwargs):
        if not isinstance(value, self.__modelclass__):
            self.error(self.INVALID_MODEL, value, model=self.__modelclass__.__name__)
            return False

        return super(ModelValidate, self)._internal_is_valid(value, *args, **kwargs)
<commit_msg>Ignore attr errors<commit_after>"""
Validators library

Complex validators
"""
from .basic import BaseValidator
from dirty_validators.basic import NotNone
from collections import OrderedDict
from dirty_models.models import BaseModel

class Chain(BaseValidator):

    """
    Use a chain of validators for one value
    """

    def __init__(self, validators=None, stop_on_fail=True, *args, **kwargs):
        super(Chain, self).__init__(*args, **kwargs)

        self.stop_on_fail = stop_on_fail
        self.validators = validators.copy() if validators is not None else []

    def _internal_is_valid(self, value, *args, **kwargs):
        result = True
        for validator in self.validators:
            if not validator.is_valid(value, *args, **kwargs):
                self.messages.update(validator.messages)
                result = False
                if self.stop_on_fail:
                    return False
        return result

class Some(BaseValidator):

    """
    Pass some validators for one value
    """

    def __init__(self, validators=None, *args, **kwargs):
        super(Some, self).__init__(*args, **kwargs)

        self.validators = validators.copy() if validators is not None else []

    def _internal_is_valid(self, value, *args, **kwargs):
        messages = {}
        for validator in self.validators:
            if validator.is_valid(value, *args, **kwargs):
                return True
            messages.update(validator.messages)

        self.messages.update(messages)
        return False

class ComplexValidator(BaseValidator):

    """
    Base for validator which inject context
    """

    def is_valid(self, value, *args, **kwargs):
        context = kwargs.get('context', [])
        context.append(value)
        kwargs['context'] = context

        result = super(ComplexValidator, self).is_valid(value, *args, **kwargs)

        context.pop()
        return result

    def import_messages(self, prefix, messages):
        base_messages = {}
        for key, message in messages.items():
            if isinstance(message, dict):
                self.messages[str(prefix) + "." + str(key)] = message
            else:
                base_messages[key] = message

        if len(base_messages):
            self.messages[prefix] = base_messages

class ListValidator(ComplexValidator):

    """
    Validate items on list
    """

    def __init__(self, validator, stop_on_fail=True, *args, **kwargs):
        super(ListValidator, self).__init__(*args, **kwargs)

        self.validator = validator
        self.stop_on_fail = stop_on_fail

class AllItems(ListValidator):

    """
    Validate all items on list
    """

    def _internal_is_valid(self, value, *args, **kwargs):
        result = True
        for item_index in range(len(value)):
            if not self.validator.is_valid(value[item_index], *args, **kwargs):
                self.import_messages(item_index, self.validator.messages)
                result = False
                if self.stop_on_fail:
                    return False
        return result

class SomeItems(ListValidator):

    """
    Validate some items on list
    """

    TOO_MANY_VALID_ITEMS = 'tooManyValidItems'
    TOO_FEW_VALID_ITEMS = 'tooFewValidItems'

    error_messages = {
        TOO_MANY_VALID_ITEMS: "Too many items pass validation",
        TOO_FEW_VALID_ITEMS: "Too few items pass validation"
    }

    def __init__(self, min=1, max=-1, *args, **kwargs):
        super(SomeItems, self).__init__(*args, **kwargs)
        assert min != -1 or max != -1, 'At least one of `min` or `max` must be specified.'
        assert max == -1 or min <= max, '`min` cannot be more than `max`.'
        self.min = min
        self.max = max

        self.message_values.update({"min": self.min, "max": self.max})

    def _internal_is_valid(self, value, *args, **kwargs):
        item_pass = 0
        for item_index in range(len(value)):
            if not self.validator.is_valid(value[item_index], *args, **kwargs):
                self.import_messages(item_index, self.validator.messages)
            else:
                item_pass += 1
                if self.stop_on_fail and max != -1 and item_pass > self.max:
                    self.error(self.TOO_MANY_VALID_ITEMS, value)
                    return False

        if max != -1 and item_pass > self.max:
            self.error(self.TOO_MANY_VALID_ITEMS, value)
            return False

        if min != -1 and item_pass < self.min:
            self.error(self.TOO_FEW_VALID_ITEMS, value)
            return False

        self.messages = {}
        return True

class ItemLimitedOccuerrences(BaseValidator):

    """
    Validate whether item in list are distincts
    """

    TOO_MANY_ITEM_OCCURRENCES = 'tooManyItemOccurrences'
    TOO_FEW_ITEM_OCCURRENCES = 'tooFewItemOccurrences'

    error_messages = {
        TOO_MANY_ITEM_OCCURRENCES: "Item '$value' is repeated to many times. Limit to $max_occ.",
        TOO_FEW_ITEM_OCCURRENCES: "Item '$value' is not enough repeated. Limit to $min_occ.",
    }

    def __init__(self, min_occ=0, max_occ=1, *args, **kwargs):
        super(ItemLimitedOccuerrences, self).__init__(*args, **kwargs)
        self.min_occ = min_occ
        self.max_occ = max_occ

        self.message_values['min_occ'] = self.min_occ
        self.message_values['max_occ'] = self.max_occ

    def _get_checking_value(self, value):
        """
        It must be override on descendant validators
        """
        return value

    def _internal_is_valid(self, value, *args, **kwargs):

        def add_occurrence(val, counter):
            if val not in counter:
                counter[val] = 0

            counter[val] += 1

        counter = {}

        for item in value:
            val = self._get_checking_value(item)
            add_occurrence(val, counter)

            if counter[val] > self.max_occ:
                self.error(self.TOO_MANY_ITEM_OCCURRENCES, val)
                return False

        for val, occ in counter.items():
            if occ < self.min_occ:
                self.error(self.TOO_FEW_ITEM_OCCURRENCES, val)
                return False

        return True

def get_field_value_from_context(field_name, context_list):
    """
    Helper to get field value from string path.
    String '<context>' is used to go up on context stack. It just
    can be used at the beginning of path: <context>.<context>.field_name_1
    """
    field_path = field_name.split('.')
    context_index = -1
    while field_path[0] == '<context>':
        context_index -= 1
        field_path.pop(0)

    try:
        field_value = context_list[context_index]

        while len(field_path):
            field = field_path.pop(0)
            if isinstance(field_value, (list, tuple)):
                if field.isdigit():
                    field = int(field)
                field_value = field_value[field]
            elif isinstance(field_value, dict):
                try:
                    field_value = field_value[field]
                except KeyError:
                    if field.isdigit():
                        field = int(field)
                        field_value = field_value[field]
                    else:
                        field_value = None

            else:
                field_value = getattr(field_value, field)

        return field_value
    except (IndexError, AttributeError, KeyError):
        return None

class IfField(BaseValidator):

    """
    Conditional validator. It run validators if a specific field value pass validations.
    """

    NEEDS_VALIDATE = 'needsValidate'

    error_messages = {
        NEEDS_VALIDATE: "Some validate error due to field '$field_name' has value '$field_value'.",
    }

    def __init__(self, validator, field_name, field_validator=None,
                 run_if_none=False, add_check_info=True, *args, **kwargs):
        super(IfField, self).__init__(*args, **kwargs)

        self.validator = validator
        self.field_name = field_name
        self.field_validator = field_validator
        self.run_if_none = run_if_none
        self.add_check_info = add_check_info

        self.message_values['field_name'] = field_name

    def _internal_is_valid(self, value, *args, **kwargs):
        field_value = get_field_value_from_context(self.field_name, kwargs.get('context', []))

        if (self.run_if_none or field_value is not None) and \
                (self.field_validator is None or self.field_validator.is_valid(field_value, *args, **kwargs)) and \
                not self.validator.is_valid(value, *args, **kwargs):
            self.messages.update(self.validator.messages)
            if self.add_check_info:
                self.error(self.NEEDS_VALIDATE, value, field_value=field_value)
            return False

        return True

class BaseSpec(ComplexValidator):

    """
    Base class to use spec
    """

    def __init__(self, spec=None, stop_on_fail=True, *args, **kwargs):
        super(BaseSpec, self).__init__(*args, **kwargs)

        if spec is not None:
            self.spec = spec.copy()

        try:
            self.spec
        except AttributeError:
            self.spec = OrderedDict()

        self.stop_on_fail = stop_on_fail

    def _internal_is_valid(self, value, *args, **kwargs):
        result = True
        for field_name, validator in self.spec.items():
            field_value = self.get_field_value(field_name, value, kwargs)

            if not validator.is_valid(field_value, *args, **kwargs):
                self.import_messages(field_name, validator.messages)
                result = False
                if self.stop_on_fail:
                    return False

        return result

class DictValidate(BaseSpec):

    INVALID_TYPE = 'notDict'

    error_messages = {
        INVALID_TYPE: "'$value' is not a dictionary",
    }

    def get_field_value(self, field_name, value, kwargs):
        return value.get(field_name, None)

    def _internal_is_valid(self, value, *args, **kwargs):
        if not isinstance(value, dict):
            self.error(self.INVALID_TYPE, value)
            return False
        return super(DictValidate, self)._internal_is_valid(value, *args, **kwargs)

class Required(Chain):

    REQUIRED = 'required'

    error_messages = {
        REQUIRED: "Value is required and can not be empty",
    }

    def __init__(self, empty_validator=None, *args, **kwargs):
        self.empty_validator = empty_validator or NotNone()
        super(Required, self).__init__(*args, **kwargs)

    def _internal_is_valid(self, value, *args, **kwargs):
        if not self.empty_validator.is_valid(value):
            self.error(self.REQUIRED, value)
            return False

        return super(Required, self)._internal_is_valid(value, *args, **kwargs)

class Optional(Chain):

    def __init__(self, empty_validator=None, *args, **kwargs):
        self.empty_validator = empty_validator or NotNone()
        super(Optional, self).__init__(*args, **kwargs)

    def _internal_is_valid(self, value, *args, **kwargs):
        if not self.empty_validator.is_valid(value):
            return True

        return Chain._internal_is_valid(self, value, *args, **kwargs)

class ModelValidateMetaclass(type):

    @classmethod
    def __prepare__(metacls, name, bases):  # No keywords in this case
        return OrderedDict()

    def __new__(cls, name, bases, classdict):

        result = super(ModelValidateMetaclass, cls).__new__(
            cls, name, bases, classdict)

        spec = OrderedDict([(field, validator) for field, validator in classdict.items()
                            if hasattr(validator, 'is_valid')])

        setattr(result, 'spec', spec)
        return result

class ModelValidate(BaseSpec, metaclass=ModelValidateMetaclass):

    __modelclass__ = BaseModel

    INVALID_MODEL = 'notModel'

    error_messages = {
        INVALID_MODEL: "'$value' is not an instance of $model",
    }

    def __init__(self, spec=None, *args, **kwargs):
        self.spec = self.spec.copy()
        if spec is not None:
            self.spec.update(spec)

        super(ModelValidate, self).__init__(*args, **kwargs)

    def get_field_value(self, field_name, value, kwargs):
        try:
            kwargs['is_modified'] = value.is_modified_field(field_name)
        except:
            pass
        try:
            return getattr(value, field_name)
        except AttributeError:
            return None

    def _internal_is_valid(self, value, *args, **kwargs):
        if not isinstance(value, self.__modelclass__):
            self.error(self.INVALID_MODEL, value, model=self.__modelclass__.__name__)
            return False

        return super(ModelValidate, self)._internal_is_valid(value, *args, **kwargs)
<|endoftext|>


<commit_before>"""
App configuration for ``icekit_events`` app.
"""

# Register signal handlers, but avoid interacting with the database.
# See: https://docs.djangoproject.com/en/1.8/ref/applications/#django.apps.AppConfig.ready

from django.apps import AppConfig
from django.utils.module_loading import autodiscover_modules

class AppConfig(AppConfig):
    name = '_'.join(__name__.split('.')[:-1])
    label = 'icekit_events'
    verbose_name = "Events"

    def ready(self):
        # look through installed apps to see what event types are registered
        autodiscover_modules('event_type_plugins')
<commit_msg>Register base event for ICEKitURLField<commit_after>"""
App configuration for ``icekit_events`` app.
"""

# Register signal handlers, but avoid interacting with the database.
# See: https://docs.djangoproject.com/en/1.8/ref/applications/#django.apps.AppConfig.ready

from django.apps import AppConfig
from django.utils.module_loading import autodiscover_modules

from any_urlfield.forms import SimpleRawIdWidget

from icekit.fields import ICEkitURLField

class AppConfig(AppConfig):
    name = '_'.join(__name__.split('.')[:-1])
    label = 'icekit_events'
    verbose_name = "Events"

    def ready(self):
        # look through installed apps to see what event types are registered
        autodiscover_modules('event_type_plugins')

        from .models import EventBase
        ICEkitURLField.register_model(
            EventBase, widget=SimpleRawIdWidget(EventBase), title='Event')
<|endoftext|>