<commit_before>from __future__ import absolute_import

import cPickle
import itertools
import operator
from time import time
import pdb

from lasagne.layers import get_all_layers
from lasagne.layers import get_all_params
from lasagne.objectives import mse
from lasagne.updates import nesterov_momentum
import numpy as np
from sklearn.base import BaseEstimator
from sklearn.cross_validation import KFold
from sklearn.cross_validation import StratifiedKFold
from sklearn.metrics import accuracy_score
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import LabelEncoder
import theano
from theano import tensor as T

class _list(list):
    pass

class _dict(dict):
    def __contains__(self, key):
        return True

class ansi:
    BLUE = '\033[94m'
    GREEN = '\033[32m'
    ENDC = '\033[0m'

def negative_log_likelihood(output, prediction):
    return -T.mean(T.log(output)[T.arange(prediction.shape[0]), prediction])

class BatchIterator(object):
    def __init__(self, batch_size):
        self.batch_size = batch_size

    def __call__(self, X, y=None):
        self.X, self.y = X, y
        return self

    def __iter__(self):
        n_samples = self.X.shape[0]
        bs = self.batch_size
        for i in range((n_samples + bs - 1) / bs):
            sl = slice(i * bs, (i + 1) * bs)
            Xb = self.X[sl]
            if self.y is not None:
                yb = self.y[sl]
            else:
                yb = None
            yield self.transform(Xb, yb)

    def transform(self, Xb, yb):
        return Xb, yb

class NeuralNet(BaseEstimator):
    """A scikit-learn estimator based on Lasagne.
    """
    def __init__(
        self,
        layers,
        update=nesterov_momentum,
        loss=None,
        batch_iterator_train=BatchIterator(batch_size=128),
        batch_iterator_test=BatchIterator(batch_size=128),
        regression=False,
        max_epochs=100,
        eval_size=0.2,
        X_tensor_type=None,
        y_tensor_type=None,
        use_label_encoder=False,
        on_epoch_finished=(),
        on_training_finished=(),
        more_params=None,
        verbose=0,
        **kwargs
        ):
        if loss is None:
            loss = mse if regression else negative_log_likelihood
        if X_tensor_type is None:
            types = {
                2: T.matrix,
                3: T.tensor3,
                4: T.tensor4,
                }
            X_tensor_type = types[len(kwargs['input_shape'])]
        if y_tensor_type is None:
            y_tensor_type = T.fmatrix if regression else T.ivector

        self.layers = layers
        self.update = update
        self.loss = loss
        self.batch_iterator_train = batch_iterator_train
        self.batch_iterator_test = batch_iterator_test
        self.regression = regression
        self.max_epochs = max_epochs
        self.eval_size = eval_size
        self.X_tensor_type = X_tensor_type
        self.y_tensor_type = y_tensor_type
        self.use_label_encoder = use_label_encoder
        self.on_epoch_finished = on_epoch_finished
        self.on_training_finished = on_training_finished
        self.more_params = more_params or {}
        self.verbose = verbose

        for key in kwargs.keys():
            assert not hasattr(self, key)
        vars(self).update(kwargs)
        self._kwarg_keys = kwargs.keys()

        self.train_history_ = []

        if 'batch_iterator' in kwargs:  # BBB
            raise ValueError(
                "The 'batch_iterator' argument has been replaced. "
                "Use 'batch_iterator_train' and 'batch_iterator_test' instead."
                )

    def fit(self, X, y):
        if not self.regression and self.use_label_encoder:
            self.enc_ = LabelEncoder()
            y = self.enc_.fit_transform(y).astype(np.int32)
            self.classes_ = self.enc_.classes_

        out = getattr(self, '_output_layer', None)
        if out is None:
            out = self._output_layer = self.initialize_layers()
        if self.verbose:
            self._print_layer_info(self.get_all_layers())

        iter_funcs = self._create_iter_funcs(
            out, self.loss, self.update,
            self.X_tensor_type,
            self.y_tensor_type,
            )
        self.train_iter_, self.eval_iter_, self.predict_iter_ = iter_funcs

        try:
            self.train_loop(X, y)
        except KeyboardInterrupt:
            pdb.set_trace()
        return self

    def train_loop(self, X, y):
        X_train, X_valid, y_train, y_valid = self.train_test_split(
            X, y, self.eval_size)

        on_epoch_finished = self.on_epoch_finished
        if not isinstance(on_epoch_finished, (list, tuple)):
            on_epoch_finished = [on_epoch_finished]

        on_training_finished = self.on_training_finished
        if not isinstance(on_training_finished, (list, tuple)):
            on_training_finished = [on_training_finished]

        epoch = 0
        info = None
        best_valid_loss = np.inf
        best_train_loss = np.inf

        if self.verbose:
            print("""
 Epoch  |  Train loss  |  Valid loss  |  Train / Val  |  Valid acc  |  Dur
--------|--------------|--------------|---------------|-------------|-------\
""")

        while epoch < self.max_epochs:
            epoch += 1

            train_losses = []
            valid_losses = []
            valid_accuracies = []

            t0 = time()

            for Xb, yb in self.batch_iterator_train(X_train, y_train):
                batch_train_loss = self.train_iter_(Xb, yb)
                train_losses.append(batch_train_loss)

            for Xb, yb in self.batch_iterator_test(X_valid, y_valid):
                batch_valid_loss, accuracy = self.eval_iter_(Xb, yb)
                valid_losses.append(batch_valid_loss)
                valid_accuracies.append(accuracy)

            avg_train_loss = np.mean(train_losses)
            avg_valid_loss = np.mean(valid_losses)
            avg_valid_accuracy = np.mean(valid_accuracies)

            if avg_train_loss < best_train_loss:
                best_train_loss = avg_train_loss
            if avg_valid_loss < best_valid_loss:
                best_valid_loss = avg_valid_loss

            if self.verbose:
                best_train = best_train_loss == avg_train_loss
                best_valid = best_valid_loss == avg_valid_loss
                print(" {:>5}  |  {}{:>10.6f}{}  |  {}{:>10.6f}{}  "
                      "|  {:>11.6f}  |  {:>9}  |  {:>3.1f}s".format(
                          epoch,
                          ansi.BLUE if best_train else "",
                          avg_train_loss,
                          ansi.ENDC if best_train else "",
                          ansi.GREEN if best_valid else "",
                          avg_valid_loss,
                          ansi.ENDC if best_valid else "",
                          avg_train_loss / avg_valid_loss,
                          "{:.2f}%".format(avg_valid_accuracy * 100)
                          if not self.regression else "",
                          time() - t0,
                          ))

            info = dict(
                epoch=epoch,
                train_loss=avg_train_loss,
                valid_loss=avg_valid_loss,
                valid_accuracy=avg_valid_accuracy,
                )
            self.train_history_.append(info)
            try:
                for func in on_epoch_finished:
                    func(self, self.train_history_)
            except StopIteration:
                break

        for func in on_training_finished:
            func(self, self.train_history_)

    def predict_proba(self, X):
        probas = []
        for Xb, yb in self.batch_iterator_test(X):
            probas.append(self.predict_iter_(Xb))
        return np.vstack(probas)

    def predict(self, X):
        if self.regression:
            return self.predict_proba(X)
        else:
            y_pred = np.argmax(self.predict_proba(X), axis=1)
            if self.use_label_encoder:
                y_pred = self.enc_.inverse_transform(y_pred)
            return y_pred

    def score(self, X, y):
        score = mean_squared_error if self.regression else accuracy_score
        return float(score(self.predict(X), y))

    def train_test_split(self, X, y, eval_size):
        if self.regression:
            kf = KFold(y.shape[0], 1. / eval_size)
        else:
            kf = StratifiedKFold(y, 1. / eval_size)

        train_indices, valid_indices = iter(kf).next()
        X_train, y_train = X[train_indices], y[train_indices]
        X_valid, y_valid = X[valid_indices], y[valid_indices]
        return X_train, X_valid, y_train, y_valid

    def get_all_layers(self):
        return get_all_layers(self._output_layer)[::-1]

    def get_all_params(self):
        return get_all_params(self._output_layer)[::-1]

    def _create_iter_funcs(self, output_layer, loss_func, update, input_type,
                           output_type):
        X = input_type('x')
        y = output_type('y')
        X_batch = input_type('x_batch')
        y_batch = output_type('y_batch')

        loss_train = loss_func(
            output_layer.get_output(X_batch), y_batch)
        loss_eval = loss_func(
            output_layer.get_output(X_batch, deterministic=True), y_batch)
        predict_proba = output_layer.get_output(X_batch, deterministic=True)
        if not self.regression:
            predict = predict_proba.argmax(axis=1)
            accuracy = T.mean(T.eq(predict, y_batch))
        else:
            accuracy = loss_eval

        all_params = get_all_params(output_layer)
        update_params = self._get_params_for('update')
        updates = update(loss_train, all_params, **update_params)

        train_iter = theano.function(
            inputs=[theano.Param(X_batch), theano.Param(y_batch)],
            outputs=[loss_train],
            updates=updates,
            givens={
                X: X_batch,
                y: y_batch,
                },
            )
        eval_iter = theano.function(
            inputs=[theano.Param(X_batch), theano.Param(y_batch)],
            outputs=[loss_eval, accuracy],
            givens={
                X: X_batch,
                y: y_batch,
                },
            )
        predict_iter = theano.function(
            inputs=[theano.Param(X_batch)],
            outputs=predict_proba,
            givens={
                X: X_batch,
                },
            )

        return train_iter, eval_iter, predict_iter

    def _get_params_for(self, name):
        collected = {}
        prefix = '{}_'.format(name)

        params = vars(self)
        more_params = self.more_params

        for key, value in itertools.chain(params.items(), more_params.items()):
            if key.startswith(prefix):
                collected[key[len(prefix):]] = value

        return collected

    def load_weights_from(self, source):
        self._output_layer = self.initialize_layers()

        if isinstance(source, str):
            source = np.load(source)

        if isinstance(source, NeuralNet):
            source = source.get_all_params()

        source_weights = [
            w.get_value() if hasattr(w, 'get_value') else w for w in source]

        for w1, w2 in zip(source_weights, self.get_all_params()):
            if w1.shape != w2.get_value().shape:
                continue
            w2.set_value(w1)

    def save_weights_to(self, fname):
        weights = [w.get_value() for w in self.get_all_params()]
        with open(fname, 'wb') as f:
            cPickle.dump(weights, f, -1)

    def initialize_layers(self, layers=None):
        if layers is not None:
            self.layers = layers

        input_layer_name, input_layer_factory = self.layers[0]
        input_layer_params = self._get_params_for(input_layer_name)
        layer = input_layer_factory(**input_layer_params)

        for (layer_name, layer_factory) in self.layers[1:]:
            layer_params = self._get_params_for(layer_name)
            layer = layer_factory(layer, **layer_params)

        self._output_layer = layer
        return layer

    def _print_layer_info(self, layers):
        for layer in layers:
            output_shape = layer.get_output_shape()
            print("  {:<18}\t{:<20}\tproduces {:>7} outputs".format(
                layer.__class__.__name__,
                output_shape,
                reduce(operator.mul, output_shape[1:]),
                ))

    def get_params(self, deep=True):
        params = super(NeuralNet, self).get_params(deep=deep)

        # Incidentally, Lasagne layers have a 'get_params' too, which
        # for sklearn's 'clone' means it would treat it in a special
        # way when cloning.  Wrapping the list of layers in a custom
        # list type does the trick here, but of course it's crazy:
        params['layers'] = _list(params['layers'])
        return _dict(params)

    def _get_param_names(self):
        # This allows us to have **kwargs in __init__ (woot!):
        param_names = super(NeuralNet, self)._get_param_names()
        return param_names + self._kwarg_keys
<commit_msg>Fix #19: Don't use pdb when ^C is pressed.<commit_after>from __future__ import absolute_import

import cPickle
import itertools
import operator
from time import time
import pdb

from lasagne.layers import get_all_layers
from lasagne.layers import get_all_params
from lasagne.objectives import mse
from lasagne.updates import nesterov_momentum
import numpy as np
from sklearn.base import BaseEstimator
from sklearn.cross_validation import KFold
from sklearn.cross_validation import StratifiedKFold
from sklearn.metrics import accuracy_score
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import LabelEncoder
import theano
from theano import tensor as T

class _list(list):
    pass

class _dict(dict):
    def __contains__(self, key):
        return True

class ansi:
    BLUE = '\033[94m'
    GREEN = '\033[32m'
    ENDC = '\033[0m'

def negative_log_likelihood(output, prediction):
    return -T.mean(T.log(output)[T.arange(prediction.shape[0]), prediction])

class BatchIterator(object):
    def __init__(self, batch_size):
        self.batch_size = batch_size

    def __call__(self, X, y=None):
        self.X, self.y = X, y
        return self

    def __iter__(self):
        n_samples = self.X.shape[0]
        bs = self.batch_size
        for i in range((n_samples + bs - 1) / bs):
            sl = slice(i * bs, (i + 1) * bs)
            Xb = self.X[sl]
            if self.y is not None:
                yb = self.y[sl]
            else:
                yb = None
            yield self.transform(Xb, yb)

    def transform(self, Xb, yb):
        return Xb, yb

class NeuralNet(BaseEstimator):
    """A scikit-learn estimator based on Lasagne.
    """
    def __init__(
        self,
        layers,
        update=nesterov_momentum,
        loss=None,
        batch_iterator_train=BatchIterator(batch_size=128),
        batch_iterator_test=BatchIterator(batch_size=128),
        regression=False,
        max_epochs=100,
        eval_size=0.2,
        X_tensor_type=None,
        y_tensor_type=None,
        use_label_encoder=False,
        on_epoch_finished=(),
        on_training_finished=(),
        more_params=None,
        verbose=0,
        **kwargs
        ):
        if loss is None:
            loss = mse if regression else negative_log_likelihood
        if X_tensor_type is None:
            types = {
                2: T.matrix,
                3: T.tensor3,
                4: T.tensor4,
                }
            X_tensor_type = types[len(kwargs['input_shape'])]
        if y_tensor_type is None:
            y_tensor_type = T.fmatrix if regression else T.ivector

        self.layers = layers
        self.update = update
        self.loss = loss
        self.batch_iterator_train = batch_iterator_train
        self.batch_iterator_test = batch_iterator_test
        self.regression = regression
        self.max_epochs = max_epochs
        self.eval_size = eval_size
        self.X_tensor_type = X_tensor_type
        self.y_tensor_type = y_tensor_type
        self.use_label_encoder = use_label_encoder
        self.on_epoch_finished = on_epoch_finished
        self.on_training_finished = on_training_finished
        self.more_params = more_params or {}
        self.verbose = verbose

        for key in kwargs.keys():
            assert not hasattr(self, key)
        vars(self).update(kwargs)
        self._kwarg_keys = kwargs.keys()

        self.train_history_ = []

        if 'batch_iterator' in kwargs:  # BBB
            raise ValueError(
                "The 'batch_iterator' argument has been replaced. "
                "Use 'batch_iterator_train' and 'batch_iterator_test' instead."
                )

    def fit(self, X, y):
        if not self.regression and self.use_label_encoder:
            self.enc_ = LabelEncoder()
            y = self.enc_.fit_transform(y).astype(np.int32)
            self.classes_ = self.enc_.classes_

        out = getattr(self, '_output_layer', None)
        if out is None:
            out = self._output_layer = self.initialize_layers()
        if self.verbose:
            self._print_layer_info(self.get_all_layers())

        iter_funcs = self._create_iter_funcs(
            out, self.loss, self.update,
            self.X_tensor_type,
            self.y_tensor_type,
            )
        self.train_iter_, self.eval_iter_, self.predict_iter_ = iter_funcs

        try:
            self.train_loop(X, y)
        except KeyboardInterrupt:
            pass
        return self

    def train_loop(self, X, y):
        X_train, X_valid, y_train, y_valid = self.train_test_split(
            X, y, self.eval_size)

        on_epoch_finished = self.on_epoch_finished
        if not isinstance(on_epoch_finished, (list, tuple)):
            on_epoch_finished = [on_epoch_finished]

        on_training_finished = self.on_training_finished
        if not isinstance(on_training_finished, (list, tuple)):
            on_training_finished = [on_training_finished]

        epoch = 0
        info = None
        best_valid_loss = np.inf
        best_train_loss = np.inf

        if self.verbose:
            print("""
 Epoch  |  Train loss  |  Valid loss  |  Train / Val  |  Valid acc  |  Dur
--------|--------------|--------------|---------------|-------------|-------\
""")

        while epoch < self.max_epochs:
            epoch += 1

            train_losses = []
            valid_losses = []
            valid_accuracies = []

            t0 = time()

            for Xb, yb in self.batch_iterator_train(X_train, y_train):
                batch_train_loss = self.train_iter_(Xb, yb)
                train_losses.append(batch_train_loss)

            for Xb, yb in self.batch_iterator_test(X_valid, y_valid):
                batch_valid_loss, accuracy = self.eval_iter_(Xb, yb)
                valid_losses.append(batch_valid_loss)
                valid_accuracies.append(accuracy)

            avg_train_loss = np.mean(train_losses)
            avg_valid_loss = np.mean(valid_losses)
            avg_valid_accuracy = np.mean(valid_accuracies)

            if avg_train_loss < best_train_loss:
                best_train_loss = avg_train_loss
            if avg_valid_loss < best_valid_loss:
                best_valid_loss = avg_valid_loss

            if self.verbose:
                best_train = best_train_loss == avg_train_loss
                best_valid = best_valid_loss == avg_valid_loss
                print(" {:>5}  |  {}{:>10.6f}{}  |  {}{:>10.6f}{}  "
                      "|  {:>11.6f}  |  {:>9}  |  {:>3.1f}s".format(
                          epoch,
                          ansi.BLUE if best_train else "",
                          avg_train_loss,
                          ansi.ENDC if best_train else "",
                          ansi.GREEN if best_valid else "",
                          avg_valid_loss,
                          ansi.ENDC if best_valid else "",
                          avg_train_loss / avg_valid_loss,
                          "{:.2f}%".format(avg_valid_accuracy * 100)
                          if not self.regression else "",
                          time() - t0,
                          ))

            info = dict(
                epoch=epoch,
                train_loss=avg_train_loss,
                valid_loss=avg_valid_loss,
                valid_accuracy=avg_valid_accuracy,
                )
            self.train_history_.append(info)
            try:
                for func in on_epoch_finished:
                    func(self, self.train_history_)
            except StopIteration:
                break

        for func in on_training_finished:
            func(self, self.train_history_)

    def predict_proba(self, X):
        probas = []
        for Xb, yb in self.batch_iterator_test(X):
            probas.append(self.predict_iter_(Xb))
        return np.vstack(probas)

    def predict(self, X):
        if self.regression:
            return self.predict_proba(X)
        else:
            y_pred = np.argmax(self.predict_proba(X), axis=1)
            if self.use_label_encoder:
                y_pred = self.enc_.inverse_transform(y_pred)
            return y_pred

    def score(self, X, y):
        score = mean_squared_error if self.regression else accuracy_score
        return float(score(self.predict(X), y))

    def train_test_split(self, X, y, eval_size):
        if self.regression:
            kf = KFold(y.shape[0], 1. / eval_size)
        else:
            kf = StratifiedKFold(y, 1. / eval_size)

        train_indices, valid_indices = iter(kf).next()
        X_train, y_train = X[train_indices], y[train_indices]
        X_valid, y_valid = X[valid_indices], y[valid_indices]
        return X_train, X_valid, y_train, y_valid

    def get_all_layers(self):
        return get_all_layers(self._output_layer)[::-1]

    def get_all_params(self):
        return get_all_params(self._output_layer)[::-1]

    def _create_iter_funcs(self, output_layer, loss_func, update, input_type,
                           output_type):
        X = input_type('x')
        y = output_type('y')
        X_batch = input_type('x_batch')
        y_batch = output_type('y_batch')

        loss_train = loss_func(
            output_layer.get_output(X_batch), y_batch)
        loss_eval = loss_func(
            output_layer.get_output(X_batch, deterministic=True), y_batch)
        predict_proba = output_layer.get_output(X_batch, deterministic=True)
        if not self.regression:
            predict = predict_proba.argmax(axis=1)
            accuracy = T.mean(T.eq(predict, y_batch))
        else:
            accuracy = loss_eval

        all_params = get_all_params(output_layer)
        update_params = self._get_params_for('update')
        updates = update(loss_train, all_params, **update_params)

        train_iter = theano.function(
            inputs=[theano.Param(X_batch), theano.Param(y_batch)],
            outputs=[loss_train],
            updates=updates,
            givens={
                X: X_batch,
                y: y_batch,
                },
            )
        eval_iter = theano.function(
            inputs=[theano.Param(X_batch), theano.Param(y_batch)],
            outputs=[loss_eval, accuracy],
            givens={
                X: X_batch,
                y: y_batch,
                },
            )
        predict_iter = theano.function(
            inputs=[theano.Param(X_batch)],
            outputs=predict_proba,
            givens={
                X: X_batch,
                },
            )

        return train_iter, eval_iter, predict_iter

    def _get_params_for(self, name):
        collected = {}
        prefix = '{}_'.format(name)

        params = vars(self)
        more_params = self.more_params

        for key, value in itertools.chain(params.items(), more_params.items()):
            if key.startswith(prefix):
                collected[key[len(prefix):]] = value

        return collected

    def load_weights_from(self, source):
        self._output_layer = self.initialize_layers()

        if isinstance(source, str):
            source = np.load(source)

        if isinstance(source, NeuralNet):
            source = source.get_all_params()

        source_weights = [
            w.get_value() if hasattr(w, 'get_value') else w for w in source]

        for w1, w2 in zip(source_weights, self.get_all_params()):
            if w1.shape != w2.get_value().shape:
                continue
            w2.set_value(w1)

    def save_weights_to(self, fname):
        weights = [w.get_value() for w in self.get_all_params()]
        with open(fname, 'wb') as f:
            cPickle.dump(weights, f, -1)

    def initialize_layers(self, layers=None):
        if layers is not None:
            self.layers = layers

        input_layer_name, input_layer_factory = self.layers[0]
        input_layer_params = self._get_params_for(input_layer_name)
        layer = input_layer_factory(**input_layer_params)

        for (layer_name, layer_factory) in self.layers[1:]:
            layer_params = self._get_params_for(layer_name)
            layer = layer_factory(layer, **layer_params)

        self._output_layer = layer
        return layer

    def _print_layer_info(self, layers):
        for layer in layers:
            output_shape = layer.get_output_shape()
            print("  {:<18}\t{:<20}\tproduces {:>7} outputs".format(
                layer.__class__.__name__,
                output_shape,
                reduce(operator.mul, output_shape[1:]),
                ))

    def get_params(self, deep=True):
        params = super(NeuralNet, self).get_params(deep=deep)

        # Incidentally, Lasagne layers have a 'get_params' too, which
        # for sklearn's 'clone' means it would treat it in a special
        # way when cloning.  Wrapping the list of layers in a custom
        # list type does the trick here, but of course it's crazy:
        params['layers'] = _list(params['layers'])
        return _dict(params)

    def _get_param_names(self):
        # This allows us to have **kwargs in __init__ (woot!):
        param_names = super(NeuralNet, self)._get_param_names()
        return param_names + self._kwarg_keys
<|endoftext|>


<commit_before>from astropy_helpers.git_helpers import get_git_devstr

__all__ = [
    "__title__", "__summary__", "__uri__", "__version__", "__author__",
    "__email__", "__license__", "__copyright__",
]

__title__ = "PyNOMAD"
__summary__ = "Routines for accessing a self-hosted local copy of the USNO NOMAD stellar catalog"
__uri__ = "https://github.com/henryroe/PyNOMAD"

# VERSION should be PEP386 compatible (http://www.python.org/dev/peps/pep-0386)
__version__ = "0.1.1dev"

# Indicates if this version is a release version
RELEASE = 'dev' not in __version__

if not RELEASE:
    __version__ += get_git_devstr(False)

__author__ = "Henry Roe"
__email__ = "hroe@hroe.me"

__license__ = "MIT License"
__copyright__ = "2015 %s" % __author__
<commit_msg>removed dev from version number<commit_after>from astropy_helpers.git_helpers import get_git_devstr

__all__ = [
    "__title__", "__summary__", "__uri__", "__version__", "__author__",
    "__email__", "__license__", "__copyright__",
]

__title__ = "PyNOMAD"
__summary__ = "Routines for accessing a self-hosted local copy of the USNO NOMAD stellar catalog"
__uri__ = "https://github.com/henryroe/PyNOMAD"

# VERSION should be PEP386 compatible (http://www.python.org/dev/peps/pep-0386)
__version__ = "0.1.1"

# Indicates if this version is a release version
RELEASE = 'dev' not in __version__

if not RELEASE:
    __version__ += get_git_devstr(False)

__author__ = "Henry Roe"
__email__ = "hroe@hroe.me"

__license__ = "MIT License"
__copyright__ = "2015 %s" % __author__
<|endoftext|>


<commit_before># -*- coding: utf8 -*-
from django.conf.urls import patterns, url

urlpatterns = patterns(
    '',
    url(r'^login/$', 'nopassword.views.login', name='login'),
    url(r'^login-code/(?P<login_code>[a-zA-Z0-9]+)/$',
        'nopassword.views.login_with_code'),
    url(r'^login-code/(?P<username>[a-zA-Z0-9_@\.-]+)/(?P<login_code>[a-zA-Z0-9]+)/$',
        'nopassword.views.login_with_code_and_username'),
    url(r'^logout/$', 'nopassword.views.logout'),
)
<commit_msg>Remove deprecated patterns<commit_after># -*- coding: utf8 -*-
from django.conf.urls import url

urlpatterns = [
    url(r'^login/$', 'nopassword.views.login', name='login'),
    url(r'^login-code/(?P<login_code>[a-zA-Z0-9]+)/$',
        'nopassword.views.login_with_code'),
    url(r'^login-code/(?P<username>[a-zA-Z0-9_@\.-]+)/(?P<login_code>[a-zA-Z0-9]+)/$',
        'nopassword.views.login_with_code_and_username'),
    url(r'^logout/$', 'nopassword.views.logout'),
]
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from argparse import ArgumentParser

from topics.modules.guild_changes import endGuild, excludeFromGuild
from lib.guilds import Player, Guild
from lib.commands import database

import backup_database
import make_weekly_posts
import make_eweek_post
import update_guilds
import check_topics
import eweek_notify

scripts = {
	"backupdata":backup_database.backup,
	"mkweekly":make_weekly_posts.generate,
	"mkeweek":make_eweek_post.make,
	"updguilds":update_guilds.updateAllGuilds,
	"chtopics":check_topics.main,
	"notify":eweek_notify.notify,
}

def main():
	parser = makeParser()
	request = parser.parse_args()
	handleRequest(request)

def makeParser():
	main_parser = ArgumentParser("Guild Manager")
	script_help = "[{}]".format(", ".join(list(scripts.keys())))
	main_parser.add_argument("-s", "--script", dest="scripts",
		nargs="+", choices=scripts.keys(), metavar="", help=script_help)
	makeGuildParser(main_parser)
	return main_parser

def makeGuildParser(main_parser):
	main_subparsers = main_parser.add_subparsers()
	guilds = main_subparsers.add_parser("guilds")
	guilds_subparsers = guilds.add_subparsers(dest="guild_command")
	exclude = guilds_subparsers.add_parser("exclude")
	exclude.add_argument("id")
	setrank = guilds_subparsers.add_parser("setrank")
	setrank.add_argument("id")
	setrank.add_argument("rank", choices=["player", "vice", "head"])
	endguild = guilds_subparsers.add_parser("endguild")
	endguild.add_argument("id")

def handleRequest(request):
	if request.scripts is not None:
		for script in request.scripts:
			scripts[script]()
			print("Script '{}' finished".format(script))
	elif hasattr(request, "command"):
		if request.command == "exclude":
			player = Player(request.id)
			excludeFromGuild(player)
			print(player.get("name"), "excluded.")
		elif request.command == "setrank":
			player.guild.setPosition(request.id, request.rank)
		elif request.command == "endguild":
			endGuild(Guild(request.id))
		database.rewrite()

main()
<commit_msg>[fix] more logical lookup (in instead of hasattr)<commit_after>#!/usr/bin/env python3

from argparse import ArgumentParser

from topics.modules.guild_changes import endGuild, excludeFromGuild
from lib.guilds import Player, Guild
from lib.commands import database

import backup_database
import make_weekly_posts
import make_eweek_post
import update_guilds
import check_topics
import eweek_notify

scripts = {
	"backupdata":backup_database.backup,
	"mkweekly":make_weekly_posts.generate,
	"mkeweek":make_eweek_post.make,
	"updguilds":update_guilds.updateAllGuilds,
	"chtopics":check_topics.main,
	"notify":eweek_notify.notify,
}

def main():
	parser = makeParser()
	request = parser.parse_args()
	handleRequest(request)

def makeParser():
	main_parser = ArgumentParser("Guild Manager")
	script_help = "[{}]".format(", ".join(list(scripts.keys())))
	main_parser.add_argument("-s", "--script", dest="scripts",
		nargs="+", choices=scripts.keys(), metavar="", help=script_help)
	makeGuildParser(main_parser)
	return main_parser

def makeGuildParser(main_parser):
	main_subparsers = main_parser.add_subparsers()
	guilds = main_subparsers.add_parser("guilds")
	guilds_subparsers = guilds.add_subparsers(dest="guild_command")
	exclude = guilds_subparsers.add_parser("exclude")
	exclude.add_argument("id")
	setrank = guilds_subparsers.add_parser("setrank")
	setrank.add_argument("id")
	setrank.add_argument("rank", choices=["player", "vice", "head"])
	endguild = guilds_subparsers.add_parser("endguild")
	endguild.add_argument("id")

def handleRequest(request):
	if request.scripts is not None:
		for script in request.scripts:
			scripts[script]()
			print("Script '{}' finished".format(script))
	elif "command" in request:
		if request.command == "exclude":
			player = Player(request.id)
			excludeFromGuild(player)
			print(player.get("name"), "excluded.")
		elif request.command == "setrank":
			player.guild.setPosition(request.id, request.rank)
		elif request.command == "endguild":
			endGuild(Guild(request.id))
		database.rewrite()
	else:
		print(request)

main()
<|endoftext|>


<commit_before>#encoding: utf-8

from django.db import models
from ckeditor.fields import RichTextField
from django.template.defaultfilters import slugify
from multimedia.models import Fotos
from django.contrib.auth.models import User
from django.contrib.contenttypes import generic
from django.core.urlresolvers import reverse
from taggit.managers import TaggableManager
from multimedia.models import Adjuntos
from eventos.models import Categoria
from socios.models import Pais

# Create your models here.
class Noticias(models.Model):
    titulo = models.CharField(max_length=200)
    slug = models.SlugField(max_length=200,editable=False)
    fecha = models.DateField()
    descripcion = RichTextField('Descripción')
    fotos = generic.GenericRelation(Fotos)
    categoria= models.ManyToManyField(Categoria,
                                    null=True, blank=True)
    pais = models.ForeignKey(Pais)
    tags = TaggableManager(blank=True)
    destacada = models.BooleanField()
    adjunto = generic.GenericRelation(Adjuntos)

    autor = models.ForeignKey(User)

    def save(self, *args, **kwargs):
        self.slug = (slugify(self.titulo))
        super(Noticias, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return reverse('noticias_detalle', kwargs={'slug': self.slug})

    class Meta:
        verbose_name = 'Noticia'
        verbose_name_plural = 'Noticias'
        ordering = ('-fecha',)

    def __unicode__(self):
        return u'%s' % self.titulo

class InicioTexto(models.Model):
    nombre = models.CharField(max_length=150)
    texto = RichTextField()

    def __unicode__(self):
        return u'%s' % (self.nombre)<commit_msg>fix slug<commit_after>#encoding: utf-8

from django.db import models
from ckeditor.fields import RichTextField
from django.template.defaultfilters import slugify
from multimedia.models import Fotos
from django.contrib.auth.models import User
from django.contrib.contenttypes import generic
from django.core.urlresolvers import reverse
from taggit.managers import TaggableManager
from multimedia.models import Adjuntos
from eventos.models import Categoria
from socios.models import Pais

# Create your models here.
class Noticias(models.Model):
    titulo = models.CharField(max_length=200)
    slug = models.SlugField(max_length=200,editable=False)
    fecha = models.DateField()
    descripcion = RichTextField('Descripción')
    fotos = generic.GenericRelation(Fotos)
    categoria= models.ManyToManyField(Categoria,
                                    null=True, blank=True)
    pais = models.ForeignKey(Pais)
    tags = TaggableManager(blank=True)
    destacada = models.BooleanField()
    adjunto = generic.GenericRelation(Adjuntos)

    autor = models.ForeignKey(User)

    def save(self, *args, **kwargs):
        add = not self.pk
        super(Noticias, self).save(*args, **kwargs)
        if add:
            self.slug = (slugify(self.titulo)) + '-' + str(self.id)
            kwargs['force_insert'] = False # create() uses this, which causes error.
            super(Noticias, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return reverse('noticias_detalle', kwargs={'slug': self.slug})

    class Meta:
        verbose_name = 'Noticia'
        verbose_name_plural = 'Noticias'
        ordering = ('-fecha',)
        unique_together = ('slug',)

    def __unicode__(self):
        return u'%s' % self.titulo

class InicioTexto(models.Model):
    nombre = models.CharField(max_length=150)
    texto = RichTextField()

    def __unicode__(self):
        return u'%s' % (self.nombre)<|endoftext|>


<commit_before># python 2/3 interoperability
from __future__ import division

try:
    range = xrange
except NameError:
    pass

try:
    long
except NameError:
    long = int

import math
import types
from collections import defaultdict, OrderedDict
import numpy as np
from numba import jit
import scipy as sp
from scipy.sparse import csc_matrix, csr_matrix, coo_matrix, SparseEfficiencyWarning
import warnings

def _fix_empty_features(feature_mat):
    if feature_mat.format != 'csc':
        raise NotImplementedError
    nf = feature_mat.getnnz(axis=1)
    zf = (nf == 0)
    if zf.any():
        # add +1 dummy feature for every zero row
        # to avoid zeros on diagonal of similarity matrix
        nnz = zf.sum(dtype=long) #long is required to avoid overflow with int32 dtype
        nnz_ind = np.where(zf)[0]
        feature_mat.indptr = np.hstack([feature_mat.indptr[:-1], feature_mat.indptr[-1]+np.arange(nnz+1)])
        feature_mat.indices = np.hstack([feature_mat.indices, nnz_ind])
        feature_mat.data = np.hstack([feature_mat.data, np.ones((nnz,), dtype=feature_mat.data.dtype)])
        feature_mat._shape = (feature_mat.shape[0], feature_mat.shape[1]+nnz)
        nf[nnz_ind] = 1
    return nf

def set_diagonal_values(mat, val=1):
    # disable warning when setting diagonal elements of sparse similarity matrix
    with warnings.catch_warnings():
        warnings.simplefilter('ignore', category=SparseEfficiencyWarning)
        mat.setdiag(val)

def safe_inverse_root(d):
    if (d < 0).any():
        raise ValueError
    return np.piecewise(d, [d>0, d==0], [lambda x: x**(-0.5), lambda x: x])

def normalize_binary_features(feature_mat):
    sqsum = feature_mat.getnnz(axis=1)
    if feature_mat.format == 'csc':
        ind = feature_mat.indices.copy()
        ptr = feature_mat.indptr.copy()
        norm_data = 1 / np.sqrt(np.take(sqsum, ind))
        normed = csc_matrix((norm_data, ind, ptr), shape=feature_mat.shape)
    else:
        norm_data = safe_inverse_root(sqsum.astype(np.float64))
        normed = sp.sparse.diags(norm_data).dot(feature_mat)
    return normed

def normalize_features(feature_mat):
    if feature_mat.format == 'csc':
        ptr = feature_mat.indptr.copy()
        ind = feature_mat.indices.copy()
        sqsum = np.bincount(ind, weights=feature_mat.data**2)
        # could do np.add.at(sqsum, indices, data**2), but bincount seem faster
        norm_data = feature_mat.data / np.sqrt(np.take(sqsum, ind))
        normed = csc_matrix((norm_data, ind, ptr), shape=feature_mat.shape)
    else:
        sqsum = feature_mat.power(2).sum(axis=1).view(type=np.ndarray).squeeze()
        # avoid zero division
        norm_data = safe_inverse_root(sqsum.astype(np.float64))
        normed = sp.sparse.diags(norm_data).dot(feature_mat)
    return normed

def tfidf_transform(feature_mat):
    itemfreq = 1 + feature_mat.getnnz(axis=0)
    N = 1 + feature_mat.shape[0]
    idf = np.log(N/itemfreq)

    if feature_mat.format == 'csr':
        ind = feature_mat.indices
        ptr = feature_mat.indptr
        data = np.take(idf, ind)
        transformed = csr_matrix((data, ind, ptr), shape=feature_mat.shape)
    else:
        # safe multiplicationin in case data is not binary
        transformed = (feature_mat!=0).dot(sp.sparse.diags(idf))
    return transformed

@jit(nopython=True)
def _jaccard_similarity_inplace(sdata, indcs, pntrs, nf):
    # assumes data is symmetric => format doesn't matter
    # it doesn't enforce  1 on the diagonal!
    # so the similarity data must be prepared accordingly
    ncols = len(pntrs) - 1
    for col in range(ncols):
        lind = pntrs[col]
        rind = pntrs[col+1]
        nf_col = nf[col]
        for j in range(lind, rind):
            row = indcs[j]
            denom = nf_col + nf[row] - sdata[j]
            sdata[j] /= denom

def jaccard_similarity(F, fill_diagonal=True):
    F = (F!=0)
    nf = F.getnnz(axis=1)

    S = F.dot(F.T).astype(np.float64) # dtype needed for inplace jaccard index computation
    _jaccard_similarity_inplace(S.data, S.indices, S.indptr, nf)
    if fill_diagonal:
        # eigenvalues computation is very sensitive to roundoff errors in
        # similarity matrix; explicitly seting diagonal values is better
        # than using _fix_empty_features
        set_diagonal_values(S, 1)
    return S

def cosine_similarity(F, fill_diagonal=True, assume_binary=False):
    normalize = normalize_binary_features if assume_binary else normalize_features
    F = normalize(F)
    S = F.dot(F.T)
    if fill_diagonal:
        # eigenvalues computation is very sensitive to roundoff errors in
        # similarity matrix; explicitly seting diagonal values is better
        # than using _fix_empty_features
        set_diagonal_values(S, 1)
    return S

def cosine_tfidf_similarity(F, fill_diagonal=True):
    F = tfidf_transform(F)
    S = cosine_similarity(F, fill_diagonal=fill_diagonal, assume_binary=False)
    return S

@jit(nopython=True)
def _jaccard_similarity_weighted_tri(dat, ind, ptr, shift):
    z = dat[0] #np.float64
    # need to initialize lists with certain dtype
    data = [z]
    cols = [z]
    rows = [z]

    nrows = len(ptr) - 1
    for i in range(nrows):
        lind_i = ptr[i]
        rind_i = ptr[i+1]
        if lind_i != rind_i:
            ish = i + shift
            for j in range(ish, nrows):
                lind_j = ptr[j]
                rind_j = ptr[j+1]
                min_sum = 0
                max_sum = 0
                for k in range(lind_j, rind_j):
                    for s in range(lind_i, rind_i):
                        iind = ind[s]
                        jind = ind[k]
                        if iind == jind:
                            min_val = min(dat[s], dat[k])
                            max_val = max(dat[s], dat[k])
                            min_sum += min_val
                            max_sum += max_val
                            break
                    else:
                        max_sum += dat[k]

                for s in range(lind_i, rind_i):
                    for k in range(lind_j, rind_j):
                        iind = ind[s]
                        jind = ind[k]
                        if iind == jind:
                            break
                    else:
                        max_sum += dat[s]

                if min_sum:
                    wjac = min_sum / max_sum
                    data.append(wjac)
                    cols.append(i)
                    rows.append(j)

    return data[1:], rows[1:], cols[1:] #ignore initialization element

def jaccard_similarity_weighted(F, fill_diagonal=True):
    assert F.format == 'csr'
    if not F.has_sorted_indices:
        F.sort_indices()

    ind = F.indices
    ptr = F.indptr
    dat = F.data.astype(np.float64, copy=False) # dtype needed for jaccard computation

    shift = 1 if fill_diagonal else 0
    data, rows, cols = _jaccard_similarity_weighted_tri(dat, ind, ptr, shift)

    S = coo_matrix((data, (rows, cols)), shape=(F.shape[0],)*2).tocsc()
    S += S.T # doubles diagonal values if fill_diagonal is False

    if fill_diagonal:
        set_diagonal_values(S, 1)
    else:
        set_diagonal_values(S, np.sign(S.diagonal())) # set to 1, preserve zeros
    return S

def jaccard_similarity_weighted_dense(F, fill_diagonal=True):
    if (F.data < 0).any():
        raise ValueError

    f = F.sum(axis=1).view(type=np.ndarray).squeeze()
    fplus = f + f[:, None]
    FA = F.A
    fminus = np.abs(FA[None, :, :] - FA[:, None, :]).sum(axis=2)

    with np.errstate(invalid='ignore'):
        res = np.nan_to_num((fplus - fminus) / (fplus + fminus))

    if fill_diagonal:
        np.fill_diagonal(res, 1)
    return res

def uniquify_ordered(seq):
    # order preserving
    # https://www.peterbe.com/plog/uniqifiers-benchmark
    # http://stackoverflow.com/questions/480214/how-do-you-remove-duplicates-from-a-list-in-whilst-preserving-order
    seen = set()
    seen_add = seen.add
    return [x for x in seq if not (x in seen or seen_add(x))]

def build_indicator_matrix(labels, max_items=None):
    indices = [i for x in labels for i in x]
    indprt = np.r_[0, labels.apply(len).cumsum().values]
    data = np.ones_like(indices, dtype=np.bool)
    shape = [len(labels), max_items or indices.max()+1]
    return csr_matrix((data, indices, indprt), shape=shape)

def feature2sparse(feature_data, ranking=None, deduplicate=True):
    if deduplicate:
        feature_data = feature_data.apply(uniquify_ordered if ranking else set)

    feature_lbl = defaultdict(lambda: len(feature_lbl))
    indices = [feature_lbl[item] for items in feature_data for item in items]
    indptr = np.r_[0, feature_data.apply(len).cumsum().values]

    if ranking:
        if ranking is True:
            ranking = 'linear'

        if isinstance(ranking, str):
            if ranking.lower() == 'linear':
                data = [1/(n+1) for items in feature_data for n, item in enumerate(items)]
            elif ranking.lower() == 'exponential':
                data = [math.exp(-n) for items in feature_data for n, item in enumerate(items)]
            elif ranking.lower() == 'bag-of-features':
                raise NotImplementedError
            else:
                raise ValueError
        elif isinstance(ranking, types.FunctionType):
            data = [ranking(n) for items in feature_data for n, item in enumerate(items)]
    else:
        data = np.ones_like(indices)

    feature_mat = csr_matrix((data, indices, indptr),
                             shape=(feature_data.shape[0], len(feature_lbl)))
    if not ranking:
        feature_mat = feature_mat.tocsc()

    return feature_mat, dict(feature_lbl)

def get_features_data(meta_data, ranking=None, deduplicate=True):
    feature_mats = OrderedDict()
    feature_lbls = OrderedDict()
    features = meta_data.columns

    ranking = ranking or {}

    if ranking is True:
        ranking = 'linear'

    if isinstance(ranking, str):
        ranking = [ranking] * len(features)

    if not isinstance(ranking, dict):
        ranking = {k: v for k, v in zip(features, ranking)}

    for feature in features:
        feature_data = meta_data[feature]
        mat, lbl = feature2sparse(feature_data, ranking=ranking.get(feature, None), deduplicate=deduplicate)
        feature_mats[feature], feature_lbls[feature] = mat, lbl
    return feature_mats, feature_lbls

def _sim_func(func_type):
    if func_type.lower() == 'jaccard':
        return jaccard_similarity
    elif func_type.lower() == 'cosine':
        return cosine_similarity
    elif func_type.lower() == 'tfidf-cosine':
        return cosine_tfidf_similarity
    elif func_type.lower() == 'jaccard-weighted':
        return jaccard_similarity_weighted
    else:
        raise NotImplementedError

def one_hot_similarity(meta_data):
    raise NotImplementedError

def get_similarity_data(meta_data, similarity_type='jaccard'):
    features = meta_data.columns

    if isinstance(similarity_type, str):
        similarity_type = [similarity_type] * len(features)

    if not isinstance(similarity_type, dict):
        similarity_type = {k: v for k, v in zip(features, similarity_type)}

    similarity_mats = {}
    for feature in features:
        feature_data = meta_data[feature]
        sim_type = similarity_type[feature]
        ranking = sim_type == 'jaccard-weighted'
        get_similarity = _sim_func(sim_type)

        feature_mat, feature_lbl = feature2sparse(feature_data, ranking=ranking)
        similarity_mats[feature] = get_similarity(feature_mat)

    return similarity_mats

def combine_distribute_similarity_data(meta_data, similarity_type='jaccard', weights=None):
    # similarity_mats = get_similarity_data(meta_data, similarity_type)
    # iprob = {f: 1+np.prod(s.shape)/s.nnz for f, s in similarity_mats.iteritems()}
    # wsum = np.log(iprob.values()).sum()
    # weights = {f:w/wsum for f, w in iprob.iteritems()}
    raise NotImplementedError

def combine_similarity_data(meta_data, similarity_type='jaccard', weights=None):
    features = meta_data.columns
    num_feat = len(features)

    if isinstance(similarity_type, str):
        similarity_type = [similarity_type] * num_feat

    if not isinstance(similarity_type, dict):
        similarity_type = {k: v for k, v in zip(features, similarity_type)}

    if weights is None:
        weights = [1.0/num_feat] * num_feat

    if not isinstance(weights, dict):
        weights = {k: v for k, v in zip(features, weights)}

    similarity = csc_matrix((meta_data.shape[0],)*2)
    for feature in features:
        feature_data = meta_data[feature]
        sim_type = similarity_type[feature]
        weight = weights[feature]
        ranking = sim_type == 'jaccard-weighted'
        get_similarity = _sim_func(sim_type)

        feature_mat, feature_lbl = feature2sparse(feature_data, ranking=ranking)
        similarity += weight * get_similarity(feature_mat)

    # remove round off errors
    similarity.setdiag(1)
    similarity.data[similarity.data>1] = 1
    return similarity
<commit_msg>improve sparse features normalization routine<commit_after># python 2/3 interoperability
from __future__ import division

try:
    range = xrange
except NameError:
    pass

try:
    long
except NameError:
    long = int

import math
import types
from collections import defaultdict, OrderedDict
import numpy as np
from numba import jit
import scipy as sp
from scipy.sparse import csc_matrix, csr_matrix, coo_matrix, SparseEfficiencyWarning
import warnings

def _fix_empty_features(feature_mat):
    if feature_mat.format != 'csc':
        raise NotImplementedError
    nf = feature_mat.getnnz(axis=1)
    zf = (nf == 0)
    if zf.any():
        # add +1 dummy feature for every zero row
        # to avoid zeros on diagonal of similarity matrix
        nnz = zf.sum(dtype=long) #long is required to avoid overflow with int32 dtype
        nnz_ind = np.where(zf)[0]
        feature_mat.indptr = np.hstack([feature_mat.indptr[:-1], feature_mat.indptr[-1]+np.arange(nnz+1)])
        feature_mat.indices = np.hstack([feature_mat.indices, nnz_ind])
        feature_mat.data = np.hstack([feature_mat.data, np.ones((nnz,), dtype=feature_mat.data.dtype)])
        feature_mat._shape = (feature_mat.shape[0], feature_mat.shape[1]+nnz)
        nf[nnz_ind] = 1
    return nf

def set_diagonal_values(mat, val=1):
    # disable warning when setting diagonal elements of sparse similarity matrix
    with warnings.catch_warnings():
        warnings.simplefilter('ignore', category=SparseEfficiencyWarning)
        mat.setdiag(val)

def safe_inverse_root(d, dtype=None):
    if (d < 0).any():
        raise ValueError
    return np.power(d, -0.5, where=d>0, dtype=dtype)

def normalize_binary_features(feature_mat, dtype=None):
    sqsum = feature_mat.getnnz(axis=1)
    if feature_mat.format == 'csc':
        ind = feature_mat.indices.copy()
        ptr = feature_mat.indptr.copy()
        norm_data = 1 / np.sqrt(np.take(sqsum, ind))
        normed = csc_matrix((norm_data, ind, ptr), shape=feature_mat.shape, dtype=dtype)
    else:
        norm_data = safe_inverse_root(sqsum, dtype=dtype)
        normed = sp.sparse.diags(norm_data).dot(feature_mat)
    return normed

def normalize_features(feature_mat, dtype=None):
    if feature_mat.format == 'csc':
        ptr = feature_mat.indptr.copy()
        ind = feature_mat.indices.copy()
        sqsum = np.bincount(ind, weights=feature_mat.data**2)
        # could do np.add.at(sqsum, indices, data**2), but bincount seems faster
        norm_data = feature_mat.data / np.sqrt(np.take(sqsum, ind))
        normed = csc_matrix((norm_data, ind, ptr), shape=feature_mat.shape, dtype=dtype)
    else:
        sqsum = feature_mat.power(2).sum(axis=1).view(type=np.ndarray).squeeze()
        # avoid zero division
        norm_data = safe_inverse_root(sqsum, dtype=dtype)
        normed = sp.sparse.diags(norm_data).dot(feature_mat)
    return normed

def tfidf_transform(feature_mat):
    itemfreq = 1 + feature_mat.getnnz(axis=0)
    N = 1 + feature_mat.shape[0]
    idf = np.log(N/itemfreq)

    if feature_mat.format == 'csr':
        ind = feature_mat.indices
        ptr = feature_mat.indptr
        data = np.take(idf, ind)
        transformed = csr_matrix((data, ind, ptr), shape=feature_mat.shape)
    else:
        # safe multiplicationin in case data is not binary
        transformed = (feature_mat!=0).dot(sp.sparse.diags(idf))
    return transformed

@jit(nopython=True)
def _jaccard_similarity_inplace(sdata, indcs, pntrs, nf):
    # assumes data is symmetric => format doesn't matter
    # it doesn't enforce  1 on the diagonal!
    # so the similarity data must be prepared accordingly
    ncols = len(pntrs) - 1
    for col in range(ncols):
        lind = pntrs[col]
        rind = pntrs[col+1]
        nf_col = nf[col]
        for j in range(lind, rind):
            row = indcs[j]
            denom = nf_col + nf[row] - sdata[j]
            sdata[j] /= denom

def jaccard_similarity(F, fill_diagonal=True):
    F = (F!=0)
    nf = F.getnnz(axis=1)

    S = F.dot(F.T).astype(np.float64) # dtype needed for inplace jaccard index computation
    _jaccard_similarity_inplace(S.data, S.indices, S.indptr, nf)
    if fill_diagonal:
        # eigenvalues computation is very sensitive to roundoff errors in
        # similarity matrix; explicitly seting diagonal values is better
        # than using _fix_empty_features
        set_diagonal_values(S, 1)
    return S

def cosine_similarity(F, fill_diagonal=True, assume_binary=False):
    normalize = normalize_binary_features if assume_binary else normalize_features
    F = normalize(F)
    S = F.dot(F.T)
    if fill_diagonal:
        # eigenvalues computation is very sensitive to roundoff errors in
        # similarity matrix; explicitly seting diagonal values is better
        # than using _fix_empty_features
        set_diagonal_values(S, 1)
    return S

def cosine_tfidf_similarity(F, fill_diagonal=True):
    F = tfidf_transform(F)
    S = cosine_similarity(F, fill_diagonal=fill_diagonal, assume_binary=False)
    return S

@jit(nopython=True)
def _jaccard_similarity_weighted_tri(dat, ind, ptr, shift):
    z = dat[0] #np.float64
    # need to initialize lists with certain dtype
    data = [z]
    cols = [z]
    rows = [z]

    nrows = len(ptr) - 1
    for i in range(nrows):
        lind_i = ptr[i]
        rind_i = ptr[i+1]
        if lind_i != rind_i:
            ish = i + shift
            for j in range(ish, nrows):
                lind_j = ptr[j]
                rind_j = ptr[j+1]
                min_sum = 0
                max_sum = 0
                for k in range(lind_j, rind_j):
                    for s in range(lind_i, rind_i):
                        iind = ind[s]
                        jind = ind[k]
                        if iind == jind:
                            min_val = min(dat[s], dat[k])
                            max_val = max(dat[s], dat[k])
                            min_sum += min_val
                            max_sum += max_val
                            break
                    else:
                        max_sum += dat[k]

                for s in range(lind_i, rind_i):
                    for k in range(lind_j, rind_j):
                        iind = ind[s]
                        jind = ind[k]
                        if iind == jind:
                            break
                    else:
                        max_sum += dat[s]

                if min_sum:
                    wjac = min_sum / max_sum
                    data.append(wjac)
                    cols.append(i)
                    rows.append(j)

    return data[1:], rows[1:], cols[1:] #ignore initialization element

def jaccard_similarity_weighted(F, fill_diagonal=True):
    assert F.format == 'csr'
    if not F.has_sorted_indices:
        F.sort_indices()

    ind = F.indices
    ptr = F.indptr
    dat = F.data.astype(np.float64, copy=False) # dtype needed for jaccard computation

    shift = 1 if fill_diagonal else 0
    data, rows, cols = _jaccard_similarity_weighted_tri(dat, ind, ptr, shift)

    S = coo_matrix((data, (rows, cols)), shape=(F.shape[0],)*2).tocsc()
    S += S.T # doubles diagonal values if fill_diagonal is False

    if fill_diagonal:
        set_diagonal_values(S, 1)
    else:
        set_diagonal_values(S, np.sign(S.diagonal())) # set to 1, preserve zeros
    return S

def jaccard_similarity_weighted_dense(F, fill_diagonal=True):
    if (F.data < 0).any():
        raise ValueError

    f = F.sum(axis=1).view(type=np.ndarray).squeeze()
    fplus = f + f[:, None]
    FA = F.A
    fminus = np.abs(FA[None, :, :] - FA[:, None, :]).sum(axis=2)

    with np.errstate(invalid='ignore'):
        res = np.nan_to_num((fplus - fminus) / (fplus + fminus))

    if fill_diagonal:
        np.fill_diagonal(res, 1)
    return res

def uniquify_ordered(seq):
    # order preserving
    # https://www.peterbe.com/plog/uniqifiers-benchmark
    # http://stackoverflow.com/questions/480214/how-do-you-remove-duplicates-from-a-list-in-whilst-preserving-order
    seen = set()
    seen_add = seen.add
    return [x for x in seq if not (x in seen or seen_add(x))]

def build_indicator_matrix(labels, max_items=None):
    indices = [i for x in labels for i in x]
    indprt = np.r_[0, labels.apply(len).cumsum().values]
    data = np.ones_like(indices, dtype=np.bool)
    shape = [len(labels), max_items or indices.max()+1]
    return csr_matrix((data, indices, indprt), shape=shape)

def feature2sparse(feature_data, ranking=None, deduplicate=True):
    if deduplicate:
        feature_data = feature_data.apply(uniquify_ordered if ranking else set)

    feature_lbl = defaultdict(lambda: len(feature_lbl))
    indices = [feature_lbl[item] for items in feature_data for item in items]
    indptr = np.r_[0, feature_data.apply(len).cumsum().values]

    if ranking:
        if ranking is True:
            ranking = 'linear'

        if isinstance(ranking, str):
            if ranking.lower() == 'linear':
                data = [1/(n+1) for items in feature_data for n, item in enumerate(items)]
            elif ranking.lower() == 'exponential':
                data = [math.exp(-n) for items in feature_data for n, item in enumerate(items)]
            elif ranking.lower() == 'bag-of-features':
                raise NotImplementedError
            else:
                raise ValueError
        elif isinstance(ranking, types.FunctionType):
            data = [ranking(n) for items in feature_data for n, item in enumerate(items)]
    else:
        data = np.ones_like(indices)

    feature_mat = csr_matrix((data, indices, indptr),
                             shape=(feature_data.shape[0], len(feature_lbl)))
    if not ranking:
        feature_mat = feature_mat.tocsc()

    return feature_mat, dict(feature_lbl)

def get_features_data(meta_data, ranking=None, deduplicate=True):
    feature_mats = OrderedDict()
    feature_lbls = OrderedDict()
    features = meta_data.columns

    ranking = ranking or {}

    if ranking is True:
        ranking = 'linear'

    if isinstance(ranking, str):
        ranking = [ranking] * len(features)

    if not isinstance(ranking, dict):
        ranking = {k: v for k, v in zip(features, ranking)}

    for feature in features:
        feature_data = meta_data[feature]
        mat, lbl = feature2sparse(feature_data, ranking=ranking.get(feature, None), deduplicate=deduplicate)
        feature_mats[feature], feature_lbls[feature] = mat, lbl
    return feature_mats, feature_lbls

def _sim_func(func_type):
    if func_type.lower() == 'jaccard':
        return jaccard_similarity
    elif func_type.lower() == 'cosine':
        return cosine_similarity
    elif func_type.lower() == 'tfidf-cosine':
        return cosine_tfidf_similarity
    elif func_type.lower() == 'jaccard-weighted':
        return jaccard_similarity_weighted
    else:
        raise NotImplementedError

def one_hot_similarity(meta_data):
    raise NotImplementedError

def get_similarity_data(meta_data, similarity_type='jaccard'):
    features = meta_data.columns

    if isinstance(similarity_type, str):
        similarity_type = [similarity_type] * len(features)

    if not isinstance(similarity_type, dict):
        similarity_type = {k: v for k, v in zip(features, similarity_type)}

    similarity_mats = {}
    for feature in features:
        feature_data = meta_data[feature]
        sim_type = similarity_type[feature]
        ranking = sim_type == 'jaccard-weighted'
        get_similarity = _sim_func(sim_type)

        feature_mat, feature_lbl = feature2sparse(feature_data, ranking=ranking)
        similarity_mats[feature] = get_similarity(feature_mat)

    return similarity_mats

def combine_distribute_similarity_data(meta_data, similarity_type='jaccard', weights=None):
    # similarity_mats = get_similarity_data(meta_data, similarity_type)
    # iprob = {f: 1+np.prod(s.shape)/s.nnz for f, s in similarity_mats.iteritems()}
    # wsum = np.log(iprob.values()).sum()
    # weights = {f:w/wsum for f, w in iprob.iteritems()}
    raise NotImplementedError

def combine_similarity_data(meta_data, similarity_type='jaccard', weights=None):
    features = meta_data.columns
    num_feat = len(features)

    if isinstance(similarity_type, str):
        similarity_type = [similarity_type] * num_feat

    if not isinstance(similarity_type, dict):
        similarity_type = {k: v for k, v in zip(features, similarity_type)}

    if weights is None:
        weights = [1.0/num_feat] * num_feat

    if not isinstance(weights, dict):
        weights = {k: v for k, v in zip(features, weights)}

    similarity = csc_matrix((meta_data.shape[0],)*2)
    for feature in features:
        feature_data = meta_data[feature]
        sim_type = similarity_type[feature]
        weight = weights[feature]
        ranking = sim_type == 'jaccard-weighted'
        get_similarity = _sim_func(sim_type)

        feature_mat, feature_lbl = feature2sparse(feature_data, ranking=ranking)
        similarity += weight * get_similarity(feature_mat)

    # remove round off errors
    similarity.setdiag(1)
    similarity.data[similarity.data>1] = 1
    return similarity
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from pollirio.modules import expose
from pollirio.dbutils import *
from pollirio import choose_dest

import re
import random
import time
import exceptions

class LartsDb:
    def __init__(self):
        self.db = db_init("larts")

    def size(self):
        return run(self.db.count()).fetchone()[0]

    def random(self):
        larts = run(self.db.select()).fetchall()
        if not larts:
            return []
        else:
            return random.choice(larts)[1]

    def idlart(self, id):
        lart = run(self.db.select(self.db.c.indx == id)).fetchone()
        if lart:
            return lart[1]

    def search(self, what):
        rs = run(self.db.select(self.db.c.lart.like("%%%s%%" % what))).fetchall()
        if rs:
            return rs

    def add(self, lart):
        rs = run(self.db.insert({"lart": unicode(lart, 'utf-8')}))
        return rs.last_inserted_ids()[0]

    def delete(self, id):
        if run(self.db.select(self.db.c.indx == id)).fetchone():
            run(self.db.delete(self.db.c.indx == id))
            return 1
        return 0

class LartIgnoreDb:
    def __init__(self):
        self.db = db_init("lartignore")

    def ignored(self, src, dst):
        r = run(self.db.select(and_( \
            self.db.c.src.like("%%%s%%" % src), \
            self.db.c.dst.like("%%%s%%" % dst)
        ))).fetchall()
        if r:
            return True
        return False

    def add(self, src, dst):
        rs = run(self.db.insert({"src": src, "dst": dst}))
        return rs.last_inserted_ids()[0]

    def delete(self, src, dst):
        r = run(self.db.delete().where(and_( \
            (self.db.c.src == src),
            (self.db.c.dst == dst) \
        ))).fetchone()
        if r:
            return 1
        return 0

larts = LartsDb()
ignores = LartIgnoreDb()

@expose("lart", 1)
@expose("larta", 1)
def lart(bot, ievent):
    """lart <utente> [numero lart]"""

    if len(ievent.args):
        if ignores.ignored(ievent.nick, ievent.args[0]):
            return

    if len(ievent.args) == 1:
        lart = larts.random()
    else:
        try:
            lart = larts.idlart(int(ievent.args[1]))
        except exceptions.ValueError:
            lart = larts.random()

    if re.match('.*!.*@unaffiliated/damn3dg1rl', ievent.host) or \
       ievent.nick == 'DAMN3dg1rl':
       ievent.args[0] = ievent.nick

    if not lart:
        bot.msg(ievent.channel, "%s autolartati, non esiste quel lart!" % ievent.nick)
        return

    lart = lart.replace("$who", ievent.args[0]).encode("utf-8")
    bot.describe(ievent.channel, lart)
    return

@expose("lartignore", 1)
def lartignore(bot, ievent):
    """lartignore <utente>"""
    bot.sendLine('WHO %s' % ievent.channel)
    modes = bot.userlist[ievent.channel.lower()][ievent.nick]
    if '~' in modes or \
      '&' in modes or \
      '@' in modes or \
      ievent.nick == 'dapal':
        if len(ievent.args):
            ignores.add(ievent.args[0], ievent.nick)
            bot.msg(choose_dest(ievent), '%s: cot' % ievent.nick)
    return

@expose("lartallow", 1)
def lartllow(bot, ievent):
    """lartallow <utente>"""
    bot.sendLine('WHO %s' % ievent.channel)
    modes = bot.userlist[ievent.channel.lower()][ievent.nick]
    if '~' in modes or \
      '&' in modes or \
      '@' in modes or \
      ievent.nick == 'dapal':
        if len(ievent.args):
            ignores.delete(ievent.args[0], ievent.nick)
            bot.msg(choose_dest(ievent), '%s: cot' % ievent.nick)
    return

@expose("lartami")
@expose("autolart")
def selflart(bot, ievent):
    ievent.args = [ievent.nick] + ievent.msg.split()[1:]
    lart(bot, ievent)

@expose("lart-add", 1)
def lartadd(bot, ievent):
    """lart-add <testo contenente $who>"""
    args = ievent.msg.split(" ", 1)
    bot.sendLine('WHO %s' % ievent.channel)
    modes = bot.userlist[ievent.channel.lower()][ievent.nick]
    if '~' in modes or \
      '&' in modes or \
      '@' in modes or \
      ievent.nick == 'dapal':
        id = larts.add(args[1])
        bot.msg(choose_dest(ievent), "%s: lart %s aggiunto." % (ievent.nick, id))
    return

@expose("lart-del", 1)
def lartdel(bot, ievent):
    """lart-del <id del lart>"""
    bot.sendLine('WHO %s' % ievent.channel)
    modes = bot.userlist[ievent.channel.lower()][ievent.nick]
    if '~' in modes or \
      '&' in modes or \
      '@' in modes or \
      ievent.nick == 'dapal':
        id = int(ievent.args[0])
        if larts.delete(id):
            bot.msg(choose_dest(ievent), "%s: lart %s cancellato." % (ievent.nick, id))
        else:
            bot.msg(choose_dest(ievent), "%s: impossibile cancellare lart %s." % (ievent.nick, id))
    return

@expose("lart-search", 1)
@expose("sl", 1)
def lartsearch(bot, ievent):
    """lart-search <query>"""
    args = ievent.msg.split(" ", 1)
    if len(args) == 1:
        bot.msg(choose_dest(ievent), "%s: cosa devo cercare?" % ievent.nick)
        return

    query = args[1]
    if len(query) < 3:
        bot.msg(choose_dest(ievent), '%s: il termine di ricerca deve contenere almeno 3 caratteri.' % ievent.nick)
        return
    if query.strip() in ['$who', 'who']:
        bot.msg(choose_dest(ievent), '%s: volevi fare il furbo, eh?' % ievent.nick)
        return
    for id, lart in larts.search(query):
        bot.msg(choose_dest(ievent), "#%s: %s" % (id, lart.encode("utf-8")))
        time.sleep(0.75)
    return

@expose("lart-size")
def lartsize(bot, ievent):
    bot.msg(choose_dest(ievent), "%s: al momento conosco %s lart." % (ievent.nick, larts.size()))
    return
<commit_msg>Restrict larts on #reclute-war<commit_after># -*- coding: utf-8 -*-

from pollirio.modules import expose
from pollirio.dbutils import *
from pollirio import choose_dest

import re
import random
import time
import exceptions

class LartsDb:
    def __init__(self):
        self.db = db_init("larts")

    def size(self):
        return run(self.db.count()).fetchone()[0]

    def random(self):
        larts = run(self.db.select()).fetchall()
        if not larts:
            return []
        else:
            return random.choice(larts)[1]

    def idlart(self, id):
        lart = run(self.db.select(self.db.c.indx == id)).fetchone()
        if lart:
            return lart[1]

    def search(self, what):
        rs = run(self.db.select(self.db.c.lart.like("%%%s%%" % what))).fetchall()
        if rs:
            return rs

    def add(self, lart):
        rs = run(self.db.insert({"lart": unicode(lart, 'utf-8')}))
        return rs.last_inserted_ids()[0]

    def delete(self, id):
        if run(self.db.select(self.db.c.indx == id)).fetchone():
            run(self.db.delete(self.db.c.indx == id))
            return 1
        return 0

class LartIgnoreDb:
    def __init__(self):
        self.db = db_init("lartignore")

    def ignored(self, src, dst):
        r = run(self.db.select(and_( \
            self.db.c.src.like("%%%s%%" % src), \
            self.db.c.dst.like("%%%s%%" % dst)
        ))).fetchall()
        if r:
            return True
        return False

    def add(self, src, dst):
        rs = run(self.db.insert({"src": src, "dst": dst}))
        return rs.last_inserted_ids()[0]

    def delete(self, src, dst):
        r = run(self.db.delete().where(and_( \
            (self.db.c.src == src),
            (self.db.c.dst == dst) \
        ))).fetchone()
        if r:
            return 1
        return 0

larts = LartsDb()
ignores = LartIgnoreDb()

@expose("lart", 1)
@expose("larta", 1)
def lart(bot, ievent):
    """lart <utente> [numero lart]"""

    if len(ievent.args):
        if ignores.ignored(ievent.nick, ievent.args[0]):
            return

    if len(ievent.args) == 1:
        lart = larts.random()
    else:
        try:
            lart = larts.idlart(int(ievent.args[1]))
        except exceptions.ValueError:
            lart = larts.random()

    if re.match('.*!.*@unaffiliated/damn3dg1rl', ievent.host) or \
       ievent.nick == 'DAMN3dg1rl':
       ievent.args[0] = ievent.nick

    if not lart:
        bot.msg(ievent.channel, "%s autolartati, non esiste quel lart!" % ievent.nick)
        return

    if ievent.channel in ['#reclute-war']:
        bot.sendLine('WHO %s' % ievent.channel)
        modes = bot.userlist[ievent.channel.lower()][ievent.nick]
        if '~' not in modes and \
           '&' not in modes and \
           '@' not in modes and \
            ievent.nick != 'dapal':
             return

    lart = lart.replace("$who", ievent.args[0]).encode("utf-8")
    bot.describe(ievent.channel, lart)
    return

@expose("lartignore", 1)
def lartignore(bot, ievent):
    """lartignore <utente>"""
    bot.sendLine('WHO %s' % ievent.channel)
    modes = bot.userlist[ievent.channel.lower()][ievent.nick]
    if '~' in modes or \
      '&' in modes or \
      '@' in modes or \
      ievent.nick == 'dapal':
        if len(ievent.args):
            ignores.add(ievent.args[0], ievent.nick)
            bot.msg(choose_dest(ievent), '%s: cot' % ievent.nick)
    return

@expose("lartallow", 1)
def lartllow(bot, ievent):
    """lartallow <utente>"""
    bot.sendLine('WHO %s' % ievent.channel)
    modes = bot.userlist[ievent.channel.lower()][ievent.nick]
    if '~' in modes or \
      '&' in modes or \
      '@' in modes or \
      ievent.nick == 'dapal':
        if len(ievent.args):
            ignores.delete(ievent.args[0], ievent.nick)
            bot.msg(choose_dest(ievent), '%s: cot' % ievent.nick)
    return

@expose("lartami")
@expose("autolart")
def selflart(bot, ievent):
    ievent.args = [ievent.nick] + ievent.msg.split()[1:]
    lart(bot, ievent)

@expose("lart-add", 1)
def lartadd(bot, ievent):
    """lart-add <testo contenente $who>"""
    args = ievent.msg.split(" ", 1)
    bot.sendLine('WHO %s' % ievent.channel)
    modes = bot.userlist[ievent.channel.lower()][ievent.nick]
    if '~' in modes or \
      '&' in modes or \
      '@' in modes or \
      ievent.nick == 'dapal':
        id = larts.add(args[1])
        bot.msg(choose_dest(ievent), "%s: lart %s aggiunto." % (ievent.nick, id))
    return

@expose("lart-del", 1)
def lartdel(bot, ievent):
    """lart-del <id del lart>"""
    bot.sendLine('WHO %s' % ievent.channel)
    modes = bot.userlist[ievent.channel.lower()][ievent.nick]
    if '~' in modes or \
      '&' in modes or \
      '@' in modes or \
      ievent.nick == 'dapal':
        id = int(ievent.args[0])
        if larts.delete(id):
            bot.msg(choose_dest(ievent), "%s: lart %s cancellato." % (ievent.nick, id))
        else:
            bot.msg(choose_dest(ievent), "%s: impossibile cancellare lart %s." % (ievent.nick, id))
    return

@expose("lart-search", 1)
@expose("sl", 1)
def lartsearch(bot, ievent):
    """lart-search <query>"""
    args = ievent.msg.split(" ", 1)
    if len(args) == 1:
        bot.msg(choose_dest(ievent), "%s: cosa devo cercare?" % ievent.nick)
        return

    query = args[1]
    if len(query) < 3:
        bot.msg(choose_dest(ievent), '%s: il termine di ricerca deve contenere almeno 3 caratteri.' % ievent.nick)
        return
    if query.strip() in ['$who', 'who']:
        bot.msg(choose_dest(ievent), '%s: volevi fare il furbo, eh?' % ievent.nick)
        return
    for id, lart in larts.search(query):
        bot.msg(choose_dest(ievent), "#%s: %s" % (id, lart.encode("utf-8")))
        time.sleep(0.75)
    return

@expose("lart-size")
def lartsize(bot, ievent):
    bot.msg(choose_dest(ievent), "%s: al momento conosco %s lart." % (ievent.nick, larts.size()))
    return
<|endoftext|>


<commit_before>import re
import random

from hamper.interfaces import ChatPlugin

class NumberWang(ChatPlugin):
    name = 'numberwang'

    def setup(self, *args, **kwargs):
        super(NumberWang, self).setup(*args, **kwargs)
        self.next_numberwang()

    def next_numberwang(self):
        self.numberwang = round(abs(random.gauss(0, 500)))
        print 'New numberwang is %d' % self.numberwang

    def message(self, bot, comm):
        numbers = re.findall(r'\d+', comm['message'])
        if any(int(d) == self.numberwang for d in numbers):
            bot.reply(comm, "That's numberwang!")
            self.next_numberwang()
        if random.random() > 0.9:
            self.next_numberwang()

numberwang = NumberWang()
<commit_msg>Wanganum!<commit_after>import re
import random

from hamper.interfaces import ChatPlugin

class NumberWang(ChatPlugin):
    name = 'numberwang'

    def setup(self, *args, **kwargs):
        super(NumberWang, self).setup(*args, **kwargs)
        self.numberwang = []
        self.more_numberwangs()

    def more_numberwangs(self, goal=0.9):
        again = random.random()

        while again < goal:
            another = round(abs(random.gauss(0, 100)))
            self.numberwang.append(another)
            print 'New numberwang is %d' % another
            again = random.random()
            goal **= 2

    def message(self, bot, comm):
        numbers = [int(d) for d in re.findall(r'\d+', comm['message'])]

        for d in numbers:
            if d in self.numberwang:
                bot.reply(comm, self.success(d))
                self.numberwang = []
                self.more_numberwangs()
                break

        self.more_numberwangs(0.9 if numbers else 0.1)

    def success(self, num):
        return random.choice(
            ["That's numberwang!"] * 5 +
            ["THAT'S NUMBERWANG!"] * 3 +
            ["That's wanganum!"] * 3 +
            ["Did someone say %d? That's numberwang!" % num] * 2 +
            ["That's numberwank!"] +
            ["Love those decimals. Numberwang!"]
        )

numberwang = NumberWang()
<|endoftext|>


<commit_before>from mako.util import LRUCache
import string, unittest, time

class item:
    def __init__(self, id):
        self.id = id

    def __str__(self):
        return "item id %d" % self.id

class LRUTest(unittest.TestCase):

    def setUp(self):
        self.cache = LRUCache(10, threshold=.2)

    def print_cache(l):
        for item in l:
            print item,
        print

    def testlru(self):
        l = self.cache

        for id in range(1,20):
            time.sleep(.001)
            l[id] = item(id)

        # first couple of items should be gone
        self.assert_(not l.has_key(1))
        self.assert_(not l.has_key(2))

        # next batch over the threshold of 10 should be present
        for id in range(11,20):
            self.assert_(l.has_key(id))

        l[12]
        l[15]
        l[23] = item(23)
        l[24] = item(24)
        l[25] = item(25)

        self.assert_(not l.has_key(11))
        self.assert_(not l.has_key(13))

        for id in (25, 24, 23, 14, 12, 19, 18, 17, 16, 15):
            self.assert_(l.has_key(id))

if __name__ == "__main__":
    unittest.main()
<commit_msg>added a threading test for lru....<commit_after>from mako.util import LRUCache
import string, unittest, time, random

import thread

class item:
    def __init__(self, id):
        self.id = id

    def __str__(self):
        return "item id %d" % self.id

class LRUTest(unittest.TestCase):

    def testlru(self):
        l = LRUCache(10, threshold=.2)

        for id in range(1,20):
            time.sleep(.001)
            l[id] = item(id)

        # first couple of items should be gone
        self.assert_(not l.has_key(1))
        self.assert_(not l.has_key(2))

        # next batch over the threshold of 10 should be present
        for id in range(11,20):
            self.assert_(l.has_key(id))

        l[12]
        l[15]
        l[23] = item(23)
        l[24] = item(24)
        l[25] = item(25)

        self.assert_(not l.has_key(11))
        self.assert_(not l.has_key(13))

        for id in (25, 24, 23, 14, 12, 19, 18, 17, 16, 15):
            self.assert_(l.has_key(id))

    def disabled_test_threaded(self):
        size = 100
        threshold = .5
        all_elems = 2000
        hot_zone = range(30,100)
        cache = LRUCache(size, threshold)
        class Element(object):
            def __init__(self, id):
                self.id = id
        def get_elem():
            if random.randint(1,4) == 1:
                return hot_zone[random.randint(0, len(hot_zone) - 1)]
            else:
                return random.randint(1, all_elems)

        def request_elem():
            while True:
                id = get_elem()
                try:
                    elem = cache[id]
                except KeyError:
                    cache[id] = Element(id)
                time.sleep(random.random() / 1000)
        for x in range(0,20):
            thread.start_new_thread(request_elem, ())
        for x in range(0,5):
            time.sleep(1)
            #print "size:", len(cache)
            assert len(cache) < size + size * threshold * 2
            assert len(cache) > size - (size * .1)
        total = 0
        for h in hot_zone:
            if h in cache:
                total += 1
        #print "total hot zone in cache: " , total, "of", len(hot_zone)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before># coding: utf-8

""" Given observations of Q = v sin i for some sample of objects, model the
    distribution of v as a mixture of components and infer parameters of those
    components. Essentially a deconvolution of sin i.
"""

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Standard library
import os
import sys
import socket

# Third-party
import emcee
import matplotlib.pyplot as plt
import numpy as np
from astropy.stats import median_absolute_deviation
from astropy import log as logger
from streamteam.util import get_pool

def main(pool, path, plot=False):
    # Project
    from tiltshift.model import ln_posterior
    from tiltshift.fakedata import generate_data

    N = 256  # Number of fake data points
    J = 128  # Number of steps to use when integrating over i
    K = 16  # Number of mixture components

    base_save_path = os.path.join(path, "cache")
    save_path = os.path.join(base_save_path, "N{}_J{}_K{}".format(N,J,K))
    if not os.path.exists(save_path):
        os.makedirs(save_path)

    # -------------------------------------------------------------------------
    # Generate fake data
    #

    np.random.seed(42)
    sigma = 1.  # km/s
    v_func = np.random.lognormal
    # v_func_args = dict(mean=5., sigma=0.01)  # tight v
    v_func_args = dict(mean=5., sigma=0.25)  # wide v
    v,Q,sigma_Q = generate_data(N, v_func, v_func_kwargs=v_func_args,
                                sigma=sigma, censor="Q > 5*sigma")

    # -------------------------------------------------------------------------
    # Now try sampling
    #

    # fix the positions of the mixture components
    v_k = np.linspace(5., 250., K)
    logger.debug("v_ks: {}".format(v_k))

    nwalkers = K*4
    ndim = K
    sampler = emcee.EnsembleSampler(nwalkers, dim=ndim,
                                    lnpostfn=ln_posterior,
                                    args=(v_k,Q,sigma_Q),
                                    pool=pool)
    p0 = np.random.normal(0.5,0.05,size=(nwalkers,ndim))
    p0 = p0 * N / p0.sum(axis=1)[:,np.newaxis]
    if np.any(p0 <= 0.):
        raise ValueError("Dumby.")

    logger.debug("Running sampler to burn in...")
    # pos,prob,state = sampler.run_mcmc(p0, 1000)
    pos,prob,state = sampler.run_mcmc(p0, 100) # HACK

    bad_walkers = sampler.acceptance_fraction < 0.1
    pos[bad_walkers] = np.random.normal(np.median(pos[~bad_walkers],axis=0),
                                        median_absolute_deviation(pos[~bad_walkers],axis=0),
                                        size=(bad_walkers.sum(), ndim))

    sampler.reset()

    logger.debug("Running sampler for main sampling...")
    pos,prob,state = sampler.run_mcmc(pos, 5000)
    logger.debug("Done sampling!")

    pool.close()

    for j in range(ndim):
        plt.clf()
        for walker in sampler.chain[...,j]:
            plt.plot(walker)
        plt.ylim(0,sampler.flatchain.max())
        plt.xlabel("Step number")
        plt.savefig(os.path.join(save_path, "{}.png".format(j)))

    nbins = 25
    bins = np.linspace(v_k.min(), v_k.max(), nbins)

    fig,axes = plt.subplots(1, 2, figsize=(10,5), sharex=True)

    axes[0].hist(v, bins=bins)
    axes[0].set_xlabel(r'True $v$ [km s$^{-1}$]')

    median_a = np.median(sampler.flatchain, axis=0)
    axes[1].plot(v_k, median_a, linestyle='none', marker='o')
    axes[1].set_xlim(bins.min(), bins.max())
    axes[1].set_xlabel(r'Model')

    fig.tight_layout()
    fig.savefig(os.path.join(save_path, "compare.png"))

if __name__ == '__main__':
    from argparse import ArgumentParser
    import logging

    # Define parser object
    parser = ArgumentParser(description="")
    parser.add_argument("-v", "--verbose", action="store_true", dest="verbose",
                        default=False, help="Be chatty! (default = False)")
    parser.add_argument("-q", "--quiet", action="store_true", dest="quiet",
                        default=False, help="Be quiet! (default = False)")

    # parser.add_argument("-f", dest="field_id", default=None, required=True,
    #                     type=int, help="Field ID")
    parser.add_argument("-p", dest="plot", action="store_true", default=False,
                        help="Plot or not")
    parser.add_argument("--mpi", dest="mpi", default=False, action="store_true",
                        help="Run with MPI.")
    parser.add_argument("--threads", dest="threads", default=None, type=int,
                        help="Number of multiprocessing threads to run on.")

    args = parser.parse_args()

    # Set logger level based on verbose flags
    if args.verbose:
        logger.setLevel(logging.DEBUG)
    elif args.quiet:
        logger.setLevel(logging.ERROR)
    else:
        logger.setLevel(logging.INFO)

    pool = get_pool(mpi=args.mpi, threads=args.threads)

    if socket.gethostname() == "adrn":
        logger.info("Assuming we're on adrn (laptop)...")
        base_path = "/Users/adrian/projects/tilt-shift"
    else:
        logger.info("Assuming we're on yeti...")
        base_path = "/vega/astro/users/amp2217/projects/tilt-shift"

    sys.path.append(base_path)
    main(pool, path=base_path, plot=args.plot)
    sys.exit(0)
<commit_msg>params<commit_after># coding: utf-8

""" Given observations of Q = v sin i for some sample of objects, model the
    distribution of v as a mixture of components and infer parameters of those
    components. Essentially a deconvolution of sin i.
"""

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Standard library
import os
import sys
import socket

# Third-party
import emcee
import matplotlib.pyplot as plt
import numpy as np
from astropy.stats import median_absolute_deviation
from astropy import log as logger
from streamteam.util import get_pool

def main(pool, path, plot=False):
    # Project
    from tiltshift.model import ln_posterior
    from tiltshift.fakedata import generate_data

    N = 256  # Number of fake data points
    J = 128  # Number of steps to use when integrating over i
    K = 16  # Number of mixture components
    nsteps = 5000

    # HACK
    N = 128
    J = 64
    K = 4
    nsteps = 100

    base_save_path = os.path.join(path, "cache")
    save_path = os.path.join(base_save_path, "N{}_J{}_K{}".format(N,J,K))
    if not os.path.exists(save_path):
        os.makedirs(save_path)

    # -------------------------------------------------------------------------
    # Generate fake data
    #

    np.random.seed(42)
    sigma = 1.  # km/s
    v_func = np.random.lognormal
    # v_func_args = dict(mean=5., sigma=0.01)  # tight v
    v_func_args = dict(mean=5., sigma=0.25)  # wide v
    v,Q,sigma_Q = generate_data(N, v_func, v_func_kwargs=v_func_args,
                                sigma=sigma, censor="Q > 5*sigma")

    # -------------------------------------------------------------------------
    # Now try sampling
    #

    # fix the positions of the mixture components
    v_k = np.linspace(5., 250., K)
    logger.debug("v_ks: {}".format(v_k))

    nwalkers = K*8
    ndim = K
    sampler = emcee.EnsembleSampler(nwalkers, dim=ndim,
                                    lnpostfn=ln_posterior,
                                    args=(v_k,Q,sigma_Q),
                                    pool=pool)
    p0 = np.random.normal(0.5,0.05,size=(nwalkers,ndim))
    p0 = p0 * N / p0.sum(axis=1)[:,np.newaxis]
    if np.any(p0 <= 0.):
        raise ValueError("Dumby.")

    logger.debug("Running sampler to burn in...")
    pos,prob,state = sampler.run_mcmc(p0, nsteps//10)

    bad_walkers = sampler.acceptance_fraction < 0.1
    pos[bad_walkers] = np.random.normal(np.median(pos[~bad_walkers],axis=0),
                                        median_absolute_deviation(pos[~bad_walkers],axis=0),
                                        size=(bad_walkers.sum(), ndim))

    sampler.reset()

    logger.debug("Running sampler for main sampling...")
    pos,prob,state = sampler.run_mcmc(pos, nsteps)
    logger.debug("Done sampling!")

    pool.close()

    for j in range(ndim):
        plt.clf()
        for walker in sampler.chain[...,j]:
            plt.plot(walker)
        plt.ylim(0,sampler.flatchain.max())
        plt.xlabel("Step number")
        plt.savefig(os.path.join(save_path, "{}.png".format(j)))

    nbins = 25
    bins = np.linspace(v_k.min(), v_k.max(), nbins)

    fig,axes = plt.subplots(1, 2, figsize=(10,5), sharex=True)

    axes[0].hist(v, bins=bins)
    axes[0].set_xlabel(r'True $v$ [km s$^{-1}$]')

    median_a = np.median(sampler.flatchain, axis=0)
    axes[1].plot(v_k, median_a, linestyle='none', marker='o')
    axes[1].set_xlim(bins.min(), bins.max())
    axes[1].set_xlabel(r'Model')

    fig.tight_layout()
    fig.savefig(os.path.join(save_path, "compare.png"))

if __name__ == '__main__':
    from argparse import ArgumentParser
    import logging

    # Define parser object
    parser = ArgumentParser(description="")
    parser.add_argument("-v", "--verbose", action="store_true", dest="verbose",
                        default=False, help="Be chatty! (default = False)")
    parser.add_argument("-q", "--quiet", action="store_true", dest="quiet",
                        default=False, help="Be quiet! (default = False)")

    # parser.add_argument("-f", dest="field_id", default=None, required=True,
    #                     type=int, help="Field ID")
    parser.add_argument("-p", dest="plot", action="store_true", default=False,
                        help="Plot or not")
    parser.add_argument("--mpi", dest="mpi", default=False, action="store_true",
                        help="Run with MPI.")
    parser.add_argument("--threads", dest="threads", default=None, type=int,
                        help="Number of multiprocessing threads to run on.")

    args = parser.parse_args()

    # Set logger level based on verbose flags
    if args.verbose:
        logger.setLevel(logging.DEBUG)
    elif args.quiet:
        logger.setLevel(logging.ERROR)
    else:
        logger.setLevel(logging.INFO)

    if socket.gethostname() == "adrn":
        logger.info("Assuming we're on adrn (laptop)...")
        base_path = "/Users/adrian/projects/tilt-shift"
    else:
        logger.info("Assuming we're on yeti...")
        base_path = "/vega/astro/users/amp2217/projects/tilt-shift"

    sys.path.append(base_path)
    pool = get_pool(mpi=args.mpi, threads=args.threads)
    main(pool, path=base_path, plot=args.plot)
    sys.exit(0)
<|endoftext|>