<commit_before># -*- coding: utf-8 -*-

# tests_samplers.py

# MIT License

# Copyright (c) 2017 Rene Jean Corneille

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import os
import sys
import inspect
import unittest

currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(os.path.dirname(currentdir))
sys.path.insert(0,parentdir)

from trendpy.samplers import *

from scipy.stats import rv_continuous, gamma
from pandas import DataFrame, date_range

class TestSamplers(unittest.TestCase):

	def setUp(self):
		self.p = Parameter('test',rv_continuous,(10,10))
		self.q = Parameter('test2',rv_continuous,(1,1))
		self.P = Parameters()
		self.S = L1Filter(gamma.rvs(10,size=date_range(start='2014-01-01', end='2015-04-03',freq='D').size))

	def tearDown(self):
		self.p = None
		self.q = None
		self.P = None
		self.S = None

	def test_empty_parameters(self):
		self.assertEqual(self.P.list,{})
		self.assertEqual(self.P.hierarchy,[])

	def test_parameter_is_multivariate(self):
		self.assertTrue(self.p.is_multivariate() and not self.q.is_multivariate())

	def test_initial_current_value(self):
		self.assertIsNone(self.p.current_value)

	def test_add_parameter(self):
		self.P.append(self.p)
		self.assertTrue(self.p.name in self.P.list and self.p.name in self.P.hierarchy)

	def test_remove_all(self):
		self.P.append(self.p)
		self.P.removeAll()
		self.assertTrue(not self.p.name in self.P.list and not self.p.name in self.P.hierarchy)

	def test_hierarchy(self):
		self.P.append(self.p)
		self.P.append(self.q)
		self.assertTrue(self.P.hierarchy.index(self.p.name) < self.P.hierarchy.index(self.q.name))

	def test_parameters_before_initial_value(self):
		for name in self.S.parameters.hierarchy:
			self.assertEqual(self.S.parameters.list[name].current_value,None)

	def test_parameters_length(self):
		self.assertEqual(len(self.S.parameters),4)

	def test_size_initial_values(self):
		for name in self.S.parameters.hierarchy:
			self.S.parameters.list[name].current_value = self.S.initial_value(name)
			self.assertNotEqual(self.S.parameters.list[name].current_value,None)

	def test_distribution_parameters(self):
		for name in self.S.parameters.hierarchy:
			self.S.parameters.list[name].current_value = self.S.initial_value(name)
		for name in self.S.parameters.hierarchy:
			self.assertIsInstance(self.S.distribution_parameters(name),dict)

	def test_random_draw_size(self):
		for name in self.S.parameters.hierarchy:
			self.S.parameters.list[name].current_value = self.S.initial_value(name)
		for name in self.S.parameters.hierarchy:
			if self.S.parameters.list[name].is_multivariate():
				self.assertEqual(self.S.parameters.list[name].current_value.size,self.S.generate(name).size)

if __name__ == "__main__":
	suite = unittest.TestSuite()
	suite.addTest(unittest.makeSuite(TestSamplers))
	unittest.TextTestRunner(verbosity=2).run(suite)

<commit_msg>Update tests_samplers.py<commit_after># -*- coding: utf-8 -*-

# tests_samplers.py

# MIT License

# Copyright (c) 2017 Rene Jean Corneille

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import os
import sys
import inspect
import unittest

currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(os.path.dirname(currentdir))
sys.path.insert(0,parentdir)

from trendpy.samplers import *

from scipy.stats import rv_continuous, gamma
from pandas import DataFrame, date_range

class TestSamplers(unittest.TestCase):

	def test_empty_parameters(self):
		P = Parameters()
		self.assertEqual(P.list,{})

	def test_parameter_is_multivariate(self):
		p = Parameter('test',rv_continuous,(10,10))
		q = Parameter('test2',rv_continuous,(1,1))
		self.assertTrue(p.is_multivariate() and not q.is_multivariate())

	def test_initial_current_value(self):
		p = Parameter('test',rv_continuous,(10,10))
		self.assertIsNone(p.current_value)

	def test_add_parameter(self):
		P = Parameters()
		p = Parameter('test',rv_continuous,(10,10))
		P.append(p)
		self.assertTrue(p.name in P.list and p.name in P.hierarchy)

	def test_remove_all(self):
		P = Parameters()
		p = Parameter('test',rv_continuous,(10,10))
		P.append(p)
		P.removeAll()
		self.assertTrue(not p.name in P.list and not p.name in P.hierarchy)

	def test_hierarchy(self):
		P = Parameters()
		p = Parameter('test',rv_continuous,(10,10))
		q = Parameter('test2',rv_continuous,(1,1))
		P.append(p)
		P.append(q)
		self.assertTrue(P.hierarchy.index(p.name) < P.hierarchy.index(q.name))

	def test_parameters_before_initial_value(self):
		S = L1Filter(gamma.rvs(10,size=date_range(start='2014-01-01', end='2015-04-03',freq='D').size))
		print(S.parmeters.hierarchy)
		for name in S.parameters.hierarchy:
			self.assertEqual(S.parameters.list[name].current_value,None)

	def test_parameters_length(self):
		S = L1Filter(gamma.rvs(10,size=date_range(start='2014-01-01', end='2015-04-03',freq='D').size))
		self.assertEqual(len(S.parameters),4)

	def test_size_initial_values(self):
		S = L1Filter(gamma.rvs(10,size=date_range(start='2014-01-01', end='2015-04-03',freq='D').size))
		for name in S.parameters.hierarchy:
			S.parameters.list[name].current_value = S.initial_value(name)
			self.assertNotEqual(S.parameters.list[name].current_value,None)

	def test_distribution_parameters(self):
		S = L1Filter(gamma.rvs(10,size=date_range(start='2014-01-01', end='2015-04-03',freq='D').size))
		for name in S.parameters.hierarchy:
			S.parameters.list[name].current_value = S.initial_value(name)
		for name in S.parameters.hierarchy:
			self.assertIsInstance(S.distribution_parameters(name),dict)

	def test_random_draw_size(self):
		S = L1Filter(gamma.rvs(10,size=date_range(start='2014-01-01', end='2015-04-03',freq='D').size))
		for name in S.parameters.hierarchy:
			S.parameters.list[name].current_value = S.initial_value(name)
		for name in S.parameters.hierarchy:
			if S.parameters.list[name].is_multivariate():
				self.assertEqual(S.parameters.list[name].current_value.size,S.generate(name).size)

if __name__ == "__main__":
	suite = unittest.TestSuite()
	suite.addTest(unittest.makeSuite(TestSamplers))
	unittest.TextTestRunner(verbosity=2).run(suite)

<|endoftext|>


<commit_before>from sklearn.metrics import confusion_matrix
from sklearn.base import BaseEstimator, ClassifierMixin, clone
from sklearn.model_selection import train_test_split
from tslearn.utils import to_time_series_dataset
from tslearn.clustering import TimeSeriesKMeans
import numpy as np
from scipy.sparse import coo_matrix

class NonMyopicEarlyClassification(BaseEstimator, ClassifierMixin):
    """Early Classification modelling for time series using the model presented in [1]

    Parameters
    ----------
    n_clusters : int
        Number of clusters to form.

    base_classifier : Estimator
        Estimator (instance) to be cloned and used for classifications.

    minimum_time_stamp : int
        Earliest time at which a classification can be performed on a time series

    lamb : float
        Value of the hyper parameter lambda used during the computation of the cost function to evaluate the probability
        that a time series belongs to a cluster given the time series.

    cost_time_parameter : float
        Parameter of the cost function of time. This function is of the form : f(time) = time * cost_time_parameter

    Attributes
    --------------------
    silhouette_ : float
        The silhouette score from the clustering

    classifiers_ : list
        A list containing all the classifiers trained for the model

    clusters_ : dictionary
        Contains the times series by clusters

    pyhatyck_ : dictionary
        Contains the probabilities of being classified as class y_hat given class y and cluster ck

    indice_ck_ : list
        Contains for each clusters the indexes of the time series in the dataset

    Examples
    --------------------

    References
    --------------------
    [1] A. Dachraoui, A. Bondu & A. Cornuejols. Early classification of time series as a non myopic sequential decision
    making problem. 2015 Conference paper

    """

    def __init__(
        self,
        n_clusters,
        base_classifier,
        minimum_time_stamp,
        lamb,
        cost_time_parameter,
    ):
        super(NonMyopicEarlyClassification, self).__init__()
        self.base_classifier = base_classifier
        self.n_clusters = n_clusters
        self.minimum_time_stamp = minimum_time_stamp
        self.lamb = lamb
        self.cost_time_parameter = cost_time_parameter

    def fit(self, X, y):
        """
        This function fits classifiers that are currently multilayer perceptrons to a training set of time series and
        associated classes. A classifier is fit for each time stamp above a minimum time stamp that is an attribute of
        the class of the model. Then some probabilities are computed using clusters already computed.
        This function should be divided among 2 or 3 functions.

        Parameters
        ----------
        X: Array-like
            a dataset of time series
        y: vector
            the associated classes of the series from X_train
        """
        y_classes = np.unique(y)
        self.labels_ = sorted(set(y_classes))
        y_classes_indices = [self.labels_.index(yi) for yi in y_classes]
        y_ = y
        for idx, current_classe in enumerate(y_classes):
            y_[y_ == current_classe] = y_classes_indices[idx]

        self.cluster_ = TimeSeriesKMeans(n_clusters=self.n_clusters)
        self.classifiers_ = {
            t: clone(self.base_classifier)
            for t in range(self.minimum_time_stamp, X.shape[1] + 1)
        }
        self.__n_classes_ = len(y_classes_indices)
        self.__len_X_ = X.shape[1]
        self.pyhatyck_ = np.empty((self.__len_X_ - self.minimum_time_stamp + 1,
                                   self.n_clusters, self.__n_classes_, self.__n_classes_))
        c_k = self.cluster_.fit_predict((X))
        X_and_cluster = np.concatenate((X, c_k[:, np.newaxis]), axis=1)
        X1, X2, c_k1, c_k2, y1, y2 = train_test_split(X_and_cluster, c_k, y_, test_size=0.5)
        vector_of_ones = np.ones((len(X[:]),))
        self.pyck_ = coo_matrix(
            (vector_of_ones, (y_, c_k)),
            shape=(self.__n_classes_, self.n_clusters),
        ).toarray()
        self.pyck_ /= self.pyck_.sum(axis=0, keepdims=True)

        for t in range(self.minimum_time_stamp, self.__len_X_ + 1):
            self.classifiers_[t].fit(X1[:, :t], y1)
            for k in range(0, self.n_clusters):
                index = (c_k2 == k)
                if index.shape[0] != 0:
                    X2_current_cluster = X2[index, :t]
                    y2_current_cluster = y2[index]
                    y2_hat = self.classifiers_[t].predict(
                        X2_current_cluster[:, :t]
                    )
                    conf_matrix = confusion_matrix(
                        y2_current_cluster, y2_hat, labels=y_classes_indices, normalize="true"
                    )
                    #The diagonal of the confusion matrices must be put to 0
                    # to fullfil the criterions of the cost function
                    np.fill_diagonal(conf_matrix, 0)
                    self.pyhatyck_[t - self.minimum_time_stamp, k] = conf_matrix

    def get_cluster_probas(self, X):
        """
        This function computes the probabilities of the time series xt to be in a cluster of the model

        Parameters
        ----------
        X: vector
            a time series that may be truncated

        Returns
        -------
        vector : the probabilities of the given time series to be in the clusters of the model
        """
        diffs = X[np.newaxis, :] - self.cluster_.cluster_centers_[:, :len(X)]
        distances_clusters = np.linalg.norm(diffs, axis=(1, 2))
        minimum_distance = np.min(distances_clusters)
        average_distance = np.mean(distances_clusters)
        minimum_distance_to_average = (
            average_distance - minimum_distance
        ) / average_distance
        delta = (average_distance - distances_clusters) / average_distance
        probas = 1.0 / (
            np.exp(self.lamb * minimum_distance_to_average)
            + np.exp(self.lamb * (minimum_distance_to_average - delta))
            + 1e-6
        )
        return probas / probas.sum()

    def _expected_cost(self, X, tau):
        """
        From a incomplete series xt, this function compute the expected cost of a prediction made at time "last time of
        xt + tau"

        Parameters
        ----------
        X: vector
            a time series that may be truncated for which we want to have the cost
        tau: int
            gives the future time for which the cost is calculated

        Returns
        --------
        float : the computed cost
        """
        proba_clusters = self.get_cluster_probas(X=X)
        truncated_t = X.shape[-1]
        sum_pyhatyck = np.sum(self.pyhatyck_[truncated_t + tau - self.minimum_time_stamp], axis=1)
        sum_pyhatyck = np.transpose(sum_pyhatyck)
        sum_global = np.sum((sum_pyhatyck * self.pyck_), axis=0)
        cost = np.dot(proba_clusters, sum_global)
        return cost + self._cost_time(truncated_t + tau)

    def cost_function_minimizer(self, end_of_time, xt):
        """
        We want to minimize a function "funct" according a time series "xt" for a list of integers from 0 to
        "end_of_time"

        Parameters
        ----------
        end_of_time: int
            the highest integer at which the function to be minimized is calculated
        function_to_minimize: funct
            the function of interest
        xt: vector
            the time series related to the function "funct"

        Returns
         ------
         int : The integer in {0,...,stop} that minimizes "funct"
         float : the so fat minimum cost
        """
        costs = [self._expected_cost(xt, tau) for tau in range(end_of_time + 1)]
        tau_star = np.argmin(costs)
        min_cost = costs[tau_star]
        return tau_star, min_cost

    def _predict_single_series(self, Xi):
        """
        This function classifies a single time series xt

        Parameters
        ----------
        xt: vector
            a time series that is probably incomplete but that nonetheless we want to classify
        Returns
        -------
        int: the class which is predicted
        int : the time of the prediction
        float : the probability used for computing the cost
        float : the loss when classifying
        """
        length_Xi = Xi.shape[0]
        Xi = np.reshape(Xi, (1, length_Xi))
        time_prediction = self.minimum_time_stamp
        for t in range(self.minimum_time_stamp, self.__len_X_ + 1):
            minimum_tau, minimum_loss = self.cost_function_minimizer(
                end_of_time=length_Xi - t, xt=Xi[:, :t])
            if (time_prediction == length_Xi) or (minimum_tau == t):
                result = self.classifiers_[t].predict(Xi[:, :t])
                result_proba = self.classifiers_[t].predict_proba(Xi[:, :t])
                loss_exit = minimum_loss
                break
            else:
                time_prediction = time_prediction + 1
        return result, time_prediction, result_proba, loss_exit

    def predict(self, X):
        """
        Predicts the classes of the series of the test dataset.

        Parameters
        ----------
        X_test : Array-like
            The test dataset

        Returns
        -------
        Vector : the predicted classes
        """

        y_pred = []
        time_prediction = []
        for i in range(0, X.shape[0]):
            (
                new_classif,
                new_time,
                new_proba,
                new_cost,
            ) = self._predict_single_series(X[i])
            y_pred.append(new_classif)
            time_prediction.append(new_time)
        return y_pred, time_prediction

    def _cost_time(self, t):
        return t * self.cost_time_parameter
<commit_msg>a matter of transposition<commit_after>from sklearn.metrics import confusion_matrix
from sklearn.base import BaseEstimator, ClassifierMixin, clone
from sklearn.model_selection import train_test_split
from tslearn.utils import to_time_series_dataset, check_dims
from tslearn.clustering import TimeSeriesKMeans
import numpy as np
from scipy.sparse import coo_matrix

class NonMyopicEarlyClassification(BaseEstimator, ClassifierMixin):
    """Early Classification modelling for time series using the model presented in [1]

    Parameters
    ----------
    n_clusters : int
        Number of clusters to form.

    base_classifier : Estimator
        Estimator (instance) to be cloned and used for classifications.

    minimum_time_stamp : int
        Earliest time at which a classification can be performed on a time series

    lamb : float
        Value of the hyper parameter lambda used during the computation of the cost function to evaluate the probability
        that a time series belongs to a cluster given the time series.

    cost_time_parameter : float
        Parameter of the cost function of time. This function is of the form : f(time) = time * cost_time_parameter

    Attributes
    --------------------
    silhouette_ : float
        The silhouette score from the clustering

    classifiers_ : list
        A list containing all the classifiers trained for the model

    clusters_ : dictionary
        Contains the times series by clusters

    pyhatyck_ : dictionary
        Contains the probabilities of being classified as class y_hat given class y and cluster ck

    indice_ck_ : list
        Contains for each clusters the indexes of the time series in the dataset

    Examples
    --------------------

    References
    --------------------
    [1] A. Dachraoui, A. Bondu & A. Cornuejols. Early classification of time series as a non myopic sequential decision
    making problem. 2015 Conference paper

    """

    def __init__(
        self,
        n_clusters,
        base_classifier,
        minimum_time_stamp,
        lamb,
        cost_time_parameter,
    ):
        super(NonMyopicEarlyClassification, self).__init__()
        self.base_classifier = base_classifier
        self.n_clusters = n_clusters
        self.minimum_time_stamp = minimum_time_stamp
        self.lamb = lamb
        self.cost_time_parameter = cost_time_parameter

    def fit(self, X, y):
        """
        This function fits classifiers that are currently multilayer perceptrons to a training set of time series and
        associated classes. A classifier is fit for each time stamp above a minimum time stamp that is an attribute of
        the class of the model. Then some probabilities are computed using clusters already computed.
        This function should be divided among 2 or 3 functions.

        Parameters
        ----------
        X: Array-like
            a dataset of time series
        y: vector
            the associated classes of the series from X_train
        """

        X = check_dims(X)
        y_classes = np.unique(y)
        self.labels_ = sorted(set(y_classes))
        y_classes_indices = [self.labels_.index(yi) for yi in y_classes]
        y_ = np.copy(y)
        for idx, current_classe in enumerate(y_classes):
            y_[y_ == current_classe] = y_classes_indices[idx]

        self.cluster_ = TimeSeriesKMeans(n_clusters=self.n_clusters)
        self.classifiers_ = {
            t: clone(self.base_classifier)
            for t in range(self.minimum_time_stamp, X.shape[1] + 1)
        }
        self.__n_classes_ = len(y_classes_indices)
        self.__len_X_ = X.shape[1]
        self.pyhatyck_ = np.empty((self.__len_X_ - self.minimum_time_stamp + 1,
                                   self.n_clusters, self.__n_classes_, self.__n_classes_))
        c_k = self.cluster_.fit_predict(X)
        X1, X2, c_k1, c_k2, y1, y2 = train_test_split(X, c_k, y_, test_size=0.5)
        vector_of_ones = np.ones((len(X[:]),))
        self.pyck_ = coo_matrix(
            (vector_of_ones, (y_, c_k)),
            shape=(self.__n_classes_, self.n_clusters),
        ).toarray()
        self.pyck_ /= self.pyck_.sum(axis=0, keepdims=True)

        for t in range(self.minimum_time_stamp, self.__len_X_ + 1):
            self.classifiers_[t].fit(X1[:, :t], y1)
            for k in range(0, self.n_clusters):
                index = (c_k2 == k)
                if index.shape[0] != 0:
                    X2_current_cluster = X2[index, :t]
                    y2_current_cluster = y2[index]
                    y2_hat = self.classifiers_[t].predict(
                        X2_current_cluster[:, :t]
                    )
                    conf_matrix = confusion_matrix(
                        y2_current_cluster, y2_hat, labels=y_classes_indices, normalize="true"
                    )
                    #The diagonal of the confusion matrices must be put to 0
                    # to fullfil the criterions of the cost function
                    np.fill_diagonal(conf_matrix, 0)
                    self.pyhatyck_[t - self.minimum_time_stamp, k] = conf_matrix

    def get_cluster_probas(self, X):
        """
        This function computes the probabilities of the time series xt to be in a cluster of the model

        Parameters
        ----------
        X: vector
            a time series that may be truncated

        Returns
        -------
        vector : the probabilities of the given time series to be in the clusters of the model
        """
        diffs = X[np.newaxis, :] - self.cluster_.cluster_centers_[:, :len(X)]
        distances_clusters = np.linalg.norm(diffs, axis=(1, 2))
        minimum_distance = np.min(distances_clusters)
        average_distance = np.mean(distances_clusters)
        minimum_distance_to_average = (
            average_distance - minimum_distance
        ) / average_distance
        delta = (average_distance - distances_clusters) / average_distance
        probas = 1.0 / (
            np.exp(self.lamb * minimum_distance_to_average)
            + np.exp(self.lamb * (minimum_distance_to_average - delta))
            + 1e-6
        )
        return probas / probas.sum()

    def _expected_cost(self, X, tau):
        """
        From a incomplete series xt, this function compute the expected cost of a prediction made at time "last time of
        xt + tau"

        Parameters
        ----------
        X: vector
            a time series that may be truncated for which we want to have the cost
        tau: int
            gives the future time for which the cost is calculated

        Returns
        --------
        float : the computed cost
        """
        proba_clusters = self.get_cluster_probas(X=X)
        truncated_t = X.shape[0]
        sum_pyhatyck = np.sum(self.pyhatyck_[truncated_t + tau - self.minimum_time_stamp], axis=1)
        sum_pyhatyck = np.transpose(sum_pyhatyck)
        sum_global = np.sum((sum_pyhatyck * self.pyck_), axis=0)
        cost = np.dot(proba_clusters, sum_global)
        return cost + self._cost_time(truncated_t + tau)

    def cost_function_minimizer(self, end_of_time, xt):
        """
        We want to minimize a function "funct" according a time series "xt" for a list of integers from 0 to
        "end_of_time"

        Parameters
        ----------
        end_of_time: int
            the highest integer at which the function to be minimized is calculated
        function_to_minimize: funct
            the function of interest
        xt: vector
            the time series related to the function "funct"

        Returns
         ------
         int : The integer in {0,...,stop} that minimizes "funct"
         float : the so fat minimum cost
        """
        costs = [self._expected_cost(xt, tau) for tau in range(end_of_time + 1)]
        tau_star = np.argmin(costs)
        min_cost = costs[tau_star]
        return tau_star, min_cost

    def _predict_single_series(self, Xi):
        """
        This function classifies a single time series xt

        Parameters
        ----------
        xt: vector
            a time series that is probably incomplete but that nonetheless we want to classify
        Returns
        -------
        int: the class which is predicted
        int : the time of the prediction
        float : the probability used for computing the cost
        float : the loss when classifying
        """

        time_prediction = self.minimum_time_stamp
        for t in range(self.minimum_time_stamp, self.__len_X_ + 1):
            minimum_tau, minimum_loss = self.cost_function_minimizer(
                end_of_time=self.__len_X_ - t, xt=Xi[:t])
            if (t == self.__len_X_) or (minimum_tau == t):
                #Without the transpose here the predict fails miserably
                Xi = np.transpose(Xi)
                result = self.classifiers_[t].predict(Xi[:t])
                result_proba = self.classifiers_[t].predict_proba(Xi[:t])
                loss_exit = minimum_loss
                break
        return result, time_prediction, result_proba, loss_exit

    def predict(self, X):
        """
        Predicts the classes of the series of the test dataset.

        Parameters
        ----------
        X_test : Array-like
            The test dataset

        Returns
        -------
        Vector : the predicted classes
        """

        X = check_dims(X)
        y_pred = []
        time_prediction = []
        print(X.shape)
        for i in range(0, X.shape[0]):
            (
                new_classif,
                new_time,
                new_proba,
                new_cost,
            ) = self._predict_single_series(X[i])
            y_pred.append(new_classif)
            time_prediction.append(new_time)
        print(y_pred)
        return y_pred, time_prediction

    def _cost_time(self, t):
        return t * self.cost_time_parameter
<|endoftext|>


<commit_before>
'''

Implementation of Modified Velocity Centroids (Lazarian & Esquivel, 03)

'''

import numpy as np
import scipy.ndimage as nd
from itertools import izip
from ..psds import pspec
import statsmodels.formula.api as sm
from pandas import Series, DataFrame

try:
    from scipy.fftpack import fft2, fftshift
except ImportError:
    from numpy.fft import fft2, fftshift

class MVC(object):

    """
    Implementation of Modified Velocity Centroids (Lazarian & Esquivel, 03)

    INPUTS
    ------

    centroid   - array
                 Normalized first moment array

    moment0  - array
                     Intensity array

    linewidth   - array
                  Normalized second moment array

    distances   - list
                  List of distances vector lengths
                  If None, MVC is computed using the distances in the distance
                  array.

    FUNCTIONS
    ---------

    correlation_function - computes the MVC function

    run_mvc - calls correlation_function for each distance

    compute_pspec - computes the power spectrum

    compute_radial_pspec - computes the radially-averaged power spectrum

    OUTPUTS
    -------

    correlation_array - array
                           Resultant 2D array from mvc

    correlation_spectrum - array
                           1D array of average value at each distance

    ps1D - array
           1D array of the radially-averaged power spectrum

    ps2D - array
           2D array of the power spectrum

    freq - spatial frequencies used to compute ps1D

    """

    def __init__(self, centroid, moment0, linewidth, header, distances=None):
        # super(MVC, self).__init__()
        self.centroid = centroid
        self.moment0 = moment0
        self.linewidth = linewidth

        # Get rid of nans.
        self.centroid[np.isnan(self.centroid)] = 0.0
        self.moment0[np.isnan(self.moment0)] = 0.0
        self.linewidth[np.isnan(self.linewidth)] = 0.0
        self.degperpix = np.abs(header["CDELT2"])

        assert self.centroid.shape == self.moment0.shape
        assert self.centroid.shape == self.linewidth.shape
        self.shape = self.centroid.shape

        self.center_pixel = []
        for i_shape in self.shape:
            if i_shape % 2. != 0:
                self.center_pixel.append(int((i_shape - 1) / 2.))
            else:
                self.center_pixel.append(int(i_shape / 2.))

        self.center_pixel = tuple(self.center_pixel)
        centering_array = np.ones(self.shape)
        centering_array[self.center_pixel] = 0

        self.distance_array = nd.distance_transform_edt(centering_array)

        if not distances:
            self.distances = np.unique(
                self.distance_array[np.nonzero(self.distance_array)])
        else:
            assert isinstance(distances, list)
            self.distances = distances

        self.correlation_array = np.ones(self.shape)
        self.correlation_spectrum = np.zeros((1, len(self.distances)))
        self.ps2D = None
        self.ps1D = None
        self.freq = None

    def correlation_function(self, points):
        '''

        Function which performs the correlation

        '''

        center_centroid = (
            self.centroid[self.center_pixel] * self.moment0[self.center_pixel])
        center_intensity = self.moment0[self.center_pixel]

        for i, j in izip(points[0], points[1]):
            first_term = (
                center_centroid - self.centroid[i, j] *
                self.moment0[i, j]) ** 2.
            second_term = self.linewidth[
                i, j] * (center_intensity - self.moment0[i, j]) ** 2.

            self.correlation_array[i, j] = first_term - second_term

        return self

    def compute_pspec(self):
        '''
        Compute the power spectrum of MVC
        '''

        mvc_fft = fft2(self.centroid.astype("f8")) - \
            self.linewidth * fft2(self.moment0.astype("f8"))
        mvc_fft = fftshift(mvc_fft)

        self.ps2D = np.abs(mvc_fft) ** 2.

        return self

    def compute_radial_pspec(self, return_index=True, wavenumber=False,
                             return_stddev=False, azbins=1, binsize=1.0,
                             view=False, **kwargs):
        '''

        Computes the radially averaged power spectrum
        Based on Adam Ginsburg's code

        '''

        self.freq, self.ps1D = pspec(self.ps2D, return_index=return_index,
                                     wavenumber=wavenumber,
                                     return_stddev=return_stddev,
                                     azbins=azbins, binsize=binsize,
                                     view=view, **kwargs)

        return self

    def run(self, distances=None, phys_units=False, verbose=False):
        '''
        Full computation of MVC
        '''

        if not distances:
            pass
        else:
            assert isinstance(distances, list)
            self.distances = distances

        for n, distance in enumerate(self.distances):
            points = np.where(self.distance_array == distance)
            self.correlation_function(points)
            self.correlation_spectrum[0, n] = np.sum(
                self.correlation_array[points]) / len(points[0])

        self.compute_pspec()
        self.compute_radial_pspec(logspacing=True)

        if phys_units:
            self.freqs *= self.degperpix ** -1

        if verbose:
            import matplotlib.pyplot as p
            p.subplot(121)
            p.imshow(
                np.log10(self.ps2D), origin="lower", interpolation="nearest")
            p.colorbar()
            p.subplot(122)
            p.loglog(self.freq, self.ps1D, "bD-")
            p.show()

        return self

class MVC_distance(object):

    """

    Distance metric for MVC and wrapper for whole analysis

    INPUTS
    ------

    data1 - dictionary
            dictionary containing necessary property arrays

    data2 - dictionary
            dictionary containing necessary property arrays

    distances - list
                passed to MVC class

    FUNCTIONS
    ---------

    mvc_distance - computes the distance metric for 2 datasets

    OUTPUTS
    -------

    distance - float
               value of distance metric

    results  - statsmodels class
               results of the linear fit

    """

    def __init__(self, data1, data2, distances=None, fiducial_model=None):
        # super(mvc_distance, self).__init__()

        self.shape1 = data1["centroid"][0].shape
        self.shape2 = data2["centroid"][0].shape

        if fiducial_model is not None:
            self.mvc1 = fiducial_model
        else:
            self.mvc1 = MVC(data1["centroid"][0] * data1["centroid_error"][0] ** 2.,
                            data1["moment0"][0] * data1["moment0_error"][0] ** 2.,
                            data1["linewidth"][0] * data1["linewidth_error"][0] ** 2.,
                            data1["centroid"][1], distances=distances)
            self.mvc1.run()

        self.mvc2 = MVC(data2["centroid"][0] * data2["centroid_error"][0] ** 2.,
                        data2["moment0"][0] * data2["moment0_error"][0] ** 2.,
                        data2["linewidth"][0] * data2["linewidth_error"][0] ** 2.,
                        data2["centroid"][1], distances=distances)
        self.mvc2.run()

        self.results = None
        self.distance = None

    def distance_metric(self, verbose=False):
        '''

        Implements the distance metric for 2 MVC transforms.
        We fit the linear portion of the transform to represent the powerlaw
        A statistical comparison is used on the powerlaw indexes.

        '''

        # Clipping from 8 pixels to half the box size
        # Noise effects dominate outside this region
        clip_mask1 = np.zeros((self.mvc1.freq.shape))
        for i, x in enumerate(self.mvc1.freq):
            if x > 8.0 and x < self.shape1[0] / 2.:
                clip_mask1[i] = 1
        clip_freq1 = self.mvc1.freq[np.where(clip_mask1 == 1)]
        clip_ps1D1 = self.mvc1.ps1D[np.where(clip_mask1 == 1)]

        clip_mask2 = np.zeros((self.mvc2.freq.shape))
        for i, x in enumerate(self.mvc2.freq):
            if x > 8.0 and x < self.shape2[0] / 2.:
                clip_mask2[i] = 1
        clip_freq2 = self.mvc2.freq[np.where(clip_mask2 == 1)]
        clip_ps1D2 = self.mvc2.ps1D[np.where(clip_mask2 == 1)]

        dummy = [0] * len(clip_freq1) + [1] * len(clip_freq2)
        x = np.concatenate((np.log10(clip_freq1), np.log10(clip_freq2)))
        regressor = x.T * dummy

        log_ps1D = np.concatenate((np.log10(clip_ps1D1), np.log10(clip_ps1D2)))

        d = {"dummy": Series(dummy), "scales": Series(
            x), "log_ps1D": Series(log_ps1D), "regressor": Series(regressor)}

        df = DataFrame(d)

        model = sm.ols(
            formula="log_ps1D ~ dummy + scales + regressor", data=df)

        self.results = model.fit()

        self.distance = np.abs(self.results.tvalues["regressor"])

        if verbose:

            print self.results.summary()

            import matplotlib.pyplot as p
            p.plot(np.log10(clip_freq1), np.log10(clip_ps1D1), "bD",
                   np.log10(clip_freq2), np.log10(clip_ps1D2), "gD")
            p.plot(df["scales"][:len(clip_freq1)],
                   self.results.fittedvalues[:len(clip_freq1)], "b",
                   df["scales"][-len(clip_freq2):],
                   self.results.fittedvalues[-len(clip_freq2):], "g")
            p.grid(True)
            p.xlabel("log K")
            p.ylabel("MVC Power (K)")
            p.show()

        return self
<commit_msg>Set nans to minimum.<commit_after>
'''

Implementation of Modified Velocity Centroids (Lazarian & Esquivel, 03)

'''

import numpy as np
import scipy.ndimage as nd
from itertools import izip
from ..psds import pspec
import statsmodels.formula.api as sm
from pandas import Series, DataFrame

try:
    from scipy.fftpack import fft2, fftshift
except ImportError:
    from numpy.fft import fft2, fftshift

class MVC(object):

    """
    Implementation of Modified Velocity Centroids (Lazarian & Esquivel, 03)

    INPUTS
    ------

    centroid   - array
                 Normalized first moment array

    moment0  - array
                     Intensity array

    linewidth   - array
                  Normalized second moment array

    distances   - list
                  List of distances vector lengths
                  If None, MVC is computed using the distances in the distance
                  array.

    FUNCTIONS
    ---------

    correlation_function - computes the MVC function

    run_mvc - calls correlation_function for each distance

    compute_pspec - computes the power spectrum

    compute_radial_pspec - computes the radially-averaged power spectrum

    OUTPUTS
    -------

    correlation_array - array
                           Resultant 2D array from mvc

    correlation_spectrum - array
                           1D array of average value at each distance

    ps1D - array
           1D array of the radially-averaged power spectrum

    ps2D - array
           2D array of the power spectrum

    freq - spatial frequencies used to compute ps1D

    """

    def __init__(self, centroid, moment0, linewidth, header, distances=None):
        # super(MVC, self).__init__()
        self.centroid = centroid
        self.moment0 = moment0
        self.linewidth = linewidth

        # Get rid of nans.
        self.centroid[np.isnan(self.centroid)] = np.nanmin(self.centroid)
        self.moment0[np.isnan(self.moment0)] = np.nanmin(self.moment0)
        self.linewidth[np.isnan(self.linewidth)] = np.nanmin(self.linewidth)
        self.degperpix = np.abs(header["CDELT2"])

        assert self.centroid.shape == self.moment0.shape
        assert self.centroid.shape == self.linewidth.shape
        self.shape = self.centroid.shape

        self.center_pixel = []
        for i_shape in self.shape:
            if i_shape % 2. != 0:
                self.center_pixel.append(int((i_shape - 1) / 2.))
            else:
                self.center_pixel.append(int(i_shape / 2.))

        self.center_pixel = tuple(self.center_pixel)
        centering_array = np.ones(self.shape)
        centering_array[self.center_pixel] = 0

        self.distance_array = nd.distance_transform_edt(centering_array)

        if not distances:
            self.distances = np.unique(
                self.distance_array[np.nonzero(self.distance_array)])
        else:
            assert isinstance(distances, list)
            self.distances = distances

        self.correlation_array = np.ones(self.shape)
        self.correlation_spectrum = np.zeros((1, len(self.distances)))
        self.ps2D = None
        self.ps1D = None
        self.freq = None

    def correlation_function(self, points):
        '''

        Function which performs the correlation

        '''

        center_centroid = (
            self.centroid[self.center_pixel] * self.moment0[self.center_pixel])
        center_intensity = self.moment0[self.center_pixel]

        for i, j in izip(points[0], points[1]):
            first_term = (
                center_centroid - self.centroid[i, j] *
                self.moment0[i, j]) ** 2.
            second_term = self.linewidth[
                i, j] * (center_intensity - self.moment0[i, j]) ** 2.

            self.correlation_array[i, j] = first_term - second_term

        return self

    def compute_pspec(self):
        '''
        Compute the power spectrum of MVC
        '''

        mvc_fft = fft2(self.centroid.astype("f8")) - \
            self.linewidth * fft2(self.moment0.astype("f8"))
        mvc_fft = fftshift(mvc_fft)

        self.ps2D = np.abs(mvc_fft) ** 2.

        return self

    def compute_radial_pspec(self, return_index=True, wavenumber=False,
                             return_stddev=False, azbins=1, binsize=1.0,
                             view=False, **kwargs):
        '''

        Computes the radially averaged power spectrum
        Based on Adam Ginsburg's code

        '''

        self.freq, self.ps1D = pspec(self.ps2D, return_index=return_index,
                                     wavenumber=wavenumber,
                                     return_stddev=return_stddev,
                                     azbins=azbins, binsize=binsize,
                                     view=view, **kwargs)

        return self

    def run(self, distances=None, phys_units=False, verbose=False):
        '''
        Full computation of MVC
        '''

        if not distances:
            pass
        else:
            assert isinstance(distances, list)
            self.distances = distances

        for n, distance in enumerate(self.distances):
            points = np.where(self.distance_array == distance)
            self.correlation_function(points)
            self.correlation_spectrum[0, n] = np.sum(
                self.correlation_array[points]) / len(points[0])

        self.compute_pspec()
        self.compute_radial_pspec(logspacing=True)

        if phys_units:
            self.freqs *= self.degperpix ** -1

        if verbose:
            import matplotlib.pyplot as p
            p.subplot(121)
            p.imshow(
                np.log10(self.ps2D), origin="lower", interpolation="nearest")
            p.colorbar()
            p.subplot(122)
            p.loglog(self.freq, self.ps1D, "bD-")
            p.show()

        return self

class MVC_distance(object):

    """

    Distance metric for MVC and wrapper for whole analysis

    INPUTS
    ------

    data1 - dictionary
            dictionary containing necessary property arrays

    data2 - dictionary
            dictionary containing necessary property arrays

    distances - list
                passed to MVC class

    FUNCTIONS
    ---------

    mvc_distance - computes the distance metric for 2 datasets

    OUTPUTS
    -------

    distance - float
               value of distance metric

    results  - statsmodels class
               results of the linear fit

    """

    def __init__(self, data1, data2, distances=None, fiducial_model=None):
        # super(mvc_distance, self).__init__()

        self.shape1 = data1["centroid"][0].shape
        self.shape2 = data2["centroid"][0].shape

        if fiducial_model is not None:
            self.mvc1 = fiducial_model
        else:
            self.mvc1 = MVC(data1["centroid"][0] * data1["centroid_error"][0] ** 2.,
                            data1["moment0"][0] * data1["moment0_error"][0] ** 2.,
                            data1["linewidth"][0] * data1["linewidth_error"][0] ** 2.,
                            data1["centroid"][1], distances=distances)
            self.mvc1.run()

        self.mvc2 = MVC(data2["centroid"][0] * data2["centroid_error"][0] ** 2.,
                        data2["moment0"][0] * data2["moment0_error"][0] ** 2.,
                        data2["linewidth"][0] * data2["linewidth_error"][0] ** 2.,
                        data2["centroid"][1], distances=distances)
        self.mvc2.run()

        self.results = None
        self.distance = None

    def distance_metric(self, verbose=False):
        '''

        Implements the distance metric for 2 MVC transforms.
        We fit the linear portion of the transform to represent the powerlaw
        A statistical comparison is used on the powerlaw indexes.

        '''

        # Clipping from 8 pixels to half the box size
        # Noise effects dominate outside this region
        clip_mask1 = np.zeros((self.mvc1.freq.shape))
        for i, x in enumerate(self.mvc1.freq):
            if x > 8.0 and x < self.shape1[0] / 2.:
                clip_mask1[i] = 1
        clip_freq1 = self.mvc1.freq[np.where(clip_mask1 == 1)]
        clip_ps1D1 = self.mvc1.ps1D[np.where(clip_mask1 == 1)]

        clip_mask2 = np.zeros((self.mvc2.freq.shape))
        for i, x in enumerate(self.mvc2.freq):
            if x > 8.0 and x < self.shape2[0] / 2.:
                clip_mask2[i] = 1
        clip_freq2 = self.mvc2.freq[np.where(clip_mask2 == 1)]
        clip_ps1D2 = self.mvc2.ps1D[np.where(clip_mask2 == 1)]

        dummy = [0] * len(clip_freq1) + [1] * len(clip_freq2)
        x = np.concatenate((np.log10(clip_freq1), np.log10(clip_freq2)))
        regressor = x.T * dummy

        log_ps1D = np.concatenate((np.log10(clip_ps1D1), np.log10(clip_ps1D2)))

        d = {"dummy": Series(dummy), "scales": Series(
            x), "log_ps1D": Series(log_ps1D), "regressor": Series(regressor)}

        df = DataFrame(d)

        model = sm.ols(
            formula="log_ps1D ~ dummy + scales + regressor", data=df)

        self.results = model.fit()

        self.distance = np.abs(self.results.tvalues["regressor"])

        if verbose:

            print self.results.summary()

            import matplotlib.pyplot as p
            p.plot(np.log10(clip_freq1), np.log10(clip_ps1D1), "bD",
                   np.log10(clip_freq2), np.log10(clip_ps1D2), "gD")
            p.plot(df["scales"][:len(clip_freq1)],
                   self.results.fittedvalues[:len(clip_freq1)], "b",
                   df["scales"][-len(clip_freq2):],
                   self.results.fittedvalues[-len(clip_freq2):], "g")
            p.grid(True)
            p.xlabel("log K")
            p.ylabel("MVC Power (K)")
            p.show()

        return self
<|endoftext|>


<commit_before>from txircd.module_interface import Command, ICommand, ModuleData
from txircd.user import LocalUser
from txircd.utils import ircLower, splitMessage
from zope.interface import implements

class Service(ModuleData):
    """A service is a specific kind of module, with some pre-made stuff to assist in
    creating a pseudo-user that can do useful things in response to commands."""

    @property
    def nick(self):
        """The name to use for the service user. Defaults to the module"s name."""
        return self.name

    # Long description of what this service does and how to use it.
    # Should not go into detail of particular commands (this info is generated from per-command help)
    help = ""

    # An optional dict mapping user commands to (priority, service command)
    # For example, {"PASS": (20, "LOGIN")} will alias the irc command PASS
    # to the service command "LOGIN" with the same params.
    # If you give None instead of a service command, it will instead expect the command as an arg,
    # eg. {"MYSERV": (20, None)} wil alias the command MYSERV to send a message to your service.
    user_cmd_aliases = {}

    def serviceCommands(self):
        """Should return a dict mapping commands to tuple (handler, admin_only, summary, help).
        Commands MUST be upper case.
        handler: The function to be called in response to this command.
        admin_only: Flag. When true, only admins for this service can use this command.
        summary: A one-line summary for what the command does.
                 Set this to None to make the command not show up in the command list.
        help: Full-length usage help for the command.
        """
        return {}

    def getIdentityInfo(self):
        """Optional method that should return identity info for the service user
        as a dict with keys "host", "ident" and "gecos".
        """
        return {}

    def _serviceCommands(self):
        """For internal use. Gets custom serviceCommands and then adds in default ones."""
        commands = {
            "HELP": (self.handleHelp, False, "Get help on specific commands",
                     ("This command can be run with no arguments to get a help summary and command list, "
                     "or you can give it specific commands to get extra info on them - for example, "
                     "\x02/msg {} HELP HELP\x02 will print this message.").format(self.nick)
                    ),
        }
        commands.update(self.serviceCommands())
        return commands

    def hookIRCd(self, ircd):
        self.ircd = ircd

    def load(self):
        self.user = LocalUser(self.ircd, "127.0.0.1")
        self.user.setSendMsgFunc(self.handleMessage)
        self.user.changeNick(self.nick)
        info = self.getIdentityInfo()
        self.user.changeIdent(info.get("ident", self.nick))
        self.user.changeGecos(info.get("gecos", self.nick))
        self.user.changeHost(info.get("host", self.ircd.name))
        self.user.register("NICK")

    def unload(self):
        self.user.disconnect("Unloading module")

    def actions(self):
        return [("localnickcollision", 20, self.handleNickCollision)]

    def handleNickCollision(self, localUser, remoteUser, fromServer):
        if localUser == self.user:
            remoteUser.changeNick(remoteUser.uuid)
            return False

    def handleMessage(self, user, command, *params, **kw):
        """Handle any messages directly addressed to us."""
        if command != "PRIVMSG":
            return
        if "to" in kw:
            return # message was a channel message
        if "sourceuser" not in kw:
            return # we don't know who it's from?
        if not params:
            return
        text = params[0]
        if text.startswith(":"):
            text = text[1:] # strip leading :
        service_params = filter(None, text.split())
        if not service_params:
            return
        command = service_params.pop(0)
        self.handleCommand(kw["sourceuser"], command, service_params)

    def userCommands(self):
        class AliasCommand(Command):
            implements(ICommand)
            def __init__(self, service, service_cmd):
                self.service = service
                self.service_cmd = service_cmd
            def parseParams(self, source, params, prefix, tags):
                if self.service_cmd:
                    command = self.service_cmd
                else:
                    if not params:
                        return None
                    command, params = params[0], params[1:]
                return {"command": command, "params": params}
            def execute(self, source, data):
                self.service.handleCommand(source, data["command"], data["params"])
        return [(user_cmd, priority, AliasCommand(self, service_cmd))
                for user_cmd, (priority, service_cmd) in self.user_cmd_aliases.items()]

    def handleCommand(self, user, command, params):
        command = command.upper()
        commands = self._serviceCommands()
        if command in commands:
            handler, admin_only, summary, long_help = commands[command]
            if not admin_only or self.isAdmin(user):
                handler(user, params)
                return
        self.tellUser(user, "Unknown command \x02{}\x02. Use \x1f/msg {} HELP\x1f for help.".format(command, self.name))

    def handleHelp(self, user, params):
        if not params:
            self.tellUser(user, self.help, split=True)
            self.tellUser(user, "")
            for command, (handler, admin_only, summary, long_help) in sorted(self._serviceCommands().items()):
                if admin_only and not self.isAdmin(user):
                    continue
                if summary is None:
                    continue
                self.tellUser(user, "\x02{}\x02: {}".format(command, summary))
            self.tellUser(user, "")
            self.tellUser(user, "*** End of help")
            return
        command = params[0].upper()
        if command in self._serviceCommands():
            handler, admin_only, summary, long_help = self._serviceCommands()[command]
            if not admin_only or self.isAdmin(user):
                self.tellUser(user, "*** Help for \x02{}\x02:".format(command))
                self.tellUser(user, long_help, split=True)
                self.tellUser(user, "*** End of help for \x02{}\x02".format(command))
                return
        self.tellUser(user, "No help available for \x02{}\x02".format(command))

    def isAdmin(self, user):
        return self.ircd.runActionUntilValue("userhasoperpermission", user, "service-admin-{}".format(ircLower(self.name)))

    def tellUser(self, user, message, split=False):
        if user.uuid not in self.ircd.users:
            return # user has disconnected
        chunks = splitMessage(message, 80) if split else [message]
        for chunk in chunks:
            user.sendMessage("NOTICE", ":{}".format(chunk), sourceuser=self.user)<commit_msg>Don't remove already removed users on services unload when we're shutting down the IRCd<commit_after>from txircd.module_interface import Command, ICommand, ModuleData
from txircd.user import LocalUser
from txircd.utils import ircLower, splitMessage
from zope.interface import implements

class Service(ModuleData):
    """A service is a specific kind of module, with some pre-made stuff to assist in
    creating a pseudo-user that can do useful things in response to commands."""

    @property
    def nick(self):
        """The name to use for the service user. Defaults to the module"s name."""
        return self.name

    # Long description of what this service does and how to use it.
    # Should not go into detail of particular commands (this info is generated from per-command help)
    help = ""

    # An optional dict mapping user commands to (priority, service command)
    # For example, {"PASS": (20, "LOGIN")} will alias the irc command PASS
    # to the service command "LOGIN" with the same params.
    # If you give None instead of a service command, it will instead expect the command as an arg,
    # eg. {"MYSERV": (20, None)} wil alias the command MYSERV to send a message to your service.
    user_cmd_aliases = {}

    def serviceCommands(self):
        """Should return a dict mapping commands to tuple (handler, admin_only, summary, help).
        Commands MUST be upper case.
        handler: The function to be called in response to this command.
        admin_only: Flag. When true, only admins for this service can use this command.
        summary: A one-line summary for what the command does.
                 Set this to None to make the command not show up in the command list.
        help: Full-length usage help for the command.
        """
        return {}

    def getIdentityInfo(self):
        """Optional method that should return identity info for the service user
        as a dict with keys "host", "ident" and "gecos".
        """
        return {}

    def _serviceCommands(self):
        """For internal use. Gets custom serviceCommands and then adds in default ones."""
        commands = {
            "HELP": (self.handleHelp, False, "Get help on specific commands",
                     ("This command can be run with no arguments to get a help summary and command list, "
                     "or you can give it specific commands to get extra info on them - for example, "
                     "\x02/msg {} HELP HELP\x02 will print this message.").format(self.nick)
                    ),
        }
        commands.update(self.serviceCommands())
        return commands

    def hookIRCd(self, ircd):
        self.ircd = ircd

    def load(self):
        self.user = LocalUser(self.ircd, "127.0.0.1")
        self.user.setSendMsgFunc(self.handleMessage)
        self.user.changeNick(self.nick)
        info = self.getIdentityInfo()
        self.user.changeIdent(info.get("ident", self.nick))
        self.user.changeGecos(info.get("gecos", self.nick))
        self.user.changeHost(info.get("host", self.ircd.name))
        self.user.register("NICK")

    def unload(self):
        if user.uuid in self.ircd.users: # If we're shutting down the IRCd, the users dict has already been emptied
            self.user.disconnect("Unloading module")

    def actions(self):
        return [("localnickcollision", 20, self.handleNickCollision)]

    def handleNickCollision(self, localUser, remoteUser, fromServer):
        if localUser == self.user:
            remoteUser.changeNick(remoteUser.uuid)
            return False

    def handleMessage(self, user, command, *params, **kw):
        """Handle any messages directly addressed to us."""
        if command != "PRIVMSG":
            return
        if "to" in kw:
            return # message was a channel message
        if "sourceuser" not in kw:
            return # we don't know who it's from?
        if not params:
            return
        text = params[0]
        if text.startswith(":"):
            text = text[1:] # strip leading :
        service_params = filter(None, text.split())
        if not service_params:
            return
        command = service_params.pop(0)
        self.handleCommand(kw["sourceuser"], command, service_params)

    def userCommands(self):
        class AliasCommand(Command):
            implements(ICommand)
            def __init__(self, service, service_cmd):
                self.service = service
                self.service_cmd = service_cmd
            def parseParams(self, source, params, prefix, tags):
                if self.service_cmd:
                    command = self.service_cmd
                else:
                    if not params:
                        return None
                    command, params = params[0], params[1:]
                return {"command": command, "params": params}
            def execute(self, source, data):
                self.service.handleCommand(source, data["command"], data["params"])
        return [(user_cmd, priority, AliasCommand(self, service_cmd))
                for user_cmd, (priority, service_cmd) in self.user_cmd_aliases.items()]

    def handleCommand(self, user, command, params):
        command = command.upper()
        commands = self._serviceCommands()
        if command in commands:
            handler, admin_only, summary, long_help = commands[command]
            if not admin_only or self.isAdmin(user):
                handler(user, params)
                return
        self.tellUser(user, "Unknown command \x02{}\x02. Use \x1f/msg {} HELP\x1f for help.".format(command, self.name))

    def handleHelp(self, user, params):
        if not params:
            self.tellUser(user, self.help, split=True)
            self.tellUser(user, "")
            for command, (handler, admin_only, summary, long_help) in sorted(self._serviceCommands().items()):
                if admin_only and not self.isAdmin(user):
                    continue
                if summary is None:
                    continue
                self.tellUser(user, "\x02{}\x02: {}".format(command, summary))
            self.tellUser(user, "")
            self.tellUser(user, "*** End of help")
            return
        command = params[0].upper()
        if command in self._serviceCommands():
            handler, admin_only, summary, long_help = self._serviceCommands()[command]
            if not admin_only or self.isAdmin(user):
                self.tellUser(user, "*** Help for \x02{}\x02:".format(command))
                self.tellUser(user, long_help, split=True)
                self.tellUser(user, "*** End of help for \x02{}\x02".format(command))
                return
        self.tellUser(user, "No help available for \x02{}\x02".format(command))

    def isAdmin(self, user):
        return self.ircd.runActionUntilValue("userhasoperpermission", user, "service-admin-{}".format(ircLower(self.name)))

    def tellUser(self, user, message, split=False):
        if user.uuid not in self.ircd.users:
            return # user has disconnected
        chunks = splitMessage(message, 80) if split else [message]
        for chunk in chunks:
            user.sendMessage("NOTICE", ":{}".format(chunk), sourceuser=self.user)<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""
Convert txtfnnl match, bioner, and norm output to RankLib input.
"""

import os
import sys
from collections import namedtuple, defaultdict

GREEK_ORDINALS = set(range(ord('Α'), ord('ω') + 1))

GREEK_LOWER = {
    "alpha": "α",
    "beta": "β",
    "gamma": "γ",
    "delta": "δ",
    "epsilon": "ε",
    "zeta": "ζ",
    "eta": "η",
    "theta": "θ",
    "iota": "ι",
    "kappa": "κ",
    "lambda": "λ",
    "mu": "μ",
    "nu": "ν",
    "xi": "ξ",
    "omicron": "ο",
    "pi": "π",
    "rho": "ρ",
    "sigma": "σ",
    "tau": "τ",
    "upsilon": "υ",
    "ypsilon": "υ",
    "phi": "φ",
    "chi": "χ",
    "psi": "ψ",
    "omega": "ω",
}

GREEK_UPPER = {
    "Alpha": "Α",
    "Beta": "Β",
    "Gamma": "Γ",
    "Delta": "Δ",
    "Epsilon": "Ε",
    "Zeta": "Ζ",
    "Eta": "Η",
    "Theta": "Θ",
    "Iota": "Ι",
    "Kappa": "Κ",
    "Lambda": "Λ",
    "Mu": "Μ",
    "Nu": "Ν",
    "Xi": "Ξ",
    "Omicron": "Ο",
    "Pi": "Π",
    "Rho": "Ρ",
    "Sigma": "Σ",
    "Tau": "Τ",
    "Upsilon": "Υ",
    "Ypsilon": "Υ",
    "Phi": "Φ",
    "Chi": "Χ",
    "Psi": "Ψ",
    "Omega": "Ω",
}

GREEK2LATIN = {v: k for k, v in GREEK_UPPER.items()}
GREEK2LATIN.update({v: k for k, v in GREEK_LOWER.items()})

# 0:before, 1:prefix, 2:match, 3:suffix, 4:after, 5:pos, 6:offset, 7:gid, 8:sim, 9:name, 10:taxon, 11:entrez
GENE_ITEMS = ["before", "prefix", "match", "suffix", "after", "pos", "offset", "gid", "sim", "name", "taxon", "entrez"]
OFFSET = 6
MENTION = 2
GID = 7
KEYS = [9, 10, 11]
INTEGERS = [7, 10, 11]
Gene = namedtuple('Gene', GENE_ITEMS)

def ParseGeneLines(filepath):
    for line in open(filepath):
        items = line.split('\t')
        try:
            start, end = items[OFFSET].split(":")
        except IndexError:
            print(filepath, file=sys.stderr)
            print(line, file=sys.stderr)
            print("offset:", items[OFFSET], file=sys.stderr)
            raise
        items[OFFSET] = (int(start), int(end))
        for i in KEYS:
            txt = items[i]
            items[i] = txt[txt.find('"') + 1:txt.rfind('"')]
        for i in INTEGERS:
            try:
                items[i] = int(items[i])
            except ValueError:
                print(filepath, file=sys.stderr)
                print(line, file=sys.stderr)
                print(items[i], i, file=sys.stderr)
                raise
        yield (items[GID], items[MENTION]), Gene(*items)

def ParseNerLines(filepath):
    # 0:match, 1:offset, 2:type 3:conf
    for line in open(filepath):
        items = line.split('\t')
        start, end = items[1].split(":")
        yield (int(start), int(end)), items[2]

def ParseTaxonLines(filepath):
    # 0:match, 1:offset, 2:tid 3:sim
    for line in open(filepath):
        items = line.split('\t')
        start, end = items[1].split(":")
        yield int(items[2]), (int(start), int(end))

def ParseGold(filepath):
    for line in open(filepath):
        file_id, entrez = line.split('\t')
        yield file_id, int(entrez)

def ParseLinkout(filepath):
    for line in open(filepath):
        gid, count = line.split()
        yield int(gid), int(count)

def ParseCounters(filepath):
    for line in open(filepath):
        try:
            gid, sym, refc, globc = line.split('\t')
        except ValueError:
            print(filepath, '"{}"'.format(line))
            raise
        yield int(gid), sym, int(refc), int(globc)

def JoinData(article_id, genes, taxa, entities, gold, links, symbols, references):
    sym_counts = defaultdict(set)
    for gid, data in genes.items():
        for sym, mentions in data.items():
            sym_counts[sym].update(m.offset for m in mentions)
    sym_counts = dict(sym_counts)
    mention_counts = 0

    for gid, data in genes.items():
        count_GID = sum(len(m) for m in data.values())

        for sym, mentions in data.items():
            name = mentions[0].name
            count_SYM = len(sym_counts[sym])
            count_GIDSYM = len(mentions)
            count_links = links[gid] if gid in links else 0
            try:
                count_sym = symbols[name]
            except KeyError:
                if any(ord(i) in GREEK_ORDINALS for i in name):
                    replaced = []
                    for i in name:
                        if i in GREEK2LATIN:
                            replaced.append(GREEK2LATIN[i])
                        else:
                            replaced.append(i)
                    name = ''.join(replaced)
                    try:
                        count_sym = symbols[name]
                    except KeyError:
                        print('unknown name "{}" in mention "{}"'.format(name, gid), file=sys.stderr)
                        continue
                else:
                    print('unknown name "{}" in mention "{}"'.format(name, gid), file=sys.stderr)
                    continue
            try:
                count_refs = references[gid][name]
            except KeyError:
                if gid not in references:
                    print('unknown gid "{}" with name "{}"'.format(gid, name), file=sys.stderr)
                else:
                    print('unknown name "{}" for gid "{}"'.format(name, gid), file=sys.stderr)
                continue

            try:
                count_tids = len(taxa[mentions[0].taxon])
            except KeyError:
                print('unknown taxon "{}" in mention "{}"'.format(mentions[0].taxon, gid), file=sys.stderr)
                continue

            for mention in mentions:
                offset = mention.offset
                # distance tid <-> mention as 1/distance
                taxon_distance = sys.maxsize
                for taxon_offset in taxa[mention.taxon]:
                    taxon_distance = min(taxon_distance,
                                         min(abs(offset[0] - taxon_offset[1]),
                                             abs(taxon_offset[0] - offset[1])))
                taxon_distance = 1.0 / taxon_distance

                # is mention an entity, and which?
                entity_type = None
                for entity_offset in entities:
                    if offset[0] >= entity_offset[0] and offset[1] <= entity_offset[1]:
                        entity_type = entities[entity_offset]
                        break

                e = [
                    1.0 if entity_type is None else 0.0,
                    1.0 if entity_type == 'cell_line' else 0.0,
                    1.0 if entity_type == 'cell_type' else 0.0,
                    1.0 if entity_type == 'DNA' else 0.0,
                    1.0 if entity_type == 'protein' else 0.0,
                    1.0 if entity_type == 'RNA' else 0.0,
                ]

                # (mixed!) data per SYM
                # 1=none, 2=cell_line, 3=cell_type, 4=DNA, 5=protein, 6=RNA, 7=sim, 8=tax_distance
                yield mention.entrez in gold, article_id, e[0], e[1], e[2], e[3], e[4], e[5], float(mention.sim), taxon_distance
                mention_counts += 1

            # (integer only!) data per GID
            # 9=GIDc, 10=SYMc, 11=GIDSYMc, 12=links, 13=syms, 14=refs, 15=tids
            yield [mention_counts, count_GID, count_SYM, count_GIDSYM, count_links, count_sym, count_refs, count_tids]

def WriteLines(result_generator):
    sym_data = []
    gid_data = []
    for features in result_generator:
        if type(features[0]) is bool:
            sym_data.append(features)
        else:
            gid_data.append(features)

    # normalize GID data (all integers!)
    for i in range(1, len(gid_data[0])):
        m = max(r[i] for r in gid_data)
        if m == 0:
            print("all vaules zero in position", i+1, file=sys.stderr)
            for r in gid_data:
                r[i] = 0.0
        else:
            for r in gid_data:
                r[i] /= m

    base = len(sym_data[0]) - 1
    gid_iter = iter(gid_data)
    gid_feats = next(gid_iter)
    gid_feats_str = ' '.join('{}:{:.8f}'.format(i+base, f) for i, f in enumerate(gid_feats[1:]))
    for cnt, r in enumerate(sym_data):
        while cnt >= gid_feats[0]:
            gid_feats = next(gid_iter)
            gid_feats_str = ' '.join('{}:{:.8f}'.format(i+base, f) for i, f in enumerate(gid_feats[1:]))
        print(int(r[0]), 'qid:{}'.format(r[1]), ' '.join('{}:{:.8f}'.format(i+1, f) for i, f in enumerate(r[2:])), gid_feats_str)

def Process(gene_dir, taxon_dir, ner_dir, gold_file, counter_file, linkout_file):
    gold = defaultdict(set)
    for article_id, entrez_id in ParseGold(gold_file):
        gold[article_id].add(entrez_id)
    print("parsed {} gold items".format(len(gold)), file=sys.stderr)
    gold = dict(gold)

    link_counts = dict(ParseLinkout(linkout_file))
    print("parsed {} linkout items".format(len(link_counts)), file=sys.stderr)

    sym_counts = {}
    ref_counts = defaultdict(dict)
    for gid, sym, refc, symc in ParseCounters(counter_file):
        if sym not in sym_counts:
            sym_counts[sym] = symc
        ref_counts[gid][sym] = refc
    print("parsed {} refcount items".format(len(ref_counts)), file=sys.stderr)
    ref_counts = dict(ref_counts)

    for filepath in os.listdir(gene_dir):
        article_id = filepath[:filepath.rfind('.')]

        if article_id not in gold:
            #print("skipping", article_id, file=sys.stderr)
            continue

        print("processing", article_id, file=sys.stderr)

        genes = {}
        for (gid, mention), gene in ParseGeneLines(os.path.join(gene_dir, filepath)):
            if gid in genes:
                if mention in genes[gid]:
                    genes[gid][mention].append(gene)
                else:
                    genes[gid][mention] = [gene]
            else:
                genes[gid] = {mention: [gene]}

        taxa = defaultdict(set)
        for tid, offset in ParseTaxonLines(os.path.join(taxon_dir, filepath)):
            taxa[tid].add(offset)
        taxa = dict(taxa)

        entities = dict(ParseNerLines(os.path.join(ner_dir, filepath)))
        WriteLines(JoinData(article_id, genes, taxa, entities, gold[article_id], link_counts, sym_counts, ref_counts))

if __name__ == '__main__':
    # gene_dir, taxon_dir, ner_dir, gold_file, counter_file, refcount_file
    Process(*sys.argv[1:])
<commit_msg>added mention ambiguity as feature<commit_after>#!/usr/bin/env python3
"""
Convert txtfnnl match, bioner, and norm output to RankLib input.
"""

import os
import sys
from collections import namedtuple, defaultdict

GREEK_ORDINALS = set(range(ord('Α'), ord('ω') + 1))

GREEK_LOWER = {
    "alpha": "α",
    "beta": "β",
    "gamma": "γ",
    "delta": "δ",
    "epsilon": "ε",
    "zeta": "ζ",
    "eta": "η",
    "theta": "θ",
    "iota": "ι",
    "kappa": "κ",
    "lambda": "λ",
    "mu": "μ",
    "nu": "ν",
    "xi": "ξ",
    "omicron": "ο",
    "pi": "π",
    "rho": "ρ",
    "sigma": "σ",
    "tau": "τ",
    "upsilon": "υ",
    "ypsilon": "υ",
    "phi": "φ",
    "chi": "χ",
    "psi": "ψ",
    "omega": "ω",
}

GREEK_UPPER = {
    "Alpha": "Α",
    "Beta": "Β",
    "Gamma": "Γ",
    "Delta": "Δ",
    "Epsilon": "Ε",
    "Zeta": "Ζ",
    "Eta": "Η",
    "Theta": "Θ",
    "Iota": "Ι",
    "Kappa": "Κ",
    "Lambda": "Λ",
    "Mu": "Μ",
    "Nu": "Ν",
    "Xi": "Ξ",
    "Omicron": "Ο",
    "Pi": "Π",
    "Rho": "Ρ",
    "Sigma": "Σ",
    "Tau": "Τ",
    "Upsilon": "Υ",
    "Ypsilon": "Υ",
    "Phi": "Φ",
    "Chi": "Χ",
    "Psi": "Ψ",
    "Omega": "Ω",
}

GREEK2LATIN = {v: k for k, v in GREEK_UPPER.items()}
GREEK2LATIN.update({v: k for k, v in GREEK_LOWER.items()})

# 0:before, 1:prefix, 2:match, 3:suffix, 4:after, 5:pos, 6:offset, 7:gid, 8:sim, 9:name, 10:taxon, 11:entrez
GENE_ITEMS = ["before", "prefix", "match", "suffix", "after", "pos", "offset", "gid", "sim", "name", "taxon", "entrez"]
OFFSET = 6
MENTION = 2
GID = 7
KEYS = [9, 10, 11]
INTEGERS = [7, 10, 11]
Gene = namedtuple('Gene', GENE_ITEMS)

def ParseGeneLines(filepath):
    for line in open(filepath):
        items = line.split('\t')
        try:
            start, end = items[OFFSET].split(":")
        except IndexError:
            print(filepath, file=sys.stderr)
            print(line, file=sys.stderr)
            print("offset:", items[OFFSET], file=sys.stderr)
            raise
        items[OFFSET] = (int(start), int(end))
        for i in KEYS:
            txt = items[i]
            items[i] = txt[txt.find('"') + 1:txt.rfind('"')]
        for i in INTEGERS:
            try:
                items[i] = int(items[i])
            except ValueError:
                print(filepath, file=sys.stderr)
                print(line, file=sys.stderr)
                print(items[i], i, file=sys.stderr)
                raise
        yield (items[GID], items[MENTION]), Gene(*items)

def ParseNerLines(filepath):
    # 0:match, 1:offset, 2:type 3:conf
    for line in open(filepath):
        items = line.split('\t')
        start, end = items[1].split(":")
        yield (int(start), int(end)), items[2]

def ParseTaxonLines(filepath):
    # 0:match, 1:offset, 2:tid 3:sim
    for line in open(filepath):
        items = line.split('\t')
        start, end = items[1].split(":")
        yield int(items[2]), (int(start), int(end))

def ParseGold(filepath):
    for line in open(filepath):
        file_id, entrez = line.split('\t')
        yield file_id, int(entrez)

def ParseLinkout(filepath):
    for line in open(filepath):
        gid, count = line.split()
        yield int(gid), int(count)

def ParseCounters(filepath):
    for line in open(filepath):
        try:
            gid, sym, refc, globc = line.split('\t')
        except ValueError:
            print(filepath, '"{}"'.format(line))
            raise
        yield int(gid), sym, int(refc), int(globc)

def JoinData(article_id, genes, taxa, entities, gold, links, symbols, references):
    sym_counts = defaultdict(set)
    ambiguity_counts = defaultdict(int)
    for gid, data in genes.items():
        for sym, mentions in data.items():
            sym_counts[sym].update(m.offset for m in mentions)
            for m in mentions:
                ambiguity_counts[m.offset] += 1

    sym_counts = dict(sym_counts)
    mention_counts = 0

    for gid, data in genes.items():
        count_GID = sum(len(m) for m in data.values())

        for sym, mentions in data.items():
            name = mentions[0].name
            count_SYM = len(sym_counts[sym])
            count_GIDSYM = len(mentions)
            count_links = links[gid] if gid in links else 0
            try:
                count_sym = symbols[name]
            except KeyError:
                if any(ord(i) in GREEK_ORDINALS for i in name):
                    replaced = []
                    for i in name:
                        if i in GREEK2LATIN:
                            replaced.append(GREEK2LATIN[i])
                        else:
                            replaced.append(i)
                    name = ''.join(replaced)
                    try:
                        count_sym = symbols[name]
                    except KeyError:
                        print('unknown name "{}" in mention "{}"'.format(name, gid), file=sys.stderr)
                        continue
                else:
                    print('unknown name "{}" in mention "{}"'.format(name, gid), file=sys.stderr)
                    continue
            try:
                count_refs = references[gid][name]
            except KeyError:
                if gid not in references:
                    print('unknown gid "{}" with name "{}"'.format(gid, name), file=sys.stderr)
                else:
                    print('unknown name "{}" for gid "{}"'.format(name, gid), file=sys.stderr)
                continue

            try:
                count_tids = len(taxa[mentions[0].taxon])
            except KeyError:
                print('unknown taxon "{}" in mention "{}"'.format(mentions[0].taxon, gid), file=sys.stderr)
                continue

            for mention in mentions:
                offset = mention.offset
                # distance tid <-> mention as 1/distance
                taxon_distance = sys.maxsize
                for taxon_offset in taxa[mention.taxon]:
                    taxon_distance = min(taxon_distance,
                                         min(abs(offset[0] - taxon_offset[1]),
                                             abs(taxon_offset[0] - offset[1])))
                taxon_distance = 1.0 / taxon_distance

                # is mention an entity, and which?
                entity_type = None
                for entity_offset in entities:
                    if offset[0] >= entity_offset[0] and offset[1] <= entity_offset[1]:
                        entity_type = entities[entity_offset]
                        break

                e = [
                    1.0 if entity_type is None else 0.0,
                    1.0 if entity_type == 'cell_line' else 0.0,
                    1.0 if entity_type == 'cell_type' else 0.0,
                    1.0 if entity_type == 'DNA' else 0.0,
                    1.0 if entity_type == 'protein' else 0.0,
                    1.0 if entity_type == 'RNA' else 0.0,
                ]

                # (mixed!) data per SYM
                # 1=none, 2=cell_line, 3=cell_type, 4=DNA, 5=protein, 6=RNA, 7=sim, 8=tax_distance, 9=mention_ambiguity
                yield mention.entrez in gold, article_id, e[0], e[1], e[2], e[3], e[4], e[5], float(mention.sim), taxon_distance, 1.0 / ambiguity_counts[mention.offset]
                mention_counts += 1

            # (integer only!) data per GID
            # 10=GIDc, 11=SYMc, 12=GIDSYMc, 13=links, 14=syms, 15=refs, 16=tids
            yield [mention_counts, count_GID, count_SYM, count_GIDSYM, count_links, count_sym, count_refs, count_tids]

def WriteLines(result_generator):
    sym_data = []
    gid_data = []
    for features in result_generator:
        if type(features[0]) is bool:
            sym_data.append(features)
        else:
            gid_data.append(features)

    # normalize GID data (all integers!)
    for i in range(1, len(gid_data[0])):
        m = max(r[i] for r in gid_data)
        if m == 0:
            print("all vaules zero in position", i+1, file=sys.stderr)
            for r in gid_data:
                r[i] = 0.0
        else:
            for r in gid_data:
                r[i] /= m

    base = len(sym_data[0]) - 1
    gid_iter = iter(gid_data)
    gid_feats = next(gid_iter)
    gid_feats_str = ' '.join('{}:{:.8f}'.format(i+base, f) for i, f in enumerate(gid_feats[1:]))
    for cnt, r in enumerate(sym_data):
        while cnt >= gid_feats[0]:
            gid_feats = next(gid_iter)
            gid_feats_str = ' '.join('{}:{:.8f}'.format(i+base, f) for i, f in enumerate(gid_feats[1:]))
        print(int(r[0]), 'qid:{}'.format(r[1]), ' '.join('{}:{:.8f}'.format(i+1, f) for i, f in enumerate(r[2:])), gid_feats_str)

def Process(gene_dir, taxon_dir, ner_dir, gold_file, counter_file, linkout_file):
    gold = defaultdict(set)
    for article_id, entrez_id in ParseGold(gold_file):
        gold[article_id].add(entrez_id)
    print("parsed {} gold items".format(len(gold)), file=sys.stderr)
    gold = dict(gold)

    link_counts = dict(ParseLinkout(linkout_file))
    print("parsed {} linkout items".format(len(link_counts)), file=sys.stderr)

    sym_counts = {}
    ref_counts = defaultdict(dict)
    for gid, sym, refc, symc in ParseCounters(counter_file):
        if sym not in sym_counts:
            sym_counts[sym] = symc
        ref_counts[gid][sym] = refc
    print("parsed {} refcount items".format(len(ref_counts)), file=sys.stderr)
    ref_counts = dict(ref_counts)

    for filepath in os.listdir(gene_dir):
        article_id = filepath[:filepath.rfind('.')]

        if article_id not in gold:
            #print("skipping", article_id, file=sys.stderr)
            continue

        print("processing", article_id, file=sys.stderr)

        genes = {}
        for (gid, mention), gene in ParseGeneLines(os.path.join(gene_dir, filepath)):
            if gid in genes:
                if mention in genes[gid]:
                    genes[gid][mention].append(gene)
                else:
                    genes[gid][mention] = [gene]
            else:
                genes[gid] = {mention: [gene]}

        taxa = defaultdict(set)
        for tid, offset in ParseTaxonLines(os.path.join(taxon_dir, filepath)):
            taxa[tid].add(offset)
        taxa = dict(taxa)

        entities = dict(ParseNerLines(os.path.join(ner_dir, filepath)))
        WriteLines(JoinData(article_id, genes, taxa, entities, gold[article_id], link_counts, sym_counts, ref_counts))

if __name__ == '__main__':
    # gene_dir, taxon_dir, ner_dir, gold_file, counter_file, refcount_file
    Process(*sys.argv[1:])
<|endoftext|>


<commit_before>#!/usr/bin/env python

# EXAMPLE-npyscreenreactor-client.py

# Copyright (c) 2015 Mark Tearle <mark@tearle.com>
# See LICENSE for details.

# This examples connects to localhost on port 5000
#
# This can either be
#    EXAMPLE-npyscreenreactor-server.py
# or
#    netcat:
#
#        nc -l 127.0.0.1 5000

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

import curses

import npyscreen
import npyscreenreactor

from twisted.internet import protocol
from twisted.protocols.basic import LineReceiver

class TestProtocol(LineReceiver):
    """This is just about the simplest possible protocol"""

    def __init__(self, app, *args, **keywords):
        self.app = app

    def connectionMade(self):
        line='Connection made.\n\n'
        self.app.lines_to_buffer(line)

    def lineReceived(self,line):
	self.app.line_to_buffer(line)

class TestClientFactory(protocol.ClientFactory):
    def __init__(self,app):
        self.app = app
	app.set_factory(self)

    def startedConnecting(self, connector):
        line='Started connecting'
	self.app.connector = connector
	#self.app.line_to_buffer(line)

    def buildProtocol(self, addr):
        line='Building protocol'
        self.instance = self.protocol(self.app)
	self.instance.delimiter = "\n"
	self.app.set_instance(self.instance)
	self.instance.setLineMode()
        return self.instance

    def clientConnectionLost(self, connector, reason):
	self.app.connector = connector
        line="""
Lost connection.
Reason: %s

Type ^Q to exit, hit ENTER to (re) connect
""" % reason
        self.app.lines_to_buffer(line)

    def clientConnectionFailed(self, connector, reason):
	self.app.connector = connector
        line="""
Connection failed.
Reason: %s

Type ^Q to exit, hit ENTER to (re) connect
""" % reason
        self.app.lines_to_buffer(line)

class EditorFormExample(npyscreen.FormMutt):
    MAIN_WIDGET_CLASS = npyscreen.BufferPager

    def __init__(self, *args, **keywords):
	super(npyscreen.FormMutt, self).__init__(*args, **keywords)
	self.wCommand.add_handlers({
			curses.ascii.NL : self.do_line,
			curses.ascii.CR : self.do_line,
			"^Q" : self.exit_application,
		})
	self.wMain.add_handlers({
			"^Q" : self.exit_application,
		})

    def do_line(self,name):
	if self.parentApp.connector.state == "disconnected":
		self.parentApp.connector.connect()
	elif self.parentApp.connector.state == "connecting":
		pass
	else:
		self.parentApp.process_line()
		self.display()

    def afterEditing(self):
        self.parentApp.switchForm(None)

    def exit_application(self,name):
        self.parentApp.line_to_buffer("Quitting...")
        self.parentApp.switchForm(None)

class TestApp(npyscreen.StandardApp):
    def onCleanExit(self):
        npyscreen.notify_wait("Goodbye!")

    def onStart(self):
	self.connector = None

	# connection details
	host = "127.0.0.1"
	port = 5000

	intro="""
%s

Welcome to the twisted npyscreen reactor client example.

Type ^Q to exit, hit ENTER to (re) connect

Connecting to %s on %s
...
""" % (__file__, host, port)

        factory = TestClientFactory(App)
        factory.protocol = TestProtocol
        self.reactor.connectTCP(host,port,factory)
        self.F = self.addForm('MAIN', EditorFormExample)
        self.F.wStatus1.value = "Status Line "
        self.F.wStatus2.value = "Enter text to send ...."
	self.lines_to_buffer(intro)
	# set initial focus
	# how to do this?

    def when_exit(self,val):
        self.parentApp.switchForm(None)

    def lines_to_buffer(self, lines):
	for line in lines.split("\n"):
		self.line_to_buffer(line)

    def line_to_buffer(self, line):
	self.F.wMain.buffer((line,))
	self.F.display()

    def process_line(self):
	self.instance.sendLine(self.F.wCommand.value)
	self.line_to_buffer(self.F.wCommand.value)
	self.F.wCommand.value = ""
	self.F.display()

    def set_reactor(self,reactor):
	self.reactor = reactor

    def set_factory(self,factory):
	self.factory = factory

    def set_instance(self,instance):
	self.instance = instance

if __name__ == "__main__":
    App = TestApp()
    reactor = npyscreenreactor.install()
    reactor.registerNpyscreenApp(App)
    App.set_reactor(reactor)
    try:
        reactor.run()
    finally:
        reactor.stop()

<commit_msg>Make indentation consistently spaces in EXAMPLE-npyscreenreactor-client.py<commit_after>#!/usr/bin/env python

# EXAMPLE-npyscreenreactor-client.py

# Copyright (c) 2015 Mark Tearle <mark@tearle.com>
# See LICENSE for details.

# This examples connects to localhost on port 5000
#
# This can either be
#    EXAMPLE-npyscreenreactor-server.py
# or
#    netcat:
#
#        nc -l 127.0.0.1 5000

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

import curses

import npyscreen
import npyscreenreactor

from twisted.internet import protocol
from twisted.protocols.basic import LineReceiver

class TestProtocol(LineReceiver):
    """This is just about the simplest possible protocol"""

    def __init__(self, app, *args, **keywords):
        self.app = app

    def connectionMade(self):
        line='Connection made.\n\n'
        self.app.lines_to_buffer(line)

    def lineReceived(self,line):
        self.app.line_to_buffer(line)

class TestClientFactory(protocol.ClientFactory):
    def __init__(self,app):
        self.app = app
        app.set_factory(self)

    def startedConnecting(self, connector):
        line='Started connecting'
        self.app.connector = connector
        #self.app.line_to_buffer(line)

    def buildProtocol(self, addr):
        line='Building protocol'
        self.instance = self.protocol(self.app)
        self.instance.delimiter = "\n"
        self.app.set_instance(self.instance)
        self.instance.setLineMode()
        return self.instance

    def clientConnectionLost(self, connector, reason):
        self.app.connector = connector
        line="""
Lost connection.
Reason: %s

Type ^Q to exit, hit ENTER to (re) connect
""" % reason
        self.app.lines_to_buffer(line)

    def clientConnectionFailed(self, connector, reason):
        self.app.connector = connector
        line="""
Connection failed.
Reason: %s

Type ^Q to exit, hit ENTER to (re) connect
""" % reason
        self.app.lines_to_buffer(line)

class EditorFormExample(npyscreen.FormMutt):
    MAIN_WIDGET_CLASS = npyscreen.BufferPager

    def __init__(self, *args, **keywords):
        super(npyscreen.FormMutt, self).__init__(*args, **keywords)
        self.wCommand.add_handlers({
                        curses.ascii.NL : self.do_line,
                        curses.ascii.CR : self.do_line,
                        "^Q" : self.exit_application,
                })
        self.wMain.add_handlers({
                        "^Q" : self.exit_application,
                })

    def do_line(self,name):
        if self.parentApp.connector.state == "disconnected":
                self.parentApp.connector.connect()
        elif self.parentApp.connector.state == "connecting":
                pass
        else:
                self.parentApp.process_line()
                self.display()

    def afterEditing(self):
        self.parentApp.switchForm(None)

    def exit_application(self,name):
        self.parentApp.line_to_buffer("Quitting...")
        self.parentApp.switchForm(None)

class TestApp(npyscreen.StandardApp):
    def onCleanExit(self):
        npyscreen.notify_wait("Goodbye!")

    def onStart(self):
        self.connector = None

        # connection details
        host = "127.0.0.1"
        port = 5000

        intro="""
%s

Welcome to the twisted npyscreen reactor client example.

Type ^Q to exit, hit ENTER to (re) connect

Connecting to %s on %s
...
""" % (__file__, host, port)

        factory = TestClientFactory(App)
        factory.protocol = TestProtocol
        self.reactor.connectTCP(host,port,factory)
        self.F = self.addForm('MAIN', EditorFormExample)
        self.F.wStatus1.value = "Status Line "
        self.F.wStatus2.value = "Enter text to send ...."
        self.lines_to_buffer(intro)
        # set initial focus
        # how to do this?

    def when_exit(self,val):
        self.parentApp.switchForm(None)

    def lines_to_buffer(self, lines):
        for line in lines.split("\n"):
                self.line_to_buffer(line)

    def line_to_buffer(self, line):
        self.F.wMain.buffer((line,))
        self.F.display()

    def process_line(self):
        self.instance.sendLine(self.F.wCommand.value)
        self.line_to_buffer(self.F.wCommand.value)
        self.F.wCommand.value = ""
        self.F.display()

    def set_reactor(self,reactor):
        self.reactor = reactor

    def set_factory(self,factory):
        self.factory = factory

    def set_instance(self,instance):
        self.instance = instance

if __name__ == "__main__":
    App = TestApp()
    reactor = npyscreenreactor.install()
    reactor.registerNpyscreenApp(App)
    App.set_reactor(reactor)
    try:
        reactor.run()
    finally:
        reactor.stop()

<|endoftext|>