<commit_before>from .connectionpools import ConnectionPool
from .cloudvolume import CloudVolume
from .provenance import DataLayerProvenance
from .skeletonservice import PrecomputedSkeleton, SkeletonEncodeError, SkeletonDecodeError
from .storage import Storage
from .threaded_queue import ThreadedQueue
from .txrx import EmptyVolumeException, EmptyRequestException, AlignmentError
from .volumecutout import VolumeCutout

from . import secrets
from . import txrx

__version__ = '0.33.3'
<commit_msg>Version 0.33.4<commit_after>from .connectionpools import ConnectionPool
from .cloudvolume import CloudVolume
from .provenance import DataLayerProvenance
from .skeletonservice import PrecomputedSkeleton, SkeletonEncodeError, SkeletonDecodeError
from .storage import Storage
from .threaded_queue import ThreadedQueue
from .txrx import EmptyVolumeException, EmptyRequestException, AlignmentError
from .volumecutout import VolumeCutout

from . import secrets
from . import txrx

__version__ = '0.33.4'
<|endoftext|>


<commit_before>"""Unit tests for CLTK"""

__author__ = 'Kyle P. Johnson <kyle@kyle-p-johnson.com>'
__license__ = 'MIT License. See LICENSE.'

from cltk.corpus.classical_greek.beta_to_unicode import Replacer
from cltk.stem.classical_latin.j_and_v_converter import JVReplacer
from cltk.tokenize.sentence_tokenizer_greek import tokenize_latin_sentences
from nltk.tokenize.punkt import PunktWordTokenizer
import os
import unittest

class TestSequenceFunctions(unittest.TestCase):  # pylint: disable=R0904
    """Class for unittest"""

    def test_latin_i_u_transform(self):
        """Test conversion of j to i and v to u"""
        j = JVReplacer()
        trans = j.replace('vem jam VEL JAM')
        self.assertEqual(trans, 'uem iam UEL IAM')

    def test_latin_stopwords(self):
        """Filter Latin stopwords"""
        from cltk.stop.classical_latin.stops import STOPS_LIST
        sentence = 'Quo usque tandem abutere, Catilina, patientia nostra?'
        lowered = sentence.lower()
        tokens = PunktWordTokenizer().tokenize(lowered)
        no_stops = [w for w in tokens if w not in STOPS_LIST]
        target_list = ['usque', 'tandem', 'abutere', ',', 'catilina', ',',
                       'patientia', 'nostra', '?']
        self.assertEqual(no_stops, target_list)

    def test_greek_stopwords(self):
        """Filter Greek stopwords."""
        from cltk.stop.classical_greek.stops_unicode import STOPS_LIST
        sentence = 'Ἅρπαγος δὲ καταστρεψάμενος Ἰωνίην ἐποιέετο στρατηίην \
        ἐπὶ Κᾶρας καὶ Καυνίους καὶ Λυκίους, ἅμα ἀγόμενος καὶ Ἴωνας καὶ \
        Αἰολέας.'
        lowered = sentence.lower()
        tokens = PunktWordTokenizer().tokenize(lowered)
        no_stops = [w for w in tokens if w not in STOPS_LIST]
        target_list = ['ἅρπαγος', 'καταστρεψάμενος', 'ἰωνίην', 'ἐποιέετο',
                       'στρατηίην', 'κᾶρας', 'καυνίους', 'λυκίους', ',',
                       'ἅμα', 'ἀγόμενος', 'ἴωνας', 'αἰολέας.']
        self.assertEqual(no_stops, target_list)

    def test_greek_betacode_to_unicode(self):
        """Test conversion of Beta Code to Unicode.
        Note: assertEqual appears to not be correctly comparing certain
        characters (ά and ί, at least).
        """
        beta_example = r"""O(/PWS OU)=N MH\ TAU)TO\ """
        replacer = Replacer()
        unicode = replacer.beta_code(beta_example)
        target_unicode = 'ὅπως οὖν μὴ ταὐτὸ '
        self.assertEqual(unicode, target_unicode)

    def test_sentence_tokenizer_greek(self):
        """Reads Greek sentence tokenizer pickle, tokenizes into a list, and
        checks against a known good list.
        """
        try:
            cltk_data_dir_rel = '~/cltk_data'
            cltk_data_dir_abs = os.path.expanduser(cltk_data_dir_rel)
            greek_sentence_tokenizer_dir = os.path.join(cltk_data_dir_abs,
                                                        'compiled/sentence_tokens_greek')
            pickle_file = 'greek.pickle'
            pickle_file_path = os.path.join(greek_sentence_tokenizer_dir, pickle_file)
            try:
                os.path.exists(pickle_file_path)
                sentences = 'εἰ δὲ καὶ τῷ ἡγεμόνι πιστεύσομεν ὃν ἂν Κῦρος διδῷ, τί κωλύει καὶ τὰ ἄκρα ἡμῖν κελεύειν Κῦρον προκαταλαβεῖν; ἐγὼ γὰρ ὀκνοίην μὲν ἂν εἰς τὰ πλοῖα ἐμβαίνειν ἃ ἡμῖν δοίη, μὴ ἡμᾶς ταῖς τριήρεσι καταδύσῃ, φοβοίμην δ᾽ ἂν τῷ ἡγεμόνι ὃν δοίη ἕπεσθαι, μὴ ἡμᾶς ἀγάγῃ ὅθεν οὐκ ἔσται ἐξελθεῖν· βουλοίμην δ᾽ ἂν ἄκοντος ἀπιὼν Κύρου λαθεῖν αὐτὸν ἀπελθών· ὃ οὐ δυνατόν ἐστιν.'  # pylint: disable=C0301
                good_tokenized_sentences = ['εἰ δὲ καὶ τῷ ἡγεμόνι πιστεύσομεν ὃν ἂν Κῦρος διδῷ, τί κωλύει καὶ τὰ ἄκρα ἡμῖν κελεύειν Κῦρον προκαταλαβεῖν;', 'ἐγὼ γὰρ ὀκνοίην μὲν ἂν εἰς τὰ πλοῖα ἐμβαίνειν ἃ ἡμῖν δοίη, μὴ ἡμᾶς ταῖς τριήρεσι καταδύσῃ, φοβοίμην δ᾽ ἂν τῷ ἡγεμόνι ὃν δοίη ἕπεσθαι, μὴ ἡμᾶς ἀγάγῃ ὅθεν οὐκ ἔσται ἐξελθεῖν· βουλοίμην δ᾽ ἂν ἄκοντος ἀπιὼν Κύρου λαθεῖν αὐτὸν ἀπελθών· ὃ οὐ δυνατόν ἐστιν.']  # pylint: disable=C0301
                tokenized_sentences = tokenize_latin_sentences(sentences)
                self.assertEqual(tokenized_sentences, good_tokenized_sentences)
            except:
                pass
        except:
            pass

    def test_sentence_tokenizer_latin(self):
        """Reads Latin sentence tokenizer pickle, tokenizes into a list, and
        checks against a known good list.
        """
        try:
            cltk_data_dir_rel = '~/cltk_data'
            cltk_data_dir_abs = os.path.expanduser(cltk_data_dir_rel)
            latin_sentence_tokenizer_dir = os.path.join(cltk_data_dir_abs,
                                                        'compiled/sentence_tokens_latin')
            pickle_file = 'latin.pickle'
            pickle_file_path = os.path.join(latin_sentence_tokenizer_dir, pickle_file)
            try:
                os.path.exists(pickle_file_path)
                sentences = "Num qui exsules restituti? Unum aiebat, praeterea neminem. Num immunitates datae? 'Nullae', respondebat. Assentiri etiam nos Ser. Sulpicio, clarissimo viro, voluit, ne qua tabula post Idus Martias ullius decreti Caesaris aut beneficii figeretur. Multa praetereo, eaque praeclara; ad singulare enim M. Antoni factum festinat oratio. Dictaturam, quae iam vim regiae potestatis obsederat, funditus ex re publica sustulit; de qua re ne sententias quidem diximus. Scriptum senatus consultum, quod fieri vellet, attulit; quo recitato, auctoritatem eius summo studio secuti sumus eique amplissimis verbis per senatus consultum gratias egimus."  # pylint: disable=C0301
                good_tokenized_sentences = ['Num qui exsules restituti?', 'Unum aiebat, praeterea neminem.', 'Num immunitates datae?', "'Nullae', respondebat.", 'Assentiri etiam nos Ser.', 'Sulpicio, clarissimo viro, voluit, ne qua tabula post Idus Martias ullius decreti Caesaris aut beneficii figeretur.', 'Multa praetereo, eaque praeclara; ad singulare enim M. Antoni factum festinat oratio.', 'Dictaturam, quae iam vim regiae potestatis obsederat, funditus ex re publica sustulit; de qua re ne sententias quidem diximus.', 'Scriptum senatus consultum, quod fieri vellet, attulit; quo recitato, auctoritatem eius summo studio secuti sumus eique amplissimis verbis per senatus consultum gratias egimus.']  # pylint: disable=C0301
                tokenized_sentences = tokenize_latin_sentences(sentences)
                self.assertEqual(tokenized_sentences, good_tokenized_sentences)
            except:
                pass
        except:
            pass

if __name__ == '__main__':
    unittest.main()
<commit_msg>fix tes<commit_after>"""Unit tests for CLTK"""

__author__ = 'Kyle P. Johnson <kyle@kyle-p-johnson.com>'
__license__ = 'MIT License. See LICENSE.'

from cltk.corpus.classical_greek.beta_to_unicode import Replacer
from cltk.stem.classical_latin.j_and_v_converter import JVReplacer
from cltk.tokenize.sentence_tokenizer_greek import tokenize_greek_sentences
from cltk.tokenize.sentence_tokenizer_latin import tokenize_latin_sentences
from nltk.tokenize.punkt import PunktWordTokenizer
import os
import unittest

class TestSequenceFunctions(unittest.TestCase):  # pylint: disable=R0904
    """Class for unittest"""

    def test_latin_i_u_transform(self):
        """Test conversion of j to i and v to u"""
        j = JVReplacer()
        trans = j.replace('vem jam VEL JAM')
        self.assertEqual(trans, 'uem iam UEL IAM')

    def test_latin_stopwords(self):
        """Filter Latin stopwords"""
        from cltk.stop.classical_latin.stops import STOPS_LIST
        sentence = 'Quo usque tandem abutere, Catilina, patientia nostra?'
        lowered = sentence.lower()
        tokens = PunktWordTokenizer().tokenize(lowered)
        no_stops = [w for w in tokens if w not in STOPS_LIST]
        target_list = ['usque', 'tandem', 'abutere', ',', 'catilina', ',',
                       'patientia', 'nostra', '?']
        self.assertEqual(no_stops, target_list)

    def test_greek_stopwords(self):
        """Filter Greek stopwords."""
        from cltk.stop.classical_greek.stops_unicode import STOPS_LIST
        sentence = 'Ἅρπαγος δὲ καταστρεψάμενος Ἰωνίην ἐποιέετο στρατηίην \
        ἐπὶ Κᾶρας καὶ Καυνίους καὶ Λυκίους, ἅμα ἀγόμενος καὶ Ἴωνας καὶ \
        Αἰολέας.'
        lowered = sentence.lower()
        tokens = PunktWordTokenizer().tokenize(lowered)
        no_stops = [w for w in tokens if w not in STOPS_LIST]
        target_list = ['ἅρπαγος', 'καταστρεψάμενος', 'ἰωνίην', 'ἐποιέετο',
                       'στρατηίην', 'κᾶρας', 'καυνίους', 'λυκίους', ',',
                       'ἅμα', 'ἀγόμενος', 'ἴωνας', 'αἰολέας.']
        self.assertEqual(no_stops, target_list)

    def test_greek_betacode_to_unicode(self):
        """Test conversion of Beta Code to Unicode.
        Note: assertEqual appears to not be correctly comparing certain
        characters (ά and ί, at least).
        """
        beta_example = r"""O(/PWS OU)=N MH\ TAU)TO\ """
        replacer = Replacer()
        unicode = replacer.beta_code(beta_example)
        target_unicode = 'ὅπως οὖν μὴ ταὐτὸ '
        self.assertEqual(unicode, target_unicode)

    def test_sentence_tokenizer_greek(self):
        """Reads Greek sentence tokenizer pickle, tokenizes into a list, and
        checks against a known good list.
        """
        try:
            cltk_data_dir_rel = '~/cltk_data'
            cltk_data_dir_abs = os.path.expanduser(cltk_data_dir_rel)
            greek_sentence_tokenizer_dir = os.path.join(cltk_data_dir_abs,
                                                        'compiled/sentence_tokens_greek')
            pickle_file = 'greek.pickle'
            pickle_file_path = os.path.join(greek_sentence_tokenizer_dir, pickle_file)
            try:
                os.path.exists(pickle_file_path)
                sentences = 'εἰ δὲ καὶ τῷ ἡγεμόνι πιστεύσομεν ὃν ἂν Κῦρος διδῷ, τί κωλύει καὶ τὰ ἄκρα ἡμῖν κελεύειν Κῦρον προκαταλαβεῖν; ἐγὼ γὰρ ὀκνοίην μὲν ἂν εἰς τὰ πλοῖα ἐμβαίνειν ἃ ἡμῖν δοίη, μὴ ἡμᾶς ταῖς τριήρεσι καταδύσῃ, φοβοίμην δ᾽ ἂν τῷ ἡγεμόνι ὃν δοίη ἕπεσθαι, μὴ ἡμᾶς ἀγάγῃ ὅθεν οὐκ ἔσται ἐξελθεῖν· βουλοίμην δ᾽ ἂν ἄκοντος ἀπιὼν Κύρου λαθεῖν αὐτὸν ἀπελθών· ὃ οὐ δυνατόν ἐστιν.'  # pylint: disable=C0301
                good_tokenized_sentences = ['εἰ δὲ καὶ τῷ ἡγεμόνι πιστεύσομεν ὃν ἂν Κῦρος διδῷ, τί κωλύει καὶ τὰ ἄκρα ἡμῖν κελεύειν Κῦρον προκαταλαβεῖν;', 'ἐγὼ γὰρ ὀκνοίην μὲν ἂν εἰς τὰ πλοῖα ἐμβαίνειν ἃ ἡμῖν δοίη, μὴ ἡμᾶς ταῖς τριήρεσι καταδύσῃ, φοβοίμην δ᾽ ἂν τῷ ἡγεμόνι ὃν δοίη ἕπεσθαι, μὴ ἡμᾶς ἀγάγῃ ὅθεν οὐκ ἔσται ἐξελθεῖν· βουλοίμην δ᾽ ἂν ἄκοντος ἀπιὼν Κύρου λαθεῖν αὐτὸν ἀπελθών· ὃ οὐ δυνατόν ἐστιν.']  # pylint: disable=C0301
                tokenized_sentences = tokenize_greek_sentences(sentences)
                self.assertEqual(tokenized_sentences, good_tokenized_sentences)
            except:
                pass
        except:
            pass

    def test_sentence_tokenizer_latin(self):
        """Reads Latin sentence tokenizer pickle, tokenizes into a list, and
        checks against a known good list.
        """
        try:
            cltk_data_dir_rel = '~/cltk_data'
            cltk_data_dir_abs = os.path.expanduser(cltk_data_dir_rel)
            latin_sentence_tokenizer_dir = os.path.join(cltk_data_dir_abs,
                                                        'compiled/sentence_tokens_latin')
            pickle_file = 'latin.pickle'
            pickle_file_path = os.path.join(latin_sentence_tokenizer_dir, pickle_file)
            try:
                os.path.exists(pickle_file_path)
                sentences = "Num qui exsules restituti? Unum aiebat, praeterea neminem. Num immunitates datae? 'Nullae', respondebat. Assentiri etiam nos Ser. Sulpicio, clarissimo viro, voluit, ne qua tabula post Idus Martias ullius decreti Caesaris aut beneficii figeretur. Multa praetereo, eaque praeclara; ad singulare enim M. Antoni factum festinat oratio. Dictaturam, quae iam vim regiae potestatis obsederat, funditus ex re publica sustulit; de qua re ne sententias quidem diximus. Scriptum senatus consultum, quod fieri vellet, attulit; quo recitato, auctoritatem eius summo studio secuti sumus eique amplissimis verbis per senatus consultum gratias egimus."  # pylint: disable=C0301
                good_tokenized_sentences = ['Num qui exsules restituti?', 'Unum aiebat, praeterea neminem.', 'Num immunitates datae?', "'Nullae', respondebat.", 'Assentiri etiam nos Ser.', 'Sulpicio, clarissimo viro, voluit, ne qua tabula post Idus Martias ullius decreti Caesaris aut beneficii figeretur.', 'Multa praetereo, eaque praeclara; ad singulare enim M. Antoni factum festinat oratio.', 'Dictaturam, quae iam vim regiae potestatis obsederat, funditus ex re publica sustulit; de qua re ne sententias quidem diximus.', 'Scriptum senatus consultum, quod fieri vellet, attulit; quo recitato, auctoritatem eius summo studio secuti sumus eique amplissimis verbis per senatus consultum gratias egimus.']  # pylint: disable=C0301
                tokenized_sentences = tokenize_latin_sentences(sentences)
                self.assertEqual(tokenized_sentences, good_tokenized_sentences)
            except:
                pass
        except:
            pass

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import sys
import tempfile
import gzip
import numpy as np
import pandas as pd
from aclust import aclust
from .plotting import plot_dmr, plot_hbar, plot_continuous
from clustercorr import feature_gen, cluster_to_dataframe, clustered_model

xopen = lambda f: gzip.open(f) if f.endswith('.gz') else open(f)

def is_numeric(pd_series):
    if np.issubdtype(pd_series.dtype, int) or \
        np.issubdtype(pd_series.dtype, float):
        return len(pd_series.unique()) > 2
    return False

def run_model(cluster, covs, model, X, outlier_sds, liptak, bumping, gee_args,
        skat):
    # we turn the cluster list into a pandas dataframe with columns
    # of samples and rows of probes. these must match our covariates
    cluster_df = cluster_to_dataframe(cluster, columns=covs.index)

        # now we want to test a model on our clustered dataset.
    res = clustered_model(covs, cluster_df, model, X=X, gee_args=gee_args,
                liptak=liptak, bumping=bumping, skat=skat,
                outlier_sds=outlier_sds)
    res['chrom'] = cluster[0].group
    res['start'] = cluster[0].start
    res['end'] = cluster[-1].end
    res['n_probes'] = len(cluster)
    return res

def distX(dmr, expr):
    strand = "-" if expr.strand == "-" else "+"
    dmr['distance'] = 0
    if dmr['end'] < expr.start:
        dmr['distance'] = expr.start - dmr['end']
        # dmr is left of gene. that means it is upstream if strand is +
        # we use "-" for upstream
        if strand == "+":
            dmr['distance'] *= -1

    elif dmr['start'] > expr.end:
        dmr['distance'] = dmr['start'] - expr.end
        # dmr is right of gene. that is upstream if strand is -
        # use - for upstream
        if strand == "-":
            dmr['distance'] *= -1
    dmr['Xstart'], dmr['Xend'], dmr['Xstrand'] = expr.start, expr.end, expr.strand

def clustermodel(fcovs, fmeth, model, max_dist=500, linkage='complete',
        rho_min=0.3, min_clust_size=2, sep="\t",
        X=None, X_locs=None, X_dist=None,
        outlier_sds=None,
        liptak=False, bumping=False, gee_args=(), skat=False, png_path=None):
    # an iterable of feature objects
    feature_iter = feature_gen(fmeth, rho_min=rho_min)
    assert min_clust_size > 1

    cluster_gen = (c for c in aclust(feature_iter, max_dist=max_dist,
                                     max_skip=1, linkage=linkage)
                    if len(c) >= min_clust_size)

    covs = pd.read_table(fcovs, index_col=0, sep=sep)
    covariate = model.split("~")[1].split("+")[0].strip()
    X_file = X

    if not X_locs is None: # read expression into memory and pull out subsets as needed.
        X_locs = pd.read_table(xopen(X_locs), index_col="probe")
        X = pd.read_table(xopen(X), index_col=0)
        X_probes = set(X.index)

    for cluster in cluster_gen:

        if not X_locs is None:
            fh = tempfile.NamedTemporaryFile(delete=True)
            chrom = cluster[0].group
            start, end = cluster[0].start, cluster[-1].end
            probe_locs = X_locs[((X_locs.ix[:, 0] == chrom) &
                             (X_locs.ix[:, 1] < end + X_dist) &
                             (X_locs.ix[:, 2] > start - X_dist))]
            probes = [p for p in probe_locs.index if p in X_probes]
            if len(probes) == 0: continue
            subset = X.ix[probes, :]
            subset.to_csv(fh.name, sep="\t", index=True, index_label="probe",
                            float_format="%.4f")
            fh.flush()
            X_file = fh.name

        res = run_model(cluster, covs, model, X_file, outlier_sds, liptak, bumping, gee_args,
                skat)

        if not X is None:
            if not X_locs is None:
                fh.close()
            # got a pandas dataframe
            df = res
            for i, row in df.iterrows():
                row = dict(row)
                if not X_locs is None:
                    distX(row, probe_locs.ix[row['X'], :])
                yield row

            continue

        if res['p'] < 1e-4 and png_path:
            cluster_df = cluster_to_dataframe(cluster, columns=covs.index)
            plot_res(res, png_path, covs, covariate, cluster_df)
        yield res

def plot_res(res, png_path, covs, covariate, cluster_df):
    from matplotlib import pyplot as plt
    from mpltools import style
    style.use('ggplot')

    region = "{chrom}_{start}_{end}".format(**res)
    if png_path.endswith('show'):
        png = None
    elif png_path.endswith(('.png', '.pdf')):
        png = "%s.%s%s" % (png_path[:-4], region, png_path[-4:])
    elif png_path:
        png = "%s.%s.png" % (png_path.rstrip("."), region)

    if is_numeric(getattr(covs, covariate)):
        f = plot_continuous(covs, cluster_df, covariate, res['chrom'], res, png)
    else:
        f = plt.figure(figsize=(11, 4))
        ax = f.add_subplot(1, 1, 1)
        if 'spaghetti' in png_path:
            plot_dmr(covs, cluster_df, covariate, res['chrom'], res, png)
        else:
            plot_hbar(covs, cluster_df, covariate, res['chrom'], res, png)
        plt.title('p-value: %.3g %s: %.3f' % (res['p'], covariate, res['coef']))
    f.set_tight_layout(True)
    if png:
        plt.savefig(png)
    else:
        plt.show()

def main_example():
    fcovs = "clustercorr/tests/example-covariates.txt"
    fmeth = "clustercorr/tests/example-methylation.txt.gz"
    model = "methylation ~ disease + gender"

    for cluster_p in clustermodel(fcovs, fmeth, model):
        if cluster_p['p'] < 1e-5:
            print cluster_p

def add_modelling_args(p):
    mp = p.add_argument_group('modeling choices (choose one or specify a '
            'mixed-model using lme4 syntax)')
    group = mp.add_mutually_exclusive_group()

    group.add_argument('--skat', action='store_true')
    group.add_argument('--gee-args',
                       help='comma-delimited correlation-structure, variable')
    group.add_argument('--liptak', action="store_true")
    group.add_argument('--bumping', action="store_true")

    p.add_argument('model',
                   help="model in R syntax, e.g. 'methylation ~ disease'")
    p.add_argument('covs', help="tab-delimited file of covariates: shape is "
                   "n_samples * n_covariates")
    p.add_argument('methylation', help="tab-delimited file of methylation"
                   " rows of this file must match the columns of `covs`"
                   " shape is n_probes * n_samples")

def add_expression_args(p):
    ep = p.add_argument_group('optional expression parameters')
    ep.add_argument('--X', help='file with same sample columns as methylation, '
            'rows of probes and values of some measurement (likely expression)'
            ' this will perform a methyl-eQTL--for each DMR, it will test '
            'againts all rows in this methylation array. As such, it is best'
            ' to run this on subsets of data, e.g. only looking for cis '
            'relationships')
    ep.add_argument('--X-locs', help="BED file with locations of probes from"
            " the first column in --X. Should have a 'probe' column header")
    ep.add_argument('--X-dist', type=int, help="only look at cis interactions"
            " between X and methylation sites with this as the maximum",
            default=100000)

def add_clustering_args(p):
    cp = p.add_argument_group('clustering parameters')
    cp.add_argument('--rho-min', type=float, default=0.3,
                   help="minimum correlation to merge 2 probes")
    cp.add_argument('--min-cluster-size', type=int, default=2,
                    help="minimum cluster size on which to run model: "
                   "must be at least 2")
    cp.add_argument('--linkage', choices=['single', 'complete'],
                    default='complete', help="linkage method")

    cp.add_argument('--max-dist', default=500, type=int,
                    help="maximum distance beyond which a probe can not be"
                    " added to a cluster")

def main(args=sys.argv[1:]):
    import argparse
    p = argparse.ArgumentParser(__doc__)
    add_modelling_args(p)
    add_clustering_args(p)

    p.add_argument('--png-path',
                   help="path to save a png of regions with low p-values. If "
                   "this ends with 'show', each plot will be shown in a window"
                   " if this contains the string 'spaghetti', it will draw a "
                   "a spaghetti plot, otherwise, it's a histogram plot")
    p.add_argument('--outlier-sds', type=float, default=30,
            help="remove points that are more than this many standard "
                 "deviations away from the mean (only usable with GEEs"
                 " and mixed-models which allow missing data")

    add_expression_args(p)

    a = p.parse_args(args)

    if a.gee_args is not None:
        method = 'gee:' + a.gee_args
        a.gee_args = a.gee_args.split(",")
    else:
        if a.liptak: method = 'liptak'
        elif a.bumping: method = 'bumping'
        elif a.skat: method = 'skat'
        else:
            assert "|" in a.model
            method = "mixed-model"

    fmt = "{chrom}\t{start}\t{end}\t{coef}\t{p}\t{n_probes}\t{model}\t{method}"
    if a.X_locs:
        fmt += "\t{Xstart}\t{Xend}\t{Xstrand}\t{distance}"
    print "#" + fmt.replace("}", "").replace("{", "")
    for c in clustermodel(a.covs, a.methylation, a.model,
                          max_dist=a.max_dist,
                          linkage=a.linkage,
                          rho_min=a.rho_min,
                          min_clust_size=a.min_cluster_size,
                          liptak=a.liptak,
                          bumping=a.bumping,
                          gee_args=a.gee_args,
                          skat=a.skat,
                          X=a.X,
                          X_locs=a.X_locs,
                          X_dist=a.X_dist,
                          outlier_sds=a.outlier_sds,
                          png_path=a.png_path):
        c['method'] = method
        print fmt.format(**c)

if __name__ == "__main__":
    import sys
    if len(sys.argv) > 1 and sys.argv[1] == "example":
        sys.exit(main_example())
    if len(sys.argv) > 1 and sys.argv[1] == "simulate":
        from . import simulate
        sys.exit(simulate.main(sys.argv[2:]))
    if len(sys.argv) > 1 and sys.argv[1] == "regional":
        sys.exit(region.main(sys.argv[2:]))

    main()
<commit_msg>framework for specifying input regions<commit_after>import sys
import tempfile
import gzip
import numpy as np
import pandas as pd
from aclust import aclust
from .plotting import plot_dmr, plot_hbar, plot_continuous
from clustercorr import feature_gen, cluster_to_dataframe, clustered_model

xopen = lambda f: gzip.open(f) if f.endswith('.gz') else open(f)

def is_numeric(pd_series):
    if np.issubdtype(pd_series.dtype, int) or \
        np.issubdtype(pd_series.dtype, float):
        return len(pd_series.unique()) > 2
    return False

def run_model(cluster, covs, model, X, outlier_sds, liptak, bumping, gee_args,
        skat):
    # we turn the cluster list into a pandas dataframe with columns
    # of samples and rows of probes. these must match our covariates
    cluster_df = cluster_to_dataframe(cluster, columns=covs.index)

        # now we want to test a model on our clustered dataset.
    res = clustered_model(covs, cluster_df, model, X=X, gee_args=gee_args,
                liptak=liptak, bumping=bumping, skat=skat,
                outlier_sds=outlier_sds)
    res['chrom'] = cluster[0].group
    res['start'] = cluster[0].start
    res['end'] = cluster[-1].end
    res['n_probes'] = len(cluster)
    return res

def distX(dmr, expr):
    strand = "-" if expr.strand == "-" else "+"
    dmr['distance'] = 0
    if dmr['end'] < expr.start:
        dmr['distance'] = expr.start - dmr['end']
        # dmr is left of gene. that means it is upstream if strand is +
        # we use "-" for upstream
        if strand == "+":
            dmr['distance'] *= -1

    elif dmr['start'] > expr.end:
        dmr['distance'] = dmr['start'] - expr.end
        # dmr is right of gene. that is upstream if strand is -
        # use - for upstream
        if strand == "-":
            dmr['distance'] *= -1
    dmr['Xstart'], dmr['Xend'], dmr['Xstrand'] = expr.start, expr.end, expr.strand

def clustermodel(fcovs, fmeth, model, max_dist=500, linkage='complete',
        rho_min=0.3, min_clust_size=2, sep="\t",
        X=None, X_locs=None, X_dist=None,
        outlier_sds=None,
        liptak=False, bumping=False, gee_args=(), skat=False, png_path=None):
    # an iterable of feature objects
    feature_iter = feature_gen(fmeth, rho_min=rho_min)
    assert min_clust_size > 1

    cluster_gen = (c for c in aclust(feature_iter, max_dist=max_dist,
                                     max_skip=1, linkage=linkage)
                    if len(c) >= min_clust_size)

    covs = pd.read_table(fcovs, index_col=0, sep=sep)
    covariate = model.split("~")[1].split("+")[0].strip()
    X_file = X

    if not X_locs is None: # read expression into memory and pull out subsets as needed.
        X_locs = pd.read_table(xopen(X_locs), index_col="probe")
        X = pd.read_table(xopen(X), index_col=0)
        X_probes = set(X.index)

    for cluster in cluster_gen:

        if not X_locs is None:
            fh = tempfile.NamedTemporaryFile(delete=True)
            chrom = cluster[0].group
            start, end = cluster[0].start, cluster[-1].end
            probe_locs = X_locs[((X_locs.ix[:, 0] == chrom) &
                             (X_locs.ix[:, 1] < end + X_dist) &
                             (X_locs.ix[:, 2] > start - X_dist))]
            probes = [p for p in probe_locs.index if p in X_probes]
            if len(probes) == 0: continue
            subset = X.ix[probes, :]
            subset.to_csv(fh.name, sep="\t", index=True, index_label="probe",
                            float_format="%.4f")
            fh.flush()
            X_file = fh.name

        res = run_model(cluster, covs, model, X_file, outlier_sds, liptak, bumping, gee_args,
                skat)

        if not X is None:
            if not X_locs is None:
                fh.close()
            # got a pandas dataframe
            df = res
            for i, row in df.iterrows():
                row = dict(row)
                if not X_locs is None:
                    distX(row, probe_locs.ix[row['X'], :])
                yield row

            continue

        if res['p'] < 1e-4 and png_path:
            cluster_df = cluster_to_dataframe(cluster, columns=covs.index)
            plot_res(res, png_path, covs, covariate, cluster_df)
        yield res

def plot_res(res, png_path, covs, covariate, cluster_df):
    from matplotlib import pyplot as plt
    from mpltools import style
    style.use('ggplot')

    region = "{chrom}_{start}_{end}".format(**res)
    if png_path.endswith('show'):
        png = None
    elif png_path.endswith(('.png', '.pdf')):
        png = "%s.%s%s" % (png_path[:-4], region, png_path[-4:])
    elif png_path:
        png = "%s.%s.png" % (png_path.rstrip("."), region)

    if is_numeric(getattr(covs, covariate)):
        f = plot_continuous(covs, cluster_df, covariate, res['chrom'], res, png)
    else:
        f = plt.figure(figsize=(11, 4))
        ax = f.add_subplot(1, 1, 1)
        if 'spaghetti' in png_path:
            plot_dmr(covs, cluster_df, covariate, res['chrom'], res, png)
        else:
            plot_hbar(covs, cluster_df, covariate, res['chrom'], res, png)
        plt.title('p-value: %.3g %s: %.3f' % (res['p'], covariate, res['coef']))
    f.set_tight_layout(True)
    if png:
        plt.savefig(png)
    else:
        plt.show()

def main_example():
    fcovs = "clustercorr/tests/example-covariates.txt"
    fmeth = "clustercorr/tests/example-methylation.txt.gz"
    model = "methylation ~ disease + gender"

    for cluster_p in clustermodel(fcovs, fmeth, model):
        if cluster_p['p'] < 1e-5:
            print cluster_p

def add_modelling_args(p):
    mp = p.add_argument_group('modeling choices (choose one or specify a '
            'mixed-model using lme4 syntax)')
    group = mp.add_mutually_exclusive_group()

    group.add_argument('--skat', action='store_true')
    group.add_argument('--gee-args',
                       help='comma-delimited correlation-structure, variable')
    group.add_argument('--liptak', action="store_true")
    group.add_argument('--bumping', action="store_true")

    p.add_argument('model',
                   help="model in R syntax, e.g. 'methylation ~ disease'")
    p.add_argument('covs', help="tab-delimited file of covariates: shape is "
                   "n_samples * n_covariates")
    p.add_argument('methylation', help="tab-delimited file of methylation"
                   " rows of this file must match the columns of `covs`"
                   " shape is n_probes * n_samples")

def add_expression_args(p):
    ep = p.add_argument_group('optional expression parameters')
    ep.add_argument('--X', help='file with same sample columns as methylation, '
            'rows of probes and values of some measurement (likely expression)'
            ' this will perform a methyl-eQTL--for each DMR, it will test '
            'againts all rows in this methylation array. As such, it is best'
            ' to run this on subsets of data, e.g. only looking for cis '
            'relationships')
    ep.add_argument('--X-locs', help="BED file with locations of probes from"
            " the first column in --X. Should have a 'probe' column header")
    ep.add_argument('--X-dist', type=int, help="only look at cis interactions"
            " between X and methylation sites with this as the maximum",
            default=100000)

def add_clustering_args(p):
    cp = p.add_argument_group('clustering parameters')
    cp.add_argument('--rho-min', type=float, default=0.3,
                   help="minimum correlation to merge 2 probes")
    cp.add_argument('--min-cluster-size', type=int, default=2,
                    help="minimum cluster size on which to run model: "
                   "must be at least 2")
    cp.add_argument('--linkage', choices=['single', 'complete'],
                    default='complete', help="linkage method")

    cp.add_argument('--max-dist', default=500, type=int,
                    help="maximum distance beyond which a probe can not be"
                    " added to a cluster")

def add_misc_args(p):
    p.add_argument('--png-path',
                   help="path to save a png of regions with low p-values. If "
                   "this ends with 'show', each plot will be shown in a window"
                   " if this contains the string 'spaghetti', it will draw a "
                   "a spaghetti plot, otherwise, it's a histogram plot")
    p.add_argument('--outlier-sds', type=float, default=30,
            help="remove points that are more than this many standard "
                 "deviations away from the mean (only usable with GEEs"
                 " and mixed-models which allow missing data")

def get_method(a):
    if a.gee_args is not None:
        method = 'gee:' + a.gee_args
        a.gee_args = a.gee_args.split(",")
    else:
        if a.liptak: method = 'liptak'
        elif a.bumping: method = 'bumping'
        elif a.skat: method = 'skat'
        else:
            assert "|" in a.model
            method = "mixed-model"
    return method

def regional_main(args=sys.argv[1:]):
    import argparse
    p = argparse.ArgumentParser(__doc__)
    add_modelling_args(p)
    add_misc_args(p)
    add_expression_args(p)

    p.add_argument('--regions', required=True, help="BED file of regions to "
            "test", metavar="BED")

    a = p.parse_args(args)
    method = get_method(a)

    fmt = "{chrom}\t{start}\t{end}\t{coef}\t{p}\t{n_probes}\t{model}\t{method}"
    if a.X_locs:
        fmt += "\t{Xstart}\t{Xend}\t{Xstrand}\t{distance}"

def main(args=sys.argv[1:]):
    import argparse
    p = argparse.ArgumentParser(__doc__)

    add_modelling_args(p)
    add_clustering_args(p)
    add_misc_args(p)
    add_expression_args(p)

    a = p.parse_args(args)
    method = get_method(a)

    fmt = "{chrom}\t{start}\t{end}\t{coef}\t{p}\t{n_probes}\t{model}\t{method}"
    if a.X_locs:
        fmt += "\t{Xstart}\t{Xend}\t{Xstrand}\t{distance}"
    print "#" + fmt.replace("}", "").replace("{", "")
    for c in clustermodel(a.covs, a.methylation, a.model,
                          max_dist=a.max_dist,
                          linkage=a.linkage,
                          rho_min=a.rho_min,
                          min_clust_size=a.min_cluster_size,
                          liptak=a.liptak,
                          bumping=a.bumping,
                          gee_args=a.gee_args,
                          skat=a.skat,
                          X=a.X,
                          X_locs=a.X_locs,
                          X_dist=a.X_dist,
                          outlier_sds=a.outlier_sds,
                          png_path=a.png_path):
        c['method'] = method
        print fmt.format(**c)

if __name__ == "__main__":
    import sys
    if len(sys.argv) > 1 and sys.argv[1] == "example":
        sys.exit(main_example())
    if len(sys.argv) > 1 and sys.argv[1] == "simulate":
        from . import simulate
        sys.exit(simulate.main(sys.argv[2:]))

    # want to specify existing regions, not use found ones.
    if len(sys.argv) > 1 and sys.argv[1] == "regional":
        sys.exit(regional_main(sys.argv[2:]))

    main()
<|endoftext|>


<commit_before>from datetime import time
from django.contrib.auth.models import User
from django.db import models
from django.utils import timezone

class Event(models.Model):
    belongs_to = models.ForeignKey(User, related_name='my_events', null=True, blank=True, default=None)
    start_date = models.DateTimeField(default=timezone.now)
    end_date = models.DateTimeField(default=timezone.now)
    event_text = models.TextField(default='')

    requested_swap = models.BooleanField(default=False)
    agreed_swap = models.ForeignKey('Event', related_name='to_swap_agree', default=None, null=True, blank=True)
    approved_swap = models.BooleanField(default=False)

    #swap_request_time = models.DateTimeField(default=None, null=True, blank=True)
    #transfer_request_time = models.DateTimeField(default=None, null=True, blank=True)

    requested_transfer = models.BooleanField(default=False)
    approved_transfer = models.BooleanField(default=False)

    to_swap_events = models.ManyToManyField('Event', default=None, null=True, blank=True)
    to_transfer_events = models.ManyToManyField(User, related_name='to_pickup_events', default=None, null=True,
                                                blank=True)
    to_change_user = models.ForeignKey(User, related_name='transfer_events', default=None, null=True, blank=True)

    def __unicode__(self):
        return self.event_text + ' - ' + self.belongs_to.get_full_name()

    @property
    def transfer_check(self):
        return self.requested_transfer

    def json(self):
        return {
            'id': self.id,
            'title': str(self.event_text + " for " + self.belongs_to.get_full_name()),
            'start': self.start_date.strftime('%Y-%m-%dT%X-05:00'),
            'end': self.end_date.strftime('%Y-%m-%dT%X-05:00'),
            'user_id': self.belongs_to.id,
            'to_swap': self.requested_swap,
            'to_transfer': self.requested_transfer,
        }

class PreviousTransfers(models.Model):
    from_user = models.ForeignKey(User, related_name='from_prev_transfer')
    to_user = models.ForeignKey(User, related_name='to_prev_transfer')
    event = models.ForeignKey(Event)
    approve_time = models.DateTimeField(default=timezone.now)

class UserProfile(models.Model):
    user = models.OneToOneField(User, unique=True)
    role = models.CharField(max_length=140, default='employee')
    qualification = models.CharField(max_length=140, default='RN')
    organization = models.CharField(max_length=140, default='UHN')
    first_time = models.BooleanField(default=True)

    @property
    def is_admin(self):
        return self.role == 'admin'

    def __unicode__(self):
        return u'Profile of user: %s' % self.user.username

legend_hr = {'TD': (time(hour=11, minute=0, second=0, microsecond=0), time(hour=23, minute=0, second=0, microsecond=0)),
             'TN': (time(hour=19, minute=0, second=0, microsecond=0), time(hour=9, minute=0, second=0, microsecond=0)),
             'TE': (time(hour=12, minute=0, second=0, microsecond=0), time(hour=21, minute=0, second=0, microsecond=0)),
             'D': (time(hour=7, minute=0, second=0, microsecond=0), time(hour=19, minute=0, second=0, microsecond=0)),
             'N': (time(hour=19, minute=0, second=0, microsecond=0), time(hour=7, minute=0, second=0, microsecond=0))}<commit_msg>changes<commit_after>from datetime import time
from django.contrib.auth.models import User
from django.db import models
from django.utils import timezone

class Event(models.Model):
    belongs_to = models.ForeignKey(User, related_name='my_events', null=True, blank=True, default=None)
    start_date = models.DateTimeField(default=timezone.now)
    end_date = models.DateTimeField(default=timezone.now)
    event_text = models.TextField(default='')

    requested_swap = models.BooleanField(default=False)
    agreed_swap = models.ForeignKey('Event', related_name='to_swap_agree', default=None, null=True, blank=True)
    approved_swap = models.BooleanField(default=False)

    swap_request_time = models.DateTimeField(default=None, null=True, blank=True)
    transfer_request_time = models.DateTimeField(default=None, null=True, blank=True)

    requested_transfer = models.BooleanField(default=False)
    approved_transfer = models.BooleanField(default=False)

    to_swap_events = models.ManyToManyField('Event', default=None, null=True, blank=True)
    to_transfer_events = models.ManyToManyField(User, related_name='to_pickup_events', default=None, null=True,
                                                blank=True)
    to_change_user = models.ForeignKey(User, related_name='transfer_events', default=None, null=True, blank=True)

    def __unicode__(self):
        return self.event_text + ' - ' + self.belongs_to.get_full_name()

    @property
    def transfer_check(self):
        return self.requested_transfer

    def json(self):
        return {
            'id': self.id,
            'title': str(self.event_text + " for " + self.belongs_to.get_full_name()),
            'start': self.start_date.strftime('%Y-%m-%dT%X-05:00'),
            'end': self.end_date.strftime('%Y-%m-%dT%X-05:00'),
            'user_id': self.belongs_to.id,
            'to_swap': self.requested_swap,
            'to_transfer': self.requested_transfer,
        }

class PreviousTransfers(models.Model):
    from_user = models.ForeignKey(User, related_name='from_prev_transfer')
    to_user = models.ForeignKey(User, related_name='to_prev_transfer')
    event = models.ForeignKey(Event)
    approve_time = models.DateTimeField(default=timezone.now)

class UserProfile(models.Model):
    user = models.OneToOneField(User, unique=True)
    role = models.CharField(max_length=140, default='employee')
    qualification = models.CharField(max_length=140, default='RN')
    organization = models.CharField(max_length=140, default='UHN')
    first_time = models.BooleanField(default=True)

    @property
    def is_admin(self):
        return self.role == 'admin'

    def __unicode__(self):
        return u'Profile of user: %s' % self.user.username

legend_hr = {'TD': (time(hour=11, minute=0, second=0, microsecond=0), time(hour=23, minute=0, second=0, microsecond=0)),
             'TN': (time(hour=19, minute=0, second=0, microsecond=0), time(hour=9, minute=0, second=0, microsecond=0)),
             'TE': (time(hour=12, minute=0, second=0, microsecond=0), time(hour=21, minute=0, second=0, microsecond=0)),
             'D': (time(hour=7, minute=0, second=0, microsecond=0), time(hour=19, minute=0, second=0, microsecond=0)),
             'N': (time(hour=19, minute=0, second=0, microsecond=0), time(hour=7, minute=0, second=0, microsecond=0))}<|endoftext|>


<commit_before># Copyright 2015 Infoblox Inc.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import re

from infoblox_client import exceptions as ibc_exc
from neutron_lib import constants as n_const

from networking_infoblox.neutron.common import constants as const

class PatternBuilder(object):

    def __init__(self, ib_context):
        self.ib_cxt = ib_context
        self.grid_config = self.ib_cxt.grid_config

    def get_hostname(self, ip_address, instance_name=None, port_id=None,
                     device_owner=None, device_id=None, port_name=None,
                     external=False, port_tenant_id=None, tenant_name=None):
        """Build fqdn based on patterns for network type and device owner.

        Two types of host and domain patterns exist:
        - for external network (optional);
        - for private network (default);
        If pattern for external network is not set, default one is used.
        If device owner is a known one (like dhcp_port, routern interface
        etc.), then per owner patterns are used. Floating ip can be exception
        from this rule if VM is associated with it and instance name is
        present in the pattern.
        """
        if external:
            host_pattern = (self.grid_config.external_host_name_pattern or
                            self.grid_config.default_host_name_pattern)
            domain_pattern = (self.grid_config.external_domain_name_pattern or
                              self.grid_config.default_domain_name_pattern)
        else:
            host_pattern = self.grid_config.default_host_name_pattern
            domain_pattern = self.grid_config.default_domain_name_pattern

        if device_owner in const.NEUTRON_DEVICE_OWNER_TO_PATTERN_MAP and (
                device_owner != n_const.DEVICE_OWNER_FLOATINGIP or not
                instance_name or "{instance_name}" not in host_pattern):
            host_pattern = (
                const.NEUTRON_DEVICE_OWNER_TO_PATTERN_MAP[device_owner])

        pattern = [host_pattern, domain_pattern]
        pattern = '.'.join(el.strip('.') for el in pattern if el)
        return self._build(pattern, ip_address, instance_name, port_id,
                           device_id, port_name=port_name,
                           port_tenant_id=port_tenant_id,
                           tenant_name=tenant_name)

    def get_zone_name_pattern(self, subnet_name=None, is_external=False):
        pattern = self.grid_config.default_domain_name_pattern
        if is_external and self.grid_config.external_domain_name_pattern:
            pattern = self.grid_config.external_domain_name_pattern
        return pattern

    def get_zone_name(self, subnet_name=None, tenant_name=None,
                      port_tenant_id=None, is_external=False):
        pattern = self.get_zone_name_pattern(subnet_name, is_external)
        return self._build(pattern, subnet_name=subnet_name,
                           tenant_name=tenant_name,
                           port_tenant_id=port_tenant_id)

    def _build(self, pattern, ip_address=None, instance_name=None,
               port_id=None, device_id=None, subnet_name=None,
               port_name=None, port_tenant_id=None, tenant_name=None):
        self._validate_pattern(pattern)

        subnet = self.ib_cxt.subnet
        network = self.ib_cxt.network
        if not subnet_name:
            subnet_name = (subnet['name'] if subnet.get('name')
                           else subnet['id'])
        network_name = (network['name'] if network.get('name')
                        else network['id'])

        pattern_dict = {
            'network_id': subnet['network_id'],
            'network_name': network_name,
            'tenant_id': port_tenant_id or self.ib_cxt.tenant_id,
            'tenant_name': tenant_name or self.ib_cxt.tenant_name,
            'subnet_name': subnet_name,
            'subnet_id': subnet['id']
        }

        if port_id:
            pattern_dict['port_id'] = port_id

        if device_id:
            pattern_dict['instance_id'] = device_id
            if instance_name:
                pattern_dict['instance_name'] = re.sub("[^A-Za-z0-9-]", "-",
                                                       instance_name.strip())
            else:
                # During port_creation for instance_name is not available,
                # so set it to instance_id
                pattern_dict['instance_name'] = pattern_dict['instance_id']

        if ip_address:
            octets = ip_address.split('.')
            ip_addr = ip_address.replace('.', '-').replace(':', '-')
            pattern_dict['ip_address'] = ip_addr
            for i in range(len(octets)):
                octet_key = 'ip_address_octet{i}'.format(i=(i + 1))
                pattern_dict[octet_key] = octets[i]

        if port_name:
            pattern_dict['port_name'] = port_name
        elif 'ip_address' in pattern_dict:
            pattern_dict['port_name'] = pattern_dict['ip_address']
        else:
            pattern_dict['port_name'] = port_id

        try:
            # Validate grid config pattern with pattern_dict to
            # restrict user to use only those pattern variable in
            # grid config pattern which is available in pattern_dict
            self._validate_pattern_struct(pattern, pattern_dict)
            fqdn = pattern.format(**pattern_dict)
        except (KeyError, IndexError) as e:
            raise ibc_exc.InfobloxConfigException(
                msg="Invalid pattern %s" % e)
        return fqdn

    @staticmethod
    def _validate_pattern_struct(pattern, pattern_dict):
        # This function fetches all variables from grid config
        # pattern and validate the list with supported pattern
        # variables, raises a KeyError if validation fails.
        input_pattern_list = re.findall("\{(.*?)\}", pattern)
        invalid_pattern = set(input_pattern_list) - set(pattern_dict)
        if invalid_pattern:
            raise KeyError(list(invalid_pattern))

    @staticmethod
    def _validate_pattern(pattern):
        invalid_values = ['..']
        for val in invalid_values:
            if val in pattern:
                error_message = "Invalid pattern value {0}".format(val)
                raise ibc_exc.InfobloxConfigException(msg=error_message)
<commit_msg>Fixed exceptions when uppercase name used for creating resources in openstack<commit_after># Copyright 2015 Infoblox Inc.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import re

from infoblox_client import exceptions as ibc_exc
from neutron_lib import constants as n_const

from networking_infoblox.neutron.common import constants as const

class PatternBuilder(object):

    def __init__(self, ib_context):
        self.ib_cxt = ib_context
        self.grid_config = self.ib_cxt.grid_config

    def get_hostname(self, ip_address, instance_name=None, port_id=None,
                     device_owner=None, device_id=None, port_name=None,
                     external=False, port_tenant_id=None, tenant_name=None):
        """Build fqdn based on patterns for network type and device owner.

        Two types of host and domain patterns exist:
        - for external network (optional);
        - for private network (default);
        If pattern for external network is not set, default one is used.
        If device owner is a known one (like dhcp_port, routern interface
        etc.), then per owner patterns are used. Floating ip can be exception
        from this rule if VM is associated with it and instance name is
        present in the pattern.
        """
        if external:
            host_pattern = (self.grid_config.external_host_name_pattern or
                            self.grid_config.default_host_name_pattern)
            domain_pattern = (self.grid_config.external_domain_name_pattern or
                              self.grid_config.default_domain_name_pattern)
        else:
            host_pattern = self.grid_config.default_host_name_pattern
            domain_pattern = self.grid_config.default_domain_name_pattern

        if device_owner in const.NEUTRON_DEVICE_OWNER_TO_PATTERN_MAP and (
                device_owner != n_const.DEVICE_OWNER_FLOATINGIP or not
                instance_name or "{instance_name}" not in host_pattern):
            host_pattern = (
                const.NEUTRON_DEVICE_OWNER_TO_PATTERN_MAP[device_owner])

        pattern = [host_pattern, domain_pattern]
        pattern = '.'.join(el.strip('.') for el in pattern if el)
        return self._build(pattern, ip_address, instance_name, port_id,
                           device_id, port_name=port_name,
                           port_tenant_id=port_tenant_id,
                           tenant_name=tenant_name)

    def get_zone_name_pattern(self, subnet_name=None, is_external=False):
        pattern = self.grid_config.default_domain_name_pattern
        if is_external and self.grid_config.external_domain_name_pattern:
            pattern = self.grid_config.external_domain_name_pattern
        return pattern

    def get_zone_name(self, subnet_name=None, tenant_name=None,
                      port_tenant_id=None, is_external=False):
        pattern = self.get_zone_name_pattern(subnet_name, is_external)
        return self._build(pattern, subnet_name=subnet_name,
                           tenant_name=tenant_name,
                           port_tenant_id=port_tenant_id)

    def _build(self, pattern, ip_address=None, instance_name=None,
               port_id=None, device_id=None, subnet_name=None,
               port_name=None, port_tenant_id=None, tenant_name=None):
        self._validate_pattern(pattern)

        subnet = self.ib_cxt.subnet
        network = self.ib_cxt.network
        if not subnet_name:
            subnet_name = (subnet['name'] if subnet.get('name')
                           else subnet['id'])
        network_name = (network['name'] if network.get('name')
                        else network['id'])

        pattern_dict = {
            'network_id': subnet['network_id'],
            'network_name': network_name,
            'tenant_id': port_tenant_id or self.ib_cxt.tenant_id,
            'tenant_name': tenant_name or self.ib_cxt.tenant_name,
            'subnet_name': subnet_name,
            'subnet_id': subnet['id']
        }

        if port_id:
            pattern_dict['port_id'] = port_id

        if device_id:
            pattern_dict['instance_id'] = device_id
            if instance_name:
                pattern_dict['instance_name'] = re.sub("[^A-Za-z0-9-]", "-",
                                                       instance_name.strip())
            else:
                # During port_creation for instance_name is not available,
                # so set it to instance_id
                pattern_dict['instance_name'] = pattern_dict['instance_id']

        if ip_address:
            octets = ip_address.split('.')
            ip_addr = ip_address.replace('.', '-').replace(':', '-')
            pattern_dict['ip_address'] = ip_addr
            for i in range(len(octets)):
                octet_key = 'ip_address_octet{i}'.format(i=(i + 1))
                pattern_dict[octet_key] = octets[i]

        if port_name:
            pattern_dict['port_name'] = port_name
        elif 'ip_address' in pattern_dict:
            pattern_dict['port_name'] = pattern_dict['ip_address']
        else:
            pattern_dict['port_name'] = port_id

        try:
            # Validate grid config pattern with pattern_dict to
            # restrict user to use only those pattern variable in
            # grid config pattern which is available in pattern_dict
            self._validate_pattern_struct(pattern, pattern_dict)
            fqdn = pattern.format(**pattern_dict)
        except (KeyError, IndexError) as e:
            raise ibc_exc.InfobloxConfigException(
                msg="Invalid pattern %s" % e)
        # Return fqdn as lowercase string as NIOS creates all resources
        # in lowercases.
        return fqdn.lower()

    @staticmethod
    def _validate_pattern_struct(pattern, pattern_dict):
        # This function fetches all variables from grid config
        # pattern and validate the list with supported pattern
        # variables, raises a KeyError if validation fails.
        input_pattern_list = re.findall("\{(.*?)\}", pattern)
        invalid_pattern = set(input_pattern_list) - set(pattern_dict)
        if invalid_pattern:
            raise KeyError(list(invalid_pattern))

    @staticmethod
    def _validate_pattern(pattern):
        invalid_values = ['..']
        for val in invalid_values:
            if val in pattern:
                error_message = "Invalid pattern value {0}".format(val)
                raise ibc_exc.InfobloxConfigException(msg=error_message)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import sys
sys.path.append("..")
import josecommon as jcommon
import jcoin.josecoin as jcoin

import aiohttp
import json
import urllib.parse
import traceback

from random import SystemRandom
random = SystemRandom()

IMAGE_LIMIT = 14

class JoseImages(jcommon.Extension):
    def __init__(self, cl):
        jcommon.Extension.__init__(self, cl)

    async def ext_load(self):
        return True, ''

    async def ext_unload(self):
        return True, ''

    async def get_json(self, url):
        r = await aiohttp.request('GET', url)
        r = await r.text()
        r = json.loads(r)
        return r

    async def json_api(self, cxt, config):
        '''
        {
            'search_term': stuff to search,
            'index_url': where are the latest posts,
            'search_url': search for stuff(defaults to index_url),
            'show_url': get individual posts,
            'post_key': where to get the post image URL,
            'limit_key': what is the tag to limit results,
            'search_key': what is the tag to search in search_url,

            'list_key': where are the array of posts,
            'search_key': same thing as list_key but in search,
        }
        '''
        search_term = config.get('search_term')
        index_url = config.get('index_url')
        search_url = config.get('search_url', index_url)
        show_url = config.get('show_url')
        post_key = config.get('post_key')
        rspec = config.get('rspec')
        limit_key = config.get('limit_key', 'limit')
        search_key = config.get('search_key', 'tags')
        list_key = config.get('list_key', None)
        search_url_key = config.get('search_url_key', None)
        random_flag = False

        if search_term == '-latest':
            # get latest
            await cxt.say('`[img.json] most recent posts`')
            url = '%s?%s=%s' % (index_url, limit_key, IMAGE_LIMIT)
        elif search_term == '-random':
            # random id
            await cxt.say('`[img.json] random ID`')
            random_flag = True
            url = '%s?%s=%s' % (index_url, limit_key, 1)
        else:
            # normally, search tags
            await cxt.say('`[img.json] tags: %r`' % search_term)
            url = '%s?%s=%s&%s=%s' % (search_url, limit_key, IMAGE_LIMIT,\
                search_key, search_term)
            if search_key:
                list_key = search_url_key

        self.logger.info("image: json_api: %r", url)
        response = await self.get_json(url)

        if list_key:
            response = response[list_key]

        post = None
        if rspec:
            post = random.choice(response[rspec])
        elif random_flag:
            most_recent_id = response[0]['id']
            random_id = random.randint(1, most_recent_id)
            if not show_url:
                await cxt.say("`[img.json] API doesn't support individual posts`")
                return
            random_post_url = '%s?id=%s' % (show_url, random_id)
            post = await self.get_json(random_post_url)
        else:
            if len(response) < 1:
                await cxt.say("`[img.json] No results found.`")
                return
            post = random.choice(response)

        post_url = post[post_key]
        if ('hypnohub' in post_url) or ('derpi' in post_url):
            post_url = post_url.replace('//', '/')
            post_url = 'http:/%s' % post_url

        await cxt.say('ID: %s, URL: %s' % (str(post['id']), str(post_url)))
        return

    async def porn_routine(self, cxt):
        res = await jcoin.jcoin_control(self.current.author.id, jcommon.PORN_PRICE)
        if not res[0]:
            await cxt.say("PermError: %s" % res[1])
            return False
        return True

    async def c_hypno(self, message, args, cxt):
        access = await self.porn_routine(cxt)
        if access:
            await self.json_api(cxt, {
                'search_term': ' '.join(args[1:]),
                'index_url': 'http://hypnohub.net/post/index.json',
                'post_key': 'file_url',
            })

    async def c_yandere(self, message, args, cxt):
        access = await self.porn_routine(cxt)
        if access:
            await self.json_api(cxt, {
                'search_term': ' '.join(args[1:]),
                'index_url': 'https://yande.re/post.json',
                'post_key': 'file_url',
            })

    async def c_e621(self, message, args, cxt):
        access = await self.porn_routine(cxt)
        if access:
            await self.json_api(cxt, {
                'search_term': ' '.join(args[1:]),
                'index_url': 'https://e621.net/post/index.json',
                'show_url': 'https://e621.net/post/show.json',
                'post_key': 'file_url',
            })

    async def c_derpibooru(self, message, args, cxt):
        access = await self.porn_routine(cxt)
        if access:
            await self.json_api(cxt, {
                'search_term': ' '.join(args[1:]),
                'search_url': 'http://derpibooru.org/search.json',
                'index_url': 'http://derpibooru.org/images.json',
                # 'show_url': 'derpibooru.org/%d.json',
                'post_key': 'image',
                'limit_key': 'page',
                'search_key': 'q',
                'list_key': 'images',
                'search_url_key': 'search',
            })

    async def c_derpi(self, message, args, cxt):
        await self.c_derpibooru(message, args, cxt)

    async def c_urban(self, message, args, cxt):
        term = ' '.join(args[1:])

        url = 'http://api.urbandictionary.com/v0/define?term=%s' % urllib.parse.quote(term)
        resp = await aiohttp.request('GET', url)
        content = await resp.text()
        r = json.loads(content)

        try:
            if len(r['list']) < 1:
                await cxt.say("*no free definitions*")
                return
            await cxt.say(self.codeblock('', '%s:\n%s' % (term, r['list'][0]['definition'])))
        except Exception as e:
            await self.debug('```%s```'%traceback.format_exc())
        return
<commit_msg>jimages: show error when parsing json response<commit_after>#!/usr/bin/env python3

import sys
sys.path.append("..")
import josecommon as jcommon
import jcoin.josecoin as jcoin

import aiohttp
import json
import urllib.parse
import traceback

from random import SystemRandom
random = SystemRandom()

IMAGE_LIMIT = 14

class JoseImages(jcommon.Extension):
    def __init__(self, cl):
        jcommon.Extension.__init__(self, cl)

    async def ext_load(self):
        return True, ''

    async def ext_unload(self):
        return True, ''

    async def get_json(self, url):
        r = await aiohttp.request('GET', url)
        r = await r.text()
        try:
            r = json.loads(r)
        except:
            return None
        return r

    async def json_api(self, cxt, config):
        '''
        {
            'search_term': stuff to search,
            'index_url': where are the latest posts,
            'search_url': search for stuff(defaults to index_url),
            'show_url': get individual posts,
            'post_key': where to get the post image URL,
            'limit_key': what is the tag to limit results,
            'search_key': what is the tag to search in search_url,

            'list_key': where are the array of posts,
            'search_key': same thing as list_key but in search,
        }
        '''
        search_term = config.get('search_term')
        index_url = config.get('index_url')
        search_url = config.get('search_url', index_url)
        show_url = config.get('show_url')
        post_key = config.get('post_key')
        rspec = config.get('rspec')
        limit_key = config.get('limit_key', 'limit')
        search_key = config.get('search_key', 'tags')
        list_key = config.get('list_key', None)
        search_url_key = config.get('search_url_key', None)
        random_flag = False

        if search_term == '-latest':
            # get latest
            await cxt.say('`[img.json] most recent posts`')
            url = '%s?%s=%s' % (index_url, limit_key, IMAGE_LIMIT)
        elif search_term == '-random':
            # random id
            await cxt.say('`[img.json] random ID`')
            random_flag = True
            url = '%s?%s=%s' % (index_url, limit_key, 1)
        else:
            # normally, search tags
            await cxt.say('`[img.json] tags: %r`' % search_term)
            url = '%s?%s=%s&%s=%s' % (search_url, limit_key, IMAGE_LIMIT,\
                search_key, search_term)
            if search_key:
                list_key = search_url_key

        self.logger.info("image: json_api: %r", url)
        response = await self.get_json(url)

        if not response:
            await cxt.say("`[img.json] Error parsing JSON response`")
            return

        if list_key:
            response = response[list_key]

        post = None
        if rspec:
            post = random.choice(response[rspec])
        elif random_flag:
            most_recent_id = response[0]['id']
            random_id = random.randint(1, most_recent_id)
            if not show_url:
                await cxt.say("`[img.json] API doesn't support individual posts`")
                return
            random_post_url = '%s?id=%s' % (show_url, random_id)
            post = await self.get_json(random_post_url)
        else:
            if len(response) < 1:
                await cxt.say("`[img.json] No results found.`")
                return
            post = random.choice(response)

        post_url = post[post_key]
        if ('hypnohub' in post_url) or ('derpi' in post_url):
            post_url = post_url.replace('//', '/')
            post_url = 'http:/%s' % post_url

        await cxt.say('ID: %s, URL: %s' % (str(post['id']), str(post_url)))
        return

    async def porn_routine(self, cxt):
        res = await jcoin.jcoin_control(self.current.author.id, jcommon.PORN_PRICE)
        if not res[0]:
            await cxt.say("PermError: %s" % res[1])
            return False
        return True

    async def c_hypno(self, message, args, cxt):
        access = await self.porn_routine(cxt)
        if access:
            await self.json_api(cxt, {
                'search_term': ' '.join(args[1:]),
                'index_url': 'http://hypnohub.net/post/index.json',
                'post_key': 'file_url',
            })

    async def c_yandere(self, message, args, cxt):
        access = await self.porn_routine(cxt)
        if access:
            await self.json_api(cxt, {
                'search_term': ' '.join(args[1:]),
                'index_url': 'https://yande.re/post.json',
                'post_key': 'file_url',
            })

    async def c_e621(self, message, args, cxt):
        access = await self.porn_routine(cxt)
        if access:
            await self.json_api(cxt, {
                'search_term': ' '.join(args[1:]),
                'index_url': 'https://e621.net/post/index.json',
                'show_url': 'https://e621.net/post/show.json',
                'post_key': 'file_url',
            })

    async def c_derpibooru(self, message, args, cxt):
        access = await self.porn_routine(cxt)
        if access:
            await self.json_api(cxt, {
                'search_term': ' '.join(args[1:]),
                'search_url': 'http://derpibooru.org/search.json',
                'index_url': 'http://derpibooru.org/images.json',
                # 'show_url': 'derpibooru.org/%d.json',
                'post_key': 'image',
                'limit_key': 'page',
                'search_key': 'q',
                'list_key': 'images',
                'search_url_key': 'search',
            })

    async def c_derpi(self, message, args, cxt):
        await self.c_derpibooru(message, args, cxt)

    async def c_urban(self, message, args, cxt):
        term = ' '.join(args[1:])

        url = 'http://api.urbandictionary.com/v0/define?term=%s' % urllib.parse.quote(term)
        resp = await aiohttp.request('GET', url)
        content = await resp.text()
        r = json.loads(content)

        try:
            if len(r['list']) < 1:
                await cxt.say("*no free definitions*")
                return
            await cxt.say(self.codeblock('', '%s:\n%s' % (term, r['list'][0]['definition'])))
        except Exception as e:
            await self.debug('```%s```'%traceback.format_exc())
        return
<|endoftext|>


<commit_before>from sublime_plugin import WindowCommand

from ..git_command import GitCommand

class GsAmendCommand(WindowCommand, GitCommand):
    def run(self):
        self.window.run_command("gs_commit", {"amend": True})

class GsQuickStageCurrentFileAndAmendCommand(GsAmendCommand, GitCommand):
    def run(self):
        self.git("add", "--", self.file_path)
        self.window.status_message("staged {}".format(self.get_rel_path(self.file_path)))
        super().run()
<commit_msg>Snake case and define `__all__` for `amend.py`<commit_after>from sublime_plugin import WindowCommand

from ..git_command import GitCommand

__all__ = (
    "gs_amend",
    "gs_quick_stage_current_file_and_amend",
)

class gs_amend(WindowCommand, GitCommand):
    def run(self):
        self.window.run_command("gs_commit", {"amend": True})

class gs_quick_stage_current_file_and_amend(gs_amend, GitCommand):
    def run(self):
        self.git("add", "--", self.file_path)
        self.window.status_message("staged {}".format(self.get_rel_path(self.file_path)))
        super().run()
<|endoftext|>


<commit_before>"""
Eigenvector centrality.
"""
#    Copyright (C) 2004-2013 by
#    Aric Hagberg <hagberg@lanl.gov>
#    Dan Schult <dschult@colgate.edu>
#    Pieter Swart <swart@lanl.gov>
#    All rights reserved.
#    BSD license.
import networkx as nx
__author__ = "\n".join(['Aric Hagberg (aric.hagberg@gmail.com)',
                        'Pieter Swart (swart@lanl.gov)',
                        'Sasha Gutfraind (ag362@cornell.edu)'])
__all__ = ['eigenvector_centrality',
           'eigenvector_centrality_numpy']

def eigenvector_centrality(G, max_iter=100, tol=1.0e-6, nstart=None,
                           weight='weight'):
    """Compute the eigenvector centrality for the graph G.

    Uses the power method to find the eigenvector for the
    largest eigenvalue of the adjacency matrix of G.

    Parameters
    ----------
    G : graph
      A networkx graph

    max_iter : interger, optional
      Maximum number of iterations in power method.

    tol : float, optional
      Error tolerance used to check convergence in power method iteration.

    nstart : dictionary, optional
      Starting value of eigenvector iteration for each node.

    weight : None or string, optional
      If None, all edge weights are considered equal.
      Otherwise holds the name of the edge attribute used as weight.

    Returns
    -------
    nodes : dictionary
       Dictionary of nodes with eigenvector centrality as the value.

    Examples
    --------
    >>> G = nx.path_graph(4)
    >>> centrality = nx.eigenvector_centrality(G)
    >>> print(['%s %0.2f'%(node,centrality[node]) for node in centrality])
    ['0 0.37', '1 0.60', '2 0.60', '3 0.37']

    Notes
    ------
    The eigenvector calculation is done by the power iteration method
    and has no guarantee of convergence.  The iteration will stop
    after max_iter iterations or an error tolerance of
    number_of_nodes(G)*tol has been reached.

    For directed graphs this is "left" eigevector centrality which corresponds
    to the in-edges in the graph.  For out-edges eigenvector centrality
    first reverse the graph with G.reverse().

    See Also
    --------
    eigenvector_centrality_numpy
    pagerank
    hits
    """
    from math import sqrt
    if type(G) == nx.MultiGraph or type(G) == nx.MultiDiGraph:
        raise nx.NetworkXException("Not defined for multigraphs.")

    if len(G) == 0:
        raise nx.NetworkXException("Empty graph.")

    if nstart is None:
        # choose starting vector with entries of 1/len(G)
        x = dict([(n,1.0/len(G)) for n in G])
    else:
        x = nstart
    # normalize starting vector
    s = 1.0/sum(x.values())
    for k in x:
        x[k] *= s
    nnodes = G.number_of_nodes()
    # make up to max_iter iterations
    for i in range(max_iter):
        xlast = x
        x = dict.fromkeys(xlast, 0)
        # do the multiplication y^T = x^T A
        for n in x:
            for nbr in G[n]:
                x[nbr] += xlast[n] * G[n][nbr].get(weight, 1)
        # normalize vector
        try:
            s = 1.0/sqrt(sum(v**2 for v in x.values()))
        # this should never be zero?
        except ZeroDivisionError:
            s = 1.0
        for n in x:
            x[n] *= s
        # check convergence
        err = sum([abs(x[n]-xlast[n]) for n in x])
        if err < nnodes*tol:
            return x

    raise nx.NetworkXError("""eigenvector_centrality():
power iteration failed to converge in %d iterations."%(i+1))""")

def eigenvector_centrality_numpy(G, weight='weight'):
    """Compute the eigenvector centrality for the graph G.

    Parameters
    ----------
    G : graph
      A networkx graph

    weight : None or string, optional
      If None, all edge weights are considered equal.
      Otherwise holds the name of the edge attribute used as weight.

    Returns
    -------
    nodes : dictionary
       Dictionary of nodes with eigenvector centrality as the value.

    Examples
    --------
    >>> G = nx.path_graph(4)
    >>> centrality = nx.eigenvector_centrality_numpy(G)
    >>> print(['%s %0.2f'%(node,centrality[node]) for node in centrality])
    ['0 0.37', '1 0.60', '2 0.60', '3 0.37']

    Notes
    ------
    This algorithm uses the NumPy eigenvalue solver.

    For directed graphs this is "left" eigevector centrality which corresponds
    to the in-edges in the graph.  For out-edges eigenvector centrality
    first reverse the graph with G.reverse().

    See Also
    --------
    eigenvector_centrality
    pagerank
    hits
    """
    import numpy as np
    if len(G) == 0:
        raise nx.NetworkXException('Empty graph.')

    A = nx.adj_matrix(G, nodelist=G.nodes(), weight='weight').todense().T
    eigenvalues,eigenvectors = np.linalg.eig(A)
    # eigenvalue indices in reverse sorted order
    ind = eigenvalues.argsort()[::-1]
    # eigenvector of largest eigenvalue at ind[0], normalized
    largest = np.array(eigenvectors[:,ind[0]]).flatten().real
    norm = np.sign(largest.sum())*np.linalg.norm(largest)
    centrality = dict(zip(G,map(float,largest/norm)))
    return centrality

# fixture for nose tests
def setup_module(module):
    from nose import SkipTest
    try:
        import numpy
        import scipy
    except:
        raise SkipTest("SciPy not available")
<commit_msg>Switch eigenvector_centrality_numpy to use sparse matrix solver Addresses #969<commit_after>"""
Eigenvector centrality.
"""
#    Copyright (C) 2004-2014 by
#    Aric Hagberg <hagberg@lanl.gov>
#    Dan Schult <dschult@colgate.edu>
#    Pieter Swart <swart@lanl.gov>
#    All rights reserved.
#    BSD license.
import networkx as nx
__author__ = "\n".join(['Aric Hagberg (aric.hagberg@gmail.com)',
                        'Pieter Swart (swart@lanl.gov)',
                        'Sasha Gutfraind (ag362@cornell.edu)'])
__all__ = ['eigenvector_centrality',
           'eigenvector_centrality_numpy']

def eigenvector_centrality(G, max_iter=100, tol=1.0e-6, nstart=None,
                           weight='weight'):
    """Compute the eigenvector centrality for the graph G.

    Uses the power method to find the eigenvector for the
    largest eigenvalue of the adjacency matrix of G.

    Parameters
    ----------
    G : graph
      A networkx graph

    max_iter : interger, optional
      Maximum number of iterations in power method.

    tol : float, optional
      Error tolerance used to check convergence in power method iteration.

    nstart : dictionary, optional
      Starting value of eigenvector iteration for each node.

    weight : None or string, optional
      If None, all edge weights are considered equal.
      Otherwise holds the name of the edge attribute used as weight.

    Returns
    -------
    nodes : dictionary
       Dictionary of nodes with eigenvector centrality as the value.

    Examples
    --------
    >>> G = nx.path_graph(4)
    >>> centrality = nx.eigenvector_centrality(G)
    >>> print(['%s %0.2f'%(node,centrality[node]) for node in centrality])
    ['0 0.37', '1 0.60', '2 0.60', '3 0.37']

    Notes
    ------
    The eigenvector calculation is done by the power iteration method
    and has no guarantee of convergence.  The iteration will stop
    after max_iter iterations or an error tolerance of
    number_of_nodes(G)*tol has been reached.

    For directed graphs this is "left" eigevector centrality which corresponds
    to the in-edges in the graph.  For out-edges eigenvector centrality
    first reverse the graph with G.reverse().

    See Also
    --------
    eigenvector_centrality_numpy
    pagerank
    hits
    """
    from math import sqrt
    if type(G) == nx.MultiGraph or type(G) == nx.MultiDiGraph:
        raise nx.NetworkXException("Not defined for multigraphs.")

    if len(G) == 0:
        raise nx.NetworkXException("Empty graph.")

    if nstart is None:
        # choose starting vector with entries of 1/len(G)
        x = dict([(n,1.0/len(G)) for n in G])
    else:
        x = nstart
    # normalize starting vector
    s = 1.0/sum(x.values())
    for k in x:
        x[k] *= s
    nnodes = G.number_of_nodes()
    # make up to max_iter iterations
    for i in range(max_iter):
        xlast = x
        x = dict.fromkeys(xlast, 0)
        # do the multiplication y^T = x^T A
        for n in x:
            for nbr in G[n]:
                x[nbr] += xlast[n] * G[n][nbr].get(weight, 1)
        # normalize vector
        try:
            s = 1.0/sqrt(sum(v**2 for v in x.values()))
        # this should never be zero?
        except ZeroDivisionError:
            s = 1.0
        for n in x:
            x[n] *= s
        # check convergence
        err = sum([abs(x[n]-xlast[n]) for n in x])
        if err < nnodes*tol:
            return x

    raise nx.NetworkXError("""eigenvector_centrality():
power iteration failed to converge in %d iterations."%(i+1))""")

def eigenvector_centrality_numpy(G, weight='weight'):
    """Compute the eigenvector centrality for the graph G.

    Parameters
    ----------
    G : graph
      A networkx graph

    weight : None or string, optional
      The name of the edge attribute used as weight.
      If None, all edge weights are considered equal.

    Returns
    -------
    nodes : dictionary
       Dictionary of nodes with eigenvector centrality as the value.

    Examples
    --------
    >>> G = nx.path_graph(4)
    >>> centrality = nx.eigenvector_centrality_numpy(G)
    >>> print(['%s %0.2f'%(node,centrality[node]) for node in centrality])
    ['0 0.37', '1 0.60', '2 0.60', '3 0.37']

    Notes
    ------
    This algorithm uses the SciPy sparse eigenvalue solver (ARPACK) to
    find the largest eigenvalue/eigenvector pair.

    For directed graphs this is "left" eigevector centrality which corresponds
    to the in-edges in the graph.  For out-edges eigenvector centrality
    first reverse the graph with G.reverse().

    See Also
    --------
    eigenvector_centrality
    pagerank
    hits
    """
    import scipy as sp
    from scipy.sparse import linalg
    if len(G) == 0:
        raise nx.NetworkXException('Empty graph.')
    M = nx.to_scipy_sparse_matrix(G, nodelist=G.nodes(), weight=weight,
                                  dtype=float)
    eigenvalue, eigenvector = linalg.eigs(M.T, k=1, which='LR')
    largest = eigenvector.flatten().real
    norm = sp.sign(largest.sum())*sp.linalg.norm(largest)
    centrality = dict(zip(G,map(float,largest/norm)))
    return centrality

# fixture for nose tests
def setup_module(module):
    from nose import SkipTest
    try:
        import scipy
    except:
        raise SkipTest("SciPy not available")
<|endoftext|>


<commit_before>from sublime_plugin import WindowCommand

from ..git_command import GitCommand

__all__ = (
    "gs_amend",
    "gs_quick_stage_current_file_and_amend",
)

class gs_amend(WindowCommand, GitCommand):
    def run(self):
        self.window.run_command("gs_commit", {"amend": True})

class gs_quick_stage_current_file_and_amend(gs_amend, GitCommand):
    def run(self):
        self.git("add", "--", self.file_path)
        self.window.status_message("staged {}".format(self.get_rel_path(self.file_path)))
        super().run()
<commit_msg>Guard `gs_quick_stage_current_file_and_amend` against unnamed buffers<commit_after>from sublime_plugin import WindowCommand

from ..git_command import GitCommand

__all__ = (
    "gs_amend",
    "gs_quick_stage_current_file_and_amend",
)

class gs_amend(WindowCommand, GitCommand):
    def run(self):
        self.window.run_command("gs_commit", {"amend": True})

class gs_quick_stage_current_file_and_amend(gs_amend, GitCommand):
    def run(self):
        file_path = self.file_path
        if not file_path:
            self.window.status_message("Cannot staged unnamed buffer.")
            return

        self.git("add", "--", file_path)
        self.window.status_message("staged {}".format(self.get_rel_path(file_path)))
        super().run()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# This file is part of mnemosyne
#
# Copyright (C) 2013 Daniel Lombraña González
#
# Mnemosyne is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Mnemosyne is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Mnemosyne. If not, see <http://www.gnu.org/licenses/>.

"""
    Mnemosyne
    ~~~~~~~~

    Mnemosyne model package
"""
from flask.ext.sqlalchemy import SQLAlchemy

db = SQLAlchemy()

class DomainObject(object):
    def dictize(self):
        tmp = {}
        for col in self.__table__.c:
            if col.name == 'created':
                tmp[col.name] = getattr(self, col.name).isoformat()
            elif col.name == 'keywords':
                tmp[col.name] = [k.strip() for k in getattr(self, col.name).split(",")]
            else:
                tmp[col.name] = getattr(self, col.name)
        return tmp

    def _save(self):
        db.session.add(self)
        db.session.commit()
<commit_msg>Docstrings<commit_after># -*- coding: utf-8 -*-
# This file is part of mnemosyne
#
# Copyright (C) 2013 Daniel Lombraña González
#
# Mnemosyne is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Mnemosyne is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Mnemosyne. If not, see <http://www.gnu.org/licenses/>.
"""
Generic Model package.

This exports:
    - DomainObject a base class that all models inherit

"""
from flask.ext.sqlalchemy import SQLAlchemy

db = SQLAlchemy()

class DomainObject(object):

    """Generic class to manage database objects."""

    def dictize(self):
        """
        Convert a database object into a dictionary.

        Return value:
            dict - The converted object into a dictionary

        """
        tmp = {}
        for col in self.__table__.c:
            if col.name == 'created':
                tmp[col.name] = getattr(self, col.name).isoformat()
            elif col.name == 'keywords':
                tmp[col.name] = [k.strip() for k in
                                 getattr(self, col.name).split(",")]
            else:
                tmp[col.name] = getattr(self, col.name)
        return tmp

    def _save(self):
        """Save current object in the database."""
        db.session.add(self)
        db.session.commit()
<|endoftext|>