<commit_before># -*- coding: utf-8 -*-

"""Selector structure."""

#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

import numpy as np

from ...utils.array import _as_array, regular_subset, get_excerpts
from ._utils import _unique, _spikes_in_clusters, _spikes_per_cluster

#------------------------------------------------------------------------------
# Utility functions
#------------------------------------------------------------------------------

def _concat(l):
    return np.sort(np.hstack(l))

#------------------------------------------------------------------------------
# Selector class
#------------------------------------------------------------------------------

class Selector(object):
    """Object representing a selection of spikes or clusters."""
    def __init__(self, spike_clusters,
                 n_spikes_max=None,
                 excerpt_size=None,
                 ):
        self._spike_clusters = spike_clusters
        self._n_spikes_max = n_spikes_max
        self._excerpt_size = excerpt_size
        self._selected_spikes = np.array([], dtype=np.int64)

    @property
    def n_spikes_max(self):
        """Maximum number of spikes allowed in the selection."""
        return self._n_spikes_max

    @n_spikes_max.setter
    def n_spikes_max(self, value):
        """Change the maximum number of spikes allowed."""
        self._n_spikes_max = value
        # Update the selected spikes accordingly.
        self.selected_spikes = self.subset_spikes()
        if self._n_spikes_max is not None:
            assert len(self._selected_spikes) <= self._n_spikes_max

    @property
    def excerpt_size(self):
        """Maximum number of spikes allowed in the selection."""
        return self._excerpt_size

    @excerpt_size.setter
    def excerpt_size(self, value):
        """Change the excerpt size."""
        self._excerpt_size = value
        # Update the selected spikes accordingly.
        self.selected_spikes = self.subset_spikes()

    def subset_spikes(self,
                      spikes=None,
                      n_spikes_max=None,
                      excerpt_size=None,
                      ):
        """Prune the current selection to get at most `n_spikes_max` spikes.

        Parameters
        ----------

        spikes : array-like
            Array of spike ids to subset from. By default, this is
            `selector.selected_spikes`.
        n_spikes_max : int or None
            Maximum number of spikes allowed in the selection.
        excerpt_size : int or None
            If None, the method returns a regular strided selection.
            Otherwise, returns a regular selection of contiguous chunks
            with the specified chunk size.

        """
        # Default arguments.
        if spikes is None:
            spikes = self._selected_spikes
        if spikes is None or len(spikes) == 0:
            return spikes
        if n_spikes_max is None:
            n_spikes_max = self._n_spikes_max or len(spikes)
        if excerpt_size is None:
            excerpt_size = self._excerpt_size
        # Nothing to do if there are less spikes than the maximum number.
        if len(spikes) <= n_spikes_max:
            return spikes
        # Take a regular or chunked subset of the spikes.
        if excerpt_size is None:
            return regular_subset(spikes, n_spikes_max)
        else:
            n_excerpts = n_spikes_max // excerpt_size
            return get_excerpts(spikes,
                                n_excerpts=n_excerpts,
                                excerpt_size=excerpt_size,
                                )

    def _subset_spikes_clusters(self, clusters):
        """Take a subselection of spikes belonging to a set of clusters.

        This method ensures that the same number of spikes is chosen
        for every spike.

        """
        if len(clusters) == 0:
            return []
        # Take all spikes from the selected clusters.
        spikes = _spikes_in_clusters(self._spike_clusters, clusters)
        # Get the selection parameters.
        n_spikes_max = self._n_spikes_max or len(spikes)
        excerpt_size = self._excerpt_size
        # Do nothing if there are less spikes than the maximum number.
        if len(spikes) <= n_spikes_max:
            return spikes
        # Group the spikes per cluster.
        spc = _spikes_per_cluster(spikes, self._spike_clusters[spikes])
        n_clusters = len(clusters)
        n_spikes_max_c = n_spikes_max // n_clusters
        # Take a regular or chunked subset of the spikes.
        if excerpt_size is None:
            return _concat([regular_subset(spc[cluster], n_spikes_max_c)
                            for cluster in clusters])
        else:
            n_excerpts = n_spikes_max_c // excerpt_size
            return _concat([get_excerpts(spc[cluster],
                                         n_excerpts=n_excerpts,
                                         excerpt_size=excerpt_size,
                                         )
                            for cluster in clusters])

    @property
    def selected_spikes(self):
        """Ids of the selected spikes."""
        return self._selected_spikes

    @selected_spikes.setter
    def selected_spikes(self, value):
        """Explicitely select some spikes.

        The selection is automatically pruned to ensure that less than
        `n_spikes_max` spikes are selected.

        """
        value = _as_array(value)
        # Make sure there are less spikes than n_spikes_max.
        self._selected_spikes = self.subset_spikes(value)

    @property
    def selected_clusters(self):
        """Cluster ids appearing in the current spike selection."""
        return _unique(self._spike_clusters[self._selected_spikes])

    @selected_clusters.setter
    def selected_clusters(self, value):
        """Select some clusters.

        This will select less than `n_spikes_max` spikes belonging to
        those clusters.

        """
        value = _as_array(value)
        # Make sure there are less spikes than n_spikes_max.
        self.selected_spikes = self._subset_spikes_clusters(value)

    @property
    def n_spikes(self):
        return len(self._selected_spikes)

    @property
    def n_clusters(self):
        return len(self.selected_clusters)

    def on_cluster(self, up=None):
        """Callback method called when the clustering has changed.

        This currently does nothing, i.e. the spike selection remains
        unchanged when merges and splits occur.

        """
        # TODO
        pass
<commit_msg>Rename subset method in selector.<commit_after># -*- coding: utf-8 -*-

"""Selector structure."""

#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

import numpy as np

from ...utils.array import _as_array, regular_subset, get_excerpts
from ._utils import _unique, _spikes_in_clusters, _spikes_per_cluster

#------------------------------------------------------------------------------
# Utility functions
#------------------------------------------------------------------------------

def _concat(l):
    return np.sort(np.hstack(l))

#------------------------------------------------------------------------------
# Selector class
#------------------------------------------------------------------------------

class Selector(object):
    """Object representing a selection of spikes or clusters."""
    def __init__(self, spike_clusters,
                 n_spikes_max=None,
                 excerpt_size=None,
                 ):
        self._spike_clusters = spike_clusters
        self._n_spikes_max = n_spikes_max
        self._excerpt_size = excerpt_size
        self._selected_spikes = np.array([], dtype=np.int64)

    @property
    def n_spikes_max(self):
        """Maximum number of spikes allowed in the selection."""
        return self._n_spikes_max

    @n_spikes_max.setter
    def n_spikes_max(self, value):
        """Change the maximum number of spikes allowed."""
        self._n_spikes_max = value
        # Update the selected spikes accordingly.
        self.selected_spikes = self.subset_spikes()
        if self._n_spikes_max is not None:
            assert len(self._selected_spikes) <= self._n_spikes_max

    @property
    def excerpt_size(self):
        """Maximum number of spikes allowed in the selection."""
        return self._excerpt_size

    @excerpt_size.setter
    def excerpt_size(self, value):
        """Change the excerpt size."""
        self._excerpt_size = value
        # Update the selected spikes accordingly.
        self.selected_spikes = self.subset_spikes()

    def subset_spikes(self,
                      spikes=None,
                      n_spikes_max=None,
                      excerpt_size=None,
                      ):
        """Prune the current selection to get at most `n_spikes_max` spikes.

        Parameters
        ----------

        spikes : array-like
            Array of spike ids to subset from. By default, this is
            `selector.selected_spikes`.
        n_spikes_max : int or None
            Maximum number of spikes allowed in the selection.
        excerpt_size : int or None
            If None, the method returns a regular strided selection.
            Otherwise, returns a regular selection of contiguous chunks
            with the specified chunk size.

        """
        # Default arguments.
        if spikes is None:
            spikes = self._selected_spikes
        if spikes is None or len(spikes) == 0:
            return spikes
        if n_spikes_max is None:
            n_spikes_max = self._n_spikes_max or len(spikes)
        if excerpt_size is None:
            excerpt_size = self._excerpt_size
        # Nothing to do if there are less spikes than the maximum number.
        if len(spikes) <= n_spikes_max:
            return spikes
        # Take a regular or chunked subset of the spikes.
        if excerpt_size is None:
            return regular_subset(spikes, n_spikes_max)
        else:
            n_excerpts = n_spikes_max // excerpt_size
            return get_excerpts(spikes,
                                n_excerpts=n_excerpts,
                                excerpt_size=excerpt_size,
                                )

    def subset_spikes_clusters(self, clusters):
        """Take a subselection of spikes belonging to a set of clusters.

        This method ensures that the same number of spikes is chosen
        for every spike.

        """
        if len(clusters) == 0:
            return []
        # Take all spikes from the selected clusters.
        spikes = _spikes_in_clusters(self._spike_clusters, clusters)
        # Get the selection parameters.
        n_spikes_max = self._n_spikes_max or len(spikes)
        excerpt_size = self._excerpt_size
        # Do nothing if there are less spikes than the maximum number.
        if len(spikes) <= n_spikes_max:
            return spikes
        # Group the spikes per cluster.
        spc = _spikes_per_cluster(spikes, self._spike_clusters[spikes])
        n_clusters = len(clusters)
        n_spikes_max_c = n_spikes_max // n_clusters
        # Take a regular or chunked subset of the spikes.
        if excerpt_size is None:
            return _concat([regular_subset(spc[cluster], n_spikes_max_c)
                            for cluster in clusters])
        else:
            n_excerpts = n_spikes_max_c // excerpt_size
            return _concat([get_excerpts(spc[cluster],
                                         n_excerpts=n_excerpts,
                                         excerpt_size=excerpt_size,
                                         )
                            for cluster in clusters])

    @property
    def selected_spikes(self):
        """Ids of the selected spikes."""
        return self._selected_spikes

    @selected_spikes.setter
    def selected_spikes(self, value):
        """Explicitely select some spikes.

        The selection is automatically pruned to ensure that less than
        `n_spikes_max` spikes are selected.

        """
        value = _as_array(value)
        # Make sure there are less spikes than n_spikes_max.
        self._selected_spikes = self.subset_spikes(value)

    @property
    def selected_clusters(self):
        """Cluster ids appearing in the current spike selection."""
        return _unique(self._spike_clusters[self._selected_spikes])

    @selected_clusters.setter
    def selected_clusters(self, value):
        """Select some clusters.

        This will select less than `n_spikes_max` spikes belonging to
        those clusters.

        """
        value = _as_array(value)
        # Make sure there are less spikes than n_spikes_max.
        self.selected_spikes = self.subset_spikes_clusters(value)

    @property
    def n_spikes(self):
        return len(self._selected_spikes)

    @property
    def n_clusters(self):
        return len(self.selected_clusters)

    def on_cluster(self, up=None):
        """Callback method called when the clustering has changed.

        This currently does nothing, i.e. the spike selection remains
        unchanged when merges and splits occur.

        """
        # TODO
        pass
<|endoftext|>


<commit_before>import logging

logger = logging.getLogger(__name__)

def get_identifiers_url(db_name, db_id):
    """Return an identifiers.org URL for a given database name and ID.

    Parameters
    ----------
    db_name : str
        An internal database name: HGNC, UP, CHEBI, etc.
    db_id : str
        An identifier in the given database.

    Returns
    -------
    url : str
        An identifiers.org URL corresponding to the given database name and ID.
    """
    identifiers_url = 'http://identifiers.org/'
    if db_name == 'UP':
        url = identifiers_url + 'uniprot/%s' % db_id
    elif db_name == 'HGNC':
        url = identifiers_url + 'hgnc/HGNC:%s' % db_id
    elif db_name == 'IP':
        url = identifiers_url + 'interpro/%s' % db_id
    elif db_name == 'IPR':
        url = identifiers_url + 'interpro/%s' % db_id
    elif db_name == 'CHEBI':
        url = identifiers_url + 'chebi/%s' % db_id
    elif db_name == 'NCIT':
        url = identifiers_url + 'ncit/%s' % db_id
    elif db_name == 'GO':
        url = identifiers_url + 'go/%s' % db_id
    elif db_name in ('PUBCHEM', 'PCID'):  # Assuming PCID = PubChem compound ID
        if db_id.startswith('PUBCHEM:'):
            db_id = db_id[8:]
        elif db_id.startswith('PCID:'):
            db_id = db_id[5:]
        url = identifiers_url + 'pubchem.compound/%s' % db_id
    elif db_name == 'PF':
        url = identifiers_url + 'pfam/%s' % db_id
    elif db_name == 'MIRBASEM':
        url = identifiers_url + 'mirbase.mature/%s' % db_id
    elif db_name == 'MIRBASE':
        url = identifiers_url + 'mirbase/%s' % db_id
    elif db_name == 'MESH':
        url = identifiers_url + 'mesh/%s' % db_id
    elif db_name == 'EGID':
        url = 'ncbigene:%s' % db_id
    elif db_name == 'HMDB':
        url = identifiers_url + 'hmdb/%s' % db_id
    elif db_name == 'LINCS':
        if db_id.startswith('LSM-'):  # Lincs Small Molecule ID
            url = identifiers_url + 'lincs.smallmolecule:%s' % db_id
        elif db_id.startswith('LCL-'):  # Lincs Cell Line ID
            url = identifiers_url + 'lincs.cell:%s' % db_id
        else:  # Assume LINCS Protein
            url = identifiers_url + 'lincs.protein:%s' % db_id
    elif db_name == 'HMS-LINCS':
        url = identifiers_url + 'lincs.protein:%s' % db_id

    # Special cases with no identifiers entry
    elif db_name == 'FPLX':
        url = 'http://identifiers.org/fplx/%s' % db_id
    elif db_name == 'NXPFA':
        url = 'https://www.nextprot.org/term/FA-%s' % db_id
    elif db_name in ('UN', 'WDI', 'FAO'):
        url = 'https://github.com/clulab/eidos/wiki/JSON-LD#Grounding/%s' % \
                db_id
    elif db_name == 'HUME':
        url = ('https://github.com/BBN-E/Hume/blob/master/resource/ontologies/'
               'hume_ontology/%s' % db_id)
    elif db_name == 'CWMS':
        url = 'http://trips.ihmc.us/%s' % db_id
    elif db_name == 'SOFIA':
        url = 'http://cs.cmu.edu/sofia/%s' % db_id
    elif db_name == 'TEXT':
        return None
    else:
        logger.warning('Unhandled name space %s' % db_name)
        url = None
    return url
<commit_msg>Added URL generation for SCHEM, SCOMP and SFAM (BEL).<commit_after>import logging

logger = logging.getLogger(__name__)

def get_identifiers_url(db_name, db_id):
    """Return an identifiers.org URL for a given database name and ID.

    Parameters
    ----------
    db_name : str
        An internal database name: HGNC, UP, CHEBI, etc.
    db_id : str
        An identifier in the given database.

    Returns
    -------
    url : str
        An identifiers.org URL corresponding to the given database name and ID.
    """
    identifiers_url = 'http://identifiers.org/'
    if db_name == 'UP':
        url = identifiers_url + 'uniprot/%s' % db_id
    elif db_name == 'HGNC':
        url = identifiers_url + 'hgnc/HGNC:%s' % db_id
    elif db_name == 'IP':
        url = identifiers_url + 'interpro/%s' % db_id
    elif db_name == 'IPR':
        url = identifiers_url + 'interpro/%s' % db_id
    elif db_name == 'CHEBI':
        url = identifiers_url + 'chebi/%s' % db_id
    elif db_name == 'NCIT':
        url = identifiers_url + 'ncit/%s' % db_id
    elif db_name == 'GO':
        url = identifiers_url + 'go/%s' % db_id
    elif db_name in ('PUBCHEM', 'PCID'):  # Assuming PCID = PubChem compound ID
        if db_id.startswith('PUBCHEM:'):
            db_id = db_id[8:]
        elif db_id.startswith('PCID:'):
            db_id = db_id[5:]
        url = identifiers_url + 'pubchem.compound/%s' % db_id
    elif db_name == 'PF':
        url = identifiers_url + 'pfam/%s' % db_id
    elif db_name == 'MIRBASEM':
        url = identifiers_url + 'mirbase.mature/%s' % db_id
    elif db_name == 'MIRBASE':
        url = identifiers_url + 'mirbase/%s' % db_id
    elif db_name == 'MESH':
        url = identifiers_url + 'mesh/%s' % db_id
    elif db_name == 'EGID':
        url = 'ncbigene:%s' % db_id
    elif db_name == 'HMDB':
        url = identifiers_url + 'hmdb/%s' % db_id
    elif db_name == 'LINCS':
        if db_id.startswith('LSM-'):  # Lincs Small Molecule ID
            url = identifiers_url + 'lincs.smallmolecule:%s' % db_id
        elif db_id.startswith('LCL-'):  # Lincs Cell Line ID
            url = identifiers_url + 'lincs.cell:%s' % db_id
        else:  # Assume LINCS Protein
            url = identifiers_url + 'lincs.protein:%s' % db_id
    elif db_name == 'HMS-LINCS':
        url = identifiers_url + 'lincs.protein:%s' % db_id

    # Special cases with no identifiers entry
    elif db_name == 'SCHEM':
        url = 'https://arty.scai.fraunhofer.de/artifactory/bel/namespace/selventa-legacy-chemicals/selventa-legacy-chemicals-20150601.belns'
    elif db_name == 'SCOMP':
        url = 'https://arty.scai.fraunhofer.de/artifactory/bel/namespace/selventa-named-complexes/selventa-named-complexes-20150601.belns'
    elif db_name == 'SFAM':
        url = 'https://arty.scai.fraunhofer.de/artifactory/bel/namespace/selventa-protein-families/selventa-protein-families-20150601.belns'
    elif db_name == 'FPLX':
        url = 'http://identifiers.org/fplx/%s' % db_id
    elif db_name == 'NXPFA':
        url = 'https://www.nextprot.org/term/FA-%s' % db_id
    elif db_name in ('UN', 'WDI', 'FAO'):
        url = 'https://github.com/clulab/eidos/wiki/JSON-LD#Grounding/%s' % \
                db_id
    elif db_name == 'HUME':
        url = ('https://github.com/BBN-E/Hume/blob/master/resource/ontologies/'
               'hume_ontology/%s' % db_id)
    elif db_name == 'CWMS':
        url = 'http://trips.ihmc.us/%s' % db_id
    elif db_name == 'SOFIA':
        url = 'http://cs.cmu.edu/sofia/%s' % db_id
    elif db_name == 'TEXT':
        return None
    else:
        logger.warning('Unhandled name space %s' % db_name)
        url = None
    return url
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""Test plugin system."""

#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

import os.path as op

from pytest import yield_fixture, raises

from ..plugin import (IPluginRegistry,
                      IPlugin,
                      get_plugin,
                      discover_plugins,
                      get_all_plugins,
                      )
from .._misc import _write_text
from ..config import load_master_config

#------------------------------------------------------------------------------
# Fixtures
#------------------------------------------------------------------------------

@yield_fixture
def no_native_plugins():
    # Save the plugins.
    plugins = IPluginRegistry.plugins
    IPluginRegistry.plugins = []
    yield
    IPluginRegistry.plugins = plugins

@yield_fixture(params=[(False, 'my_plugins/plugin.py'),
                       (True, 'plugins/plugin.py'),
                       ])
def plugin(no_native_plugins, temp_user_dir, request):
    path = op.join(temp_user_dir, request.param[1])
    contents = """
        from phy import IPlugin
        class MyPlugin(IPlugin):
            pass
    """
    _write_text(path, contents)
    yield temp_user_dir, request.param[0], request.param[1]

def _write_my_plugins_dir_in_config(temp_user_dir):
    # Now, we specify the path to the plugin in the phy config file.
    config_contents = """
       c = get_config()
       c.Plugins.dirs = ['%s']
    """
    _write_text(op.join(temp_user_dir, 'phy_config.py'),
                config_contents % op.join(temp_user_dir, 'my_plugins/'))

#------------------------------------------------------------------------------
# Tests
#------------------------------------------------------------------------------

def test_plugin_1(no_native_plugins):
    class MyPlugin(IPlugin):
        pass

    assert IPluginRegistry.plugins == [(MyPlugin,)]
    assert get_plugin('myplugin').__name__ == 'MyPlugin'

    with raises(ValueError):
        get_plugin('unknown')

def test_discover_plugins(tempdir, no_native_plugins):
    path = op.join(tempdir, 'my_plugin.py')
    contents = '''from phy import IPlugin\nclass MyPlugin(IPlugin): pass'''
    _write_text(path, contents)

    plugins = discover_plugins([tempdir])
    assert plugins
    assert plugins[0][0].__name__ == 'MyPlugin'

def test_get_all_plugins(plugin):
    temp_user_dir, in_default_dir, path = plugin
    n_builtin_plugins = 0

    plugins = get_all_plugins()

    def _assert_loaded():
        assert len(plugins) == n_builtin_plugins + 1
        p = plugins[-1]
        assert p.__name__ == 'MyPlugin'

    if in_default_dir:
        # Create a plugin in the default plugins directory: it will be
        # discovered and automatically loaded by get_all_plugins().
        _assert_loaded()
    else:
        assert len(plugins) == n_builtin_plugins

        # This time, we write the custom plugins path in the config file.
        _write_my_plugins_dir_in_config(temp_user_dir)

        # We reload all plugins with the master config object.
        config = load_master_config()
        plugins = get_all_plugins(config)

        # This time, the plugin should be found.
        _assert_loaded()
<commit_msg>Fix path bug on Windows<commit_after># -*- coding: utf-8 -*-

"""Test plugin system."""

#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

import os.path as op

from pytest import yield_fixture, raises

from ..plugin import (IPluginRegistry,
                      IPlugin,
                      get_plugin,
                      discover_plugins,
                      get_all_plugins,
                      )
from .._misc import _write_text
from ..config import load_master_config

#------------------------------------------------------------------------------
# Fixtures
#------------------------------------------------------------------------------

@yield_fixture
def no_native_plugins():
    # Save the plugins.
    plugins = IPluginRegistry.plugins
    IPluginRegistry.plugins = []
    yield
    IPluginRegistry.plugins = plugins

@yield_fixture(params=[(False, 'my_plugins/plugin.py'),
                       (True, 'plugins/plugin.py'),
                       ])
def plugin(no_native_plugins, temp_user_dir, request):
    path = op.join(temp_user_dir, request.param[1])
    contents = """
        from phy import IPlugin
        class MyPlugin(IPlugin):
            pass
    """
    _write_text(path, contents)
    yield temp_user_dir, request.param[0], request.param[1]

def _write_my_plugins_dir_in_config(temp_user_dir):
    # Now, we specify the path to the plugin in the phy config file.
    config_contents = """
       c = get_config()
       c.Plugins.dirs = [r'%s']
    """
    _write_text(op.join(temp_user_dir, 'phy_config.py'),
                config_contents % op.join(temp_user_dir, 'my_plugins/'))

#------------------------------------------------------------------------------
# Tests
#------------------------------------------------------------------------------

def test_plugin_1(no_native_plugins):
    class MyPlugin(IPlugin):
        pass

    assert IPluginRegistry.plugins == [(MyPlugin,)]
    assert get_plugin('myplugin').__name__ == 'MyPlugin'

    with raises(ValueError):
        get_plugin('unknown')

def test_discover_plugins(tempdir, no_native_plugins):
    path = op.join(tempdir, 'my_plugin.py')
    contents = '''from phy import IPlugin\nclass MyPlugin(IPlugin): pass'''
    _write_text(path, contents)

    plugins = discover_plugins([tempdir])
    assert plugins
    assert plugins[0][0].__name__ == 'MyPlugin'

def test_get_all_plugins(plugin):
    temp_user_dir, in_default_dir, path = plugin
    n_builtin_plugins = 0

    plugins = get_all_plugins()

    def _assert_loaded():
        assert len(plugins) == n_builtin_plugins + 1
        p = plugins[-1]
        assert p.__name__ == 'MyPlugin'

    if in_default_dir:
        # Create a plugin in the default plugins directory: it will be
        # discovered and automatically loaded by get_all_plugins().
        _assert_loaded()
    else:
        assert len(plugins) == n_builtin_plugins

        # This time, we write the custom plugins path in the config file.
        _write_my_plugins_dir_in_config(temp_user_dir)

        # We reload all plugins with the master config object.
        config = load_master_config()
        plugins = get_all_plugins(config)

        # This time, the plugin should be found.
        _assert_loaded()
<|endoftext|>


<commit_before>result = []
with open("./../../../2016_cd_2a_2.txt", 'r') as f:
    content = f.readlines()
    #print(content)
    #print(len(content))
    # 逐 element 處理
    for i in range(len(content)):
        for line in content[i].splitlines():
            #print(content[i].splitlines())
            result.append(list(line.split(",")))
g.es(result)<commit_msg>Delete cdw2_1.py<commit_after><|endoftext|>


<commit_before>import codecs
import json
import os
import sys

import click
import crayons
import delegator
import pexpect
import toml
import _pipfile as pipfile

from .project import Project
from .utils import convert_deps_from_pip, convert_deps_to_pip

__version__ = '0.1.5'

project = Project()

def ensure_latest_pip():

    # Ensure that pip is installed.
    c = delegator.run('pip install pip')

    # Check if version is out of date.
    if 'however' in c.err:
        # If version is out of date, update.
        click.echo(crayons.yellow('Pip is out of date... updating to latest.'))
        c = delegator.run('pip install pip --upgrade', block=False)
        click.echo(crayons.blue(c.out))

def do_where(virtualenv=False, bare=True):
    """Executes the where functionality."""

    if not virtualenv:
        location = project.pipfile_location

        if not bare:
            click.echo('Pipfile found at {}. Considering this to be the project home.'.format(crayons.green(location)))
        else:
            click.echo(location)

    else:
        location = project.virtualenv_location

        if not bare:
            click.echo('Virtualenv location: {}'.format(crayons.green(location)))
        else:
            click.echo(location)

def do_install_dependencies(dev=False, only=False, bare=False):
    """"Executes the install functionality."""

    # Load the Pipfile.
    p = pipfile.load(project.pipfile_location)
    lockfile = json.loads(p.freeze())

    # Install default dependencies, always.
    deps = lockfile['default'] if not only else {}

    # Add development deps if --dev was passed.
    if dev:
        deps.update(lockfile['develop'])

    # Convert the deps to pip-compatbile arguments.
    deps = convert_deps_to_pip(deps)

    # Actually install each dependency into the virtualenv.
    for package_name in deps:

        if not bare:
            click.echo('Installing {}...'.format(crayons.green(package_name)))

        c = delegator.run('{} install {}'.format(which_pip(), package_name),)

        if not bare:
            click.echo(crayons.blue(c.out))

def do_freeze():
    """Executes the freeze functionality."""

    click.echo(crayons.yellow('Assuring all dependencies from Pipfile are installed...'))

    # Purge the virtualenv, for development dependencies.
    do_purge(bare=True)

    click.echo(crayons.yellow('Freezing development dependencies...'))

    # Install only development dependencies.
    do_install_dependencies(dev=True, only=True, bare=True)

    # Load the Pipfile and generate a lockfile.
    p = pipfile.load(project.pipfile_location)
    lockfile = json.loads(p.freeze())

    # Pip freeze development dependencies.
    c = delegator.run('{} freeze'.format(which_pip()))

    # Add Development dependencies to lockfile.
    for dep in c.out.split('\n'):
        if dep:
            lockfile['develop'].update(convert_deps_from_pip(dep))

    # Purge the virtualenv.
    do_purge(bare=True)

    click.echo(crayons.yellow('Freezing default dependencies...'))

    # Install only development dependencies.
    do_install_dependencies(bare=True)

    # Pip freeze default dependencies.
    c = delegator.run('{} freeze'.format(which_pip()))

    # Add default dependencies to lockfile.
    for dep in c.out.split('\n'):
        if dep:
            lockfile['default'].update(convert_deps_from_pip(dep))

    with open(project.lockfile_location, 'w') as f:
        f.write(json.dumps(lockfile, indent=4, separators=(',', ': ')))

    click.echo(crayons.yellow('Note: ') + 'your project now has only default packages installed.')
    click.echo('To install dev-packages, run: $ {}'.format(crayons.red('pipenv init --dev')))

def activate_virtualenv(source=True):
    """Returns the string to activate a virtualenv."""
    if source:
        return 'source {}/bin/activate'.format(project.virtualenv_location)
    else:
        return '{}/bin/activate'.format(project.virtualenv_location)

def do_activate_virtualenv(bare=False):
    """Executes the activate virtualenv functionality."""
    if not bare:
        click.echo('To activate this project\'s virtualenv, run the following:\n $ {}'.format(crayons.red('pipenv shell')))
    else:
        click.echo(activate_virtualenv())

def do_purge(bare=False):
    """Executes the purge functionality."""
    freeze = delegator.run('{} freeze'.format(which_pip())).out
    installed = freeze.split()

    if not bare:
        click.echo('Found {} dirty packages installed, purging...'.format(len(installed)))
    command = '{} uninstall {} -y'.format(which_pip(), ' '.join(installed))
    c = delegator.run(command)

    if not bare:
        click.echo(crayons.blue(c.out))

        click.echo(crayons.yellow('Virtualenv now purged and fresh!'))

def which_pip():
    """Returns the location of virtualenv-installed pip."""
    return os.sep.join([project.virtualenv_location] + ['bin/pip'])

def which_python():
    """Returns the location of virtualenv-installed Python."""
    return os.sep.join([project.virtualenv_location] + ['bin/python'])

@click.group()
@click.version_option(prog_name=crayons.yellow('pipenv'), version=__version__)
def cli(*args, **kwargs):
    # Ensure that pip is installed and up-to-date.
    ensure_latest_pip()

@click.command()
@click.option('--dev', '-d', is_flag=True, default=False)
def init(dev=False):

    # Assert Pipfile exists.
    if not project.pipfile_exists:

        click.echo(crayons.yellow('Creating a Pipfile for this project...'))

        # Create the pipfile if it doesn't exist.
        project.create_pipfile()

        # Create the Pipfile.freeze too.
        click.echo(crayons.yellow('Creating a Pipfile.lock as well...'))
        do_freeze()

    # Display where the Project is established.
    do_where(bare=False)
    click.echo(crayons.yellow('Creating a virtualenv for this project...'))

    # Actually create the virtualenv.
    c = delegator.run('virtualenv {} --prompt=({})'.format(project.virtualenv_location, project.name), block=False)
    click.echo(crayons.blue(c.out))

    # Say where the virtualenv is.
    do_where(virtualenv=True, bare=False)

    # Write out the lockfile if it doesn't exist.
    if project.lockfile_exists:

        # Open the lockfile.
        with codecs.open(project.lockfile_location, 'r') as f:
            lockfile = json.load(f)

        # Update the lockfile if it is out-of-date.
        p = pipfile.load(project.pipfile_location)

        # Check that the hash of the Lockfile matches the lockfile's hash.
        if not lockfile['_meta']['Pipfile-sha256'] == p.hash:
            click.echo(crayons.red('Pipfile.lock out of date, updating...'))

            do_freeze()

        click.echo(crayons.yellow('Installing dependencies from Pipfile.lock...'))

    else:

        # Load the pipfile.
        click.echo(crayons.yellow('Installing dependencies from Pipfile...'))
        p = pipfile.load(project.pipfile_location)
        lockfile = json.loads(p.freeze())

    do_install_dependencies(dev=dev)

    # Write out the lockfile if it doesn't exist.
    if not project.lockfile_exists:
        click.echo(crayons.yellow('Pipfile.lock not found, creating...'))
        with codecs.open(project.lockfile_location, 'w', 'utf-8') as f:
            f.write(p.freeze())

    # Activate virtualenv instructions.
    do_activate_virtualenv()

@click.command()
@click.option('--virtualenv', '--venv', '-v', is_flag=True, default=False)
@click.option('--bare', '-b', is_flag=True, default=False)
def where(venv=False, bare=False):
    do_where(venv, bare)

@click.command()
@click.argument('package_name')
@click.option('--dev','-d', is_flag=True, default=False)
def install(package_name, dev=False):
    click.echo('Installing {}...'.format(crayons.green(package_name)))

    c = delegator.run('{} install {}'.format(which_pip(), package_name))
    click.echo(crayons.blue(c.out))

    click.echo('Adding {} to Pipfile...'.format(crayons.green(package_name)))
    project.add_package_to_pipfile(package_name, dev)

@click.command()
@click.argument('package_name')
def uninstall(package_name):
    click.echo('Un-installing {}...'.format(crayons.green(package_name)))

    c = delegator.run('{} uninstall {} -y'.format(which_pip(), package_name))
    click.echo(crayons.blue(c.out))

    click.echo('Removing {} from Pipfile...'.format(crayons.green(package_name)))
    project.remove_package_from_pipfile(package_name)

@click.command()
def lock():
    do_freeze()

@click.command()
@click.argument('args', nargs=-1)
def python(args):
    # Spawn the Python process, and iteract with it.
    c = pexpect.spawn('{} {}'.format(which_python(), ' '.join(args)))
    c.interact()

@click.command()
def shell():
    # Spawn the Python process, and iteract with it.pip
    click.echo(crayons.yellow('Spawning virtualenv shell.'))

    shell = os.environ['SHELL']
    c = pexpect.spawn("{} -c '. {}; exec {} -i'".format(shell, activate_virtualenv(source=False), shell))
    c.send(activate_virtualenv() + '\n')

    # Interact with the new shell.
    c.interact()

@click.command()
@click.option('--bare', '-b', is_flag=True, default=False)
def venv(bare=False):
    do_activate_virtualenv(bare=bare)

@click.command()
def purge():
    do_purge()

@click.command()
def check():
    click.echo(crayons.yellow('Checking PEP 508 requirements...'))

    # Load the Pipfile.
    p = pipfile.load(project.pipfile_location)

    # Assert the given requirements.
    p.assert_requirements()

# Install click commands.
cli.add_command(init)
cli.add_command(where)
cli.add_command(install)
cli.add_command(uninstall)
cli.add_command(lock)
cli.add_command(python)
cli.add_command(venv)
cli.add_command(purge)
cli.add_command(check)
cli.add_command(shell)

if __name__ == '__main__':
    cli()
<commit_msg>overall improvements<commit_after>import codecs
import json
import os
import sys

import click
import crayons
import delegator
import pexpect
import toml
import _pipfile as pipfile

from .project import Project
from .utils import convert_deps_from_pip, convert_deps_to_pip

__version__ = '0.1.5'

project = Project()

def ensure_latest_pip():
    # Ensure that pip is installed.
    c = delegator.run('{} install pip'.format(which_pip()))

    # Check if version is out of date.
    if 'however' in c.err:
        # If version is out of date, update.
        click.echo(crayons.yellow('Pip is out of date... updating to latest.'))
        c = delegator.run('{} install pip --upgrade'.format(which_pip()), block=False)
        click.echo(crayons.blue(c.out))

def do_where(virtualenv=False, bare=True):
    """Executes the where functionality."""

    if not virtualenv:
        location = project.pipfile_location

        if not bare:
            click.echo('Pipfile found at {}. Considering this to be the project home.'.format(crayons.green(location)))
        else:
            click.echo(location)

    else:
        location = project.virtualenv_location

        if not bare:
            click.echo('Virtualenv location: {}'.format(crayons.green(location)))
        else:
            click.echo(location)

def do_install_dependencies(dev=False, only=False, bare=False):
    """"Executes the install functionality."""

    # Load the Pipfile.
    p = pipfile.load(project.pipfile_location)
    lockfile = json.loads(p.freeze())

    # Install default dependencies, always.
    deps = lockfile['default'] if not only else {}

    # Add development deps if --dev was passed.
    if dev:
        deps.update(lockfile['develop'])

    # Convert the deps to pip-compatbile arguments.
    deps = convert_deps_to_pip(deps)

    # Actually install each dependency into the virtualenv.
    for package_name in deps:

        if not bare:
            click.echo('Installing {}...'.format(crayons.green(package_name)))

        c = delegator.run('{} install {}'.format(which_pip(), package_name),)

        if not bare:
            click.echo(crayons.blue(c.out))

def do_lock():
    """Executes the freeze functionality."""

    click.echo(crayons.yellow('Assuring all dependencies from Pipfile are installed...'))

    # Purge the virtualenv, for development dependencies.
    do_purge(bare=True)

    click.echo(crayons.yellow('Freezing development dependencies...'))

    # Install only development dependencies.
    do_install_dependencies(dev=True, only=True, bare=True)

    # Load the Pipfile and generate a lockfile.
    p = pipfile.load(project.pipfile_location)
    lockfile = json.loads(p.freeze())

    # Pip freeze development dependencies.
    c = delegator.run('{} freeze'.format(which_pip()))

    # Add Development dependencies to lockfile.
    for dep in c.out.split('\n'):
        if dep:
            lockfile['develop'].update(convert_deps_from_pip(dep))

    # Purge the virtualenv.
    do_purge(bare=True)

    click.echo(crayons.yellow('Freezing default dependencies...'))

    # Install only development dependencies.
    do_install_dependencies(bare=True)

    # Pip freeze default dependencies.
    c = delegator.run('{} freeze'.format(which_pip()))

    # Add default dependencies to lockfile.
    for dep in c.out.split('\n'):
        if dep:
            lockfile['default'].update(convert_deps_from_pip(dep))

    with open(project.lockfile_location, 'w') as f:
        f.write(json.dumps(lockfile, indent=4, separators=(',', ': ')))

    click.echo(crayons.yellow('Note: ') + 'your project now has only default packages installed.')
    click.echo('To install dev-packages, run: $ {}'.format(crayons.red('pipenv init --dev')))

def activate_virtualenv(source=True):
    """Returns the string to activate a virtualenv."""
    if source:
        return 'source {}/bin/activate'.format(project.virtualenv_location)
    else:
        return '{}/bin/activate'.format(project.virtualenv_location)

def do_activate_virtualenv(bare=False):
    """Executes the activate virtualenv functionality."""
    if not bare:
        click.echo('To activate this project\'s virtualenv, run the following:\n $ {}'.format(crayons.red('pipenv shell')))
    else:
        click.echo(activate_virtualenv())

def do_purge(bare=False):
    """Executes the purge functionality."""
    freeze = delegator.run('{} freeze'.format(which_pip())).out
    installed = freeze.split()

    if not bare:
        click.echo('Found {} installed package(s), purging...'.format(len(installed)))
    command = '{} uninstall {} -y'.format(which_pip(), ' '.join(installed))
    c = delegator.run(command)

    if not bare:
        click.echo(crayons.blue(c.out))

        click.echo(crayons.yellow('Virtualenv now purged and fresh!'))

def do_init(dev=False):
    """Executes the init functionality."""

    # Assert Pipfile exists.
    if not project.pipfile_exists:

        click.echo(crayons.yellow('Creating a Pipfile for this project...'))

        # Create the pipfile if it doesn't exist.
        project.create_pipfile()

        # Create the Pipfile.freeze too.
        click.echo(crayons.yellow('Creating a Pipfile.lock as well...'))
        do_lock()

    # Display where the Project is established.
    do_where(bare=False)
    click.echo(crayons.yellow('Creating a virtualenv for this project...'))

    # Actually create the virtualenv.
    c = delegator.run(['virtualenv', project.virtualenv_location, '--prompt=({})'.format(project.name)], block=False)
    click.echo(crayons.blue(c.out))

    # Say where the virtualenv is.
    do_where(virtualenv=True, bare=False)

    # Write out the lockfile if it doesn't exist.
    if project.lockfile_exists:

        # Open the lockfile.
        with codecs.open(project.lockfile_location, 'r') as f:
            lockfile = json.load(f)

        # Update the lockfile if it is out-of-date.
        p = pipfile.load(project.pipfile_location)

        # Check that the hash of the Lockfile matches the lockfile's hash.
        if not lockfile['_meta']['Pipfile-sha256'] == p.hash:
            click.echo(crayons.red('Pipfile.lock out of date, updating...'))

            do_lock()

        click.echo(crayons.yellow('Installing dependencies from Pipfile.lock...'))

    else:

        # Load the pipfile.
        click.echo(crayons.yellow('Installing dependencies from Pipfile...'))
        p = pipfile.load(project.pipfile_location)
        lockfile = json.loads(p.freeze())

    do_install_dependencies(dev=dev)

    # Write out the lockfile if it doesn't exist.
    if not project.lockfile_exists:
        click.echo(crayons.yellow('Pipfile.lock not found, creating...'))
        with codecs.open(project.lockfile_location, 'w', 'utf-8') as f:
            f.write(p.freeze())

    # Activate virtualenv instructions.
    do_activate_virtualenv()

def which_pip():
    """Returns the location of virtualenv-installed pip."""
    return os.sep.join([project.virtualenv_location] + ['bin/pip'])

def which_python():
    """Returns the location of virtualenv-installed Python."""
    return os.sep.join([project.virtualenv_location] + ['bin/python'])

@click.group()
@click.version_option(prog_name=crayons.yellow('pipenv'), version=__version__)
def cli(*args, **kwargs):
    # Ensure that pip is installed and up-to-date.
    # ensure_latest_pip()
    pass

@click.command()
@click.option('--dev', '-d', is_flag=True, default=False)
def init(dev=False):
    do_init(dev=dev)

@click.command()
@click.option('--virtualenv', '--venv', '-v', is_flag=True, default=False)
@click.option('--bare', '-b', is_flag=True, default=False)
def where(venv=False, bare=False):
    do_where(venv, bare)

@click.command()
@click.argument('package_name')
@click.option('--dev','-d', is_flag=True, default=False)
def install(package_name, dev=False):
    click.echo('Installing {}...'.format(crayons.green(package_name)))

    c = delegator.run([which_pip(), 'install', str(package_name)])
    click.echo(crayons.blue(c.out))

    # Ensure that package was successfully installed.
    try:
        assert c.return_code == 0
    except AssertionError:
        click.echo('{} An error occured while installing {}'.format(crayons.red('Error: '), crayons.green(package_name)))
        sys.exit(1)

    if dev:
        click.echo('Adding {} to Pipfile\'s [dev-packages]...'.format(crayons.green(package_name)))
    else:
        click.echo('Adding {} to Pipfile\'s [packages]...'.format(crayons.green(package_name)))

    # Add the package to the Pipfile.
    project.add_package_to_pipfile(package_name, dev)

@click.command()
@click.argument('package_name')
def uninstall(package_name):
    click.echo('Un-installing {}...'.format(crayons.green(package_name)))

    c = delegator.run('{} uninstall {} -y'.format(which_pip(), package_name))
    click.echo(crayons.blue(c.out))

    click.echo('Removing {} from Pipfile...'.format(crayons.green(package_name)))
    project.remove_package_from_pipfile(package_name)

@click.command()
def lock():
    do_lock()

@click.command()
@click.argument('args', nargs=-1)
def python(args):
    # Spawn the Python process, and iteract with it.
    c = pexpect.spawn('{} {}'.format(which_python(), ' '.join(args)))
    c.interact()

@click.command()
def shell():
    # Spawn the Python process, and iteract with it.pip

    shell = os.environ['SHELL']
    click.echo(crayons.yellow('Spawning virtualenv shell ({}).'.format(crayons.red(shell))))

    c = pexpect.spawn("{} -c '. {}; exec {} -i'".format(shell, activate_virtualenv(source=False), shell))
    c.send(activate_virtualenv() + '\n')

    # Interact with the new shell.
    c.interact()

@click.command()
@click.option('--bare', '-b', is_flag=True, default=False)
def venv(bare=False):
    do_activate_virtualenv(bare=bare)

@click.command()
def purge():
    do_purge()

@click.command()
def check():
    click.echo(crayons.yellow('Checking PEP 508 requirements...'))

    # Load the Pipfile.
    p = pipfile.load(project.pipfile_location)

    # Assert the given requirements.
    p.assert_requirements()

@click.command()
@click.option('--dev','-d', is_flag=True, default=False)
def update(dev=False):

    # Update pip to latest version.
    ensure_latest_pip()

    click.echo(crayons.yellow('Updating all dependencies from Pipfile...'))

    do_purge()
    do_init(dev=dev)

    click.echo(crayons.yellow('All dependencies are now up-to-date!'))

# Install click commands.
cli.add_command(init)
cli.add_command(where)
cli.add_command(install)
cli.add_command(uninstall)
cli.add_command(update)
cli.add_command(lock)
cli.add_command(python)
cli.add_command(venv)
cli.add_command(purge)
cli.add_command(check)
cli.add_command(shell)

if __name__ == '__main__':
    cli()
<|endoftext|>


<commit_before>import json
import os
import redis
import time

from contextlib import contextmanager
from redis.exceptions import ConnectionError as RedisConnectionError

class ConnectionError(Exception):
    '''
    Represents connection errors that can occur when talking to the ArchiveBot
    control node, Redis or otherwise.
    '''

    pass

@contextmanager
def conn(controller):
    try:
        if not controller.connected():
            controller.connect()
        yield
    except RedisConnectionError as e:
        controller.disconnect()
        raise ConnectionError(str(e)) from e

def candidate_queues(named_queues, pipeline_nick, ao_only):
    '''
    Generates names of queues that this pipeline will check for work.
    '''

    def applies(q):
        return q.replace('pending:', '') in pipeline_nick

    if ao_only:
        return ['pending-ao']
    else:
        matches = [q for q in named_queues if applies(q)]
        matches.append('pending-ao')
        matches.append('pending')

        return matches

class Control(object):
    '''
    Handles communication to and from the ArchiveBot control server.

    If a message cannot be processed due to a connection error, the Redis
    connection is closed and deleted.  An archivebot.control.ConnectionError
    is also raised.
    '''

    def __init__(self, redis_url, log_channel, pipeline_channel):
        self.log_channel = log_channel
        self.pipeline_channel = pipeline_channel
        self.bytes_outstanding = 0
        self.items_downloaded_outstanding = 0
        self.items_queued_outstanding = 0
        self.redis_url = redis_url

        self.connect()

    def connected(self):
        return self.redis is not None

    def connect(self):
        if self.redis_url is None:
            raise ConnectionError('self.redis_url not set')

        self.redis = redis.StrictRedis.from_url(self.redis_url,
                decode_responses=True)

        self.register_scripts()

    def disconnect(self):
        self.redis = None

    def register_scripts(self):
        self.mark_done_script = self.redis.register_script(MARK_DONE_SCRIPT)
        self.mark_aborted_script = self.redis.register_script(MARK_ABORTED_SCRIPT)
        self.log_script = self.redis.register_script(LOGGER_SCRIPT)

    def all_named_pending_queues(self):
        with conn(self):
            pipelines = set()

            for name in self.redis.scan_iter('pending:*'):
                pipelines.add(name)

            return pipelines

    def reserve_job(self, pipeline_id, pipeline_nick, ao_only):
        named_queues = self.all_named_pending_queues()

        for queue in candidate_queues(named_queues, pipeline_nick, ao_only):
            ident = self.dequeue_item(queue)

            if ident:
                return self.complete_reservation(ident, pipeline_id)

        return None, None

    def dequeue_item(self, queue):
        with conn(self):
            return self.redis.rpoplpush(queue, 'working')

    def complete_reservation(self, ident, pipeline_id):
        with conn(self):
            self.redis.hmset(ident, dict(
                started_at=time.time(),
                pipeline_id=pipeline_id
            ))

            return ident, self.redis.hgetall(ident)

    def heartbeat(self, ident):
        try:
            with conn(self):
                self.redis.hincrby(ident, 'heartbeat', 1)
        except ConnectionError:
            pass

    def is_aborted(self, ident):
        with conn(self):
            return self.redis.hget(ident, 'aborted')

    def mark_done(self, item, expire_time):
        with conn(self):
            self.mark_done_script(keys=[item['ident']], args=[expire_time,
                self.log_channel, int(time.time()), json.dumps(item['info']),
                item['log_key']])

    def mark_aborted(self, ident):
        with conn(self):
            self.mark_aborted_script(keys=[ident], args=[self.log_channel])

    def update_bytes_downloaded(self, ident, size):
        try:
            with conn(self):
                self.bytes_outstanding += size
                self.redis.hincrby(ident, 'bytes_downloaded',
                        self.bytes_outstanding)
                self.bytes_outstanding = 0
        except ConnectionError:
            pass

    def update_items_downloaded(self, count):
        self.items_downloaded_outstanding += count

    def update_items_queued(self, count):
        self.items_queued_outstanding += count

    def flush_item_counts(self, ident):
        try:
            with conn(self):
                self.redis.hincrby(ident, 'items_downloaded',
                        self.items_downloaded_outstanding)
                self.items_downloaded_outstanding = 0

                self.redis.hincrby(ident, 'items_queued',
                        self.items_queued_outstanding)
                self.items_queued_outstanding = 0
        except ConnectionError:
            pass

    def pipeline_report(self, pipeline_id, report):
        try:
            with conn(self):
                self.redis.hmset(pipeline_id, report)
                self.redis.sadd('pipelines', pipeline_id)
                self.redis.publish(self.pipeline_channel, pipeline_id)
        except ConnectionError:
            pass

    def unregister_pipeline(self, pipeline_id):
        try:
            with conn(self):
                self.redis.delete(pipeline_id)
                self.redis.srem('pipelines', pipeline_id)
                self.redis.publish(self.pipeline_channel, pipeline_id)
        except ConnectionError:
            pass

    def log(self, packet, ident, log_key):
        try:
            with conn(self):
                self.log_script(keys=[ident], args=[json.dumps(packet),
                    self.log_channel, log_key])
        except ConnectionError:
            pass

    def get_url_file(self, ident):
        try:
            with conn(self):
                return self.redis.hget(ident, 'url_file')
        except ConnectionError:
            pass

    def get_settings(self, ident):
        with conn(self):
            data = self.redis.hmget(ident, 'delay_min', 'delay_max',
                    'concurrency',
                    'settings_age',
                    'abort_requested',
                    'suppress_ignore_reports',
                    'ignore_patterns_set_key')

            result = dict(
                    delay_min=data[0],
                    delay_max=data[1],
                    concurrency=data[2],
                    age=data[3],
                    abort_requested=data[4],
                    suppress_ignore_reports=data[5]
                    )

            if data[6]:
                result['ignore_patterns'] = self.redis.smembers(data[6])
            else:
                result['ignore_patterns'] = []

            return result

# ------------------------------------------------------------------------------

MARK_DONE_SCRIPT = '''
local ident = KEYS[1]
local expire_time = ARGV[1]
local log_channel = ARGV[2]
local finished_at = ARGV[3]
local info = ARGV[4]
local log_key = ARGV[5]

redis.call('hmset', ident, 'finished_at', finished_at)
redis.call('lrem', 'working', 1, ident)

local was_aborted = redis.call('hget', ident, 'aborted')

-- If the job was aborted, we ignore the given expire time.  Instead, we set a
-- much shorter expire time -- one that's long enough for (most) subscribers
-- to read a message, but short enough to not cause undue suffering in the
-- case of retrying an aborted job.
if was_aborted then
    redis.call('incr', 'jobs_aborted')
    redis.call('expire', ident, 5)
    redis.call('expire', log_key, 5)
    redis.call('expire', ident..'_ignores', 5)
else
    redis.call('incr', 'jobs_completed')
    redis.call('expire', ident, expire_time)
    redis.call('expire', log_key, expire_time)
    redis.call('expire', ident..'_ignores', expire_time)
end

redis.call('rpush', 'finish_notifications', info)
redis.call('publish', log_channel, ident)
'''

MARK_ABORTED_SCRIPT = '''
local ident = KEYS[1]
local log_channel = ARGV[1]

redis.call('hset', ident, 'aborted', 'true')
redis.call('publish', log_channel, ident)
'''

LOGGER_SCRIPT = '''
local ident = KEYS[1]
local message = ARGV[1]
local log_channel = ARGV[2]
local log_key = ARGV[3]

local nextseq = redis.call('hincrby', ident, 'log_score', 1)

redis.call('zadd', log_key, nextseq, message)
redis.call('publish', log_channel, ident)
'''

# vim:ts=4:sw=4:et:tw=78
<commit_msg>Use threading to make shipping logs into redis async with a shared queue<commit_after>import json
import os
import redis
import time

from queue import Queue
from threading import Thread
from contextlib import contextmanager
from redis.exceptions import ConnectionError as RedisConnectionError

class ConnectionError(Exception):
    '''
    Represents connection errors that can occur when talking to the ArchiveBot
    control node, Redis or otherwise.
    '''

    pass

@contextmanager
def conn(controller):
    try:
        if not controller.connected():
            controller.connect()
        yield
    except RedisConnectionError as e:
        controller.disconnect()
        raise ConnectionError(str(e)) from e

def candidate_queues(named_queues, pipeline_nick, ao_only):
    '''
    Generates names of queues that this pipeline will check for work.
    '''

    def applies(q):
        return q.replace('pending:', '') in pipeline_nick

    if ao_only:
        return ['pending-ao']
    else:
        matches = [q for q in named_queues if applies(q)]
        matches.append('pending-ao')
        matches.append('pending')

        return matches

class Control(object):
    '''
    Handles communication to and from the ArchiveBot control server.

    If a message cannot be processed due to a connection error, the Redis
    connection is closed and deleted.  An archivebot.control.ConnectionError
    is also raised.
    '''

    def __init__(self, redis_url, log_channel, pipeline_channel):
        self.log_channel = log_channel
        self.pipeline_channel = pipeline_channel
        self.bytes_outstanding = 0
        self.items_downloaded_outstanding = 0
        self.items_queued_outstanding = 0
        self.redis_url = redis_url
        self.log_queue = Queue()

        self.connect()

        log_thread = Thread(target=self.ship_logs, args=self)
        log_thread.daemon = True
        log_thread.start()

    def connected(self):
        return self.redis is not None

    def connect(self):
        if self.redis_url is None:
            raise ConnectionError('self.redis_url not set')

        self.redis = redis.StrictRedis.from_url(self.redis_url,
                decode_responses=True)

        self.register_scripts()

    def disconnect(self):
        self.redis = None

    def register_scripts(self):
        self.mark_done_script = self.redis.register_script(MARK_DONE_SCRIPT)
        self.mark_aborted_script = self.redis.register_script(MARK_ABORTED_SCRIPT)
        self.log_script = self.redis.register_script(LOGGER_SCRIPT)

    def all_named_pending_queues(self):
        with conn(self):
            pipelines = set()

            for name in self.redis.scan_iter('pending:*'):
                pipelines.add(name)

            return pipelines

    def reserve_job(self, pipeline_id, pipeline_nick, ao_only):
        named_queues = self.all_named_pending_queues()

        for queue in candidate_queues(named_queues, pipeline_nick, ao_only):
            ident = self.dequeue_item(queue)

            if ident:
                return self.complete_reservation(ident, pipeline_id)

        return None, None

    def dequeue_item(self, queue):
        with conn(self):
            return self.redis.rpoplpush(queue, 'working')

    def complete_reservation(self, ident, pipeline_id):
        with conn(self):
            self.redis.hmset(ident, dict(
                started_at=time.time(),
                pipeline_id=pipeline_id
            ))

            return ident, self.redis.hgetall(ident)

    def heartbeat(self, ident):
        try:
            with conn(self):
                self.redis.hincrby(ident, 'heartbeat', 1)
        except ConnectionError:
            pass

    def is_aborted(self, ident):
        with conn(self):
            return self.redis.hget(ident, 'aborted')

    def mark_done(self, item, expire_time):
        with conn(self):
            self.mark_done_script(keys=[item['ident']], args=[expire_time,
                self.log_channel, int(time.time()), json.dumps(item['info']),
                item['log_key']])

    def mark_aborted(self, ident):
        with conn(self):
            self.mark_aborted_script(keys=[ident], args=[self.log_channel])

    def update_bytes_downloaded(self, ident, size):
        try:
            with conn(self):
                self.bytes_outstanding += size
                self.redis.hincrby(ident, 'bytes_downloaded',
                        self.bytes_outstanding)
                self.bytes_outstanding = 0
        except ConnectionError:
            pass

    def update_items_downloaded(self, count):
        self.items_downloaded_outstanding += count

    def update_items_queued(self, count):
        self.items_queued_outstanding += count

    def flush_item_counts(self, ident):
        try:
            with conn(self):
                self.redis.hincrby(ident, 'items_downloaded',
                        self.items_downloaded_outstanding)
                self.items_downloaded_outstanding = 0

                self.redis.hincrby(ident, 'items_queued',
                        self.items_queued_outstanding)
                self.items_queued_outstanding = 0
        except ConnectionError:
            pass

    def pipeline_report(self, pipeline_id, report):
        try:
            with conn(self):
                self.redis.hmset(pipeline_id, report)
                self.redis.sadd('pipelines', pipeline_id)
                self.redis.publish(self.pipeline_channel, pipeline_id)
        except ConnectionError:
            pass

    def unregister_pipeline(self, pipeline_id):
        try:
            with conn(self):
                self.redis.delete(pipeline_id)
                self.redis.srem('pipelines', pipeline_id)
                self.redis.publish(self.pipeline_channel, pipeline_id)
        except ConnectionError:
            pass

    # This function is a thread used to asynchronously ship logs to redis for
    # this job, in a daemonic thread
    def ship_logs(self):
        while True:
            entry = self.log_queue.get() #infinite blocking
            try:
                with conn(self):
                    self.log_script(keys=entry.keys, args=entry.args)
            except ConnectionError:
                pass

            self.log_queue.task_done()

    def log(self, packet, ident, log_key):
        self.log_queue.put({'keys': [ident],
                       'args': [json.dumps(packet), self.log_channel, log_key]
                      })

    def get_url_file(self, ident):
        try:
            with conn(self):
                return self.redis.hget(ident, 'url_file')
        except ConnectionError:
            pass

    def get_settings(self, ident):
        with conn(self):
            data = self.redis.hmget(ident, 'delay_min', 'delay_max',
                    'concurrency',
                    'settings_age',
                    'abort_requested',
                    'suppress_ignore_reports',
                    'ignore_patterns_set_key')

            result = dict(
                    delay_min=data[0],
                    delay_max=data[1],
                    concurrency=data[2],
                    age=data[3],
                    abort_requested=data[4],
                    suppress_ignore_reports=data[5]
                    )

            if data[6]:
                result['ignore_patterns'] = self.redis.smembers(data[6])
            else:
                result['ignore_patterns'] = []

            return result

# ------------------------------------------------------------------------------

MARK_DONE_SCRIPT = '''
local ident = KEYS[1]
local expire_time = ARGV[1]
local log_channel = ARGV[2]
local finished_at = ARGV[3]
local info = ARGV[4]
local log_key = ARGV[5]

redis.call('hmset', ident, 'finished_at', finished_at)
redis.call('lrem', 'working', 1, ident)

local was_aborted = redis.call('hget', ident, 'aborted')

-- If the job was aborted, we ignore the given expire time.  Instead, we set a
-- much shorter expire time -- one that's long enough for (most) subscribers
-- to read a message, but short enough to not cause undue suffering in the
-- case of retrying an aborted job.
if was_aborted then
    redis.call('incr', 'jobs_aborted')
    redis.call('expire', ident, 5)
    redis.call('expire', log_key, 5)
    redis.call('expire', ident..'_ignores', 5)
else
    redis.call('incr', 'jobs_completed')
    redis.call('expire', ident, expire_time)
    redis.call('expire', log_key, expire_time)
    redis.call('expire', ident..'_ignores', expire_time)
end

redis.call('rpush', 'finish_notifications', info)
redis.call('publish', log_channel, ident)
'''

MARK_ABORTED_SCRIPT = '''
local ident = KEYS[1]
local log_channel = ARGV[1]

redis.call('hset', ident, 'aborted', 'true')
redis.call('publish', log_channel, ident)
'''

LOGGER_SCRIPT = '''
local ident = KEYS[1]
local message = ARGV[1]
local log_channel = ARGV[2]
local log_key = ARGV[3]

local nextseq = redis.call('hincrby', ident, 'log_score', 1)

redis.call('zadd', log_key, nextseq, message)
redis.call('publish', log_channel, ident)
'''

# vim:ts=4:sw=4:et:tw=78
<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 29-06-2017 15:05
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This is a template pipeline for refactoring out things from final pipelines as I identify how they're gonna look like
"""

# App imports
import config_manager
from toolbox import general

# The config manager singleton is just an example that only makes sense for specialized pipeline modules, not really for
# this template
__configuration_file = None
__configuration_manager = None

def __get_configuration_manager():
    global __configuration_manager
    if __configuration_manager is None:
        __configuration_manager = DirectorConfigurationManager(general.read_json(__configuration_file),
                                                               __configuration_file)
    return __configuration_manager

class DirectorConfigurationManager(config_manager.ConfigurationManager):
    # Command line arguments special characters
    _CONFIG_COMMAND_LINE_ARGUMENT_PARAMETER_SEPARATOR = ','
    _CONFIG_COMMAND_LINE_ARGUMENT_PARAMETER_ASSIGNMENT_CHAR = '='

    def __init__(self, configuration_object, configuration_file, pipeline_arguments):
        super(DirectorConfigurationManager, self).__init__(configuration_object, configuration_file)
        self.__pipeline_arguments = pipeline_arguments
        self.__pipeline_arguments_object = None
        # Logger the pythonist way
        self._logger = config_manager.get_app_config_manager().get_logger_for(
            "{}.{}".format(__name__, type(self).__name__))

    def _get_pipeline_arguments(self):
        return self.__pipeline_arguments

    def _get_allowed_configuration_keys(self):
        # Allowed configuration keys set is empty by default
        return {}

    def _get_pipeline_arguments_object(self):
        if self.__pipeline_arguments_object is None:
            self.__pipeline_arguments_object = self._process_pipeline_arguments()
        return self.__pipeline_arguments_object

    def _process_pipeline_arguments(self):
        if self._get_pipeline_arguments():
            if self.__pipeline_arguments_object:
                self._logger.error("DUPLICATED CALL for processing command line arguments for this pipeline, IGNORED")
            else:
                self.__pipeline_arguments_object = {}
                self._logger.debug("Processing pipeline command line arguments")
                allowed_keys = self._get_allowed_configuration_keys()
                for command_line_parameter in self._get_pipeline_arguments().split(
                        self._CONFIG_COMMAND_LINE_ARGUMENT_PARAMETER_SEPARATOR):
                    key, value = command_line_parameter.split(
                        self._CONFIG_COMMAND_LINE_ARGUMENT_PARAMETER_ASSIGNMENT_CHAR)
                    if key not in allowed_keys:
                        self._logger.error(
                            "INVALID KEY '{}' while parsing pipeline arguments, parameter '{}' SKIPPED"
                                .format(key, command_line_parameter))
                        continue
                    if key in self.__pipeline_arguments_object:
                        self._logger.error(
                            "DUPLICATED KEY '{}' while parsing pipeline arguments, parameter '{}' SKIPPED"
                                .format(key, command_line_parameter))
                        continue
                    self.__pipeline_arguments_object[key] = value
                    self._logger.debug("Pipeline argument '{}' parsed and set with value '{}'".format(key, value))
        else:
            self._logger.warning("This pipeline was provided with NO COMMAND LINE ARGUMENTS")
        return self.__pipeline_arguments_object

    def _get_value_for_pipeline_argument_key(self, key, default=None):
        if key in self._get_pipeline_arguments_object():
            return self._get_pipeline_arguments_object()[key]
        else:
            return default

class Director:
    """
    This is the director of the pipeline
    """

    def __init__(self, runner_id=None):
        logger_name = __name__
        if runner_id:
            logger_name = runner_id
        self.__logger = config_manager.get_app_config_manager().get_logger_for(logger_name)

    def _before(self):
        """
        This method implements some logic that is run before running the main pipeline director.
        Subclasses implementing the pipeline logic, can override this method for establishing pre-pipeline logic.
        :return: it returns True if there was no problem, False otherwise
        """
        self._get_logger().debug("No behaviour has been defined for 'before' running the pipeline")
        return True

    def run(self):
        """
        Pipeline template director algorithm, it executes 'before' logic, then the 'pipeline' logic, and 'after' logic
        once the pipeline is finished
        :return: True if the pipeline has been successful, False otherwise
        """
        if not self._before():
            self._get_logger().error("The logic executed BEFORE running the pipeline has FAILED")
            return False
        if not self._run_pipeline():
            self._get_logger().error("The PIPELINE execution has FAILED")
            return False
        if not self._after():
            self._get_logger().error("The logic executed AFTER running the pipeline has FAILED")
            return False
        self._get_logger().info("SUCCESSFUL Pipeline execution")
        return True

    def _run_pipeline(self):
        """
        This abstract method must be implemented by the subclasses with the strategy / director logic that drives the
        pipeline they are performing
        :return: True if the pipeline is successful, False otherwise
        """
        raise NotImplementedError("Implement your main pipeline logic here")

    def _after(self):
        """
        This method implements the logic that must be executed after the pipeline workflow has run.
        Subclasses implementing the pipeline logic, can override this method for establishing post-pipeline logic.
        :return: True if success, False otherwise
        """
        self._get_logger().debug("No behaviour has been defined for 'after' running the pipeline")
        return True

    def _get_logger(self):
        """
        Get the logger to be used for logging messages
        :return: it returns the logger set for the current instance of the class
        """
        return self.__logger

    def _set_logger(self, new_logger):
        """
        This method allows subclasses to change the default logger created when instantiating an implementation of this
        abstract class.
        :param new_logger: new logger to set up as the default logger for a particular implementation subclass instance
        :return: the newly set logger
        """
        self.__logger = new_logger
        return self._get_logger()

class PogoBasedPipelineDirector(Director):
    """
    Abstract class pipeline director class for those pipelines running PoGo in PRIDE
    """
    def __init__(self, runner_id=None):
        super().__init__(runner_id)

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<commit_msg>grab ensembl service and data downloader in template pipelines<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 29-06-2017 15:05
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This is a template pipeline for refactoring out things from final pipelines as I identify how they're gonna look like
"""

# App imports
import config_manager
import ensembl.service
import ensembl.data_downloader
from toolbox import general

# The config manager singleton is just an example that only makes sense for specialized pipeline modules, not really for
# this template
__configuration_file = None
__configuration_manager = None

def __get_configuration_manager():
    global __configuration_manager
    if __configuration_manager is None:
        __configuration_manager = DirectorConfigurationManager(general.read_json(__configuration_file),
                                                               __configuration_file)
    return __configuration_manager

class DirectorConfigurationManager(config_manager.ConfigurationManager):
    # Command line arguments special characters
    _CONFIG_COMMAND_LINE_ARGUMENT_PARAMETER_SEPARATOR = ','
    _CONFIG_COMMAND_LINE_ARGUMENT_PARAMETER_ASSIGNMENT_CHAR = '='

    def __init__(self, configuration_object, configuration_file, pipeline_arguments):
        super(DirectorConfigurationManager, self).__init__(configuration_object, configuration_file)
        self.__pipeline_arguments = pipeline_arguments
        self.__pipeline_arguments_object = None
        # Logger the pythonist way
        self._logger = config_manager.get_app_config_manager().get_logger_for(
            "{}.{}".format(__name__, type(self).__name__))

    def _get_pipeline_arguments(self):
        return self.__pipeline_arguments

    def _get_allowed_configuration_keys(self):
        # Allowed configuration keys set is empty by default
        return {}

    def _get_pipeline_arguments_object(self):
        if self.__pipeline_arguments_object is None:
            self.__pipeline_arguments_object = self._process_pipeline_arguments()
        return self.__pipeline_arguments_object

    def _process_pipeline_arguments(self):
        if self._get_pipeline_arguments():
            if self.__pipeline_arguments_object:
                self._logger.error("DUPLICATED CALL for processing command line arguments for this pipeline, IGNORED")
            else:
                self.__pipeline_arguments_object = {}
                self._logger.debug("Processing pipeline command line arguments")
                allowed_keys = self._get_allowed_configuration_keys()
                for command_line_parameter in self._get_pipeline_arguments().split(
                        self._CONFIG_COMMAND_LINE_ARGUMENT_PARAMETER_SEPARATOR):
                    key, value = command_line_parameter.split(
                        self._CONFIG_COMMAND_LINE_ARGUMENT_PARAMETER_ASSIGNMENT_CHAR)
                    if key not in allowed_keys:
                        self._logger.error(
                            "INVALID KEY '{}' while parsing pipeline arguments, parameter '{}' SKIPPED"
                                .format(key, command_line_parameter))
                        continue
                    if key in self.__pipeline_arguments_object:
                        self._logger.error(
                            "DUPLICATED KEY '{}' while parsing pipeline arguments, parameter '{}' SKIPPED"
                                .format(key, command_line_parameter))
                        continue
                    self.__pipeline_arguments_object[key] = value
                    self._logger.debug("Pipeline argument '{}' parsed and set with value '{}'".format(key, value))
        else:
            self._logger.warning("This pipeline was provided with NO COMMAND LINE ARGUMENTS")
        return self.__pipeline_arguments_object

    def _get_value_for_pipeline_argument_key(self, key, default=None):
        if key in self._get_pipeline_arguments_object():
            return self._get_pipeline_arguments_object()[key]
        else:
            return default

class Director:
    """
    This is the director of the pipeline
    """

    def __init__(self, runner_id=None):
        logger_name = __name__
        if runner_id:
            logger_name = runner_id
        self.__logger = config_manager.get_app_config_manager().get_logger_for(logger_name)

    def _before(self):
        """
        This method implements some logic that is run before running the main pipeline director.
        Subclasses implementing the pipeline logic, can override this method for establishing pre-pipeline logic.
        :return: it returns True if there was no problem, False otherwise
        """
        self._get_logger().debug("No behaviour has been defined for 'before' running the pipeline")
        return True

    def run(self):
        """
        Pipeline template director algorithm, it executes 'before' logic, then the 'pipeline' logic, and 'after' logic
        once the pipeline is finished
        :return: True if the pipeline has been successful, False otherwise
        """
        if not self._before():
            self._get_logger().error("The logic executed BEFORE running the pipeline has FAILED")
            return False
        if not self._run_pipeline():
            self._get_logger().error("The PIPELINE execution has FAILED")
            return False
        if not self._after():
            self._get_logger().error("The logic executed AFTER running the pipeline has FAILED")
            return False
        self._get_logger().info("SUCCESSFUL Pipeline execution")
        return True

    def _run_pipeline(self):
        """
        This abstract method must be implemented by the subclasses with the strategy / director logic that drives the
        pipeline they are performing
        :return: True if the pipeline is successful, False otherwise
        """
        raise NotImplementedError("Implement your main pipeline logic here")

    def _after(self):
        """
        This method implements the logic that must be executed after the pipeline workflow has run.
        Subclasses implementing the pipeline logic, can override this method for establishing post-pipeline logic.
        :return: True if success, False otherwise
        """
        self._get_logger().debug("No behaviour has been defined for 'after' running the pipeline")
        return True

    def _get_logger(self):
        """
        Get the logger to be used for logging messages
        :return: it returns the logger set for the current instance of the class
        """
        return self.__logger

    def _set_logger(self, new_logger):
        """
        This method allows subclasses to change the default logger created when instantiating an implementation of this
        abstract class.
        :param new_logger: new logger to set up as the default logger for a particular implementation subclass instance
        :return: the newly set logger
        """
        self.__logger = new_logger
        return self._get_logger()

class PogoBasedPipelineDirector(Director):
    """
    Abstract class pipeline director class for those pipelines running PoGo in PRIDE
    """
    def __init__(self, runner_id=None):
        super().__init__(runner_id)

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<|endoftext|>


<commit_before>from django.conf.urls import include, url
from pivot.views import (
    HomeView,
    PivotTemplateView,
    CourseGPAView,
    MajorGPAView,
    FeedbackView,
    Resources
)
from pivot.views.data_api import (
    MajorCourse,
    DataMap,
    StatusLookup,
    StudentData
)

urlpatterns = [
    url(r'^$', MajorGPAView.as_view(), name='majorgpa'),
    url(r'^course-gpa$', CourseGPAView.as_view(), name='coursegpa'),
    url(r'^major-gpa$', MajorGPAView.as_view(), name='majorgpa'),
    url(r'^feedback$', FeedbackView.as_view(), name='feedback'),
    url(r'^resources$', Resources.as_view(), name='resources'),
    url(r'^template$', PivotTemplateView.as_view(), name='pivottemplate'),
    url(r'^api/v1/major_course$', MajorCourse.as_view(), name='majorcourse'),
    url(r'^api/v1/data_map', DataMap.as_view(), name='datamap'),
    url(r'^api/v1/status_lookup', StatusLookup.as_view(), name='statuslookup'),
    url(r'^api/v1/student_data', StudentData.as_view(), name='studentdata'),
    url(r'^$', HomeView.as_view(), name='home'),

]
<commit_msg>Makes major-gpa the homepage<commit_after>from django.conf.urls import include, url
from django.views.generic.base import RedirectView
from pivot.views import (
    HomeView,
    PivotTemplateView,
    CourseGPAView,
    MajorGPAView,
    FeedbackView,
    Resources
)
from pivot.views.data_api import (
    MajorCourse,
    DataMap,
    StatusLookup,
    StudentData
)

urlpatterns = [
    # Home
    url(r'^$', RedirectView.as_view(url='/major-gpa')),
    url(r'^course-gpa$', CourseGPAView.as_view(), name='coursegpa'),
    url(r'^major-gpa$', MajorGPAView.as_view(), name='majorgpa'),
    url(r'^feedback$', FeedbackView.as_view(), name='feedback'),
    url(r'^resources$', Resources.as_view(), name='resources'),
    url(r'^template$', PivotTemplateView.as_view(), name='pivottemplate'),
    url(r'^api/v1/major_course$', MajorCourse.as_view(), name='majorcourse'),
    url(r'^api/v1/data_map', DataMap.as_view(), name='datamap'),
    url(r'^api/v1/status_lookup', StatusLookup.as_view(), name='statuslookup'),
    url(r'^api/v1/student_data', StudentData.as_view(), name='studentdata'),
    url(r'^$', HomeView.as_view(), name='home'),

]
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from ..core import Module, Blob
from ..dataclasses import Table
from .hdf5 import HDF5Header

import km3io
import numpy as np
from collections import defaultdict

USR_MC_TRACKS_KEYS = [b'energy_lost_in_can', b'bx', b'by', b'ichan', b'cc']

class EventPump(Module):
    def configure(self):
        self._filename = self.require("filename")
        self._split_hits = self.get("split_hits", default=True)
        self.skip_hits = self.get("skip_hits", default=False)
        self.skip_mc_hits = self.get("skip_mc_hits", default=False)
        self.skip_mc_tracks = self.get("skip_mc_tracks", default=False)
        self.skip_header = self.get("skip_header", default=False)

        self._reader = km3io.OfflineReader(self._filename)

        self.header = None
        self.raw_header = None
        if not self.skip_header:
            if self._reader.header is not None:
                self.header = HDF5Header.from_km3io(self._reader.header)
                self.raw_header = self._generate_raw_header()

        self.blobs = self._blob_generator()

    def process(self, blob=None):
        return next(self.blobs)

    def __iter__(self):
        return self

    def __next__(self):
        return next(self.blobs)

    def __getitem__(self, item):
        if not isinstance(item, int):
            raise TypeError("Only integer indices are supported.")
        return self._parse_event(self._reader.events[item], Blob())

    def get_number_of_blobs(self):
        return len(self._reader.events)

    def _blob_generator(self):
        for event in self._reader.events:
            blob = Blob()

            self._parse_event(event, blob)
            yield blob

    def _parse_event(self, event, blob):
        blob["RawHeader"] = self.raw_header
        blob["Header"] = self.header

        blob['Event'] = event
        if not self.skip_hits and event.n_hits > 0:
            blob['Hits'] = self._parse_hits(event.hits)
        if not self.skip_mc_hits and event.n_mc_hits > 0:
            blob['McHits'] = self._parse_mc_hits(event.mc_hits)
        if not self.skip_mc_tracks and event.n_mc_tracks > 0:
            blob['McTracks'] = self._parse_mc_tracks(event.mc_tracks)
        return blob

    def _generate_raw_header(self):
        header = {}
        raw_header = {"field_names": [], "parameter": [], "field_values": []}
        for n, x in self._reader._fobj['Head'
                                       ]._map_3c_string_2c_string_3e_.items():
            header[n.decode("utf-8")] = x.decode("utf-8").strip()
        for attribute, fields in km3io.definitions.mc_header.items():
            values = header.get(attribute, '').split()
            if not values:
                continue
            raw_header["field_values"].append(" ".join(map(str, values)))
            raw_header["field_names"].append(" ".join(fields))
            raw_header["parameter"].append(attribute)
        return Table(
            raw_header,
            h5loc="/raw_header",
            name="RawHeader",
            h5singleton=True
        )

    def _parse_eventinfo(self, event):
        wgt1, wgt2, wgt3, wgt4 = self._parse_wgts(event.w)
        tab_data = {
            'event_id': event.id,
            'run_id': event.run_id,
            'weight_w1': wgt1,
            'weight_w2': wgt2,
            'weight_w3': wgt3,
            'weight_w4': wgt4,
            'timestamp': event.t_sec,
            'nanoseconds': event.t_ns,
            'mc_time': event.mc_t,
            'trigger_mask': event.trigger_mask,
            'trigger_counter': event.trigger_counter,
            'overlays': event.overlays,
            'det_id': event.det_id,
            'frame_index': event.frame_index,
            'mc_run_id': event.mc_run_id,
            'group_id': self.group_id,
        }
        info = Table(tab_data, h5loc='/event_info', name='EventInfo')
        return info

    @staticmethod
    def _parse_wgts(wgt):
        if len(wgt) == 3:
            wgt1, wgt2, wgt3 = wgt
            wgt4 = np.nan
        elif len(wgt) == 4:
            # what the hell is w4?
            wgt1, wgt2, wgt3, wgt4 = wgt
        else:
            wgt1 = wgt2 = wgt3 = wgt4 = np.nan
        return wgt1, wgt2, wgt3, wgt4

    def _parse_usr_to_dct(self, mc_tracks):
        dct = defaultdict(list)
        for k in USR_KEYS:
            dec_key = k.decode('utf_8')
            for i in range(mc_tracks.usr_names.shape[0]):
                value = np.nan
                if k in mc_tracks.usr_names[i]:
                    mask = mc_tracks.usr_names[i] == k
                    value = mc_tracks.usr[i][mask][0]
                dct[dec_key].append(value)
        return dct

    def _parse_mc_tracks(self, mc_tracks):
        dct = {
            'dir_x': mc_tracks.dir_x,
            'dir_y': mc_tracks.dir_y,
            'dir_z': mc_tracks.dir_z,
            'pos_x': mc_tracks.pos_x,
            'pos_y': mc_tracks.pos_y,
            'pos_z': mc_tracks.pos_z,
            'energy': mc_tracks.E,
            'time': mc_tracks.t,
            'type': mc_tracks.type,
            'id': mc_tracks.id,
            'length': mc_tracks.len
        }
        dct.update(self._parse_usr_to_dct(mc_tracks))
        return Table(dct, name='McTracks', h5loc='/mc_tracks')

    def _parse_mc_hits(self, mc_hits):
        return Table({
            "a": mc_hits.a,
            "origin": mc_hits.origin,
            "pmt_id": mc_hits.pmt_id,
            "time": mc_hits.t
        },
                     h5loc="/mc_hits",
                     split_h5=self._split_hits,
                     name='McHits')

    def _parse_hits(self, hits):
        return Table({
            "channel_id": hits.channel_id,
            "dom_id": hits.dom_id,
            "time": hits.t,
            "tot": hits.tot,
            "triggered": hits.trig
        },
                     h5loc="/hits",
                     split_h5=self._split_hits,
                     name='Hits')
<commit_msg>Apply suggestion to km3pipe/io/offline.py<commit_after>#!/usr/bin/env python3

from ..core import Module, Blob
from ..dataclasses import Table
from .hdf5 import HDF5Header

import km3io
import numpy as np
from collections import defaultdict

USR_MC_TRACKS_KEYS = [b'energy_lost_in_can', b'bx', b'by', b'ichan', b'cc']

class EventPump(Module):
    def configure(self):
        self._filename = self.require("filename")
        self._split_hits = self.get("split_hits", default=True)
        self.skip_hits = self.get("skip_hits", default=False)
        self.skip_mc_hits = self.get("skip_mc_hits", default=False)
        self.skip_mc_tracks = self.get("skip_mc_tracks", default=False)
        self.skip_header = self.get("skip_header", default=False)

        self._reader = km3io.OfflineReader(self._filename)

        self.header = None
        self.raw_header = None
        if not self.skip_header:
            if self._reader.header is not None:
                self.header = HDF5Header.from_km3io(self._reader.header)
                self.raw_header = self._generate_raw_header()

        self.blobs = self._blob_generator()

    def process(self, blob=None):
        return next(self.blobs)

    def __iter__(self):
        return self

    def __next__(self):
        return next(self.blobs)

    def __getitem__(self, item):
        if not isinstance(item, int):
            raise TypeError("Only integer indices are supported.")
        return self._parse_event(self._reader.events[item], Blob())

    def get_number_of_blobs(self):
        return len(self._reader.events)

    def _blob_generator(self):
        for event in self._reader.events:
            blob = Blob()

            self._parse_event(event, blob)
            yield blob

    def _parse_event(self, event, blob):
        blob["RawHeader"] = self.raw_header
        blob["Header"] = self.header

        blob['Event'] = event
        if not self.skip_hits and event.n_hits > 0:
            blob['Hits'] = self._parse_hits(event.hits)
        if not self.skip_mc_hits and event.n_mc_hits > 0:
            blob['McHits'] = self._parse_mc_hits(event.mc_hits)
        if not self.skip_mc_tracks and event.n_mc_tracks > 0:
            blob['McTracks'] = self._parse_mc_tracks(event.mc_tracks)
        return blob

    def _generate_raw_header(self):
        header = {}
        raw_header = {"field_names": [], "parameter": [], "field_values": []}
        for n, x in self._reader._fobj['Head'
                                       ]._map_3c_string_2c_string_3e_.items():
            header[n.decode("utf-8")] = x.decode("utf-8").strip()
        for attribute, fields in km3io.definitions.mc_header.items():
            values = header.get(attribute, '').split()
            if not values:
                continue
            raw_header["field_values"].append(" ".join(map(str, values)))
            raw_header["field_names"].append(" ".join(fields))
            raw_header["parameter"].append(attribute)
        return Table(
            raw_header,
            h5loc="/raw_header",
            name="RawHeader",
            h5singleton=True
        )

    def _parse_eventinfo(self, event):
        wgt1, wgt2, wgt3, wgt4 = self._parse_wgts(event.w)
        tab_data = {
            'event_id': event.id,
            'run_id': event.run_id,
            'weight_w1': wgt1,
            'weight_w2': wgt2,
            'weight_w3': wgt3,
            'weight_w4': wgt4,
            'timestamp': event.t_sec,
            'nanoseconds': event.t_ns,
            'mc_time': event.mc_t,
            'trigger_mask': event.trigger_mask,
            'trigger_counter': event.trigger_counter,
            'overlays': event.overlays,
            'det_id': event.det_id,
            'frame_index': event.frame_index,
            'mc_run_id': event.mc_run_id,
            'group_id': self.group_id,
        }
        info = Table(tab_data, h5loc='/event_info', name='EventInfo')
        return info

    @staticmethod
    def _parse_wgts(wgt):
        if len(wgt) == 3:
            wgt1, wgt2, wgt3 = wgt
            wgt4 = np.nan
        elif len(wgt) == 4:
            # what the hell is w4?
            wgt1, wgt2, wgt3, wgt4 = wgt
        else:
            wgt1 = wgt2 = wgt3 = wgt4 = np.nan
        return wgt1, wgt2, wgt3, wgt4

    def _parse_usr_to_dct(self, mc_tracks):
        dct = defaultdict(list)
        for k in USR_MC_TRACKS_KEYS:
            dec_key = k.decode('utf_8')
            for i in range(mc_tracks.usr_names.shape[0]):
                value = np.nan
                if k in mc_tracks.usr_names[i]:
                    mask = mc_tracks.usr_names[i] == k
                    value = mc_tracks.usr[i][mask][0]
                dct[dec_key].append(value)
        return dct

    def _parse_mc_tracks(self, mc_tracks):
        dct = {
            'dir_x': mc_tracks.dir_x,
            'dir_y': mc_tracks.dir_y,
            'dir_z': mc_tracks.dir_z,
            'pos_x': mc_tracks.pos_x,
            'pos_y': mc_tracks.pos_y,
            'pos_z': mc_tracks.pos_z,
            'energy': mc_tracks.E,
            'time': mc_tracks.t,
            'type': mc_tracks.type,
            'id': mc_tracks.id,
            'length': mc_tracks.len
        }
        dct.update(self._parse_usr_to_dct(mc_tracks))
        return Table(dct, name='McTracks', h5loc='/mc_tracks')

    def _parse_mc_hits(self, mc_hits):
        return Table({
            "a": mc_hits.a,
            "origin": mc_hits.origin,
            "pmt_id": mc_hits.pmt_id,
            "time": mc_hits.t
        },
                     h5loc="/mc_hits",
                     split_h5=self._split_hits,
                     name='McHits')

    def _parse_hits(self, hits):
        return Table({
            "channel_id": hits.channel_id,
            "dom_id": hits.dom_id,
            "time": hits.t,
            "tot": hits.tot,
            "triggered": hits.trig
        },
                     h5loc="/hits",
                     split_h5=self._split_hits,
                     name='Hits')
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function
__author__ = 'katharine'

import os
import subprocess

from pebble_tool.exceptions import MissingSDK
from pebble_tool.util import get_persist_dir
from .manager import SDKManager

pebble_platforms = ('aplite', 'basalt', 'chalk')

SDK_VERSION = '3'

def sdk_path():
    path = sdk_manager.current_path
    if path is None:
        print("No SDK installed; installing the latest one...")
        sdk_manager.install_remote_sdk("latest")
        print("Installed SDK {}.".format(sdk_manager.get_current_sdk()))
        path = sdk_manager.current_path
    if not os.path.exists(os.path.join(path, 'pebble', 'waf')):
        raise MissingSDK("SDK unavailable; can't run this command.")
    return path

sdk_manager = SDKManager()

def sdk_version():
    return sdk_manager.get_current_sdk()

def get_sdk_persist_dir(platform, sdk_version=None):
    dir = os.path.join(get_persist_dir(), sdk_version or sdk_version(), platform)
    if not os.path.exists(dir):
        os.makedirs(dir)
    return dir

def add_tools_to_path():
    override = os.getenv('PEBBLE_TOOLCHAIN_PATH', None)
    if override is not None:
        os.environ['PATH'] = "{}:{}".format(override, os.environ['PATH'])
<commit_msg>Resolve name shadowing conflict.<commit_after>from __future__ import absolute_import, print_function
__author__ = 'katharine'

import os
import subprocess

from pebble_tool.exceptions import MissingSDK
from pebble_tool.util import get_persist_dir
from .manager import SDKManager

pebble_platforms = ('aplite', 'basalt', 'chalk')

SDK_VERSION = '3'

def sdk_path():
    path = sdk_manager.current_path
    if path is None:
        print("No SDK installed; installing the latest one...")
        sdk_manager.install_remote_sdk("latest")
        print("Installed SDK {}.".format(sdk_manager.get_current_sdk()))
        path = sdk_manager.current_path
    if not os.path.exists(os.path.join(path, 'pebble', 'waf')):
        raise MissingSDK("SDK unavailable; can't run this command.")
    return path

sdk_manager = SDKManager()

def sdk_version():
    return sdk_manager.get_current_sdk()

def get_sdk_persist_dir(platform, for_sdk_version=None):
    dir = os.path.join(get_persist_dir(), for_sdk_version or sdk_version(), platform)
    if not os.path.exists(dir):
        os.makedirs(dir)
    return dir

def add_tools_to_path():
    override = os.getenv('PEBBLE_TOOLCHAIN_PATH', None)
    if override is not None:
        os.environ['PATH'] = "{}:{}".format(override, os.environ['PATH'])
<|endoftext|>