<commit_before>"""
Multiple Plugin Agent for the New Relic Platform

"""
import helper
import importlib
import json
import logging
import os
import requests
import socket
import sys
import Queue as queue
import threading
import time

from newrelic_plugin_agent import __version__
from newrelic_plugin_agent import plugins

LOGGER = logging.getLogger(__name__)

class NewRelicPluginAgent(helper.Controller):
    """The NewRelicPluginAgent class implements a agent that polls plugins
    every minute and reports the state to NewRelic.

    """
    IGNORE_KEYS = ['license_key', 'proxy', 'endpoint',
                   'poll_interval', 'wake_interval']
    MAX_METRICS_PER_REQUEST = 10000
    PLATFORM_URL = 'https://platform-api.newrelic.com/platform/v1/metrics'
    WAKE_INTERVAL = 60

    def __init__(self, args, operating_system):
        """Initialize the NewRelicPluginAgent object.

        :param argparse.Namespace args: Command line arguments
        :param str operating_system: The operating_system name

        """
        super(NewRelicPluginAgent, self).__init__(args, operating_system)
        self.derive_last_interval = dict()
        self.endpoint = self.PLATFORM_URL
        self.http_headers = {'Accept': 'application/json',
                             'Content-Type': 'application/json'}
        self.last_interval_start = None
        self.min_max_values = dict()
        self._wake_interval = (self.config.application.get('wake_interval') or
                               self.config.application.get('poll_interval') or
                               self.WAKE_INTERVAL)
        self.next_wake_interval = int(self._wake_interval)
        self.publish_queue = queue.Queue()
        self.threads = list()
        info = tuple([__version__] + list(self.system_platform))
        LOGGER.info('Agent v%s initialized, %s %s v%s', *info)

    def setup(self):
        """Setup the internal state for the controller class. This is invoked
        on Controller.run().

        Items requiring the configuration object should be assigned here due to
        startup order of operations.

        """
        if hasattr(self.config.application, 'endpoint'):
            self.endpoint = self.config.application.endpoint
        self.http_headers['X-License-Key'] = self.license_key
        self.last_interval_start = time.time()

    @property
    def agent_data(self):
        """Return the agent data section of the NewRelic Platform data payload

        :rtype: dict

        """
        return {'host': socket.gethostname(),
                'pid': os.getpid(),
                'version': __version__}

    @property
    def license_key(self):
        """Return the NewRelic license key from the configuration values.

        :rtype: str

        """
        return self.config.application.license_key

    def poll_plugin(self, plugin_name, plugin, config):
        """Kick off a background thread to run the processing task.

        :param newrelic_plugin_agent.plugins.base.Plugin plugin: The plugin
        :param dict config: The config for the plugin

        """

        if not isinstance(config, (list, tuple)):
            config = [config]

        for instance in config:
            thread = threading.Thread(target=self.thread_process,
                                      kwargs={'config': instance,
                                              'name': plugin_name,
                                              'plugin': plugin,
                                              'poll_interval':
                                                  int(self._wake_interval)})
            thread.run()
            self.threads.append(thread)

    def process(self):
        """This method is called after every sleep interval. If the intention
        is to use an IOLoop instead of sleep interval based daemon, override
        the run method.

        """
        start_time = time.time()
        self.start_plugin_polling()

        # Sleep for a second while threads are running
        while self.threads_running:
            time.sleep(1)

        self.threads = list()
        self.send_data_to_newrelic()
        duration = time.time() - start_time
        self.next_wake_interval = self._wake_interval - duration
        if self.next_wake_interval < 1:
            LOGGER.warning('Poll interval took greater than %i seconds',
                           duration)
            self.next_wake_interval = int(self._wake_interval)
        LOGGER.info('Stats processed in %.2f seconds, next wake in %i seconds',
                    duration, self.next_wake_interval)

    def process_min_max_values(self, component):
        """Agent keeps track of previous values, so compute the differences for
        min/max values.

        :param dict component: The component to calc min/max values for

        """
        guid = component['guid']
        name = component['name']

        if guid not in self.min_max_values.keys():
            self.min_max_values[guid] = dict()

        if name not in self.min_max_values[guid].keys():
            self.min_max_values[guid][name] = dict()

        for metric in component['metrics']:
            min_val, max_val = self.min_max_values[guid][name].get(metric,
                                                                   (None, None))
            value = component['metrics'][metric]['total']
            if min_val is not None and min_val > value:
                min_val = value

            if max_val is None or max_val < value:
                max_val = value

            if component['metrics'][metric]['min'] is None:
                component['metrics'][metric]['min'] = min_val or value

            if component['metrics'][metric]['max'] is None:
                component['metrics'][metric]['max'] = max_val

            self.min_max_values[guid][name][metric] = min_val, max_val

    @property
    def proxies(self):
        """Return the proxy used to access NewRelic.

        :rtype: dict

        """
        if 'proxy' in self.config.application:
            return {
                'http': self.config.application['proxy'],
                'https': self.config.application['proxy']
            }
        return None

    def send_data_to_newrelic(self):
        metrics = 0
        components = list()
        while self.publish_queue.qsize():
            (name, data, last_values) = self.publish_queue.get()
            self.derive_last_interval[name] = last_values
            if isinstance(data, list):
                for component in data:
                    self.process_min_max_values(component)
                    components.append(component)
                    metrics += len(component['metrics'].keys())
                    if metrics >= self.MAX_METRICS_PER_REQUEST:
                        self.send_components(components, metrics)
                        components = list()
                        metrics = 0

            elif isinstance(data, dict):
                self.process_min_max_values(data)
                components.append(data)
                metrics += len(data['metrics'].keys())
                if metrics >= self.MAX_METRICS_PER_REQUEST:
                    self.send_components(components, metrics)
                    components = list()
                    metrics = 0

        LOGGER.debug('Done, will send remainder of %i metrics', metrics)
        self.send_components(components, metrics)

    def send_components(self, components, metrics):
        """Create the headers and payload to send to NewRelic platform as a
        JSON encoded POST body.

        """
        if not metrics:
            LOGGER.warning('No metrics to send to NewRelic this interval')
            return

        LOGGER.info('Sending %i metrics to NewRelic', metrics)
        body = {'agent': self.agent_data, 'components': components}
        LOGGER.debug(body)
        try:
            response = requests.post(self.endpoint,
                                     headers=self.http_headers,
                                     proxies=self.proxies,
                                     data=json.dumps(body, ensure_ascii=False),
                                     timeout=self.config.get('newrelic_api_timeout', 10),
                                     verify=self.config.get('verify_ssl_cert',
                                                            True))
            LOGGER.debug('Response: %s: %r',
                         response.status_code,
                         response.content.strip())
        except requests.ConnectionError as error:
            LOGGER.error('Error reporting stats: %s', error)
        except requests.Timeout as error:
            LOGGER.error('TimeoutError reporting stats: %s', error)

    @staticmethod
    def _get_plugin(plugin_path):
        """Given a qualified class name (eg. foo.bar.Foo), return the class

        :rtype: object

        """
        try:
            package, class_name = plugin_path.rsplit('.', 1)
        except ValueError:
            return None

        try:
            module_handle = importlib.import_module(package)
            class_handle = getattr(module_handle, class_name)
            return class_handle
        except ImportError:
            LOGGER.exception('Attempting to import %s', plugin_path)
            return None

    def start_plugin_polling(self):
        """Iterate through each plugin and start the polling process."""
        for plugin in [key for key in self.config.application.keys()
                       if key not in self.IGNORE_KEYS]:
            LOGGER.info('Enabling plugin: %s', plugin)
            plugin_class = None

            # If plugin is part of the core agent plugin list
            if plugin in plugins.available:
                plugin_class = self._get_plugin(plugins.available[plugin])

            # If plugin is in config and a qualified class name
            elif '.' in plugin:
                plugin_class = self._get_plugin(plugin)

            # If plugin class could not be imported
            if not plugin_class:
                LOGGER.error('Enabled plugin %s not available', plugin)
                continue

            self.poll_plugin(plugin, plugin_class,
                             self.config.application.get(plugin))

    @property
    def threads_running(self):
        """Return True if any of the child threads are alive

        :rtype: bool

        """
        for thread in self.threads:
            if thread.is_alive():
                return True
        return False

    def thread_process(self, name, plugin, config, poll_interval):
        """Created a thread process for the given name, plugin class,
        config and poll interval. Process is added to a Queue object which
        used to maintain the stack of running plugins.

        :param str name: The name of the plugin
        :param newrelic_plugin_agent.plugin.Plugin plugin: The plugin class
        :param dict config: The plugin configuration
        :param int poll_interval: How often the plugin is invoked

        """
        instance_name = "%s:%s" % (name, config.get('name', 'unnamed'))
        obj = plugin(config, poll_interval,
                     self.derive_last_interval.get(instance_name))
        obj.poll()
        self.publish_queue.put((instance_name, obj.values(),
                                obj.derive_last_interval))

    @property
    def wake_interval(self):
        """Return the wake interval in seconds as the number of seconds
        until the next minute.

        :rtype: int

        """
        return self.next_wake_interval

def main():
    helper.parser.description('The NewRelic Plugin Agent polls various '
                              'services and sends the data to the NewRelic '
                              'Platform')
    helper.parser.name('newrelic_plugin_agent')
    argparse = helper.parser.get()
    argparse.add_argument('-C',
                          action='store_true',
                          dest='configure',
                          help='Run interactive configuration')
    args = helper.parser.parse()
    if args.configure:
        print('Configuration')
        sys.exit(0)
    helper.start(NewRelicPluginAgent)

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    main()
<commit_msg>read licensekey from environment, else fallback to reading from configuration file<commit_after>"""
Multiple Plugin Agent for the New Relic Platform

"""
import helper
import importlib
import json
import logging
import os
import requests
import socket
import sys
import Queue as queue
import threading
import time

from newrelic_plugin_agent import __version__
from newrelic_plugin_agent import plugins

LOGGER = logging.getLogger(__name__)

class NewRelicPluginAgent(helper.Controller):
    """The NewRelicPluginAgent class implements a agent that polls plugins
    every minute and reports the state to NewRelic.

    """
    IGNORE_KEYS = ['license_key', 'proxy', 'endpoint',
                   'poll_interval', 'wake_interval']
    MAX_METRICS_PER_REQUEST = 10000
    PLATFORM_URL = 'https://platform-api.newrelic.com/platform/v1/metrics'
    WAKE_INTERVAL = 60

    def __init__(self, args, operating_system):
        """Initialize the NewRelicPluginAgent object.

        :param argparse.Namespace args: Command line arguments
        :param str operating_system: The operating_system name

        """
        super(NewRelicPluginAgent, self).__init__(args, operating_system)
        self.derive_last_interval = dict()
        self.endpoint = self.PLATFORM_URL
        self.http_headers = {'Accept': 'application/json',
                             'Content-Type': 'application/json'}
        self.last_interval_start = None
        self.min_max_values = dict()
        self._wake_interval = (self.config.application.get('wake_interval') or
                               self.config.application.get('poll_interval') or
                               self.WAKE_INTERVAL)
        self.next_wake_interval = int(self._wake_interval)
        self.publish_queue = queue.Queue()
        self.threads = list()
        info = tuple([__version__] + list(self.system_platform))
        LOGGER.info('Agent v%s initialized, %s %s v%s', *info)

    def setup(self):
        """Setup the internal state for the controller class. This is invoked
        on Controller.run().

        Items requiring the configuration object should be assigned here due to
        startup order of operations.

        """
        if hasattr(self.config.application, 'endpoint'):
            self.endpoint = self.config.application.endpoint
        self.http_headers['X-License-Key'] = self.license_key
        self.last_interval_start = time.time()

    @property
    def agent_data(self):
        """Return the agent data section of the NewRelic Platform data payload

        :rtype: dict

        """
        return {'host': socket.gethostname(),
                'pid': os.getpid(),
                'version': __version__}

    @property
    def license_key(self):
        """Return the NewRelic license key from the configuration values.

        :rtype: str

        """
        licensekey = os.getenv('NEWRELIC_LICENSE_KEY', self.config.application.license_key)
        return licensekey

    def poll_plugin(self, plugin_name, plugin, config):
        """Kick off a background thread to run the processing task.

        :param newrelic_plugin_agent.plugins.base.Plugin plugin: The plugin
        :param dict config: The config for the plugin

        """

        if not isinstance(config, (list, tuple)):
            config = [config]

        for instance in config:
            thread = threading.Thread(target=self.thread_process,
                                      kwargs={'config': instance,
                                              'name': plugin_name,
                                              'plugin': plugin,
                                              'poll_interval':
                                                  int(self._wake_interval)})
            thread.run()
            self.threads.append(thread)

    def process(self):
        """This method is called after every sleep interval. If the intention
        is to use an IOLoop instead of sleep interval based daemon, override
        the run method.

        """
        start_time = time.time()
        self.start_plugin_polling()

        # Sleep for a second while threads are running
        while self.threads_running:
            time.sleep(1)

        self.threads = list()
        self.send_data_to_newrelic()
        duration = time.time() - start_time
        self.next_wake_interval = self._wake_interval - duration
        if self.next_wake_interval < 1:
            LOGGER.warning('Poll interval took greater than %i seconds',
                           duration)
            self.next_wake_interval = int(self._wake_interval)
        LOGGER.info('Stats processed in %.2f seconds, next wake in %i seconds',
                    duration, self.next_wake_interval)

    def process_min_max_values(self, component):
        """Agent keeps track of previous values, so compute the differences for
        min/max values.

        :param dict component: The component to calc min/max values for

        """
        guid = component['guid']
        name = component['name']

        if guid not in self.min_max_values.keys():
            self.min_max_values[guid] = dict()

        if name not in self.min_max_values[guid].keys():
            self.min_max_values[guid][name] = dict()

        for metric in component['metrics']:
            min_val, max_val = self.min_max_values[guid][name].get(metric,
                                                                   (None, None))
            value = component['metrics'][metric]['total']
            if min_val is not None and min_val > value:
                min_val = value

            if max_val is None or max_val < value:
                max_val = value

            if component['metrics'][metric]['min'] is None:
                component['metrics'][metric]['min'] = min_val or value

            if component['metrics'][metric]['max'] is None:
                component['metrics'][metric]['max'] = max_val

            self.min_max_values[guid][name][metric] = min_val, max_val

    @property
    def proxies(self):
        """Return the proxy used to access NewRelic.

        :rtype: dict

        """
        if 'proxy' in self.config.application:
            return {
                'http': self.config.application['proxy'],
                'https': self.config.application['proxy']
            }
        return None

    def send_data_to_newrelic(self):
        metrics = 0
        components = list()
        while self.publish_queue.qsize():
            (name, data, last_values) = self.publish_queue.get()
            self.derive_last_interval[name] = last_values
            if isinstance(data, list):
                for component in data:
                    self.process_min_max_values(component)
                    components.append(component)
                    metrics += len(component['metrics'].keys())
                    if metrics >= self.MAX_METRICS_PER_REQUEST:
                        self.send_components(components, metrics)
                        components = list()
                        metrics = 0

            elif isinstance(data, dict):
                self.process_min_max_values(data)
                components.append(data)
                metrics += len(data['metrics'].keys())
                if metrics >= self.MAX_METRICS_PER_REQUEST:
                    self.send_components(components, metrics)
                    components = list()
                    metrics = 0

        LOGGER.debug('Done, will send remainder of %i metrics', metrics)
        self.send_components(components, metrics)

    def send_components(self, components, metrics):
        """Create the headers and payload to send to NewRelic platform as a
        JSON encoded POST body.

        """
        if not metrics:
            LOGGER.warning('No metrics to send to NewRelic this interval')
            return

        LOGGER.info('Sending %i metrics to NewRelic', metrics)
        body = {'agent': self.agent_data, 'components': components}
        LOGGER.debug(body)
        try:
            response = requests.post(self.endpoint,
                                     headers=self.http_headers,
                                     proxies=self.proxies,
                                     data=json.dumps(body, ensure_ascii=False),
                                     timeout=self.config.get('newrelic_api_timeout', 10),
                                     verify=self.config.get('verify_ssl_cert',
                                                            True))
            LOGGER.debug('Response: %s: %r',
                         response.status_code,
                         response.content.strip())
        except requests.ConnectionError as error:
            LOGGER.error('Error reporting stats: %s', error)
        except requests.Timeout as error:
            LOGGER.error('TimeoutError reporting stats: %s', error)

    @staticmethod
    def _get_plugin(plugin_path):
        """Given a qualified class name (eg. foo.bar.Foo), return the class

        :rtype: object

        """
        try:
            package, class_name = plugin_path.rsplit('.', 1)
        except ValueError:
            return None

        try:
            module_handle = importlib.import_module(package)
            class_handle = getattr(module_handle, class_name)
            return class_handle
        except ImportError:
            LOGGER.exception('Attempting to import %s', plugin_path)
            return None

    def start_plugin_polling(self):
        """Iterate through each plugin and start the polling process."""
        for plugin in [key for key in self.config.application.keys()
                       if key not in self.IGNORE_KEYS]:
            LOGGER.info('Enabling plugin: %s', plugin)
            plugin_class = None

            # If plugin is part of the core agent plugin list
            if plugin in plugins.available:
                plugin_class = self._get_plugin(plugins.available[plugin])

            # If plugin is in config and a qualified class name
            elif '.' in plugin:
                plugin_class = self._get_plugin(plugin)

            # If plugin class could not be imported
            if not plugin_class:
                LOGGER.error('Enabled plugin %s not available', plugin)
                continue

            self.poll_plugin(plugin, plugin_class,
                             self.config.application.get(plugin))

    @property
    def threads_running(self):
        """Return True if any of the child threads are alive

        :rtype: bool

        """
        for thread in self.threads:
            if thread.is_alive():
                return True
        return False

    def thread_process(self, name, plugin, config, poll_interval):
        """Created a thread process for the given name, plugin class,
        config and poll interval. Process is added to a Queue object which
        used to maintain the stack of running plugins.

        :param str name: The name of the plugin
        :param newrelic_plugin_agent.plugin.Plugin plugin: The plugin class
        :param dict config: The plugin configuration
        :param int poll_interval: How often the plugin is invoked

        """
        instance_name = "%s:%s" % (name, config.get('name', 'unnamed'))
        obj = plugin(config, poll_interval,
                     self.derive_last_interval.get(instance_name))
        obj.poll()
        self.publish_queue.put((instance_name, obj.values(),
                                obj.derive_last_interval))

    @property
    def wake_interval(self):
        """Return the wake interval in seconds as the number of seconds
        until the next minute.

        :rtype: int

        """
        return self.next_wake_interval

def main():
    helper.parser.description('The NewRelic Plugin Agent polls various '
                              'services and sends the data to the NewRelic '
                              'Platform')
    helper.parser.name('newrelic_plugin_agent')
    argparse = helper.parser.get()
    argparse.add_argument('-C',
                          action='store_true',
                          dest='configure',
                          help='Run interactive configuration')
    args = helper.parser.parse()
    if args.configure:
        print('Configuration')
        sys.exit(0)
    helper.start(NewRelicPluginAgent)

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    main()
<|endoftext|>


<commit_before>"""
The registry module provides tools to maintain a registry of channels.

The first thing that should happen when django starts is registration of
channels. It should happen first, because channels are required for
autocomplete widgets. And autocomplete widgets are required for forms. And
forms are required for ModelAdmin.

It looks like this:

- in ``yourapp/autocomplete_light_registry.py``, register your channels with
  ``autocomplete_light.register()``,
- in ``urls.py``, do ``autocomplete_light.autodiscover()`` **before**
  ``admin.autodiscover()``.

ChannelRegistry
    Subclass of Python's dict type with registration/unregistration methods.

registry
    Instance of ChannelRegistry.

register
    Proxy registry.register.

autodiscover
    Find channels and fill registry.
"""

from django.db import models

from .channel import ChannelBase

__all__ = ('ChannelRegistry', 'registry', 'register', 'autodiscover')

class ChannelRegistry(dict):
    """
    Dict with some shortcuts to handle a registry of channels.
    """

    def __init__(self):
        self._models = {}

    def channel_for_model(self, model):
        """Return the channel class for a given model."""
        try:
            return self._models[model]
        except KeyError:
            return

    def unregister(self, name):
        """
        Unregister a channel.
        """
        channel = self[name]
        del self[name]

        if channel.model:
            del self._models[channel.model]

    def register(self, *args, **kwargs):
        """
        Proxy registry.register_model_channel() or registry.register_channel()
        if there is no apparent model for the channel.

        Example usages::

            # Will create and register SomeModelChannel, if SomeChannel.model
            # is None (which is the case by default):
            autocomplete_light.register(SomeModel)

            # Same but using SomeChannel as base:
            autocomplete_light.register(SomeModel, SomeChannel)

            # Register a channel without model, ensure that SomeChannel.model
            # is None (which is the default):
            autocomplete_light.register(SomeChannel)

            # As of 0.5, you may also pass attributes*, ie.:
            autocomplete_light.register(SomeModel, search_field='search_names',
                result_template='somemodel_result.html')

        You may pass attributes via kwargs, only if the registry creates a
        type:

        - if no channel class is passed,
        - or if the channel class has no model attribute,
        - and if the channel classs is not generic
        """

        channel = None
        model = None

        for arg in args:
            if issubclass(arg, models.Model):
                model = arg
            elif issubclass(arg, ChannelBase):
                channel = arg

        if channel and getattr(channel, 'model'):
            model = channel.model

        if model:
            self.register_model_channel(model, channel, **kwargs)
        else:
            self.register_channel(channel)

    def register_model_channel(self, model, channel=None, channel_name=None,
        **kwargs):
        """
        Add a model to the registry, optionnaly with a given channel class.

        model
            The model class to register.

        channel
            The channel class to register the model with, default to
            ChannelBase.

        channel_name
            Register channel under channel_name, default is ModelNameChannel.

        kwargs
            Extra attributes to set to the channel class, if created by this
            method.

        Three cases are possible:

        - specify model class and ModelNameChannel will be generated extending
          ChannelBase, with attribute model=model
        - specify a model and a channel class that does not have a model
          attribute, and a ModelNameChannel will be generated, with attribute
          model=model
        - specify a model and a channel class with a model attribute, and the
          channel is directly registered

        To keep things simple, the name of a channel is it's class name.
        """
        kwargs.update({'model': model})
        if channel_name is None:
            channel_name = '%sChannel' % model.__name__

        if channel is None:
            channel = type(channel_name, (ChannelBase,), kwargs)
        elif channel.model is None:
            channel = type(channel_name, (channel,), kwargs)

        self.register_channel(channel)
        self._models[channel.model] = channel

    def register_channel(self, channel):
        """
        Register a channel without model, like a generic channel.
        """
        self[channel.__name__] = channel

def _autodiscover(registry):
    """See documentation for autodiscover (without the underscore)"""
    import copy
    from django.conf import settings
    from django.utils.importlib import import_module
    from django.utils.module_loading import module_has_submodule

    for app in settings.INSTALLED_APPS:
        mod = import_module(app)
        # Attempt to import the app's admin module.
        try:
            before_import_registry = copy.copy(registry)
            import_module('%s.autocomplete_light_registry' % app)
        except:
            # Reset the model registry to the state before the last import as
            # this import will have to reoccur on the next request and this
            # could raise NotRegistered and AlreadyRegistered exceptions
            # (see #8245).
            registry = before_import_registry

            # Decide whether to bubble up this error. If the app just
            # doesn't have an admin module, we can ignore the error
            # attempting to import it, otherwise we want it to bubble up.
            if module_has_submodule(mod, 'autocomplete_light_registry'):
                raise

registry = ChannelRegistry()

def autodiscover():
    """
    Check all apps in INSTALLED_APPS for stuff related to autocomplete_light.

    For each app, autodiscover imports app.autocomplete_light_registry if
    available, resulting in execution of register() statements in that module,
    filling registry.

    Consider a standard app called 'cities_light' with such a structure::

        cities_light/
            __init__.py
            models.py
            urls.py
            views.py
            autocomplete_light_registry.py

    With such a autocomplete_light_registry.py::

        from models import City, Country
        import autocomplete_light
        autocomplete_light.register(City)
        autocomplete_light.register(Country)

    When autodiscover() imports cities_light.autocomplete_light_registry, both
    CityChannel and CountryChannel will be registered. For details on how these
    channel classes are generated, read the documentation of
    ChannelRegistry.register.
    """
    _autodiscover(registry)

def register(*args, **kwargs):
    """Proxy registry.register"""
    return registry.register(*args, **kwargs)
<commit_msg>Update documentation about channel name<commit_after>"""
The registry module provides tools to maintain a registry of channels.

The first thing that should happen when django starts is registration of
channels. It should happen first, because channels are required for
autocomplete widgets. And autocomplete widgets are required for forms. And
forms are required for ModelAdmin.

It looks like this:

- in ``yourapp/autocomplete_light_registry.py``, register your channels with
  ``autocomplete_light.register()``,
- in ``urls.py``, do ``autocomplete_light.autodiscover()`` **before**
  ``admin.autodiscover()``.

ChannelRegistry
    Subclass of Python's dict type with registration/unregistration methods.

registry
    Instance of ChannelRegistry.

register
    Proxy registry.register.

autodiscover
    Find channels and fill registry.
"""

from django.db import models

from .channel import ChannelBase

__all__ = ('ChannelRegistry', 'registry', 'register', 'autodiscover')

class ChannelRegistry(dict):
    """
    Dict with some shortcuts to handle a registry of channels.
    """

    def __init__(self):
        self._models = {}

    def channel_for_model(self, model):
        """Return the channel class for a given model."""
        try:
            return self._models[model]
        except KeyError:
            return

    def unregister(self, name):
        """
        Unregister a channel.
        """
        channel = self[name]
        del self[name]

        if channel.model:
            del self._models[channel.model]

    def register(self, *args, **kwargs):
        """
        Proxy registry.register_model_channel() or registry.register_channel()
        if there is no apparent model for the channel.

        Example usages::

            # Will create and register SomeModelChannel, if SomeChannel.model
            # is None (which is the case by default):
            autocomplete_light.register(SomeModel)

            # Same but using SomeChannel as base:
            autocomplete_light.register(SomeModel, SomeChannel)

            # Register a channel without model, ensure that SomeChannel.model
            # is None (which is the default):
            autocomplete_light.register(SomeChannel)

            # As of 0.5, you may also pass attributes*, ie.:
            autocomplete_light.register(SomeModel, search_field='search_names',
                result_template='somemodel_result.html')

        You may pass attributes via kwargs, only if the registry creates a
        type:

        - if no channel class is passed,
        - or if the channel class has no model attribute,
        - and if the channel classs is not generic
        """

        channel = None
        model = None

        for arg in args:
            if issubclass(arg, models.Model):
                model = arg
            elif issubclass(arg, ChannelBase):
                channel = arg

        if channel and getattr(channel, 'model'):
            model = channel.model

        if model:
            self.register_model_channel(model, channel, **kwargs)
        else:
            self.register_channel(channel)

    def register_model_channel(self, model, channel=None, channel_name=None,
        **kwargs):
        """
        Add a model to the registry, optionnaly with a given channel class.

        model
            The model class to register.

        channel
            The channel class to register the model with, default to
            ChannelBase.

        channel_name
            Register channel under channel_name, default is ModelNameChannel.

        kwargs
            Extra attributes to set to the channel class, if created by this
            method.

        Three cases are possible:

        - specify model class and ModelNameChannel will be generated extending
          ChannelBase, with attribute model=model
        - specify a model and a channel class that does not have a model
          attribute, and a ModelNameChannel will be generated, with attribute
          model=model
        - specify a model and a channel class with a model attribute, and the
          channel is directly registered

        To keep things simple, the name of a channel is it's class name, which
        is usually generated. In case of conflicts, you may override the
        default channel name with the channel_name keyword argument.
        """
        kwargs.update({'model': model})
        if channel_name is None:
            channel_name = '%sChannel' % model.__name__

        if channel is None:
            channel = type(channel_name, (ChannelBase,), kwargs)
        elif channel.model is None:
            channel = type(channel_name, (channel,), kwargs)

        self.register_channel(channel)
        self._models[channel.model] = channel

    def register_channel(self, channel):
        """
        Register a channel without model, like a generic channel.
        """
        self[channel.__name__] = channel

def _autodiscover(registry):
    """See documentation for autodiscover (without the underscore)"""
    import copy
    from django.conf import settings
    from django.utils.importlib import import_module
    from django.utils.module_loading import module_has_submodule

    for app in settings.INSTALLED_APPS:
        mod = import_module(app)
        # Attempt to import the app's admin module.
        try:
            before_import_registry = copy.copy(registry)
            import_module('%s.autocomplete_light_registry' % app)
        except:
            # Reset the model registry to the state before the last import as
            # this import will have to reoccur on the next request and this
            # could raise NotRegistered and AlreadyRegistered exceptions
            # (see #8245).
            registry = before_import_registry

            # Decide whether to bubble up this error. If the app just
            # doesn't have an admin module, we can ignore the error
            # attempting to import it, otherwise we want it to bubble up.
            if module_has_submodule(mod, 'autocomplete_light_registry'):
                raise

registry = ChannelRegistry()

def autodiscover():
    """
    Check all apps in INSTALLED_APPS for stuff related to autocomplete_light.

    For each app, autodiscover imports app.autocomplete_light_registry if
    available, resulting in execution of register() statements in that module,
    filling registry.

    Consider a standard app called 'cities_light' with such a structure::

        cities_light/
            __init__.py
            models.py
            urls.py
            views.py
            autocomplete_light_registry.py

    With such a autocomplete_light_registry.py::

        from models import City, Country
        import autocomplete_light
        autocomplete_light.register(City)
        autocomplete_light.register(Country)

    When autodiscover() imports cities_light.autocomplete_light_registry, both
    CityChannel and CountryChannel will be registered. For details on how these
    channel classes are generated, read the documentation of
    ChannelRegistry.register.
    """
    _autodiscover(registry)

def register(*args, **kwargs):
    """Proxy registry.register"""
    return registry.register(*args, **kwargs)
<|endoftext|>


<commit_before>#
# MLDB-951-run-on-creation.py
# Datacratic, 2015
# This file is part of MLDB. Copyright 2015 Datacratic. All rights reserved.
#

import json
import datetime
import time

if False:
    mldb_wrapper = None
_legacy = mldb
mldb = mldb_wrapper.wrap(mldb) # noqa

def load_test_dataset():
    ds1 = mldb.create_dataset({
        'type': 'sparse.mutable',
        'id': 'dataset1'})

    row_count = 100
    for i in xrange(row_count - 1):
        # row name is x's value
        ds1.record_row(str(i), [['x', i, datetime.datetime.now()]])

    ds1.commit()

load_test_dataset()

# TRANSFORM PROCEDURE
# without first run
response = mldb.put("/v1/procedures/transform_procedure", {
    "type": "transform",
    "params": {
        "inputData": "select x + 1 as x from dataset1",
        "outputDataset": {"id": "dataset2", "type": "sparse.mutable"},
        "runOnCreation" : False
    }
})
mldb.log(response)
assert 'status' not in response.json(), \
    'not expecting a status field since there are no first run'

# test that location points to the actual created object (that was a bug
# found during testing)
response = mldb.post_async("/v1/procedures/transform_procedure/runs")
mldb.log(response)
location = response.headers['Location']
mldb.log(mldb.get(location))
id = location.split('/')[-1]
assert mldb.get("/v1/procedures/transform_procedure/runs").json()[0] \
    == id, 'expected ids to match'

# with a first run
response = mldb.put("/v1/procedures/transform_procedure", {
    "type": "transform",
    "params": {
        "inputData": {
            "select": "x + 1 as x",
            "from" : {"id": "dataset1"}
        },
        "outputDataset": {"id": "dataset3", "type": "sparse.mutable"},
        "runOnCreation" : True
    }
})
mldb.log(response.json())
assert 'firstRun' in response.json()['status'], 'expected a firstRun param'

# check that the transformed dataset is as expected
transformed_rows = mldb.get('/v1/query', q='select x from dataset3').json()
for row in transformed_rows:
    assert int(row['rowName']) + 1 == row['columns'][0][1], \
        'the transform was not applied correctly'

# with a first run async
response = mldb.put_async("/v1/procedures/transform_procedure", {
    "type": "transform",
    "params": {
        "inputData": "select x + 1 as x from dataset1",
        "outputDataset": {"id": "dataset4", "type": "sparse.mutable"},
        "runOnCreation" : True
    }
})
mldb.log(response)
assert 'firstRun' in response.json()['status'], 'expected a firstRun param'

running = True
run_url = '/v1/procedures/transform_procedure/runs/' \
    + response.json()['status']['firstRun']['id']
while(running):
    time.sleep(0.2)
    resp = mldb.get(run_url)
    if resp.json()['state'] == 'finished':
        running = False

# check that the transformed dataset is as expected
transformed_rows = mldb.get('/v1/query', q='select x from dataset4').json()
for row in transformed_rows:
    assert int(row['rowName']) + 1 == row['columns'][0][1], \
        'the transform was not applied correctly'

# checking return code for invalid runs
try:
    mldb.put("/v1/procedures/transform_procedure", {
        "type": "transform",
        "params": {
            "inputData": "select x + 1 as x from dataset1",
            # failing because beh may not be available or is not mutable
            "outputDataset": {"id": "dataset2", "type":"beh"},
            "runOnCreation" : True
        }
    })
except mldb_wrapper.ResponseException as exc:
    res = exc.response
else:
    assert False, 'should not be here'
mldb.log(res)
response = res.json()
mldb.log(response)
assert 'runError' in response['details'], 'expected a runError param'
assert 'error' in response['details']['runError'], \
    'expected an error description for the failure'
assert 'httpCode' in response['details']['runError'], 'expected an error code'

mldb.script.set_return('success')
<commit_msg>MLDB-1376 fix test<commit_after>#
# MLDB-951-run-on-creation.py
# Datacratic, 2015
# This file is part of MLDB. Copyright 2015 Datacratic. All rights reserved.
#

import json
import datetime
import time

if False:
    mldb_wrapper = None
_legacy = mldb
mldb = mldb_wrapper.wrap(mldb) # noqa

def load_test_dataset():
    ds1 = mldb.create_dataset({
        'type': 'sparse.mutable',
        'id': 'dataset1'})

    row_count = 100
    for i in xrange(row_count - 1):
        # row name is x's value
        ds1.record_row(str(i), [['x', i, datetime.datetime.now()]])

    ds1.commit()

load_test_dataset()

# TRANSFORM PROCEDURE
# without first run
response = mldb.put("/v1/procedures/transform_procedure", {
    "type": "transform",
    "params": {
        "inputData": "select x + 1 as x from dataset1",
        "outputDataset": {"id": "dataset2", "type": "sparse.mutable"},
        "runOnCreation" : False
    }
})
mldb.log(response)
assert 'status' not in response.json(), \
    'not expecting a status field since there are no first run'

# test that location points to the actual created object (that was a bug
# found during testing)
response = mldb.post_async("/v1/procedures/transform_procedure/runs")
mldb.log(response)
location = response.headers['Location']
mldb.log(mldb.get(location))
id = location.split('/')[-1]
res = mldb.get("/v1/procedures/transform_procedure/runs").json()[0]
res == id, 'expected ids to match'

# with a first run
response = mldb.put("/v1/procedures/transform_procedure", {
    "type": "transform",
    "params": {
        "inputData": {
            "select": "x + 1 as x",
            "from" : {"id": "dataset1"}
        },
        "outputDataset": {"id": "dataset3", "type": "sparse.mutable"},
        "runOnCreation" : True
    }
})
mldb.log(response.json())
assert 'firstRun' in response.json()['status'], 'expected a firstRun param'

# check that the transformed dataset is as expected
transformed_rows = mldb.get('/v1/query', q='select x from dataset3').json()
for row in transformed_rows:
    assert int(row['rowName']) + 1 == row['columns'][0][1], \
        'the transform was not applied correctly'

# with a first run async
response = mldb.put_async("/v1/procedures/transform_procedure", {
    "type": "transform",
    "params": {
        "inputData": "select x + 1 as x from dataset1",
        "outputDataset": {"id": "dataset4", "type": "sparse.mutable"},
        "runOnCreation" : True
    }
})
mldb.log(response)
assert 'firstRun' in response.json()['status'], 'expected a firstRun param'

running = True
run_url = '/v1/procedures/transform_procedure/runs/' \
    + response.json()['status']['firstRun']['id']
while(running):
    time.sleep(0.2)
    resp = mldb.get(run_url)
    if resp.json()['state'] == 'finished':
        running = False

# check that the transformed dataset is as expected
transformed_rows = mldb.get('/v1/query', q='select x from dataset4').json()
for row in transformed_rows:
    assert int(row['rowName']) + 1 == row['columns'][0][1], \
        'the transform was not applied correctly'

# checking return code for invalid runs
try:
    mldb.put("/v1/procedures/transform_procedure", {
        "type": "transform",
        "params": {
            "inputData": "select x + 1 as x from dataset1",
            # failing because beh may not be available or is not mutable
            "outputDataset": {"id": "dataset2", "type":"beh"},
            "runOnCreation" : True
        }
    })
except mldb_wrapper.ResponseException as exc:
    res = exc.response
else:
    assert False, 'should not be here'
mldb.log(res)
response = res.json()
mldb.log(response)
assert 'runError' in response['details'], 'expected a runError param'
assert 'error' in response['details']['runError'], \
    'expected an error description for the failure'
assert 'httpCode' in response['details']['runError'], 'expected an error code'

mldb.script.set_return('success')
<|endoftext|>


<commit_before>import logging
import re
import subprocess
import time

from seqr.utils.other_utils import FileStats
from seqr.utils.shell_utils import run

logger = logging.getLogger(__name__)

def _gsutil_command(command, gs_path):
    #  Anvil buckets are requester-pays and we bill them to the anvil project
    project_arg = '-u anvil-datastore ' if gs_path.startswith('gs://fc-secure') else ''
    return 'gsutil {project_arg}{command} {gs_path}'.format(
        project_arg=project_arg, command=command, gs_path=gs_path,
    )

def is_google_bucket_file_path(file_path):
    return file_path.startswith("gs://")

def does_google_bucket_file_exist(gs_path):
    try:
        run(_gsutil_command('ls', gs_path), verbose=False)
        return True
    except RuntimeError:
        return False

def get_google_bucket_file_stats(gs_path):
    gsutil_stat_output = run(_gsutil_command('stat', gs_path), verbose=False)

    """
    Example gsutil stat output:

    Creation time:          Fri, 09 Jun 2017 09:36:23 GMT
    Update time:            Fri, 09 Jun 2017 09:36:23 GMT
    Storage class:          REGIONAL
    Content-Length:         363620675
    Content-Type:           text/x-vcard
    Hash (crc32c):          SWOktA==
    Hash (md5):             fEdIumyOFR7HvULeAwXCwQ==
    ETag:                   CMae+J67sNQCEAE=
    Generation:             1497000983793478
    Metageneration:         1
    """

    if not gsutil_stat_output:
        return None

    EMPTY_MATCH_OBJ = re.match("()", "")
    DATE_FORMAT = '%a, %d %b %Y %H:%M:%S %Z'

    creation_time = (re.search("Creation.time:[\s]+(.+)", gsutil_stat_output, re.IGNORECASE) or EMPTY_MATCH_OBJ).group(1)
    update_time = (re.search("Update.time:[\s]+(.+)", gsutil_stat_output, re.IGNORECASE) or EMPTY_MATCH_OBJ).group(1)
    file_size = (re.search("Content-Length:[\s]+(.+)", gsutil_stat_output, re.IGNORECASE) or EMPTY_MATCH_OBJ).group(1)
    file_md5 = (re.search("Hash (md5):[\s]+(.+)", gsutil_stat_output, re.IGNORECASE) or EMPTY_MATCH_OBJ).group(1)

    ctime = time.mktime(time.strptime(creation_time, DATE_FORMAT))
    mtime = time.mktime(time.strptime(update_time, DATE_FORMAT))
    return FileStats(ctime=ctime, mtime=mtime, size=file_size, md5=file_md5)

def google_bucket_file_iter(gs_path, byte_range=None):
    """Iterate over lines in the given file"""
    range_arg = ' -r {}-{}'.format(byte_range[0], byte_range[1]) if byte_range else ''
    command = _gsutil_command('cat{}'.format(range_arg), gs_path)
    if gs_path.endswith("gz"):
        command += "| gunzip -c -q - "
    process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
    for line in process.stdout:
        yield line

def copy_google_bucket_file(source_path, destination_path):
    """Copy file to or from a google bucket"""

    try:
        run("gsutil -m cp -P %(source_path)s %(destination_path)s" % locals())
    except RuntimeError as e:
        raise ValueError("Failed to copy %s %s: %s" % (source_path, destination_path, e))

<commit_msg>fix anvil igv<commit_after>import logging
import re
import subprocess
import time

from seqr.utils.other_utils import FileStats
from seqr.utils.shell_utils import run

logger = logging.getLogger(__name__)

def _gsutil_command(command, gs_path):
    #  Anvil buckets are requester-pays and we bill them to the anvil project
    project_arg = '-u anvil-datastorage ' if gs_path.startswith('gs://fc-secure') else ''
    return 'gsutil {project_arg}{command} {gs_path}'.format(
        project_arg=project_arg, command=command, gs_path=gs_path,
    )

def is_google_bucket_file_path(file_path):
    return file_path.startswith("gs://")

def does_google_bucket_file_exist(gs_path):
    try:
        run(_gsutil_command('ls', gs_path), verbose=False)
        return True
    except RuntimeError:
        return False

def get_google_bucket_file_stats(gs_path):
    gsutil_stat_output = run(_gsutil_command('stat', gs_path), verbose=False)

    """
    Example gsutil stat output:

    Creation time:          Fri, 09 Jun 2017 09:36:23 GMT
    Update time:            Fri, 09 Jun 2017 09:36:23 GMT
    Storage class:          REGIONAL
    Content-Length:         363620675
    Content-Type:           text/x-vcard
    Hash (crc32c):          SWOktA==
    Hash (md5):             fEdIumyOFR7HvULeAwXCwQ==
    ETag:                   CMae+J67sNQCEAE=
    Generation:             1497000983793478
    Metageneration:         1
    """

    if not gsutil_stat_output:
        return None

    EMPTY_MATCH_OBJ = re.match("()", "")
    DATE_FORMAT = '%a, %d %b %Y %H:%M:%S %Z'

    creation_time = (re.search("Creation.time:[\s]+(.+)", gsutil_stat_output, re.IGNORECASE) or EMPTY_MATCH_OBJ).group(1)
    update_time = (re.search("Update.time:[\s]+(.+)", gsutil_stat_output, re.IGNORECASE) or EMPTY_MATCH_OBJ).group(1)
    file_size = (re.search("Content-Length:[\s]+(.+)", gsutil_stat_output, re.IGNORECASE) or EMPTY_MATCH_OBJ).group(1)
    file_md5 = (re.search("Hash (md5):[\s]+(.+)", gsutil_stat_output, re.IGNORECASE) or EMPTY_MATCH_OBJ).group(1)

    ctime = time.mktime(time.strptime(creation_time, DATE_FORMAT))
    mtime = time.mktime(time.strptime(update_time, DATE_FORMAT))
    return FileStats(ctime=ctime, mtime=mtime, size=file_size, md5=file_md5)

def google_bucket_file_iter(gs_path, byte_range=None):
    """Iterate over lines in the given file"""
    range_arg = ' -r {}-{}'.format(byte_range[0], byte_range[1]) if byte_range else ''
    command = _gsutil_command('cat{}'.format(range_arg), gs_path)
    if gs_path.endswith("gz"):
        command += "| gunzip -c -q - "
    process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
    for line in process.stdout:
        yield line

def copy_google_bucket_file(source_path, destination_path):
    """Copy file to or from a google bucket"""

    try:
        run("gsutil -m cp -P %(source_path)s %(destination_path)s" % locals())
    except RuntimeError as e:
        raise ValueError("Failed to copy %s %s: %s" % (source_path, destination_path, e))

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# ecnet/error_utils.py
# v.1.7.0
# Developed in 2018 by Travis Kessler <travis.j.kessler@gmail.com>
#
# Contains functions necessary creating, training, saving, and reusing neural
# network models
#

# Stdlib imports
import pickle
from pickle import dump, load
from functools import reduce
from os import environ, mkdir, path
from random import uniform

# 3rd party imports
import tensorflow as tf
from tensorflow import add, global_variables_initializer, matmul, nn
from tensorflow import placeholder, random_normal, reset_default_graph
from tensorflow import Session, square, train, Variable
from numpy import asarray, sqrt as nsqrt

environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

def __linear_fn(n):
    '''Linear definition for activation function dictionary
    '''
    return n

ACTIVATION_FUNCTONS = {
    'relu': nn.relu,
    'sigmoid': nn.sigmoid,
    'softmax': nn.softmax,
    'linear': __linear_fn
}

class Layer:

    def __init__(self, size, act_fn):
        '''Layer object: contains layer size (number of neurons), activation
        function

        Args:
            size (int): number of neurons in the layer
            act_fn (str): 'relu', 'sigmoid', 'softmax', 'linear'
        '''

        self.size = size
        if act_fn not in ACTIVATION_FUNCTONS.keys():
            raise ValueError(
                'Unsupported activation function {}'.format(act_fn)
            )
        self.act_fn = ACTIVATION_FUNCTONS[act_fn]

    def __len__(self):
        '''
        Layer length = number of neurons = layer.size
        '''

        return self.size

class MultilayerPerceptron:

    def __init__(self, id=0):
        '''Neural network (multilayer perceptron) - contains methods for training,
        using, saving and opening neural network models
        '''

        self.__layers = []
        self.__weights = []
        self.__biases = []
        self.__id = id
        if not path.isdir('./tmp/'):
            mkdir('./tmp/')
        reset_default_graph()

    def add_layer(self, size, act_fn):
        '''
        Add a layer to the model

        Args:
            size (int): number of neurons in the layer
            act_fn (str): activation function of the layer:
                - 'relu', 'sigmoid', 'softmax' or 'linear'
        '''

        self.__layers.append(Layer(size, act_fn))

    def connect_layers(self):
        '''Fully connects each layer added using add_layer()
        '''

        for layer in range(len(self.__layers) - 1):
            self.__weights.append(
                Variable(random_normal([
                    self.__layers[layer].size,
                    self.__layers[layer + 1].size
                ]), name='W_fc{}'.format(layer + 1))
            )
        for layer in range(1, len(self.__layers)):
            self.__biases.append(
                Variable(random_normal([
                    self.__layers[layer].size
                ]), name='B_fc{}'.format(layer))
            )

    def fit(self, x_l, y_l, learning_rate=0.1, train_epochs=500,
            keep_prob=1.0):
        '''Fits the neural network model using a set number of training iterations

        Args:
            x_l (numpy array): training inputs
            y_l (numpy array): training outputs
            learning_rate (float): learning rate during training
            train_epochs (int): number of training iterations
            keep_prob (float): probability that a neuron is retained (not
                subjected to dropout)
        '''

        if len(y_l) is 0 or len(x_l) is 0:
            raise ValueError('Learning set cannot be empty - check data split')

        x = placeholder('float', [None, self.__layers[0].size])
        y = placeholder('float', [None, self.__layers[-1].size])

        pred = self.__feed_forward(x, keep_prob=keep_prob)
        cost = square(y - pred)
        optimizer = train.AdamOptimizer(
            learning_rate=learning_rate
        ).minimize(cost)

        with Session() as sess:
            sess.run(global_variables_initializer())
            for _ in range(train_epochs):
                sess.run(optimizer, feed_dict={x: x_l, y: y_l})
            saver = train.Saver()
            saver.save(sess, './tmp/_m{}/_ckpt'.format(self.__id))
        sess.close()

    def fit_validation(self, x_l, y_l, x_v, y_v, learning_rate=0.1,
                       max_epochs=1500, keep_prob=1.0):
        '''Fits the neural network model using periodic (every 250 epochs)
        validation; if validation set performance worsens, training is complete

        Args:
            x_l (numpy array): training inputs
            y_l (numpy array): training outputs
            x_v (numpy array): validation inputs
            y_v (numpy array): validation outputs
            learning_rate (float): learning rate during training
            max_epochs (int): maximum number of training iterations
            keep_prob (float): probability that a neuron is retained (not
                subjected to dropout)
        '''

        if len(y_l) is 0 or len(x_l) is 0:
            raise ValueError(
                'Learning set cannot be empty - check data split'
            )
        if len(y_v) is 0 or len(x_v) is 0:
            raise ValueError(
                'Validation set cannot be empty - check data split'
            )

        x = placeholder('float', [None, self.__layers[0].size])
        y = placeholder('float', [None, self.__layers[-1].size])

        pred = self.__feed_forward(x, keep_prob=keep_prob)
        cost = square(y - pred)
        optimizer = train.AdamOptimizer(
            learning_rate=learning_rate
        ).minimize(cost)

        with Session() as sess:
            sess.run(global_variables_initializer())

            valid_rmse_lowest = 100
            current_epoch = 0

            while current_epoch < max_epochs:
                sess.run(optimizer, feed_dict={x: x_l, y: y_l})
                current_epoch += 1
                if current_epoch % 250 == 0:
                    valid_rmse = self.__calc_rmse(
                        sess.run(pred, feed_dict={x: x_v}), y_v
                    )
                    if valid_rmse < valid_rmse_lowest:
                        valid_rmse_lowest = valid_rmse
                    elif valid_rmse > valid_rmse_lowest + (
                        0.05 * valid_rmse_lowest
                    ):
                        break

            saver = train.Saver()
            saver.save(sess, './tmp/_m{}/_ckpt'.format(self.__id))
        sess.close()

    def use(self, x):
        '''Use the neural network on supplied input data

        Args:
            x (numpy array): supplied input data

        Returns:
            numpy array: predicted value for input data
        '''

        with Session() as sess:
            saver = train.Saver()
            saver.restore(sess, './tmp/_m{}/_ckpt'.format(self.__id))
            results = self.__feed_forward(x).eval()
        sess.close()
        return results

    def save(self, filepath):
        '''Save the neural network for later use; results in a .sess (tensorflow)
        file and a .struct (neural network architecture) file

        Args:
            filepath (str): location to save the model
        '''

        with Session() as sess:
            saver = train.Saver()
            saver.restore(sess, './tmp/_m{}/_ckpt'.format(self.__id))
            saver.save(sess, filepath)
        sess.close()
        architecture_file = open('{}.struct'.format(filepath), 'wb')
        dump(self.__layers, architecture_file)
        architecture_file.close()

    def load(self, filepath):
        '''Load a previously saved neural network

        Args:
            filepath (str): location of the saved model
        '''

        architecture_file = open('{}.struct'.format(filepath), 'rb')
        self.__layers = load(architecture_file)
        self.connect_layers()
        with Session() as sess:
            saver = train.Saver()
            saver.restore(sess, filepath)
            saver.save(sess, './tmp/_m{}/_ckpt'.format(self.__id))
        sess.close()

    def __feed_forward(self, x, keep_prob=1.0):
        '''Private method: feeds data through the neural network, returns output
        of final layer

        Args:
            x (1d numpy array): input data to feed through the network
            keep_prob (float): probability that a neuron is retained (not
                subjected to dropout)

        Returns:
            1d numpy array: result of the final layer pass-through
        '''

        output = x
        for idx, layer in enumerate(self.__layers[1:]):
            output = nn.dropout(layer.act_fn(
                add(matmul(
                    output,
                    self.__weights[idx]
                ), self.__biases[idx])
            ), keep_prob)
        return output

    def __calc_rmse(self, y_hat, y):
        '''Private method: calculates the RMSE of the validation set during
        validation training

        Args:
            y_hat (numpy array): predicted data
            y (numpy array): known data

        Returns:
            float: root mean squared error of predicted data
        '''

        try:
            return(nsqrt(((y_hat - y)**2).mean()))
        except:
            return(nsqrt(((asarray(y_hat) - asarray(y))**2).mean()))
<commit_msg>Moved training function to model.py<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# ecnet/error_utils.py
# v.1.7.0
# Developed in 2018 by Travis Kessler <travis.j.kessler@gmail.com>
#
# Contains functions necessary creating, training, saving, and reusing neural
# network models
#

# Stdlib imports
import pickle
from pickle import dump, load
from functools import reduce
from os import environ, mkdir, path
from random import uniform

# 3rd party imports
import tensorflow as tf
from tensorflow import add, global_variables_initializer, matmul, nn
from tensorflow import placeholder, random_normal, reset_default_graph
from tensorflow import Session, square, train, Variable
from numpy import asarray, sqrt as nsqrt

environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

def __linear_fn(n):
    '''Linear definition for activation function dictionary
    '''
    return n

ACTIVATION_FUNCTONS = {
    'relu': nn.relu,
    'sigmoid': nn.sigmoid,
    'softmax': nn.softmax,
    'linear': __linear_fn
}

class Layer:

    def __init__(self, size, act_fn):
        '''Layer object: contains layer size (number of neurons), activation
        function

        Args:
            size (int): number of neurons in the layer
            act_fn (str): 'relu', 'sigmoid', 'softmax', 'linear'
        '''

        self.size = size
        if act_fn not in ACTIVATION_FUNCTONS.keys():
            raise ValueError(
                'Unsupported activation function {}'.format(act_fn)
            )
        self.act_fn = ACTIVATION_FUNCTONS[act_fn]

    def __len__(self):
        '''
        Layer length = number of neurons = layer.size
        '''

        return self.size

class MultilayerPerceptron:

    def __init__(self, id=0):
        '''Neural network (multilayer perceptron) - contains methods for training,
        using, saving and opening neural network models
        '''

        self.__layers = []
        self.__weights = []
        self.__biases = []
        self.__id = id
        if not path.isdir('./tmp/'):
            mkdir('./tmp/')
        reset_default_graph()

    def add_layer(self, size, act_fn):
        '''
        Add a layer to the model

        Args:
            size (int): number of neurons in the layer
            act_fn (str): activation function of the layer:
                - 'relu', 'sigmoid', 'softmax' or 'linear'
        '''

        self.__layers.append(Layer(size, act_fn))

    def connect_layers(self):
        '''Fully connects each layer added using add_layer()
        '''

        for layer in range(len(self.__layers) - 1):
            self.__weights.append(
                Variable(random_normal([
                    self.__layers[layer].size,
                    self.__layers[layer + 1].size
                ]), name='W_fc{}'.format(layer + 1))
            )
        for layer in range(1, len(self.__layers)):
            self.__biases.append(
                Variable(random_normal([
                    self.__layers[layer].size
                ]), name='B_fc{}'.format(layer))
            )

    def fit(self, x_l, y_l, learning_rate=0.1, train_epochs=500,
            keep_prob=1.0):
        '''Fits the neural network model using a set number of training iterations

        Args:
            x_l (numpy array): training inputs
            y_l (numpy array): training outputs
            learning_rate (float): learning rate during training
            train_epochs (int): number of training iterations
            keep_prob (float): probability that a neuron is retained (not
                subjected to dropout)
        '''

        if len(y_l) is 0 or len(x_l) is 0:
            raise ValueError('Learning set cannot be empty - check data split')

        x = placeholder('float', [None, self.__layers[0].size])
        y = placeholder('float', [None, self.__layers[-1].size])

        pred = self.__feed_forward(x, keep_prob=keep_prob)
        cost = square(y - pred)
        optimizer = train.AdamOptimizer(
            learning_rate=learning_rate
        ).minimize(cost)

        with Session() as sess:
            sess.run(global_variables_initializer())
            for _ in range(train_epochs):
                sess.run(optimizer, feed_dict={x: x_l, y: y_l})
            saver = train.Saver()
            saver.save(sess, './tmp/_m{}/_ckpt'.format(self.__id))
        sess.close()

    def fit_validation(self, x_l, y_l, x_v, y_v, learning_rate=0.1,
                       max_epochs=1500, keep_prob=1.0):
        '''Fits the neural network model using periodic (every 250 epochs)
        validation; if validation set performance worsens, training is complete

        Args:
            x_l (numpy array): training inputs
            y_l (numpy array): training outputs
            x_v (numpy array): validation inputs
            y_v (numpy array): validation outputs
            learning_rate (float): learning rate during training
            max_epochs (int): maximum number of training iterations
            keep_prob (float): probability that a neuron is retained (not
                subjected to dropout)
        '''

        if len(y_l) is 0 or len(x_l) is 0:
            raise ValueError(
                'Learning set cannot be empty - check data split'
            )
        if len(y_v) is 0 or len(x_v) is 0:
            raise ValueError(
                'Validation set cannot be empty - check data split'
            )

        x = placeholder('float', [None, self.__layers[0].size])
        y = placeholder('float', [None, self.__layers[-1].size])

        pred = self.__feed_forward(x, keep_prob=keep_prob)
        cost = square(y - pred)
        optimizer = train.AdamOptimizer(
            learning_rate=learning_rate
        ).minimize(cost)

        with Session() as sess:
            sess.run(global_variables_initializer())

            valid_rmse_lowest = 100
            current_epoch = 0

            while current_epoch < max_epochs:
                sess.run(optimizer, feed_dict={x: x_l, y: y_l})
                current_epoch += 1
                if current_epoch % 250 == 0:
                    valid_rmse = self.__calc_rmse(
                        sess.run(pred, feed_dict={x: x_v}), y_v
                    )
                    if valid_rmse < valid_rmse_lowest:
                        valid_rmse_lowest = valid_rmse
                    elif valid_rmse > valid_rmse_lowest + (
                        0.05 * valid_rmse_lowest
                    ):
                        break

            saver = train.Saver()
            saver.save(sess, './tmp/_m{}/_ckpt'.format(self.__id))
        sess.close()

    def use(self, x):
        '''Use the neural network on supplied input data

        Args:
            x (numpy array): supplied input data

        Returns:
            numpy array: predicted value for input data
        '''

        with Session() as sess:
            saver = train.Saver()
            saver.restore(sess, './tmp/_m{}/_ckpt'.format(self.__id))
            results = self.__feed_forward(x).eval()
        sess.close()
        return results

    def save(self, filepath):
        '''Save the neural network for later use; results in a .sess (tensorflow)
        file and a .struct (neural network architecture) file

        Args:
            filepath (str): location to save the model
        '''

        with Session() as sess:
            saver = train.Saver()
            saver.restore(sess, './tmp/_m{}/_ckpt'.format(self.__id))
            saver.save(sess, filepath)
        sess.close()
        architecture_file = open('{}.struct'.format(filepath), 'wb')
        dump(self.__layers, architecture_file)
        architecture_file.close()

    def load(self, filepath):
        '''Load a previously saved neural network

        Args:
            filepath (str): location of the saved model
        '''

        architecture_file = open('{}.struct'.format(filepath), 'rb')
        self.__layers = load(architecture_file)
        self.connect_layers()
        with Session() as sess:
            saver = train.Saver()
            saver.restore(sess, filepath)
            saver.save(sess, './tmp/_m{}/_ckpt'.format(self.__id))
        sess.close()

    def __feed_forward(self, x, keep_prob=1.0):
        '''Private method: feeds data through the neural network, returns output
        of final layer

        Args:
            x (1d numpy array): input data to feed through the network
            keep_prob (float): probability that a neuron is retained (not
                subjected to dropout)

        Returns:
            1d numpy array: result of the final layer pass-through
        '''

        output = x
        for idx, layer in enumerate(self.__layers[1:]):
            output = nn.dropout(layer.act_fn(
                add(matmul(
                    output,
                    self.__weights[idx]
                ), self.__biases[idx])
            ), keep_prob)
        return output

    def __calc_rmse(self, y_hat, y):
        '''Private method: calculates the RMSE of the validation set during
        validation training

        Args:
            y_hat (numpy array): predicted data
            y (numpy array): known data

        Returns:
            float: root mean squared error of predicted data
        '''

        try:
            return(nsqrt(((y_hat - y)**2).mean()))
        except:
            return(nsqrt(((asarray(y_hat) - asarray(y))**2).mean()))

def train_model(validate, sets, vars, save_path):
    '''Starts a process to train a neural network

    Args:
        validate (bool): whether to use periodic validation or not
        sets (PackagedData): object housing learning, validation, test data
        vars (dict): Server model configuration variables
        save_path (str): path to where model is saved
    '''

    model = MultilayerPerceptron()
    model.add_layer(len(sets.learn_x[0]), vars['input_activation'])
    for layer in vars['hidden_layers']:
        model.add_layer(layer[0], layer[1])
    model.add_layer(len(sets.learn_y[0]), vars['output_activation'])
    model.connect_layers()
    if validate:
        model.fit_validation(
            sets.learn_x,
            sets.learn_y,
            sets.valid_x,
            sets.valid_y,
            learning_rate=vars['learning_rate'],
            max_epochs=vars['validation_max_epochs'],
            keep_prob=vars['keep_prob']
        )
    else:
        model.fit(
            sets.learn_x,
            sets.learn_y,
            learning_rate=vars['learning_rate'],
            train_epochs=vars['train_epochs'],
            keep_prob=vars['keep_prob']
        )
    model.save(save_path)
<|endoftext|>


<commit_before>import argparse, logging, random, sqlite3
from database import *
from record import *
from generator import *
from scipy import stats

def main():

    """ Main """

    #
    # Setup the logger.
    #
    logger = logging.getLogger()
    ch = logging.StreamHandler()
    formatter = logging.Formatter("%(asctime)s [%(module)s] [%(levelname)s] %(message)s")
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    #
    # Parse the arguments
    #
    parser = argparse.ArgumentParser(description="Generates synthetic data.")
    parser.add_argument('-n', dest='record_count', action='store', type=int,
                        default='100', help='number of records to generate (default=%(default)s)')
    parser.add_argument('-s', dest='start_session_id', action='store', type=int,
                        default='1', help='the starting session id (default=%(default)s)')
    parser.add_argument('-v', action="store_true", default=False, help='verbose mode')
    args = parser.parse_args()
    if args.record_count < 1:
        logger.error('Invalid record count!')
        exit()

    if args.v:
        logger.setLevel(logging.INFO)

    # Initialize database.
    db = Database()

    # Initialize the generator.
    generator = Generator(db)

    # Generate the record.
    for idx in range(0, args.record_count):
        session_id = args.start_session_id + idx
        record = generator.generate(session_id)
        if (idx % 10 == 0):
            logger.info("Generated %d records."%(idx))
        print record.to_string()

    logger.info("Complete!")

if __name__ == '__main__':
    main()<commit_msg>Verbose changes.<commit_after>import argparse, logging, random, sqlite3
from database import *
from record import *
from generator import *
from scipy import stats

def main():

    """ Main """

    #
    # Setup the logger.
    #
    logger = logging.getLogger()
    ch = logging.StreamHandler()
    formatter = logging.Formatter("%(asctime)s [%(module)s] [%(levelname)s] %(message)s")
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    #
    # Parse the arguments
    #
    parser = argparse.ArgumentParser(description="Generates synthetic data.")
    parser.add_argument('-n', dest='record_count', action='store', type=int,
                        default='100', help='number of records to generate (default=%(default)s)')
    parser.add_argument('-s', dest='start_session_id', action='store', type=int,
                        default='1', help='the starting session id (default=%(default)s)')
    parser.add_argument('-v', action="store_true", default=False, help='verbose mode')
    args = parser.parse_args()
    if args.record_count < 1:
        logger.error('Invalid record count!')
        exit()

    if args.v:
        logger.setLevel(logging.INFO)

    # Initialize database.
    db = Database()

    # Initialize the generator.
    generator = Generator(db)

    # Generate the record.
    for idx in range(0, args.record_count):
        session_id = args.start_session_id + idx
        record = generator.generate(session_id)
        if (idx % (int(args.record_count/10)) == 0):
            logger.info("Generated %d records."%(idx))
        print record.to_string()

    logger.info("Complete!")

if __name__ == '__main__':
    main()<|endoftext|>