<commit_before>"""Buffet templating plugin and render functions

The Buffet object is styled after the original Buffet module that implements
template language neutral rendering for CherryPy. This version of Buffet also
contains caching functionality that utilizes
`Beaker middleware <http://beaker.groovie.org/>`_ to provide template language
neutral caching functionality.

A customized version of
`BuffetMyghty <http://projects.dowski.com/projects/buffetmyghty>`_ is included
that provides a template API hook as the ``pylonsmyghty`` engine. This version
of BuffetMyghty disregards some of the TurboGears API spec so that traditional
Myghty template names can be used with ``/`` and file extensions.

The render functions are intended as the primary user-visible rendering commands
and hook into Buffet to make rendering content easy.

"""
import pkg_resources
import os
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from myghty.interp import Interpreter

import pylons

class BuffetError(Exception):
    pass

class Buffet(object):
    """Buffet style plug-in template rendering

    Buffet implements template language plug-in support modeled highly on the
    `Buffet Project <http://projects.dowski.com/projects/buffet>`_ from which
    this class inherits its name.

    """
    def __init__(self, default_engine=None, template_root=None,
        default_options=None, **config):
        """Initialize the Buffet renderer, and optionally set a default
        engine/options"""
        if default_options is None:
            default_options = {}
        self.default_engine = default_engine
        self.template_root = template_root
        self.default_options = default_options
        self.engines = {}
        if self.default_engine:
            self.prepare(default_engine, template_root, **config)

    def prepare(self, engine_name, template_root=None, alias=None, **config):
        """Prepare a template engine for use

        This method must be run before the `render <#render>`_ method is called
        so that the ``template_root`` and options can be set. Template engines
        can also be aliased if you wish to use multiplate configurations of the
        same template engines, or prefer a shorter name when rendering a template
        with the engine of your choice.

        """
        Engine = available_engines.get(engine_name, None)
        if not Engine:
            raise TemplateEngineMissing('Please install a plugin for '
                '"%s" to use its functionality' % engine_name)
        engine_name = alias or engine_name
        defaults = config.pop('default_options', None)
        self.engines[engine_name] = \
            dict(engine=Engine(options=config), root=template_root)

    def _update_names(self, ns):
        d = ns
        d.update(dict(
            c=pylons.c._current_obj(),
            h=pylons.h,
            request=pylons.request._current_obj(),
            g=pylons.g._current_obj(),
            session=pylons.session._current_obj(),
            render=render,
        ))
        if hasattr(d['request'], '_h'):
            d['h'] = d['request']._h
        return d

    def render(self, engine_name=None, template_name=None,
        include_pylons_variables=True, namespace=None,
        cache_key=None, cache_expire=None, cache_type=None, **options):
        """Render a template using a template engine plug-in

        To use templates it is expected that you will attach data to be used in
        the template to the ``c`` variable which is available in the controller
        and the template.

        When porting code from other projects it is sometimes easier to use an
        exisitng dictionary which can be specified with ``namespace``.

        ``engine_name``
            The name of the template engine to use, which must be
            'prepared' first.
        ``template_name``
            Name of the template to render
        ``include_pylons_variables``
            If a custom namespace is specified this determines whether Pylons
            variables are included in the namespace or not. Defaults to
            ``True``.
        ``namespace``
            A custom dictionary of names and values to be substituted in the
            template. If ``include_pylons_variables`` is ``True`` and any
            keys in ``namespace`` conflict with names of Pylons variables,
            an error is raised.

        Caching options (uses Beaker caching middleware)

        ``cache_key``
            Key to cache this copy of the template under.
        ``cache_type``
            Valid options are ``dbm``, ``file``, ``memory``, or ``ext:memcached``.
        ``cache_expire``
            Time in seconds to cache this template with this ``cache_key`` for. Or
            use 'never' to designate that the cache should never expire.

        The minimum key required to trigger caching is ``cache_expire='never'`` which
        will cache the template forever seconds with no key.

        All other keyword options are passed directly to the template engine
        used.

        """
        if not engine_name and self.default_engine:
            engine_name = self.default_engine
        engine_config = self.engines.get(engine_name)

        if not engine_config:
            raise Exception, "No engine with that name configured: %s" % engine_name

        full_path = template_name

        if engine_name != 'pylonsmyghty':
            if namespace==None:
                if include_pylons_variables is False:
                    raise BuffetError('You must specify ``namespace`` if ``include_pylons_variables`` is '
                                      'False')
                else:
                    namespace = self._update_names({})
            elif isinstance(namespace, dict):
                if include_pylons_variables is True:
                    for k in ['c','h','g','request','session', 'params']:
                        if k in namespace:
                            raise Exception('The variable %s specified in namespace conflicts '
                                'with the Pylons variable of the same name. Set ``include_pylons_variables`` '
                                'to ``False`` if you do not want to use Pylons variables in your template'%k)
                    namespace = self._update_names(namespace)
            else:
                namespace = self._update_names(namespace)

            if not engine_name.startswith('pylons'):
                full_path = os.path.join(engine_config['root'], template_name)
                full_path = full_path.replace(os.path.sep, '.').lstrip('.')
        else:
            if namespace is None:
                namespace = {}
            namespace['_global_args'] = self._update_names({})

            # If they passed in a variable thats listed in the global_args,
            # update the global args one instead of duplicating it
            interp = engine_config['engine'].interpreter
            for key in interp.global_args.keys() + interp.init_params.get('allow_globals', []):
                if key in namespace:
                    namespace['_global_args'][key] = namespace.pop(key)

        # If one of them is not None then the user did set something
        if cache_key is not None or cache_expire is not None or cache_type is not None:
            if not cache_type:
                cache_type = 'dbm'
            if not cache_key:
                cache_key = 'default'
            if cache_expire == 'never':
                cache_expire = None
            def content():
                return engine_config['engine'].render(namespace, template=full_path, **options)
            tfile = full_path
            if options.get('fragment', False):
                tfile += '_frag'
            mycache = pylons.cache.get_cache(tfile)
            content = mycache.get_value(cache_key, createfunc=content, type=cache_type,
                expiretime=cache_expire)
            return content

        return engine_config['engine'].render(namespace, template=full_path, **options)

class TemplateEngineMissing(Exception):
    pass

class MyghtyTemplatePlugin(object):
    """Myghty Template Plugin

    This Myghty Template Plugin varies from the official BuffetMyghty in that it will
    properly populate all the default Myghty variables needed and render fragments.

    """
    extension = "myt"

    def __init__(self, extra_vars_func=None, options=None):
        if options is None:
            options = {}
        myt_opts = {}
        for k, v in options.iteritems():
            if k.startswith('myghty.'):
                myt_opts[k[7:]] = v
        self.interpreter = Interpreter(**myt_opts)

    def load_template(self, template_path):
        pass

    def render(self, info, format="html", fragment=False, template=None):
        global_args = info.pop('_global_args')
        vars = info
        buf = StringIO()
        if fragment:
            self.interpreter.execute(template, request_args=vars, global_args=global_args, out_buffer=buf, disable_wrapping=True)
        else:
            self.interpreter.execute(template, request_args=vars, global_args=global_args, out_buffer=buf)
        return buf.getvalue()

available_engines = {}

for entry_point in pkg_resources.iter_entry_points('python.templating.engines'):
    Engine = entry_point.load()
    available_engines[entry_point.name] = Engine

def render(*args, **kargs):
    """Render a template and return it as a string (possibly Unicode)

    Optionally takes 3 keyword arguments to use caching supplied by Buffet.

    Example::

        >>> content = render('/my/template.myt')
        >>> print content
        >>> content = render('/my/template.myt', fragment=True)

    .. admonition:: Note

        Not all template languages support the concept of a fragment. In those template
        languages that do support the fragment option, this usually implies that the
        template will be rendered without extending or inheriting any site skin.

    """
    fragment = kargs.pop('fragment', False)
    args = list(args)
    template = args.pop()
    cache_args = dict(cache_expire=kargs.pop('cache_expire', None),
                      cache_type=kargs.pop('cache_type', None),
                      cache_key=kargs.pop('cache_key', None))
    if args:
        engine = args.pop()
        return pylons.buffet.render(engine, template, fragment=fragment, namespace=kargs, **cache_args)
    return pylons.buffet.render(template_name=template, fragment=fragment, namespace=kargs, **cache_args)

def render_response(*args, **kargs):
    """Returns the rendered response within a Response object

    See ``render`` for information on rendering.

    Example:

    .. code-block:: Python

        def view(self):
            return render_response('/my/template.myt')

    """
    return pylons.Response(render(*args, **kargs))

__pudge_all__ = ['render', 'render_response', 'Buffet', 'MyghtyTemplatePlugin']
__all__ = ['render', 'render_response', 'Buffet', 'MyghtyTemplatePlugin']
<commit_msg>[svn] Minor text formatting.<commit_after>"""Buffet templating plugin and render functions

The Buffet object is styled after the original Buffet module that implements
template language neutral rendering for CherryPy. This version of Buffet also
contains caching functionality that utilizes
`Beaker middleware <http://beaker.groovie.org/>`_ to provide template language
neutral caching functionality.

A customized version of
`BuffetMyghty <http://projects.dowski.com/projects/buffetmyghty>`_ is included
that provides a template API hook as the ``pylonsmyghty`` engine. This version
of BuffetMyghty disregards some of the TurboGears API spec so that traditional
Myghty template names can be used with ``/`` and file extensions.

The render functions are intended as the primary user-visible rendering commands
and hook into Buffet to make rendering content easy.

"""
import pkg_resources
import os
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from myghty.interp import Interpreter

import pylons

class BuffetError(Exception):
    pass

class Buffet(object):
    """Buffet style plug-in template rendering

    Buffet implements template language plug-in support modeled highly on the
    `Buffet Project <http://projects.dowski.com/projects/buffet>`_ from which
    this class inherits its name.

    """
    def __init__(self, default_engine=None, template_root=None,
        default_options=None, **config):
        """Initialize the Buffet renderer, and optionally set a default
        engine/options"""
        if default_options is None:
            default_options = {}
        self.default_engine = default_engine
        self.template_root = template_root
        self.default_options = default_options
        self.engines = {}
        if self.default_engine:
            self.prepare(default_engine, template_root, **config)

    def prepare(self, engine_name, template_root=None, alias=None, **config):
        """Prepare a template engine for use

        This method must be run before the `render <#render>`_ method is called
        so that the ``template_root`` and options can be set. Template engines
        can also be aliased if you wish to use multiplate configurations of the
        same template engines, or prefer a shorter name when rendering a template
        with the engine of your choice.

        """
        Engine = available_engines.get(engine_name, None)
        if not Engine:
            raise TemplateEngineMissing('Please install a plugin for '
                '"%s" to use its functionality' % engine_name)
        engine_name = alias or engine_name
        defaults = config.pop('default_options', None)
        self.engines[engine_name] = \
            dict(engine=Engine(options=config), root=template_root)

    def _update_names(self, ns):
        d = ns
        d.update(dict(
            c=pylons.c._current_obj(),
            h=pylons.h,
            request=pylons.request._current_obj(),
            g=pylons.g._current_obj(),
            session=pylons.session._current_obj(),
            render=render,
        ))
        if hasattr(d['request'], '_h'):
            d['h'] = d['request']._h
        return d

    def render(self, engine_name=None, template_name=None,
        include_pylons_variables=True, namespace=None,
        cache_key=None, cache_expire=None, cache_type=None, **options):
        """Render a template using a template engine plug-in

        To use templates it is expected that you will attach data to be used in
        the template to the ``c`` variable which is available in the controller
        and the template.

        When porting code from other projects it is sometimes easier to use an
        exisitng dictionary which can be specified with ``namespace``.

        ``engine_name``
            The name of the template engine to use, which must be
            'prepared' first.
        ``template_name``
            Name of the template to render
        ``include_pylons_variables``
            If a custom namespace is specified this determines whether Pylons
            variables are included in the namespace or not. Defaults to
            ``True``.
        ``namespace``
            A custom dictionary of names and values to be substituted in the
            template. If ``include_pylons_variables`` is ``True`` and any
            keys in ``namespace`` conflict with names of Pylons variables,
            an error is raised.

        Caching options (uses Beaker caching middleware)

        ``cache_key``
            Key to cache this copy of the template under.
        ``cache_type``
            Valid options are ``dbm``, ``file``, ``memory``, or ``ext:memcached``.
        ``cache_expire``
            Time in seconds to cache this template with this ``cache_key`` for. Or
            use 'never' to designate that the cache should never expire.

        The minimum key required to trigger caching is ``cache_expire='never'`` which
        will cache the template forever seconds with no key.

        All other keyword options are passed directly to the template engine
        used.

        """
        if not engine_name and self.default_engine:
            engine_name = self.default_engine
        engine_config = self.engines.get(engine_name)

        if not engine_config:
            raise Exception, "No engine with that name configured: %s" % engine_name

        full_path = template_name

        if engine_name != 'pylonsmyghty':
            if namespace==None:
                if include_pylons_variables is False:
                    raise BuffetError('You must specify ``namespace`` if ``include_pylons_variables`` is '
                                      'False')
                else:
                    namespace = self._update_names({})
            elif isinstance(namespace, dict):
                if include_pylons_variables is True:
                    for k in ['c','h','g','request','session', 'params']:
                        if k in namespace:
                            raise Exception('The variable %s specified in namespace conflicts '
                                'with the Pylons variable of the same name. Set ``include_pylons_variables`` '
                                'to ``False`` if you do not want to use Pylons variables in your template'%k)
                    namespace = self._update_names(namespace)
            else:
                namespace = self._update_names(namespace)

            if not engine_name.startswith('pylons'):
                full_path = os.path.join(engine_config['root'], template_name)
                full_path = full_path.replace(os.path.sep, '.').lstrip('.')
        else:
            if namespace is None:
                namespace = {}
            namespace['_global_args'] = self._update_names({})

            # If they passed in a variable thats listed in the global_args,
            # update the global args one instead of duplicating it
            interp = engine_config['engine'].interpreter
            for key in interp.global_args.keys() + interp.init_params.get('allow_globals', []):
                if key in namespace:
                    namespace['_global_args'][key] = namespace.pop(key)

        # If one of them is not None then the user did set something
        if cache_key is not None or cache_expire is not None or cache_type is not None:
            if not cache_type:
                cache_type = 'dbm'
            if not cache_key:
                cache_key = 'default'
            if cache_expire == 'never':
                cache_expire = None
            def content():
                return engine_config['engine'].render(namespace, template=full_path, **options)
            tfile = full_path
            if options.get('fragment', False):
                tfile += '_frag'
            mycache = pylons.cache.get_cache(tfile)
            content = mycache.get_value(cache_key, createfunc=content, type=cache_type,
                expiretime=cache_expire)
            return content

        return engine_config['engine'].render(namespace, template=full_path, **options)

class TemplateEngineMissing(Exception):
    pass

class MyghtyTemplatePlugin(object):
    """Myghty Template Plugin

    This Myghty Template Plugin varies from the official BuffetMyghty in that it will
    properly populate all the default Myghty variables needed and render fragments.

    """
    extension = "myt"

    def __init__(self, extra_vars_func=None, options=None):
        if options is None:
            options = {}
        myt_opts = {}
        for k, v in options.iteritems():
            if k.startswith('myghty.'):
                myt_opts[k[7:]] = v
        self.interpreter = Interpreter(**myt_opts)

    def load_template(self, template_path):
        pass

    def render(self, info, format="html", fragment=False, template=None):
        global_args = info.pop('_global_args')
        vars = info
        buf = StringIO()
        if fragment:
            self.interpreter.execute(template, request_args=vars,
                global_args=global_args, out_buffer=buf, disable_wrapping=True)
        else:
            self.interpreter.execute(template, request_args=vars,
                global_args=global_args, out_buffer=buf)
        return buf.getvalue()

available_engines = {}

for entry_point in pkg_resources.iter_entry_points('python.templating.engines'):
    Engine = entry_point.load()
    available_engines[entry_point.name] = Engine

def render(*args, **kargs):
    """Render a template and return it as a string (possibly Unicode)

    Optionally takes 3 keyword arguments to use caching supplied by Buffet.

    Example::

        >>> content = render('/my/template.myt')
        >>> print content
        >>> content = render('/my/template.myt', fragment=True)

    .. admonition:: Note

        Not all template languages support the concept of a fragment. In those template
        languages that do support the fragment option, this usually implies that the
        template will be rendered without extending or inheriting any site skin.

    """
    fragment = kargs.pop('fragment', False)
    args = list(args)
    template = args.pop()
    cache_args = dict(cache_expire=kargs.pop('cache_expire', None),
                      cache_type=kargs.pop('cache_type', None),
                      cache_key=kargs.pop('cache_key', None))
    if args:
        engine = args.pop()
        return pylons.buffet.render(engine, template, fragment=fragment, namespace=kargs, **cache_args)
    return pylons.buffet.render(template_name=template, fragment=fragment, namespace=kargs, **cache_args)

def render_response(*args, **kargs):
    """Returns the rendered response within a Response object

    See ``render`` for information on rendering.

    Example:

    .. code-block:: Python

        def view(self):
            return render_response('/my/template.myt')

    """
    return pylons.Response(render(*args, **kargs))

__pudge_all__ = ['render', 'render_response', 'Buffet', 'MyghtyTemplatePlugin']
__all__ = ['render', 'render_response', 'Buffet', 'MyghtyTemplatePlugin']
<|endoftext|>


<commit_before>import os
import time
import json
import pytz
import datetime
import requests
import threading
import ConfigParser
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot  as     plt
import paho.mqtt.client   as     mqtt
from   InformationFetcher import InformationFetcher
from   Template           import TemplateMatcher
from   requests.auth      import HTTPBasicAuth

SECOND =   1
MINUTE =  60 * SECOND
HOUR   =  60 * MINUTE
DAY    =  24 * HOUR
WEEK   =   7 * DAY
MONTH  =  31 * DAY
YEAR   = 365 * DAY

class Tank(threading.Thread):

    DAWN   = 0
    DAY    = 1
    SUNSET = 2
    NIGHT  = 3

    def _readConfig(self):

        if self._configMTime != os.stat(self._configFileName).st_mtime:

            print "Reread config file for tank"
            self._configMTime = os.stat(self._configFileName).st_mtime
            update = False
            stop   = False

            if not os.path.isdir(self._homeDir):
                print "Creating homeDir"
                os.makedirs(self._homeDir)

            if os.path.isfile(self._configFileName):
                self._config.read(self._configFileName)
            else:
                print "Config file not found"
                update = True

            if not self._config.has_section('MQTT'):
                print "Adding MQTT part"
                update = True
                self._config.add_section("MQTT")

            if not self._config.has_option("MQTT", "ServerAddress"):
                print "No Server Address"
                update = True
                self._config.set("MQTT", "ServerAddress", "<ServerAddress>")

            if not self._config.has_option("MQTT", "ServerPort"):
                print "No Server Port"
                update = True
                self._config.set("MQTT", "ServerPort", "1883")

            if not self._config.has_section('TANK'):
                print "Adding Tank part"
                update = True
                self._config.add_section("TANK")

            if not self._config.has_option("TANK", "Location"):
                print "No Tank Virtual Location"
                update = True
                self._config.set("TANK", "Location", "Port Of Spain")

            if not self._config.has_option("TANK", "LocationOffset"):
                print "No Tank Virtual Location Offset"
                update = True
                self._config.set("TANK", "LocationOffset", "0")

            if not self._config.has_option("TANK", "NightTemp"):
                print "No Tank Night Temperature"
                update = True
                self._config.set("TANK", "NightTemp", "23")

            if not self._config.has_option("TANK", "DayTemp"):
                print "No Tank Day Temperature"
                update = True
                self._config.set("TANK", "DayTemp", "24")

            if not self._config.has_option("TANK", "FertilizerInterval"):
                print "No Tank FertilizerInterval"
                update = True
                self._config.set("TANK", "FertilizerInterval", "3600")

            if not self._config.has_section('CHARTS'):
                print "Adding Charts part"
                update = True
                self._config.add_section("CHARTS")

            if not self._config.has_option("CHARTS", "GraphInterval"):
                print "No Tank GraphInterval"
                update = True
                self._config.set("CHARTS", "GraphInterval", "9000")

            if not self._config.has_option("CHARTS", "ChartsDir"):
                print "No ChartsDir name"
                update = True
                stop = True
                self._config.set("CHARTS", "ChartsDir", "<Chartsdir>")

            if not self._config.has_option("CHARTS", "DPI"):
                print "No DPI"
                update = True
                self._config.set("CHARTS", "DPI", "1024")

            if update:
                with open(self._configFileName, 'w') as f:
                    self._config.write(f)

            if stop:
                print "Please check config file"
                sys.exit(0)

    def _checkChartFolder(self):
        if not os.path.isdir(self._chartDir):
            print "Creating chartsDir"
            os.makedirs(self._chartDir)

    def __init__(self):
        threading.Thread.__init__(self)
        self.setDaemon(True)
        self._homeDir         = os.path.expanduser("~/.sensomatic")
        self._configFileName  = self._homeDir + '/config.ini'
        self._configMTime     = 0
        self._config          = ConfigParser.ConfigParser()
        self._readConfig()
        self._template        = TemplateMatcher()
        self._info            = InformationFetcher()
        self._mqclient        = mqtt.Client("Tank", clean_session=True)
        self._daystate        = Tank.NIGHT
        self._twitterdaystate = Tank.NIGHT
        self._lastfurtilizer  = time.time()
        self._lastcharts      = time.time()
        self._sunpercentage   = 0
        self._moonpercentage  = 0
        self._chartDir        = self._config.get("CHARTS", "ChartsDir")
        self._chartDPI        = int(self._config.get("CHARTS", "DPI"))
        self._checkChartFolder()
        plt.rcdefaults()

    def _on_connect(self, client, userdata, rc, msg):
        print "Connected Tank with result code %s" % rc
        self._mqclient.subscribe("livingroom/tank/#")

    def _on_message(self, client, userdata, msg):
        #print "Mq Received on channel %s -> %s" % (msg.topic, msg.payload)
        pass

    def _on_disconnect(self, client, userdata, msg):
        print "Disconnect MQTTRulez"

    def updateSunAndMoon(self):
        now                               = datetime.datetime.today()
        dawn, sunrise, noon, sunset, dusk = self._info.getSunTimes(self._config.get("TANK", "Location"), int(self._config.get("TANK", "LocationOffset")))
        moonPhase                         = self._info.getMoonPhase(self._config.get("TANK", "Location"))
        moonElevation, _                  = self._info.getMoonPosition()

        if (dawn < now < sunrise):
            duration = sunrise - dawn
            done     = now - dawn
            self._daystate = Tank.DAWN
            self._sunpercentage = int((done.total_seconds() / duration.total_seconds()) * 100)

        elif (sunrise < now < sunset):
            self._daystate = Tank.DAY
            self._sunpercentage = 100

        elif (sunset < now < dusk):
            duration = dusk - sunset
            done = now - sunset
            self._daystate      = Tank.SUNSET
            self._sunpercentage = int((1.0 - (done.total_seconds() / duration.total_seconds())) * 100)

        else:
            self._daystate      = Tank.NIGHT
            self._sunpercentage = 0

        # 0 = New moon, 7 = First quarter, 14 = Full moon, 21 = Last quarter
        moonphasepercentage = 0.0

        if (0 < moonPhase < 14):
            moonphasepercentage = 1.0 - ( (14.0 - (moonPhase       ) ) / 14.0)
        elif (14 < moonPhase < 28):
            moonphasepercentage =       ( (14.0 - (moonPhase - 14.0) ) / 14.0)

        if moonElevation > 0:
            self._moonpercentage = int(moonphasepercentage * (moonElevation / 90.0) * 100)
        else:
            self._moonpercentage = 0

    def publishMQTT(self):
        self._mqclient.publish("livingroom/tank/whitelight", self._sunpercentage )
        self._mqclient.publish("livingroom/tank/bluelight",  self._moonpercentage)

        if self._daystate in (Tank.DAWN, Tank.DAY, Tank.SUNSET):
            self._mqclient.publish("livingroom/tank/settemp",   self._config.get("TANK", "DayTemp"))
        else:
            self._mqclient.publish("livingroom/tank/settemp",   self._config.get("TANK", "NightTemp"))

    def publishTwitter(self):
        if self._twitterdaystate is not self._daystate:
            if self._daystate == Tank.DAWN:
                self._mqclient.publish("twitter/text", "Switching light scene to dawn and rise the light level.")
            if self._daystate == Tank.DAY:
                self._mqclient.publish("twitter/text", "Switching light scene to day.")
            if self._daystate == Tank.SUNSET:
                self._mqclient.publish("twitter/text", "Switching light scene to sunset and lover the light level.")
            if self._daystate == Tank.NIGHT:
                self._mqclient.publish("twitter/text", "Switching light scene to night.")
            self._twitterdaystate = self._daystate

    def publishFertilizer(self):
        now = time.time()
        if self._daystate == Tank.DAY:
            if (now - self._lastfurtilizer) > int(self._config.get("TANK", "FertilizerInterval")):
                self._mqclient.publish("livingroom/tank/fertilizer", 1)
                self._mqclient.publish("twitter/text", "Adding some material of natural or synthetic origin (other than liming materials). " + str(now))
                self._lastfurtilizer = now

    def getCloudantData(self, start, end):
        auth = HTTPBasicAuth(self._config.get("CLOUDANT", "username"),
                             self._config.get("CLOUDANT", "password"))
        url = "%s/usshorizon/_design/livingroom/_view/tank?descending=false&startkey=%f&endkey=%f" % (self._config.get("CLOUDANT", "serveraddress"), start, end)
        j = json.loads(requests.get(url, auth=auth).content)
        return j['rows']

    def getAll(self):
        return self.getCloudantData(time.time() - 10 * YEAR, time.time())

    def get1Year(self):
        return self.getCloudantData(time.time() - YEAR, time.time())

    def get1Month(self):
        return self.getCloudantData(time.time() - MONTH, time.time())

    def get1Week(self):
        return self.getCloudantData(time.time() - WEEK, time.time())

    def get1Day(self):
        return self.getCloudantData(time.time() -  DAY, time.time())

    def get1Hour(self):
        return self.getCloudantData(time.time() -  HOUR, time.time())

    def calculateCharts(self):
        data = self.get1Week()
        print "Data fetched"
        tz              = pytz.timezone('Europe/Berlin')
        timekey         = [None] * len(data)
        timeval         = [None] * len(data)
        watertemp       = [None] * len(data)
        watermax        = 0.0
        watermin        = 99.0
        heater          = [False] * len(data)
        heatergraph     = [23.8] * len(data)
        waterlevelgraph = [23.6] * len(data)
        percentage      = [None] * len(data)
        filtersize      = 8000

        for i in range(len(data)):
            timekey[i]   = datetime.datetime.fromtimestamp(data[i]['key'], tz=tz)
            timeval[i]   = data[i]['key']
            watertemp[i] = data[i]['value']['watertemp']
            watermax     = max(watermax, float(data[i]['value']['watertemp']))
            watermin     = min(watermin, float(data[i]['value']['watertemp']))
            if float(data[i]['value']['heater']) == 1.0:
                heater[i]      = True
                heatergraph[i] = 23.9
            if float(data[i]['value']['waterlevel']) == 1.0:
                waterlevelgraph[i] = 23.7

        for i in range(filtersize, len(heater)):
            timeon = 0.0
            for m in range(i - filtersize, i):
                if heater[m]:
                    timeon += timeval[m] - timeval[m - 1]
            percentage[i] = (timeon / (timeval[i] - timeval[i - filtersize])) * 100

        print "Calculation done"

        # Water temperature
        duration = DAY / 10
        fig = plt.figure(figsize=(32, 13), dpi=self._chartDPI, edgecolor='yellow')
        axes = fig.gca().set_ylim([23.5, watermax + 0.5])
        ax = fig.add_subplot(111)
        ax.plot(timekey[-duration:], watertemp[-duration:], 'green', timekey[-duration:], heatergraph[-duration:], 'red', timekey[-duration:], waterlevelgraph[-duration:], 'blue')
        plt.draw()
        filename = "/%d-Watertemp.png" % (int(time.time()))
        plt.savefig(self._chartDir + filename, dpi = self._chartDPI)

        # Heater Percentage
        duration = (5 * DAY) / 10
        fig = plt.figure(figsize=(32, 13), dpi=self._chartDPI, edgecolor='yellow')
        axes = fig.gca().set_ylim([0, 40])
        ax = fig.add_subplot(111)
        ax.plot(timekey[-duration:], percentage[-duration:], 'red')
        plt.draw()
        filename2 = "/%d-Heaterstat.png" % (int(time.time()))
        plt.savefig(self._chartDir + filename2, dpi = self._chartDPI)
        return filename, filename2

    def publishCharts(self):
        now = time.time()
        if (now - self._lastcharts) > int(self._config.get("CHARTS", "GraphInterval")):
            waterfilename, heaterfilename = self.calculateCharts()
            self._mqclient.publish("twitter/picture" + waterfilename, "water temperature, heater usage and water level")
            self._mqclient.publish("twitter/picture" + heaterfilename, "heater percentage")
            self._lastcharts = now

    def run(self):
        self._mqclient.connect(self._config.get("MQTT", "ServerAddress"), self._config.get("MQTT", "ServerPort"), 60)
        self._mqclient.on_connect    = self._on_connect
        self._mqclient.on_message    = self._on_message
        self._mqclient.on_disconnect = self._on_disconnect
        self._mqclient.loop_start()

        while True:
            self._readConfig()
            self.updateSunAndMoon()
            self.publishMQTT()
            self.publishTwitter()
            self.publishFertilizer()
            self.publishCharts()
            time.sleep(15)

if __name__ == '__main__':
    print "Start"
    t = Tank()
    t.start()
    time.sleep(100)
    print "End"

<commit_msg>Deactivate charts<commit_after>import os
import time
import json
import pytz
import datetime
import requests
import threading
import ConfigParser
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot  as     plt
import paho.mqtt.client   as     mqtt
from   InformationFetcher import InformationFetcher
from   Template           import TemplateMatcher
from   requests.auth      import HTTPBasicAuth

SECOND =   1
MINUTE =  60 * SECOND
HOUR   =  60 * MINUTE
DAY    =  24 * HOUR
WEEK   =   7 * DAY
MONTH  =  31 * DAY
YEAR   = 365 * DAY

class Tank(threading.Thread):

    DAWN   = 0
    DAY    = 1
    SUNSET = 2
    NIGHT  = 3

    def _readConfig(self):

        if self._configMTime != os.stat(self._configFileName).st_mtime:

            print "Reread config file for tank"
            self._configMTime = os.stat(self._configFileName).st_mtime
            update = False
            stop   = False

            if not os.path.isdir(self._homeDir):
                print "Creating homeDir"
                os.makedirs(self._homeDir)

            if os.path.isfile(self._configFileName):
                self._config.read(self._configFileName)
            else:
                print "Config file not found"
                update = True

            if not self._config.has_section('MQTT'):
                print "Adding MQTT part"
                update = True
                self._config.add_section("MQTT")

            if not self._config.has_option("MQTT", "ServerAddress"):
                print "No Server Address"
                update = True
                self._config.set("MQTT", "ServerAddress", "<ServerAddress>")

            if not self._config.has_option("MQTT", "ServerPort"):
                print "No Server Port"
                update = True
                self._config.set("MQTT", "ServerPort", "1883")

            if not self._config.has_section('TANK'):
                print "Adding Tank part"
                update = True
                self._config.add_section("TANK")

            if not self._config.has_option("TANK", "Location"):
                print "No Tank Virtual Location"
                update = True
                self._config.set("TANK", "Location", "Port Of Spain")

            if not self._config.has_option("TANK", "LocationOffset"):
                print "No Tank Virtual Location Offset"
                update = True
                self._config.set("TANK", "LocationOffset", "0")

            if not self._config.has_option("TANK", "NightTemp"):
                print "No Tank Night Temperature"
                update = True
                self._config.set("TANK", "NightTemp", "23")

            if not self._config.has_option("TANK", "DayTemp"):
                print "No Tank Day Temperature"
                update = True
                self._config.set("TANK", "DayTemp", "24")

            if not self._config.has_option("TANK", "FertilizerInterval"):
                print "No Tank FertilizerInterval"
                update = True
                self._config.set("TANK", "FertilizerInterval", "3600")

            if not self._config.has_section('CHARTS'):
                print "Adding Charts part"
                update = True
                self._config.add_section("CHARTS")

            if not self._config.has_option("CHARTS", "GraphInterval"):
                print "No Tank GraphInterval"
                update = True
                self._config.set("CHARTS", "GraphInterval", "9000")

            if not self._config.has_option("CHARTS", "ChartsDir"):
                print "No ChartsDir name"
                update = True
                stop = True
                self._config.set("CHARTS", "ChartsDir", "<Chartsdir>")

            if not self._config.has_option("CHARTS", "DPI"):
                print "No DPI"
                update = True
                self._config.set("CHARTS", "DPI", "1024")

            if update:
                with open(self._configFileName, 'w') as f:
                    self._config.write(f)

            if stop:
                print "Please check config file"
                sys.exit(0)

    def _checkChartFolder(self):
        if not os.path.isdir(self._chartDir):
            print "Creating chartsDir"
            os.makedirs(self._chartDir)

    def __init__(self):
        threading.Thread.__init__(self)
        self.setDaemon(True)
        self._homeDir         = os.path.expanduser("~/.sensomatic")
        self._configFileName  = self._homeDir + '/config.ini'
        self._configMTime     = 0
        self._config          = ConfigParser.ConfigParser()
        self._readConfig()
        self._template        = TemplateMatcher()
        self._info            = InformationFetcher()
        self._mqclient        = mqtt.Client("Tank", clean_session=True)
        self._daystate        = Tank.NIGHT
        self._twitterdaystate = Tank.NIGHT
        self._lastfurtilizer  = time.time()
        self._lastcharts      = time.time()
        self._sunpercentage   = 0
        self._moonpercentage  = 0
        self._chartDir        = self._config.get("CHARTS", "ChartsDir")
        self._chartDPI        = int(self._config.get("CHARTS", "DPI"))
        self._checkChartFolder()
        plt.rcdefaults()

    def _on_connect(self, client, userdata, rc, msg):
        print "Connected Tank with result code %s" % rc
        self._mqclient.subscribe("livingroom/tank/#")

    def _on_message(self, client, userdata, msg):
        #print "Mq Received on channel %s -> %s" % (msg.topic, msg.payload)
        pass

    def _on_disconnect(self, client, userdata, msg):
        print "Disconnect MQTTRulez"

    def updateSunAndMoon(self):
        now                               = datetime.datetime.today()
        dawn, sunrise, noon, sunset, dusk = self._info.getSunTimes(self._config.get("TANK", "Location"), int(self._config.get("TANK", "LocationOffset")))
        moonPhase                         = self._info.getMoonPhase(self._config.get("TANK", "Location"))
        moonElevation, _                  = self._info.getMoonPosition()

        if (dawn < now < sunrise):
            duration = sunrise - dawn
            done     = now - dawn
            self._daystate = Tank.DAWN
            self._sunpercentage = int((done.total_seconds() / duration.total_seconds()) * 100)

        elif (sunrise < now < sunset):
            self._daystate = Tank.DAY
            self._sunpercentage = 100

        elif (sunset < now < dusk):
            duration = dusk - sunset
            done = now - sunset
            self._daystate      = Tank.SUNSET
            self._sunpercentage = int((1.0 - (done.total_seconds() / duration.total_seconds())) * 100)

        else:
            self._daystate      = Tank.NIGHT
            self._sunpercentage = 0

        # 0 = New moon, 7 = First quarter, 14 = Full moon, 21 = Last quarter
        moonphasepercentage = 0.0

        if (0 < moonPhase < 14):
            moonphasepercentage = 1.0 - ( (14.0 - (moonPhase       ) ) / 14.0)
        elif (14 < moonPhase < 28):
            moonphasepercentage =       ( (14.0 - (moonPhase - 14.0) ) / 14.0)

        if moonElevation > 0:
            self._moonpercentage = int(moonphasepercentage * (moonElevation / 90.0) * 100)
        else:
            self._moonpercentage = 0

    def publishMQTT(self):
        self._mqclient.publish("livingroom/tank/whitelight", self._sunpercentage )
        self._mqclient.publish("livingroom/tank/bluelight",  self._moonpercentage)

        if self._daystate in (Tank.DAWN, Tank.DAY, Tank.SUNSET):
            self._mqclient.publish("livingroom/tank/settemp",   self._config.get("TANK", "DayTemp"))
        else:
            self._mqclient.publish("livingroom/tank/settemp",   self._config.get("TANK", "NightTemp"))

    def publishTwitter(self):
        if self._twitterdaystate is not self._daystate:
            if self._daystate == Tank.DAWN:
                self._mqclient.publish("twitter/text", "Switching light scene to dawn and rise the light level.")
            if self._daystate == Tank.DAY:
                self._mqclient.publish("twitter/text", "Switching light scene to day.")
            if self._daystate == Tank.SUNSET:
                self._mqclient.publish("twitter/text", "Switching light scene to sunset and lover the light level.")
            if self._daystate == Tank.NIGHT:
                self._mqclient.publish("twitter/text", "Switching light scene to night.")
            self._twitterdaystate = self._daystate

    def publishFertilizer(self):
        now = time.time()
        if self._daystate == Tank.DAY:
            if (now - self._lastfurtilizer) > int(self._config.get("TANK", "FertilizerInterval")):
                self._mqclient.publish("livingroom/tank/fertilizer", 1)
                self._mqclient.publish("twitter/text", "Adding some material of natural or synthetic origin (other than liming materials). " + str(now))
                self._lastfurtilizer = now

    def getCloudantData(self, start, end):
        auth = HTTPBasicAuth(self._config.get("CLOUDANT", "username"),
                             self._config.get("CLOUDANT", "password"))
        url = "%s/usshorizon/_design/livingroom/_view/tank?descending=false&startkey=%f&endkey=%f" % (self._config.get("CLOUDANT", "serveraddress"), start, end)
        j = json.loads(requests.get(url, auth=auth).content)
        return j['rows']

    def getAll(self):
        return self.getCloudantData(time.time() - 10 * YEAR, time.time())

    def get1Year(self):
        return self.getCloudantData(time.time() - YEAR, time.time())

    def get1Month(self):
        return self.getCloudantData(time.time() - MONTH, time.time())

    def get1Week(self):
        return self.getCloudantData(time.time() - WEEK, time.time())

    def get1Day(self):
        return self.getCloudantData(time.time() -  DAY, time.time())

    def get1Hour(self):
        return self.getCloudantData(time.time() -  HOUR, time.time())

    def calculateCharts(self):
        data = self.get1Week()
        print "Data fetched"
        tz              = pytz.timezone('Europe/Berlin')
        timekey         = [None] * len(data)
        timeval         = [None] * len(data)
        watertemp       = [None] * len(data)
        watermax        = 0.0
        watermin        = 99.0
        heater          = [False] * len(data)
        heatergraph     = [23.8] * len(data)
        waterlevelgraph = [23.6] * len(data)
        percentage      = [None] * len(data)
        filtersize      = 8000

        for i in range(len(data)):
            timekey[i]   = datetime.datetime.fromtimestamp(data[i]['key'], tz=tz)
            timeval[i]   = data[i]['key']
            watertemp[i] = data[i]['value']['watertemp']
            watermax     = max(watermax, float(data[i]['value']['watertemp']))
            watermin     = min(watermin, float(data[i]['value']['watertemp']))
            if float(data[i]['value']['heater']) == 1.0:
                heater[i]      = True
                heatergraph[i] = 23.9
            if float(data[i]['value']['waterlevel']) == 1.0:
                waterlevelgraph[i] = 23.7

        for i in range(filtersize, len(heater)):
            timeon = 0.0
            for m in range(i - filtersize, i):
                if heater[m]:
                    timeon += timeval[m] - timeval[m - 1]
            percentage[i] = (timeon / (timeval[i] - timeval[i - filtersize])) * 100

        print "Calculation done"

        # Water temperature
        duration = DAY / 10
        fig = plt.figure(figsize=(32, 13), dpi=self._chartDPI, edgecolor='yellow')
        axes = fig.gca().set_ylim([23.5, watermax + 0.5])
        ax = fig.add_subplot(111)
        ax.plot(timekey[-duration:], watertemp[-duration:], 'green', timekey[-duration:], heatergraph[-duration:], 'red', timekey[-duration:], waterlevelgraph[-duration:], 'blue')
        plt.draw()
        filename = "/%d-Watertemp.png" % (int(time.time()))
        plt.savefig(self._chartDir + filename, dpi = self._chartDPI)

        # Heater Percentage
        duration = (5 * DAY) / 10
        fig = plt.figure(figsize=(32, 13), dpi=self._chartDPI, edgecolor='yellow')
        axes = fig.gca().set_ylim([0, 40])
        ax = fig.add_subplot(111)
        ax.plot(timekey[-duration:], percentage[-duration:], 'red')
        plt.draw()
        filename2 = "/%d-Heaterstat.png" % (int(time.time()))
        plt.savefig(self._chartDir + filename2, dpi = self._chartDPI)
        return filename, filename2

    def publishCharts(self):
        now = time.time()
        if (now - self._lastcharts) > int(self._config.get("CHARTS", "GraphInterval")):
            waterfilename, heaterfilename = self.calculateCharts()
            self._mqclient.publish("twitter/picture" + waterfilename, "water temperature, heater usage and water level")
            self._mqclient.publish("twitter/picture" + heaterfilename, "heater percentage")
            self._lastcharts = now

    def run(self):
        self._mqclient.connect(self._config.get("MQTT", "ServerAddress"), self._config.get("MQTT", "ServerPort"), 60)
        self._mqclient.on_connect    = self._on_connect
        self._mqclient.on_message    = self._on_message
        self._mqclient.on_disconnect = self._on_disconnect
        self._mqclient.loop_start()

        while True:
            self._readConfig()
            self.updateSunAndMoon()
            self.publishMQTT()
            self.publishTwitter()
            self.publishFertilizer()
            #self.publishCharts()
            time.sleep(15)

if __name__ == '__main__':
    print "Start"
    t = Tank()
    t.start()
    time.sleep(100)
    print "End"

<|endoftext|>


<commit_before># Copyright (c) 2017 pymagicc authors:
#   Robert Gieseke <robert.gieseke@pik-potsdam.de>
# Free software under GNU Affero General Public License v3, see LICENSE
#
# The compiled MAGICC binary (http://www.magicc.org/download6) by Tom Wigley,
# Sarah Raper, and Malte Meinshausen included in this package is licensed under
# a Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported
# License (https://creativecommons.org/licenses/by-nc-sa/3.0/)
import logging
import subprocess

from ._version import get_versions
from .config import config as _config
from .core import MAGICC6, MAGICC7  # noqa
from .io import MAGICCData, read_scen_file  # noqa
from .scenarios import rcp26, rcp45, rcp60, rcp85, rcps, zero_emissions  # noqa

__version__ = get_versions()["version"]
del get_versions

if not _config["is_windows"]:
    wine_installed = (
        subprocess.call(
            "type wine", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
        )
        == 0
    )
    if not wine_installed:
        logging.warning("Wine is not installed")

def run(scenario, magicc_version=6, **kwargs):
    """
    Run a MAGICC scenario and return output data and (optionally) config parameters.

    As a reminder, putting ``out_parameters=1`` will cause MAGICC to write out its
    parameters into ``out/PARAMETERS.OUT`` and they will then be read into
    ``output.metadata["parameters"]`` where ``output`` is the returned object.

    Parameters
    ----------
    scenario : :obj:`pymagicc.io.MAGICCData`
        Scenario to run

    magicc_version : int
        MAGICC version to use for the run

    **kwargs
        Parameters overwriting default parameters

    Raises
    ------
    ValueError
        If the magicc_version is not available

    Returns
    -------
    output : :obj:`pymagicc.io.MAGICCData`
        Output of the run with the data in the ``df`` attribute and parameters and
        other metadata in the ``metadata attribute``
    """
    if magicc_version == 6:
        magicc_cls = MAGICC6
    elif magicc_version == 7:
        magicc_cls = MAGICC7
    else:
        raise ValueError("MAGICC version {} is not available".format(magicc_version))

    with magicc_cls() as magicc:
        results = magicc.run(scenario=scenario, **kwargs)

    return results
<commit_msg>Fix up logger instantiation<commit_after># Copyright (c) 2017 pymagicc authors:
#   Robert Gieseke <robert.gieseke@pik-potsdam.de>
# Free software under GNU Affero General Public License v3, see LICENSE
#
# The compiled MAGICC binary (http://www.magicc.org/download6) by Tom Wigley,
# Sarah Raper, and Malte Meinshausen included in this package is licensed under
# a Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported
# License (https://creativecommons.org/licenses/by-nc-sa/3.0/)
import logging
import subprocess

from ._version import get_versions
from .config import config as _config
from .core import MAGICC6, MAGICC7  # noqa
from .io import MAGICCData, read_scen_file  # noqa
from .scenarios import rcp26, rcp45, rcp60, rcp85, rcps, zero_emissions  # noqa

__version__ = get_versions()["version"]
del get_versions

logger = logging.getLogger(__name__)

if not _config["is_windows"]:
    wine_installed = (
        subprocess.call(
            "type wine", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
        )
        == 0
    )
    if not wine_installed:
        logger.warning("Wine is not installed")

def run(scenario, magicc_version=6, **kwargs):
    """
    Run a MAGICC scenario and return output data and (optionally) config parameters.

    As a reminder, putting ``out_parameters=1`` will cause MAGICC to write out its
    parameters into ``out/PARAMETERS.OUT`` and they will then be read into
    ``output.metadata["parameters"]`` where ``output`` is the returned object.

    Parameters
    ----------
    scenario : :obj:`pymagicc.io.MAGICCData`
        Scenario to run

    magicc_version : int
        MAGICC version to use for the run

    **kwargs
        Parameters overwriting default parameters

    Raises
    ------
    ValueError
        If the magicc_version is not available

    Returns
    -------
    output : :obj:`pymagicc.io.MAGICCData`
        Output of the run with the data in the ``df`` attribute and parameters and
        other metadata in the ``metadata attribute``
    """
    if magicc_version == 6:
        magicc_cls = MAGICC6
    elif magicc_version == 7:
        magicc_cls = MAGICC7
    else:
        raise ValueError("MAGICC version {} is not available".format(magicc_version))

    with magicc_cls() as magicc:
        results = magicc.run(scenario=scenario, **kwargs)

    return results
<|endoftext|>


<commit_before>#!/usr/bin/env python
import threading
import thread
import time

from vmthunder.drivers import fcg
from vmthunder.session import Session
from vmthunder.instance import Instance
from vmthunder.singleton import SingleTon
from vmthunder.openstack.common import log as logging
from vmthunder.drivers import volt

LOG = logging.getLogger(__name__)

@SingleTon
class Compute():
    def __init__(self):
        self.sessions = {}
        self.instances = {}
        self.cache_group = fcg.create_group()
        LOG.debug("creating a Compute_node")

    def heartbeat(self):
        info = volt.heartbeat()
        for each_key in self.sessions:
            for session in info:
                if self.sessions[each_key].peer_id == session['peer_id']:
                    self.sessions[each_key].adjust_for_heartbeat(session['parents'])
                    break
            if not self.sessions[each_key].has_vm():
                if self.sessions[each_key].destroy():
                    del self.sessions[each_key]

    def destroy(self, vm_name):
        if self.instances.has_key(vm_name):
            instance = self.instances[vm_name]
            #session = self.sessions[instance.volume_name]
            instance.del_vm()
            #session.destroy(vm_name)
            del self.instances[vm_name]

    def list(self):
        def build_list_object(instances):
            instance_list = []
            for instance in instances.keys():
                instance_list.append({
                    'vm_name': instances[instance].vm_name,
                })
            return dict(instances=instance_list)

        return build_list_object(self.instances)

    def create(self, volume_name, vm_name, connections, snapshot_dev):
        if vm_name not in self.instances.keys():
            LOG.debug("in compute to execute the method create")
            if not self.sessions.has_key(volume_name):
                self.sessions[volume_name] = Session(volume_name)
            session = self.sessions[volume_name]
            origin_path = session.deploy_image(connections)
            self.instances[vm_name] = Instance.factory(vm_name, session, snapshot_dev)
            LOG.debug("origin is %s" % origin_path)
            self.instances[vm_name].start_vm(origin_path)
            return self.instances[vm_name].snapshot_path

    def adjust_structure(self, volume_name, delete_connections, add_connections):
        if volume_name in self.sessions.keys():
            session = self.sessions[volume_name]
            session.adjust_structure(delete_connections, add_connections)

<commit_msg>solve a bug<commit_after>#!/usr/bin/env python
import threading
import thread
import time

from vmthunder.drivers import fcg
from vmthunder.session import Session
from vmthunder.instance import Instance
from vmthunder.singleton import SingleTon
from vmthunder.openstack.common import log as logging
from vmthunder.drivers import volt

LOG = logging.getLogger(__name__)

@SingleTon
class Compute():
    def __init__(self):
        self.sessions = {}
        self.instances = {}
        self.cache_group = fcg.create_group()
        LOG.debug("creating a Compute_node")

    def heartbeat(self):
        info = volt.heartbeat()
        to_delete_session = []
        for each_key in self.sessions:
            for session in info:
                if self.sessions[each_key].peer_id == session['peer_id']:
                    self.sessions[each_key].adjust_for_heartbeat(session['parents'])
                    break
            if not self.sessions[each_key].has_vm():
                if self.sessions[each_key].destroy():
                    to_delete_session.append(each_key)
        for key in to_delete_session:
            del self.sessions[key]

    def destroy(self, vm_name):
        if self.instances.has_key(vm_name):
            instance = self.instances[vm_name]
            #session = self.sessions[instance.volume_name]
            instance.del_vm()
            #session.destroy(vm_name)
            del self.instances[vm_name]

    def list(self):
        def build_list_object(instances):
            instance_list = []
            for instance in instances.keys():
                instance_list.append({
                    'vm_name': instances[instance].vm_name,
                })
            return dict(instances=instance_list)

        return build_list_object(self.instances)

    def create(self, volume_name, vm_name, connections, snapshot_dev):
        if vm_name not in self.instances.keys():
            LOG.debug("in compute to execute the method create")
            if not self.sessions.has_key(volume_name):
                self.sessions[volume_name] = Session(volume_name)
            session = self.sessions[volume_name]
            origin_path = session.deploy_image(connections)
            self.instances[vm_name] = Instance.factory(vm_name, session, snapshot_dev)
            LOG.debug("origin is %s" % origin_path)
            self.instances[vm_name].start_vm(origin_path)
            return self.instances[vm_name].snapshot_path

    def adjust_structure(self, volume_name, delete_connections, add_connections):
        if volume_name in self.sessions.keys():
            session = self.sessions[volume_name]
            session.adjust_structure(delete_connections, add_connections)

<|endoftext|>


<commit_before>#!/usr/bin/env python3

import datetime
import logging
import os
import random
import re
import signal
import subprocess
import sys
import telnetlib
import time

import IPy

import vrnetlab

def handle_SIGCHLD(signal, frame):
    os.waitpid(-1, os.WNOHANG)

def handle_SIGTERM(signal, frame):
    sys.exit(0)

signal.signal(signal.SIGINT, handle_SIGTERM)
signal.signal(signal.SIGTERM, handle_SIGTERM)
signal.signal(signal.SIGCHLD, handle_SIGCHLD)

TRACE_LEVEL_NUM = 9
logging.addLevelName(TRACE_LEVEL_NUM, "TRACE")
def trace(self, message, *args, **kws):
    # Yes, logger takes its '*args' as 'args'.
    if self.isEnabledFor(TRACE_LEVEL_NUM):
        self._log(TRACE_LEVEL_NUM, message, args, **kws)
logging.Logger.trace = trace

class VMX_vcp(vrnetlab.VM):
    def __init__(self, username, password, image, install_mode=False):
        super(VMX_vcp, self).__init__(username, password, disk_image=image, ram=2048)
        self.install_mode = install_mode
        self.num_nics = 0
        self.qemu_args.extend(["-drive", "if=ide,file=/vmx/vmxhdd.img"])
        self.smbios = ["type=0,vendor=Juniper",
                       "type=1,manufacturer=Juniper,product=VM-vcp_vmx2-161-re-0,version=0.1.0"]
        # add metadata image if it exists
        if os.path.exists("/vmx/metadata-usb-re.img"):
            self.qemu_args.extend(["-usb", "-usbdevice", "disk:format=raw:/vmx/metadata-usb-re.img"])

    def start(self):
        # use parent class start() function
        super(VMX_vcp, self).start()
        # add interface to internal control plane bridge
        if not self.install_mode:
            vrnetlab.run_command(["brctl", "addif", "int_cp", "vcp-int"])
            vrnetlab.run_command(["ip", "link", "set", "vcp-int", "up"])

    def gen_mgmt(self):
        """ Generate mgmt interface(s)

            We override the default function since we want a virtio NIC to the
            vFPC
        """
        # call parent function to generate first mgmt interface (e1000)
        res = super(VMX_vcp, self).gen_mgmt()
        if not self.install_mode:
            # add virtio NIC for internal control plane interface to vFPC
            res.append("-device")
            res.append("virtio-net-pci,netdev=vcp-int,mac=%s" % vrnetlab.gen_mac(1))
            res.append("-netdev")
            res.append("tap,ifname=vcp-int,id=vcp-int,script=no,downscript=no")
        return res

    def bootstrap_spin(self):
        """ This function should be called periodically to do work.

            returns False when it has failed and given up, otherwise True
        """

        if self.spins > 300:
            # too many spins with no result -> restart
            self.logger.warning("no output from serial console, restarting VCP")
            self.stop()
            self.start()
            self.spins = 0
            return

        (ridx, match, res) = self.tn.expect([b"login:", b"root@(%|:~ #)"], 1)
        if match: # got a match!
            if ridx == 0: # matched login prompt, so should login
                self.logger.info("matched login prompt")
                self.wait_write("root", wait=None)
            if ridx == 1:
                if self.install_mode:
                    self.logger.info("requesting power-off")
                    self.wait_write("cli", None)
                    self.wait_write("request system power-off", '>')
                    self.wait_write("yes", 'Power Off the system')
                    self.running = True
                    return
                # run main config!
                self.bootstrap_config()
                self.running = True
                self.tn.close()
                # calc startup time
                startup_time = datetime.datetime.now() - self.start_time
                self.logger.info("Startup complete in: %s" % startup_time)
                return

        else:
            # no match, if we saw some output from the router it's probably
            # booting, so let's give it some more time
            if res != b'':
                self.logger.trace("OUTPUT VCP: %s" % res.decode())
                # reset spins if we saw some output
                self.spins = 0

        self.spins += 1

    def bootstrap_config(self):
        """ Do the actual bootstrap config
        """
        self.wait_write("cli", None)
        self.wait_write("configure", '>', 10)
        self.wait_write("set chassis fpc 0 pic 0 number-of-ports 96")
        self.wait_write("set system services ssh")
        self.wait_write("set system services netconf ssh")
        self.wait_write("set system services netconf rfc-compliant")
        self.wait_write("set system login user %s class super-user authentication plain-text-password" % self.username)
        self.wait_write(self.password, 'New password:')
        self.wait_write(self.password, 'Retype new password:')
        self.wait_write("set system root-authentication plain-text-password")
        self.wait_write(self.password, 'New password:')
        self.wait_write(self.password, 'Retype new password:')
        self.wait_write("set interfaces fxp0 unit 0 family inet address 10.0.0.15/24")
        self.wait_write("commit")
        self.wait_write("exit")

    def wait_write(self, cmd, wait='#', timeout=None):
        """ Wait for something and then send command
        """
        if wait:
            self.logger.trace("Waiting for %s" % wait)
            while True:
                (ridx, match, res) = self.tn.expect([wait.encode(), b"Retry connection attempts"], timeout=timeout)
                if match:
                    if ridx == 0:
                        break
                    if ridx == 1:
                        self.tn.write("yes\r".encode())
            self.logger.trace("Read: %s" % res.decode())
        self.logger.debug("writing to serial console: %s" % cmd)
        self.tn.write("{}\r".format(cmd).encode())

class VMX_vfpc(vrnetlab.VM):
    def __init__(self, version):
        super(VMX_vfpc, self).__init__(None, None, disk_image = "/vmx/vfpc.img", num=1)
        self.version = version
        self.num_nics = 96

        self.nic_type = "virtio-net-pci"
        self.qemu_args.extend(["-cpu", "SandyBridge", "-M", "pc", "-smp", "3"])
        # add metadata image if it exists
        if os.path.exists("/vmx/metadata-usb-fpc0.img"):
            self.qemu_args.extend(["-usb", "-usbdevice", "disk:format=raw:/vmx/metadata-usb-fpc0.img"])

    def gen_mgmt(self):
        res = []
        # mgmt interface
        res.extend(["-device", "virtio-net-pci,netdev=mgmt,mac=%s" % vrnetlab.gen_mac(0)])
        res.extend(["-netdev", "user,id=mgmt,net=10.0.0.0/24"])
        # internal control plane interface to vFPC
        res.extend(["-device", "virtio-net-pci,netdev=vfpc-int,mac=%s" %
                    vrnetlab.gen_mac(0)])
        res.extend(["-netdev",
                    "tap,ifname=vfpc-int,id=vfpc-int,script=no,downscript=no"])

        if self.version in ('15.1F6.9', '16.1R2.11'):
            # dummy interface for some vMX versions - not sure why vFPC wants
            # it but without it we get a misalignment
            res.extend(["-device", "virtio-net-pci,netdev=dummy,mac=%s" %
                                   vrnetlab.gen_mac(0)])
            res.extend(["-netdev", "tap,ifname=vfpc-dummy,id=dummy,script=no,downscript=no"])

        return res

    def start(self):
        # use parent class start() function
        super(VMX_vfpc, self).start()
        # add interface to internal control plane bridge
        vrnetlab.run_command(["brctl", "addif", "int_cp", "vfpc-int"])
        vrnetlab.run_command(["ip", "link", "set", "vfpc-int", "up"])

    def bootstrap_spin(self):
        (ridx, match, res) = self.tn.expect([b"localhost login", b"mounting /dev/sda2 on /mnt failed"], 1)
        if match:
            if ridx == 0: # got login - vFPC start succeeded!
                self.logger.info("vFPC successfully started")
                self.running = True
            if ridx == 1: # vFPC start failed - restart it
                self.logger.info("vFPC start failed, restarting")
                self.stop()
                self.start()
        if res != b'':
            pass
            #self.logger.trace("OUTPUT VFPC: %s" % res.decode())

        return

class VMX(vrnetlab.VR):
    """ Juniper vMX router
    """

    def __init__(self, username, password):
        self.version = None
        self.version_info = []
        self.read_version()

        super(VMX, self).__init__(username, password)

        self.vms = [ VMX_vcp(username, password, "/vmx/" + self.vcp_image), VMX_vfpc(self.version) ]

        # set up bridge for connecting VCP with vFPC
        vrnetlab.run_command(["brctl", "addbr", "int_cp"])
        vrnetlab.run_command(["ip", "link", "set", "int_cp", "up"])

    def read_version(self):
        for e in os.listdir("/vmx/"):
            m = re.search("-(([0-9][0-9])\.([0-9])([A-Z])([0-9]+)\.([0-9]+))", e)
            if m:
                self.vcp_image = e
                self.version = m.group(1)
                self.version_info = [int(m.group(2)), int(m.group(3)), m.group(4), int(m.group(5)), int(m.group(6))]

class VMX_installer(VMX):
    """ VMX installer

        Will start the VMX VCP and then shut it down. Booting the VCP for the
        first time requires the VCP itself to load some config and then it will
        restart. Subsequent boots will not require this restart. By running
        this "install" when building the docker image we can decrease the
        normal startup time of the vMX.
    """
    def __init__(self, username, password):
        self.version = None
        self.version_info = []
        self.read_version()

        super(VMX, self).__init__(username, password)

        self.vms = [ VMX_vcp(username, password, "/vmx/" + self.vcp_image, install_mode=True) ]

    def install(self):
        self.logger.info("Installing VMX")
        vcp = self.vms[0]
        while not vcp.running:
            vcp.work()
        time.sleep(30)
        vcp.stop()
        self.logger.info("Installation complete")

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser(description='')
    parser.add_argument('--trace', action='store_true', help='enable trace level logging')
    parser.add_argument('--username', default='vrnetlab', help='Username')
    parser.add_argument('--password', default='VR-netlab9', help='Password')
    parser.add_argument('--install', action='store_true', help='Install vMX')
    args = parser.parse_args()

    LOG_FORMAT = "%(asctime)s: %(module)-10s %(levelname)-8s %(message)s"
    logging.basicConfig(format=LOG_FORMAT)
    logger = logging.getLogger()

    logger.setLevel(logging.DEBUG)
    if args.trace:
        logger.setLevel(1)

    if args.install:
        vr = VMX_installer(args.username, args.password)
        vr.install()
    else:
        vr = VMX(args.username, args.password)
        vr.start()
<commit_msg>Wait for VMX to shut down on install<commit_after>#!/usr/bin/env python3

import datetime
import logging
import os
import random
import re
import signal
import subprocess
import sys
import telnetlib
import time

import IPy

import vrnetlab

def handle_SIGCHLD(signal, frame):
    os.waitpid(-1, os.WNOHANG)

def handle_SIGTERM(signal, frame):
    sys.exit(0)

signal.signal(signal.SIGINT, handle_SIGTERM)
signal.signal(signal.SIGTERM, handle_SIGTERM)
signal.signal(signal.SIGCHLD, handle_SIGCHLD)

TRACE_LEVEL_NUM = 9
logging.addLevelName(TRACE_LEVEL_NUM, "TRACE")
def trace(self, message, *args, **kws):
    # Yes, logger takes its '*args' as 'args'.
    if self.isEnabledFor(TRACE_LEVEL_NUM):
        self._log(TRACE_LEVEL_NUM, message, args, **kws)
logging.Logger.trace = trace

class VMX_vcp(vrnetlab.VM):
    def __init__(self, username, password, image, install_mode=False):
        super(VMX_vcp, self).__init__(username, password, disk_image=image, ram=2048)
        self.install_mode = install_mode
        self.num_nics = 0
        self.qemu_args.extend(["-drive", "if=ide,file=/vmx/vmxhdd.img"])
        self.smbios = ["type=0,vendor=Juniper",
                       "type=1,manufacturer=Juniper,product=VM-vcp_vmx2-161-re-0,version=0.1.0"]
        # add metadata image if it exists
        if os.path.exists("/vmx/metadata-usb-re.img"):
            self.qemu_args.extend(["-usb", "-usbdevice", "disk:format=raw:/vmx/metadata-usb-re.img"])

    def start(self):
        # use parent class start() function
        super(VMX_vcp, self).start()
        # add interface to internal control plane bridge
        if not self.install_mode:
            vrnetlab.run_command(["brctl", "addif", "int_cp", "vcp-int"])
            vrnetlab.run_command(["ip", "link", "set", "vcp-int", "up"])

    def gen_mgmt(self):
        """ Generate mgmt interface(s)

            We override the default function since we want a virtio NIC to the
            vFPC
        """
        # call parent function to generate first mgmt interface (e1000)
        res = super(VMX_vcp, self).gen_mgmt()
        if not self.install_mode:
            # add virtio NIC for internal control plane interface to vFPC
            res.append("-device")
            res.append("virtio-net-pci,netdev=vcp-int,mac=%s" % vrnetlab.gen_mac(1))
            res.append("-netdev")
            res.append("tap,ifname=vcp-int,id=vcp-int,script=no,downscript=no")
        return res

    def bootstrap_spin(self):
        """ This function should be called periodically to do work.

            returns False when it has failed and given up, otherwise True
        """

        if self.spins > 300:
            # too many spins with no result -> restart
            self.logger.warning("no output from serial console, restarting VCP")
            self.stop()
            self.start()
            self.spins = 0
            return

        (ridx, match, res) = self.tn.expect([b"login:", b"root@(%|:~ #)"], 1)
        if match: # got a match!
            if ridx == 0: # matched login prompt, so should login
                self.logger.info("matched login prompt")
                self.wait_write("root", wait=None)
            if ridx == 1:
                if self.install_mode:
                    self.logger.info("requesting power-off")
                    self.wait_write("cli", None)
                    self.wait_write("request system power-off", '>')
                    self.wait_write("yes", 'Power Off the system')
                    self.running = True
                    return
                # run main config!
                self.bootstrap_config()
                self.running = True
                self.tn.close()
                # calc startup time
                startup_time = datetime.datetime.now() - self.start_time
                self.logger.info("Startup complete in: %s" % startup_time)
                return

        else:
            # no match, if we saw some output from the router it's probably
            # booting, so let's give it some more time
            if res != b'':
                self.logger.trace("OUTPUT VCP: %s" % res.decode())
                # reset spins if we saw some output
                self.spins = 0

        self.spins += 1

    def bootstrap_config(self):
        """ Do the actual bootstrap config
        """
        self.wait_write("cli", None)
        self.wait_write("configure", '>', 10)
        self.wait_write("set chassis fpc 0 pic 0 number-of-ports 96")
        self.wait_write("set system services ssh")
        self.wait_write("set system services netconf ssh")
        self.wait_write("set system services netconf rfc-compliant")
        self.wait_write("set system login user %s class super-user authentication plain-text-password" % self.username)
        self.wait_write(self.password, 'New password:')
        self.wait_write(self.password, 'Retype new password:')
        self.wait_write("set system root-authentication plain-text-password")
        self.wait_write(self.password, 'New password:')
        self.wait_write(self.password, 'Retype new password:')
        self.wait_write("set interfaces fxp0 unit 0 family inet address 10.0.0.15/24")
        self.wait_write("commit")
        self.wait_write("exit")

    def wait_write(self, cmd, wait='#', timeout=None):
        """ Wait for something and then send command
        """
        if wait:
            self.logger.trace("Waiting for %s" % wait)
            while True:
                (ridx, match, res) = self.tn.expect([wait.encode(), b"Retry connection attempts"], timeout=timeout)
                if match:
                    if ridx == 0:
                        break
                    if ridx == 1:
                        self.tn.write("yes\r".encode())
            self.logger.trace("Read: %s" % res.decode())
        self.logger.debug("writing to serial console: %s" % cmd)
        self.tn.write("{}\r".format(cmd).encode())

class VMX_vfpc(vrnetlab.VM):
    def __init__(self, version):
        super(VMX_vfpc, self).__init__(None, None, disk_image = "/vmx/vfpc.img", num=1)
        self.version = version
        self.num_nics = 96

        self.nic_type = "virtio-net-pci"
        self.qemu_args.extend(["-cpu", "SandyBridge", "-M", "pc", "-smp", "3"])
        # add metadata image if it exists
        if os.path.exists("/vmx/metadata-usb-fpc0.img"):
            self.qemu_args.extend(["-usb", "-usbdevice", "disk:format=raw:/vmx/metadata-usb-fpc0.img"])

    def gen_mgmt(self):
        res = []
        # mgmt interface
        res.extend(["-device", "virtio-net-pci,netdev=mgmt,mac=%s" % vrnetlab.gen_mac(0)])
        res.extend(["-netdev", "user,id=mgmt,net=10.0.0.0/24"])
        # internal control plane interface to vFPC
        res.extend(["-device", "virtio-net-pci,netdev=vfpc-int,mac=%s" %
                    vrnetlab.gen_mac(0)])
        res.extend(["-netdev",
                    "tap,ifname=vfpc-int,id=vfpc-int,script=no,downscript=no"])

        if self.version in ('15.1F6.9', '16.1R2.11'):
            # dummy interface for some vMX versions - not sure why vFPC wants
            # it but without it we get a misalignment
            res.extend(["-device", "virtio-net-pci,netdev=dummy,mac=%s" %
                                   vrnetlab.gen_mac(0)])
            res.extend(["-netdev", "tap,ifname=vfpc-dummy,id=dummy,script=no,downscript=no"])

        return res

    def start(self):
        # use parent class start() function
        super(VMX_vfpc, self).start()
        # add interface to internal control plane bridge
        vrnetlab.run_command(["brctl", "addif", "int_cp", "vfpc-int"])
        vrnetlab.run_command(["ip", "link", "set", "vfpc-int", "up"])

    def bootstrap_spin(self):
        (ridx, match, res) = self.tn.expect([b"localhost login", b"mounting /dev/sda2 on /mnt failed"], 1)
        if match:
            if ridx == 0: # got login - vFPC start succeeded!
                self.logger.info("vFPC successfully started")
                self.running = True
            if ridx == 1: # vFPC start failed - restart it
                self.logger.info("vFPC start failed, restarting")
                self.stop()
                self.start()
        if res != b'':
            pass
            #self.logger.trace("OUTPUT VFPC: %s" % res.decode())

        return

class VMX(vrnetlab.VR):
    """ Juniper vMX router
    """

    def __init__(self, username, password):
        self.version = None
        self.version_info = []
        self.read_version()

        super(VMX, self).__init__(username, password)

        self.vms = [ VMX_vcp(username, password, "/vmx/" + self.vcp_image), VMX_vfpc(self.version) ]

        # set up bridge for connecting VCP with vFPC
        vrnetlab.run_command(["brctl", "addbr", "int_cp"])
        vrnetlab.run_command(["ip", "link", "set", "int_cp", "up"])

    def read_version(self):
        for e in os.listdir("/vmx/"):
            m = re.search("-(([0-9][0-9])\.([0-9])([A-Z])([0-9]+)\.([0-9]+))", e)
            if m:
                self.vcp_image = e
                self.version = m.group(1)
                self.version_info = [int(m.group(2)), int(m.group(3)), m.group(4), int(m.group(5)), int(m.group(6))]

class VMX_installer(VMX):
    """ VMX installer

        Will start the VMX VCP and then shut it down. Booting the VCP for the
        first time requires the VCP itself to load some config and then it will
        restart. Subsequent boots will not require this restart. By running
        this "install" when building the docker image we can decrease the
        normal startup time of the vMX.
    """
    def __init__(self, username, password):
        self.version = None
        self.version_info = []
        self.read_version()

        super(VMX, self).__init__(username, password)

        self.vms = [ VMX_vcp(username, password, "/vmx/" + self.vcp_image, install_mode=True) ]

    def install(self):
        self.logger.info("Installing VMX")
        vcp = self.vms[0]
        while not vcp.running:
            vcp.work()

        # wait for system to shut down cleanly
        for i in range(0, 600):
            time.sleep(1)
            try:
                vcp.p.communicate(timeout=1)
            except subprocess.TimeoutExpired:
                pass
            except Exception as exc:
                # assume it's dead
                self.logger.info("Can't communicate with qemu process, assuming VM has shut down properly." + str(exc))
                break

            try:
                (ridx, match, res) = vcp.tn.expect([b"Powering system off"], 1)
                if res != b'':
                    self.logger.trace("OUTPUT VCP: %s" % res.decode())
            except Exception as exc:
                # assume it's dead
                self.logger.info("Can't communicate over serial console, assuming VM has shut down properly." + str(exc))
                break

        vcp.stop()
        self.logger.info("Installation complete")

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser(description='')
    parser.add_argument('--trace', action='store_true', help='enable trace level logging')
    parser.add_argument('--username', default='vrnetlab', help='Username')
    parser.add_argument('--password', default='VR-netlab9', help='Password')
    parser.add_argument('--install', action='store_true', help='Install vMX')
    args = parser.parse_args()

    LOG_FORMAT = "%(asctime)s: %(module)-10s %(levelname)-8s %(message)s"
    logging.basicConfig(format=LOG_FORMAT)
    logger = logging.getLogger()

    logger.setLevel(logging.DEBUG)
    if args.trace:
        logger.setLevel(1)

    if args.install:
        vr = VMX_installer(args.username, args.password)
        vr.install()
    else:
        vr = VMX(args.username, args.password)
        vr.start()
<|endoftext|>


<commit_before>"""
DB Benchmarking Application
===========================

Riak_db.py

This file handle all interactions with RiakDB during the benchmarking process.

"""

import riak

from load_settings import LocalSettings
from benchmark_template import BenchmarkDatabase

CONFIG = LocalSettings()

class Benchmark(BenchmarkDatabase):

    def __init__(self, setup=False, verbose=False):

        self.verbose = verbose

        if setup:
            self.setup('test')

    def setup(self, collection):
        """ `Setup()` handles all the necessary setup information for Riak.  It
        creates an instance of a Riak Client and defines the collection to use
        for reads and writes.

        :param collection:
        :return:
        """

        port = CONFIG.riak_port

        riak_servers = [
            CONFIG.riak_1,
            CONFIG.riak_2,
            CONFIG.riak_3,
        ]

        riak_nodes = []

        for server in riak_servers:

            riak_nodes.append({'host': str(server), 'http_port': port})

        self.client = riak.RiakClient(nodes=riak_nodes)

        self.bucket = self.client.bucket(collection)

    def write(self, data):

        entry = self.bucket.new('ID', data=data)

        entry.store()

    def read(self, index):

        read_entry = self.bucket.get('ID').data

        print read_entry<commit_msg>fixed a minor problem involving verbose output<commit_after>"""
DB Benchmarking Application
===========================

Riak_db.py

This file handle all interactions with RiakDB during the benchmarking process.

"""

import riak

from load_settings import LocalSettings
from benchmark_template import BenchmarkDatabase

CONFIG = LocalSettings()

class Benchmark(BenchmarkDatabase):

    def __init__(self, setup=False, verbose=False):

        self.verbose = verbose

        if setup:
            self.setup('test')

    def setup(self, collection):
        """ `Setup()` handles all the necessary setup information for Riak.  It
        creates an instance of a Riak Client and defines the collection to use
        for reads and writes.

        :param collection:
        :return:
        """

        port = CONFIG.riak_port

        riak_servers = [
            CONFIG.riak_1,
            CONFIG.riak_2,
            CONFIG.riak_3,
        ]

        riak_nodes = []

        for server in riak_servers:

            riak_nodes.append({'host': str(server), 'http_port': port})

        self.client = riak.RiakClient(nodes=riak_nodes)

        self.bucket = self.client.bucket(collection)

    def write(self, data):

        entry = self.bucket.new('ID', data=data)

        entry.store()

    def read(self, index):

        read_entry = self.bucket.get('ID').data

        if self.verbose:

            print read_entry<|endoftext|>


<commit_before>#!/usr/bin/env python

# Script by Steven Black
# https://github.com/StevenBlack
#
# This Python script will update the readme files in this repo.

import json
import os
import time
from string import Template

# Project Settings
BASEDIR_PATH = os.path.dirname(os.path.realpath(__file__))
README_TEMPLATE = os.path.join(BASEDIR_PATH, "readme_template.md")
README_FILENAME = "readme.md"
README_DATA_FILENAME = "readmeData.json"

def main():
    s = Template(
        "${description} | [Readme](https://github.com/StevenBlack/"
        "hosts/blob/master/${location}readme.md) | "
        "[link](https://raw.githubusercontent.com/StevenBlack/"
        "hosts/master/${location}hosts) | "
        "${fmtentries} | "
        "[link](http://sbc.io/hosts/${location}hosts)"
    )
    with open(README_DATA_FILENAME, "r") as f:
        data = json.load(f)

    keys = list(data.keys())

    # Sort by the number of en-dashes in the key
    # and then by the key string itself.
    keys.sort(key=lambda item: (item.count("-"), item))

    toc_rows = ""
    for key in keys:
        data[key]["fmtentries"] = "{:,}".format(data[key]["entries"])
        if key == "base":
            data[key]["description"] = "Unified hosts = **(adware + malware)**"
        else:
            data[key]["description"] = (
                "Unified hosts **+ " + key.replace("-", " + ") + "**"
            )

        if "\\" in data[key]["location"]:
            data[key]["location"] = data[key]["location"].replace("\\", "/")

        toc_rows += s.substitute(data[key]) + "\n"

    row_defaults = {
        "name": "",
        "description": "",
        "homeurl": "",
        "frequency": "",
        "url": "",
        "license": "",
        "issues": "",
    }

    t = Template(
        "${name} | ${description} |[link](${homeurl})"
        " | [raw](${url}) | ${frequency} | ${license} | [issues](${issues})"
    )

    for key in keys:
        extensions = key.replace("-", ", ")
        extensions_str = "* Extensions: **" + extensions + "**."
        extensions_header = "with " + extensions + " extensions"

        source_rows = ""
        source_list = data[key]["sourcesdata"]

        for source in source_list:
            this_row = {}
            this_row.update(row_defaults)
            this_row.update(source)
            source_rows += t.substitute(this_row) + "\n"

        with open(
            os.path.join(data[key]["location"], README_FILENAME), "wt", encoding="utf-8"
        ) as out:
            for line in open(README_TEMPLATE):
                line = line.replace(
                    "@GEN_DATE@", time.strftime("%B %d %Y", time.gmtime())
                )
                line = line.replace("@EXTENSIONS@", extensions_str)
                line = line.replace("@EXTENSIONS_HEADER@", extensions_header)
                line = line.replace(
                    "@NUM_ENTRIES@", "{:,}".format(data[key]["entries"])
                )
                line = line.replace(
                    "@SUBFOLDER@", os.path.join(data[key]["location"], "")
                )
                line = line.replace("@TOCROWS@", toc_rows)
                line = line.replace("@SOURCEROWS@", source_rows)
                out.write(line)

if __name__ == "__main__":
    main()
<commit_msg>Make sure we treat any readme assets as `utf-8`<commit_after>#!/usr/bin/env python

# Script by Steven Black
# https://github.com/StevenBlack
#
# This Python script will update the readme files in this repo.

import json
import os
import time
from string import Template

# Project Settings
BASEDIR_PATH = os.path.dirname(os.path.realpath(__file__))
README_TEMPLATE = os.path.join(BASEDIR_PATH, "readme_template.md")
README_FILENAME = "readme.md"
README_DATA_FILENAME = "readmeData.json"

def main():
    s = Template(
        "${description} | [Readme](https://github.com/StevenBlack/"
        "hosts/blob/master/${location}readme.md) | "
        "[link](https://raw.githubusercontent.com/StevenBlack/"
        "hosts/master/${location}hosts) | "
        "${fmtentries} | "
        "[link](http://sbc.io/hosts/${location}hosts)"
    )
    with open(README_DATA_FILENAME, "r", encoding="utf-8") as f:
        data = json.load(f)

    keys = list(data.keys())

    # Sort by the number of en-dashes in the key
    # and then by the key string itself.
    keys.sort(key=lambda item: (item.count("-"), item))

    toc_rows = ""
    for key in keys:
        data[key]["fmtentries"] = "{:,}".format(data[key]["entries"])
        if key == "base":
            data[key]["description"] = "Unified hosts = **(adware + malware)**"
        else:
            data[key]["description"] = (
                "Unified hosts **+ " + key.replace("-", " + ") + "**"
            )

        if "\\" in data[key]["location"]:
            data[key]["location"] = data[key]["location"].replace("\\", "/")

        toc_rows += s.substitute(data[key]) + "\n"

    row_defaults = {
        "name": "",
        "description": "",
        "homeurl": "",
        "frequency": "",
        "url": "",
        "license": "",
        "issues": "",
    }

    t = Template(
        "${name} | ${description} |[link](${homeurl})"
        " | [raw](${url}) | ${frequency} | ${license} | [issues](${issues})"
    )

    for key in keys:
        extensions = key.replace("-", ", ")
        extensions_str = "* Extensions: **" + extensions + "**."
        extensions_header = "with " + extensions + " extensions"

        source_rows = ""
        source_list = data[key]["sourcesdata"]

        for source in source_list:
            this_row = {}
            this_row.update(row_defaults)
            this_row.update(source)
            source_rows += t.substitute(this_row) + "\n"

        with open(
            os.path.join(data[key]["location"], README_FILENAME), "wt", encoding="utf-8"
        ) as out:
            for line in open(README_TEMPLATE, encoding="utf-8"):
                line = line.replace(
                    "@GEN_DATE@", time.strftime("%B %d %Y", time.gmtime())
                )
                line = line.replace("@EXTENSIONS@", extensions_str)
                line = line.replace("@EXTENSIONS_HEADER@", extensions_header)
                line = line.replace(
                    "@NUM_ENTRIES@", "{:,}".format(data[key]["entries"])
                )
                line = line.replace(
                    "@SUBFOLDER@", os.path.join(data[key]["location"], "")
                )
                line = line.replace("@TOCROWS@", toc_rows)
                line = line.replace("@SOURCEROWS@", source_rows)
                out.write(line)

if __name__ == "__main__":
    main()
<|endoftext|>