<commit_before>import copy

import six

from chainer import configuration
from chainer.dataset import convert
from chainer.dataset import iterator as iterator_module
from chainer import function
from chainer import link
from chainer import reporter as reporter_module
from chainer.training import extension

class Evaluator(extension.Extension):

    """Trainer extension to evaluate models on a validation set.

    This extension evaluates the current models by a given evaluation function.
    It creates a :class:`~chainer.Reporter` object to store values observed in
    the evaluation function on each iteration. The report for all iterations
    are aggregated to :class:`~chainer.DictSummary`. The collected mean values
    are further reported to the reporter object of the trainer, where the name
    of each observation is prefixed by the evaluator name. See
    :class:`~chainer.Reporter` for details in naming rules of the reports.

    Evaluator has a structure to customize similar to that of
    :class:`~chainer.training.StandardUpdater`. The main differences are:

    - There are no optimizers in an evaluator. Instead, it holds links
      to evaluate.
    - An evaluation loop function is used instead of an update function.
    - Preparation routine can be customized, which is called before each
      evaluation. It can be used, e.g., to initialize the state of stateful
      recurrent networks.

    There are two ways to modify the evaluation behavior besides setting a
    custom evaluation function. One is by setting a custom evaluation loop via
    the ``eval_func`` argument. The other is by inheriting this class and
    overriding the :meth:`evaluate` method. In latter case, users have to
    create and handle a reporter object manually. Users also have to copy the
    iterators before using them, in order to reuse them at the next time of
    evaluation. In both cases, the functions are called in testing mode
    (i.e., ``chainer.config.train`` is set to ``False``).

    This extension is called at the end of each epoch by default.

    Args:
        iterator: Dataset iterator for the validation dataset. It can also be
            a dictionary of iterators. If this is just an iterator, the
            iterator is registered by the name ``'main'``.
        target: Link object or a dictionary of links to evaluate. If this is
            just a link object, the link is registered by the name ``'main'``.
        converter: Converter function to build input arrays.
            :func:`~chainer.dataset.concat_examples` is used by default.
        device: Device to which the training data is sent. Negative value
            indicates the host memory (CPU).
        eval_hook: Function to prepare for each evaluation process. It is
            called at the beginning of the evaluation. The evaluator extension
            object is passed at each call.
        eval_func: Evaluation function called at each iteration. The target
            link to evaluate as a callable is used by default.

    Attributes:
        converter: Converter function.
        device: Device to which the training data is sent.
        eval_hook: Function to prepare for each evaluation process.
        eval_func: Evaluation function called at each iteration.

    """
    trigger = 1, 'epoch'
    default_name = 'validation'
    priority = extension.PRIORITY_WRITER

    def __init__(self, iterator, target, converter=convert.concat_examples,
                 device=None, eval_hook=None, eval_func=None):
        if isinstance(iterator, iterator_module.Iterator):
            iterator = {'main': iterator}
        self._iterators = iterator

        if isinstance(target, link.Link):
            target = {'main': target}
        self._targets = target

        self.converter = converter
        self.device = device
        self.eval_hook = eval_hook
        self.eval_func = eval_func

    def get_iterator(self, name):
        """Returns the iterator of the given name."""
        return self._iterators[name]

    def get_all_iterators(self):
        """Returns a dictionary of all iterators."""
        return dict(self._iterators)

    def get_target(self, name):
        """Returns the target link of the given name."""
        return self._targets[name]

    def get_all_targets(self):
        """Returns a dictionary of all target links."""
        return dict(self._targets)

    def __call__(self, trainer=None):
        """Executes the evaluator extension.

        Unlike usual extensions, this extension can be executed without passing
        a trainer object. This extension reports the performance on validation
        dataset using the :func:`~chainer.report` function. Thus, users can use
        this extension independently from any trainer by manually configuring
        a :class:`~chainer.Reporter` object.

        Args:
            trainer (~chainer.training.Trainer): Trainer object that invokes
                this extension. It can be omitted in case of calling this
                extension manually.

        Returns:
            dict: Result dictionary that contains mean statistics of values
                reported by the evaluation function.

        """
        # set up a reporter
        reporter = reporter_module.Reporter()
        if hasattr(self, 'name'):
            prefix = self.name + '/'
        else:
            prefix = ''
        for name, target in six.iteritems(self._targets):
            reporter.add_observer(prefix + name, target)
            reporter.add_observers(prefix + name,
                                   target.namedlinks(skipself=True))

        with reporter:
            with configuration.using_config('train', False):
                result = self.evaluate()

        reporter_module.report(result)
        return result

    def evaluate(self):
        """Evaluates the model and returns a result dictionary.

        This method runs the evaluation loop over the validation dataset. It
        accumulates the reported values to :class:`~chainer.DictSummary` and
        returns a dictionary whose values are means computed by the summary.

        Users can override this method to customize the evaluation routine.

        Returns:
            dict: Result dictionary. This dictionary is further reported via
                :func:`~chainer.report` without specifying any observer.

        """
        iterator = self._iterators['main']
        target = self._targets['main']
        eval_func = self.eval_func or target

        if self.eval_hook:
            self.eval_hook(self)
        it = copy.copy(iterator)
        summary = reporter_module.DictSummary()

        for batch in it:
            observation = {}
            with reporter_module.report_scope(
                    observation), function.no_backprop_mode():
                in_arrays = self.converter(batch, self.device)
                with function.no_backprop_mode():
                    if isinstance(in_arrays, tuple):
                        eval_func(*in_arrays)
                    elif isinstance(in_arrays, dict):
                        eval_func(**in_arrays)
                    else:
                        eval_func(in_arrays)

            summary.add(observation)

        return summary.compute_mean()
<commit_msg>Remove redundant with clause<commit_after>import copy

import six

from chainer import configuration
from chainer.dataset import convert
from chainer.dataset import iterator as iterator_module
from chainer import function
from chainer import link
from chainer import reporter as reporter_module
from chainer.training import extension

class Evaluator(extension.Extension):

    """Trainer extension to evaluate models on a validation set.

    This extension evaluates the current models by a given evaluation function.
    It creates a :class:`~chainer.Reporter` object to store values observed in
    the evaluation function on each iteration. The report for all iterations
    are aggregated to :class:`~chainer.DictSummary`. The collected mean values
    are further reported to the reporter object of the trainer, where the name
    of each observation is prefixed by the evaluator name. See
    :class:`~chainer.Reporter` for details in naming rules of the reports.

    Evaluator has a structure to customize similar to that of
    :class:`~chainer.training.StandardUpdater`. The main differences are:

    - There are no optimizers in an evaluator. Instead, it holds links
      to evaluate.
    - An evaluation loop function is used instead of an update function.
    - Preparation routine can be customized, which is called before each
      evaluation. It can be used, e.g., to initialize the state of stateful
      recurrent networks.

    There are two ways to modify the evaluation behavior besides setting a
    custom evaluation function. One is by setting a custom evaluation loop via
    the ``eval_func`` argument. The other is by inheriting this class and
    overriding the :meth:`evaluate` method. In latter case, users have to
    create and handle a reporter object manually. Users also have to copy the
    iterators before using them, in order to reuse them at the next time of
    evaluation. In both cases, the functions are called in testing mode
    (i.e., ``chainer.config.train`` is set to ``False``).

    This extension is called at the end of each epoch by default.

    Args:
        iterator: Dataset iterator for the validation dataset. It can also be
            a dictionary of iterators. If this is just an iterator, the
            iterator is registered by the name ``'main'``.
        target: Link object or a dictionary of links to evaluate. If this is
            just a link object, the link is registered by the name ``'main'``.
        converter: Converter function to build input arrays.
            :func:`~chainer.dataset.concat_examples` is used by default.
        device: Device to which the training data is sent. Negative value
            indicates the host memory (CPU).
        eval_hook: Function to prepare for each evaluation process. It is
            called at the beginning of the evaluation. The evaluator extension
            object is passed at each call.
        eval_func: Evaluation function called at each iteration. The target
            link to evaluate as a callable is used by default.

    Attributes:
        converter: Converter function.
        device: Device to which the training data is sent.
        eval_hook: Function to prepare for each evaluation process.
        eval_func: Evaluation function called at each iteration.

    """
    trigger = 1, 'epoch'
    default_name = 'validation'
    priority = extension.PRIORITY_WRITER

    def __init__(self, iterator, target, converter=convert.concat_examples,
                 device=None, eval_hook=None, eval_func=None):
        if isinstance(iterator, iterator_module.Iterator):
            iterator = {'main': iterator}
        self._iterators = iterator

        if isinstance(target, link.Link):
            target = {'main': target}
        self._targets = target

        self.converter = converter
        self.device = device
        self.eval_hook = eval_hook
        self.eval_func = eval_func

    def get_iterator(self, name):
        """Returns the iterator of the given name."""
        return self._iterators[name]

    def get_all_iterators(self):
        """Returns a dictionary of all iterators."""
        return dict(self._iterators)

    def get_target(self, name):
        """Returns the target link of the given name."""
        return self._targets[name]

    def get_all_targets(self):
        """Returns a dictionary of all target links."""
        return dict(self._targets)

    def __call__(self, trainer=None):
        """Executes the evaluator extension.

        Unlike usual extensions, this extension can be executed without passing
        a trainer object. This extension reports the performance on validation
        dataset using the :func:`~chainer.report` function. Thus, users can use
        this extension independently from any trainer by manually configuring
        a :class:`~chainer.Reporter` object.

        Args:
            trainer (~chainer.training.Trainer): Trainer object that invokes
                this extension. It can be omitted in case of calling this
                extension manually.

        Returns:
            dict: Result dictionary that contains mean statistics of values
                reported by the evaluation function.

        """
        # set up a reporter
        reporter = reporter_module.Reporter()
        if hasattr(self, 'name'):
            prefix = self.name + '/'
        else:
            prefix = ''
        for name, target in six.iteritems(self._targets):
            reporter.add_observer(prefix + name, target)
            reporter.add_observers(prefix + name,
                                   target.namedlinks(skipself=True))

        with reporter:
            with configuration.using_config('train', False):
                result = self.evaluate()

        reporter_module.report(result)
        return result

    def evaluate(self):
        """Evaluates the model and returns a result dictionary.

        This method runs the evaluation loop over the validation dataset. It
        accumulates the reported values to :class:`~chainer.DictSummary` and
        returns a dictionary whose values are means computed by the summary.

        Users can override this method to customize the evaluation routine.

        Returns:
            dict: Result dictionary. This dictionary is further reported via
                :func:`~chainer.report` without specifying any observer.

        """
        iterator = self._iterators['main']
        target = self._targets['main']
        eval_func = self.eval_func or target

        if self.eval_hook:
            self.eval_hook(self)
        it = copy.copy(iterator)
        summary = reporter_module.DictSummary()

        for batch in it:
            observation = {}
            with reporter_module.report_scope(observation):
                in_arrays = self.converter(batch, self.device)
                with function.no_backprop_mode():
                    if isinstance(in_arrays, tuple):
                        eval_func(*in_arrays)
                    elif isinstance(in_arrays, dict):
                        eval_func(**in_arrays)
                    else:
                        eval_func(in_arrays)

            summary.add(observation)

        return summary.compute_mean()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- encoding: utf-8 -*-

from pygithub3.core.utils import OrderedDict

from .base import Resource
from .users import User
from .orgs import Org

class Repo(Resource):

    _dates = ('created_at', 'pushed_at')
    _maps = {'owner': User, 'organization': Org, 'parent': 'self',
             'source': 'self'}

    def __str__(self):
        return '<Repo (%s)>' % getattr(self, 'name', '')

class Team(Resource):

    def __str__(self):
        return '<Team (%s)>' % getattr(self, 'name', '')

class Author(Resource):

    _dates = ('date')

    def __str__(self):
        return '<Author (%s)>' % getattr(self, 'name', '')

class Committer(Resource):

    _dates = ('date')

    def __str__(self):
        return '<Committer (%s)>' % getattr(self, 'name', '')

class Commit(Resource):

    _maps = {'author': Author, 'committer': Committer}
    #_maps.update({'tree': GitCommit})

    def __str__(self):
        return '<Commit (%s)>' % getattr(self, 'author', '')

class Stats(Resource):
    pass

class File(Resource):

    def __str__(self):
        return '<File (%s)>' % getattr(self, 'filename', '')

class GitCommit(Resource):

    _maps = {'author': User, 'committer': User, 'commit': Commit,
             'stats': Stats}
    _collection_maps = {'parents': 'self', 'files': File}

    def __str__(self):
        return '<GitCommit (%s)>' % getattr(self, 'sha', '')

class Comment(Resource):

    _dates = ('created_at', 'updated_at')
    _maps = {'user': User}

    def __str__(self):
        return '<Comment (%s)>' % getattr(self, 'user', '')

class Diff(Resource):

    _maps = {'base_commit': Commit}
    _collection_maps = {'commits': Commit, 'files': File}

    def __str__(self):
        return '<Diff (%s)>' % getattr(self, 'status', '')

class Tag(Resource):

    _maps = {'commit': GitCommit}

    def __str__(self):
        return '<Tag (%s)>' % getattr(self, 'name', '')

class Branch(Resource):

    _maps = {'commit': GitCommit}

    def __str__(self):
        return '<Branch (%s)>' % getattr(self, 'name', '')

class Download(Resource):

    def __str__(self):
        return '<Download (%s)>' % getattr(self, 'name', '')

    def ball_to_upload(self):
        return OrderedDict({
            'key': self.path, 'acl': self.acl, 'success_action_status': '201',
            'Filename': self.name, 'AWSAccessKeyId': self.accesskeyid,
            'Policy': self.policy, 'Signature': self.signature,
            'Content-Type': self.mime_type})

class Hook(Resource):

    _dates = ('created_at', 'pushed_at')

    def __str__(self):
        return '<Hook (%s)>' % getattr(self, 'name', '')
<commit_msg>add `updated_at` to `_dates` in Repo resource<commit_after>#!/usr/bin/env python
# -*- encoding: utf-8 -*-

from pygithub3.core.utils import OrderedDict

from .base import Resource
from .users import User
from .orgs import Org

class Repo(Resource):

    _dates = ('created_at', 'updated_at', 'pushed_at')
    _maps = {'owner': User, 'organization': Org, 'parent': 'self',
             'source': 'self'}

    def __str__(self):
        return '<Repo (%s)>' % getattr(self, 'name', '')

class Team(Resource):

    def __str__(self):
        return '<Team (%s)>' % getattr(self, 'name', '')

class Author(Resource):

    _dates = ('date')

    def __str__(self):
        return '<Author (%s)>' % getattr(self, 'name', '')

class Committer(Resource):

    _dates = ('date')

    def __str__(self):
        return '<Committer (%s)>' % getattr(self, 'name', '')

class Commit(Resource):

    _maps = {'author': Author, 'committer': Committer}
    #_maps.update({'tree': GitCommit})

    def __str__(self):
        return '<Commit (%s)>' % getattr(self, 'author', '')

class Stats(Resource):
    pass

class File(Resource):

    def __str__(self):
        return '<File (%s)>' % getattr(self, 'filename', '')

class GitCommit(Resource):

    _maps = {'author': User, 'committer': User, 'commit': Commit,
             'stats': Stats}
    _collection_maps = {'parents': 'self', 'files': File}

    def __str__(self):
        return '<GitCommit (%s)>' % getattr(self, 'sha', '')

class Comment(Resource):

    _dates = ('created_at', 'updated_at')
    _maps = {'user': User}

    def __str__(self):
        return '<Comment (%s)>' % getattr(self, 'user', '')

class Diff(Resource):

    _maps = {'base_commit': Commit}
    _collection_maps = {'commits': Commit, 'files': File}

    def __str__(self):
        return '<Diff (%s)>' % getattr(self, 'status', '')

class Tag(Resource):

    _maps = {'commit': GitCommit}

    def __str__(self):
        return '<Tag (%s)>' % getattr(self, 'name', '')

class Branch(Resource):

    _maps = {'commit': GitCommit}

    def __str__(self):
        return '<Branch (%s)>' % getattr(self, 'name', '')

class Download(Resource):

    def __str__(self):
        return '<Download (%s)>' % getattr(self, 'name', '')

    def ball_to_upload(self):
        return OrderedDict({
            'key': self.path, 'acl': self.acl, 'success_action_status': '201',
            'Filename': self.name, 'AWSAccessKeyId': self.accesskeyid,
            'Policy': self.policy, 'Signature': self.signature,
            'Content-Type': self.mime_type})

class Hook(Resource):

    _dates = ('created_at', 'pushed_at')

    def __str__(self):
        return '<Hook (%s)>' % getattr(self, 'name', '')
<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""
This api server runs both a json-rpc api and a notification rest api.
(neo.api.JSONRPC.JsonRpcApi and neo.api.REST.NotificationRestApi)

Example systemd service config: TODO
"""
import os
import argparse
import threading
from time import sleep

from logzero import logger
from twisted.internet import reactor, task, endpoints
from twisted.web.server import Site

from neo import __version__
from neo.Core.Blockchain import Blockchain
from neo.Implementations.Blockchains.LevelDB.LevelDBBlockchain import LevelDBBlockchain
from neo.Implementations.Notifications.LevelDB.NotificationDB import NotificationDB
from neo.api.JSONRPC.JsonRpcApi import JsonRpcApi
from neo.Implementations.Notifications.LevelDB.NotificationDB import NotificationDB
from neo.api.REST.NotificationRestApi import NotificationRestApi

from neo.Network.NodeLeader import NodeLeader
from neo.Settings import settings, DIR_PROJECT_ROOT
from neo.UserPreferences import preferences

# Logfile settings & setup
LOGFILE_FN = os.path.join(DIR_PROJECT_ROOT, 'api-server.log')
LOGFILE_MAX_BYTES = 5e7  # 50 MB
LOGFILE_BACKUP_COUNT = 3  # 3 logfiles history
settings.set_logfile(LOGFILE_FN, LOGFILE_MAX_BYTES, LOGFILE_BACKUP_COUNT)

def write_pid_file():
    """ Write a pid file, to easily kill the service """
    f = open('/tmp/json-rpc-api-server.pid', 'w')
    f.write(str(os.getpid()))
    f.close()

def custom_background_code():
    """ Custom code run in a background thread.

    This function is run in a daemonized thread, which means it can be instantly killed at any
    moment, whenever the main thread quits. If you need more safety, don't use a  daemonized
    thread and handle exiting this thread in another way (eg. with signals and events).
    """
    while True:
        logger.info("[%s] Block %s / %s", settings.net_name, str(Blockchain.Default().Height), str(Blockchain.Default().HeaderHeight))
        sleep(15)

def main():
    parser = argparse.ArgumentParser()

    group = parser.add_mutually_exclusive_group()
    group.add_argument("-m", "--mainnet", action="store_true", default=False,
                       help="Use MainNet instead of the default TestNet")
    group.add_argument("-p", "--privnet", action="store_true", default=False,
                       help="Use PrivNet instead of the default TestNet")
    group.add_argument("--coznet", action="store_true", default=False,
                       help="Use the CoZ network instead of the default TestNet")
    group.add_argument("-c", "--config", action="store", help="Use a specific config file")

    parser.add_argument("--port-rpc", type=int, help="port to use for the server", required=True)
    parser.add_argument("--port-rest", type=int, help="port to use for the server", required=True)

    args = parser.parse_args()

    # Setup depending on command line arguments. By default, the testnet settings are already loaded.
    if args.config:
        settings.setup(args.config)
    elif args.mainnet:
        settings.setup_mainnet()
    elif args.privnet:
        settings.setup_privnet()
    elif args.coznet:
        settings.setup_coznet()

    # Write a PID file to easily quit the service
    write_pid_file()

    # Instantiate the blockchain and subscribe to notifications
    blockchain = LevelDBBlockchain(settings.LEVELDB_PATH)
    Blockchain.RegisterBlockchain(blockchain)
    dbloop = task.LoopingCall(Blockchain.Default().PersistBlocks)
    dbloop.start(.1)

    # Disable logging smart contract events
    settings.set_log_smart_contract_events(False)

    # Start the notification db instance
    ndb = NotificationDB.instance()
    ndb.start()

    # Start a thread with custom code
    d = threading.Thread(target=custom_background_code)
    d.setDaemon(True)  # daemonizing the thread will kill it when the main thread is quit
    d.start()

    # Run
    reactor.suggestThreadPoolSize(15)
    NodeLeader.Instance().Start()

    host = "0.0.0.0"

    logger.info("Starting json-rpc api server on http://%s:%s" % (host, args.port_rpc))
    api_server_rpc = JsonRpcApi(args.port_rpc)

    logger.info("Starting notification api server on http://%s:%s" % (host, args.port_rest))
    api_server_rest = NotificationRestApi()

    endpoint_rpc = "tcp:port={0}:interface={1}".format(args.port_rpc, '0.0.0.0')
    endpoint_rest = "tcp:port={0}:interface={1}".format(args.port_rest, '0.0.0.0')

    endpoints.serverFromString(reactor, endpoint_rpc).listen(Site(api_server_rpc.app.resource()))
    endpoints.serverFromString(reactor, endpoint_rest).listen(Site(api_server_rest.app.resource()))

    api_server_rest.app.run(host, 9999)

if __name__ == "__main__":
    main()
<commit_msg>docs update<commit_after>#!/usr/bin/env python3
"""
This api server runs both a json-rpc api and a notification rest api.
(neo.api.JSONRPC.JsonRpcApi and neo.api.REST.NotificationRestApi)

Example systemd service config: TODO
"""
import os
import argparse
import threading
from time import sleep

from logzero import logger
from twisted.internet import reactor, task, endpoints
from twisted.web.server import Site

from neo import __version__
from neo.Core.Blockchain import Blockchain
from neo.Implementations.Blockchains.LevelDB.LevelDBBlockchain import LevelDBBlockchain
from neo.Implementations.Notifications.LevelDB.NotificationDB import NotificationDB
from neo.api.JSONRPC.JsonRpcApi import JsonRpcApi
from neo.Implementations.Notifications.LevelDB.NotificationDB import NotificationDB
from neo.api.REST.NotificationRestApi import NotificationRestApi

from neo.Network.NodeLeader import NodeLeader
from neo.Settings import settings, DIR_PROJECT_ROOT
from neo.UserPreferences import preferences

# Logfile settings & setup
LOGFILE_FN = os.path.join(DIR_PROJECT_ROOT, 'api-server.log')
LOGFILE_MAX_BYTES = 5e7  # 50 MB
LOGFILE_BACKUP_COUNT = 3  # 3 logfiles history
settings.set_logfile(LOGFILE_FN, LOGFILE_MAX_BYTES, LOGFILE_BACKUP_COUNT)

def write_pid_file():
    """ Write a pid file, to easily kill the service """
    f = open('/tmp/json-rpc-api-server.pid', 'w')
    f.write(str(os.getpid()))
    f.close()

def custom_background_code():
    """ Custom code run in a background thread.

    This function is run in a daemonized thread, which means it can be instantly killed at any
    moment, whenever the main thread quits. If you need more safety, don't use a  daemonized
    thread and handle exiting this thread in another way (eg. with signals and events).
    """
    while True:
        logger.info("[%s] Block %s / %s", settings.net_name, str(Blockchain.Default().Height), str(Blockchain.Default().HeaderHeight))
        sleep(15)

def main():
    parser = argparse.ArgumentParser()

    group = parser.add_mutually_exclusive_group()
    group.add_argument("-m", "--mainnet", action="store_true", default=False,
                       help="Use MainNet instead of the default TestNet")
    group.add_argument("-p", "--privnet", action="store_true", default=False,
                       help="Use PrivNet instead of the default TestNet")
    group.add_argument("--coznet", action="store_true", default=False,
                       help="Use the CoZ network instead of the default TestNet")
    group.add_argument("-c", "--config", action="store", help="Use a specific config file")

    parser.add_argument("--port-rpc", type=int, help="port to use for the server", required=True)
    parser.add_argument("--port-rest", type=int, help="port to use for the server", required=True)

    args = parser.parse_args()

    # Setup depending on command line arguments. By default, the testnet settings are already loaded.
    if args.config:
        settings.setup(args.config)
    elif args.mainnet:
        settings.setup_mainnet()
    elif args.privnet:
        settings.setup_privnet()
    elif args.coznet:
        settings.setup_coznet()

    # Write a PID file to easily quit the service
    write_pid_file()

    # Instantiate the blockchain and subscribe to notifications
    blockchain = LevelDBBlockchain(settings.LEVELDB_PATH)
    Blockchain.RegisterBlockchain(blockchain)
    dbloop = task.LoopingCall(Blockchain.Default().PersistBlocks)
    dbloop.start(.1)

    # Disable logging smart contract events
    settings.set_log_smart_contract_events(False)

    # Start the notification db instance
    ndb = NotificationDB.instance()
    ndb.start()

    # Start a thread with custom code
    d = threading.Thread(target=custom_background_code)
    d.setDaemon(True)  # daemonizing the thread will kill it when the main thread is quit
    d.start()

    # Run
    reactor.suggestThreadPoolSize(15)
    NodeLeader.Instance().Start()

    host = "0.0.0.0"

    logger.info("Starting json-rpc api server on http://%s:%s" % (host, args.port_rpc))
    logger.info("Starting notification api server on http://%s:%s" % (host, args.port_rest))

    # Setup Klein apps
    api_server_rpc = JsonRpcApi(args.port_rpc)
    api_server_rest = NotificationRestApi()

    # Setup endpoints. One port per api
    endpoint_rpc = "tcp:port={0}:interface={1}".format(args.port_rpc, '0.0.0.0')
    endpoint_rest = "tcp:port={0}:interface={1}".format(args.port_rest, '0.0.0.0')
    endpoints.serverFromString(reactor, endpoint_rpc).listen(Site(api_server_rpc.app.resource()))
    endpoints.serverFromString(reactor, endpoint_rest).listen(Site(api_server_rest.app.resource()))

    # Run the server, needs a dummy port
    api_server_rest.app.run(host, 9999)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    pygments.formatters.latex
    ~~~~~~~~~~~~~~~~~~~~~~~~~

    Formatter for LaTeX fancyvrb output.

    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

from pygments.formatter import Formatter
from pygments.token import Token, STANDARD_TYPES
from pygments.util import get_bool_opt, get_int_opt, StringIO

__all__ = ['LatexFormatter']

def escape_tex(text, commandprefix):
    return text.replace('\\', '\x00'). \
                replace('{', '\x01'). \
                replace('}', '\x02'). \
                replace('\x00', r'\%sZbs{}' % commandprefix). \
                replace('\x01', r'\%sZob{}' % commandprefix). \
                replace('\x02', r'\%sZcb{}' % commandprefix). \
                replace('^', r'\%sZca{}' % commandprefix). \
                replace('_', r'\%sZus{}' % commandprefix). \
                replace('&', r'\%sZam{}' % commandprefix). \
                replace('<', r'\%sZlt{}' % commandprefix). \
                replace('>', r'\%sZgt{}' % commandprefix). \
                replace('#', r'\%sZsh{}' % commandprefix). \
                replace('%', r'\%sZpc{}' % commandprefix). \
                replace('$', r'\%sZdl{}' % commandprefix). \
                replace('-', r'\%sZhy{}' % commandprefix). \
                replace("'", r'\%sZsq{}' % commandprefix). \
                replace('"', r'\%sZdq{}' % commandprefix). \
                replace('~', r'\%sZti{}' % commandprefix)

DOC_TEMPLATE = r'''
\documentclass{%(docclass)s}
\usepackage{fancyvrb}
\usepackage{color}
\usepackage[%(encoding)s]{inputenc}
%(preamble)s

%(styledefs)s

\begin{document}

\section*{%(title)s}

%(code)s
\end{document}
'''

## Small explanation of the mess below :)
#
# The previous version of the LaTeX formatter just assigned a command to
# each token type defined in the current style.  That obviously is
# problematic if the highlighted code is produced for a different style
# than the style commands themselves.
#
# This version works much like the HTML formatter which assigns multiple
# CSS classes to each <span> tag, from the most specific to the least
# specific token type, thus falling back to the parent token type if one
# is not defined.  Here, the classes are there too and use the same short
# forms given in token.STANDARD_TYPES.
#
# Highlighted code now only uses one custom command, which by default is
# \PY and selectable by the commandprefix option (and in addition the
# escapes \PYZat, \PYZlb and \PYZrb which haven't been renamed for
# backwards compatibility purposes).
#
# \PY has two arguments: the classes, separated by +, and the text to
# render in that style.  The classes are resolved into the respective
# style commands by magic, which serves to ignore unknown classes.
#
# The magic macros are:
# * \PY@it, \PY@bf, etc. are unconditionally wrapped around the text
#   to render in \PY@do.  Their definition determines the style.
# * \PY@reset resets \PY@it etc. to do nothing.
# * \PY@toks parses the list of classes, using magic inspired by the
#   keyval package (but modified to use plusses instead of commas
#   because fancyvrb redefines commas inside its environments).
# * \PY@tok processes one class, calling the \PY@tok@classname command
#   if it exists.
# * \PY@tok@classname sets the \PY@it etc. to reflect the chosen style
#   for its class.
# * \PY resets the style, parses the classnames and then calls \PY@do.
#
# Tip: to read this code, print it out in substituted form using e.g.
# >>> print STYLE_TEMPLATE % {'cp': 'PY'}

STYLE_TEMPLATE = r'''
\makeatletter
\def\%(cp)s@reset{\let\%(cp)s@it=\relax \let\%(cp)s@bf=\relax%%
    \let\%(cp)s@ul=\relax \let\%(cp)s@tc=\relax%%
    \let\%(cp)s@bc=\relax \let\%(cp)s@ff=\relax}
\def\%(cp)s@tok#1{\csname %(cp)s@tok@#1\endcsname}
\def\%(cp)s@toks#1+{\ifx\relax#1\empty\else%%
    \%(cp)s@tok{#1}\expandafter\%(cp)s@toks\fi}
\def\%(cp)s@do#1{\%(cp)s@bc{\%(cp)s@tc{\%(cp)s@ul{%%
    \%(cp)s@it{\%(cp)s@bf{\%(cp)s@ff{#1}}}}}}}
\def\%(cp)s#1#2{\%(cp)s@reset\%(cp)s@toks#1+\relax+\%(cp)s@do{#2}}

%(styles)s

\def\%(cp)sZbs{\char`\\}
\def\%(cp)sZus{\char`\_}
\def\%(cp)sZob{\char`\{}
\def\%(cp)sZcb{\char`\}}
\def\%(cp)sZca{\char`\^}
\def\%(cp)sZam{\char`\&}
\def\%(cp)sZlt{\char`\<}
\def\%(cp)sZgt{\char`\>}
\def\%(cp)sZsh{\char`\#}
\def\%(cp)sZpc{\char`\%%}
\def\%(cp)sZdl{\char`\$}
\def\%(cp)sZhy{\char`\-}
\def\%(cp)sZsq{\char`\'}
\def\%(cp)sZdq{\char`\"}
\def\%(cp)sZti{\char`\~}
%% for compatibility with earlier versions
\def\%(cp)sZat{@}
\def\%(cp)sZlb{[}
\def\%(cp)sZrb{]}
\makeatother
'''

def _get_ttype_name(ttype):
    fname = STANDARD_TYPES.get(ttype)
    if fname:
        return fname
    aname = ''
    while fname is None:
        aname = ttype[-1] + aname
        ttype = ttype.parent
        fname = STANDARD_TYPES.get(ttype)
    return fname + aname

class LatexFormatter(Formatter):
    r"""
    Format tokens as LaTeX code. This needs the `fancyvrb` and `color`
    standard packages.

    Without the `full` option, code is formatted as one ``Verbatim``
    environment, like this:

    .. sourcecode:: latex

        \begin{Verbatim}[commandchars=\\{\}]
        \PY{k}{def }\PY{n+nf}{foo}(\PY{n}{bar}):
            \PY{k}{pass}
        \end{Verbatim}

    The special command used here (``\PY``) and all the other macros it needs
    are output by the `get_style_defs` method.

    With the `full` option, a complete LaTeX document is output, including
    the command definitions in the preamble.

    The `get_style_defs()` method of a `LatexFormatter` returns a string
    containing ``\def`` commands defining the macros needed inside the
    ``Verbatim`` environments.

    Additional options accepted:

    `style`
        The style to use, can be a string or a Style subclass (default:
        ``'default'``).

    `full`
        Tells the formatter to output a "full" document, i.e. a complete
        self-contained document (default: ``False``).

    `title`
        If `full` is true, the title that should be used to caption the
        document (default: ``''``).

    `docclass`
        If the `full` option is enabled, this is the document class to use
        (default: ``'article'``).

    `preamble`
        If the `full` option is enabled, this can be further preamble commands,
        e.g. ``\usepackage`` (default: ``''``).

    `linenos`
        If set to ``True``, output line numbers (default: ``False``).

    `linenostart`
        The line number for the first line (default: ``1``).

    `linenostep`
        If set to a number n > 1, only every nth line number is printed.

    `verboptions`
        Additional options given to the Verbatim environment (see the *fancyvrb*
        docs for possible values) (default: ``''``).

    `commandprefix`
        The LaTeX commands used to produce colored output are constructed
        using this prefix and some letters (default: ``'PY'``).
        *New in Pygments 0.7.*

        *New in Pygments 0.10:* the default is now ``'PY'`` instead of ``'C'``.

    `texcomments`
        If set to ``True``, enables LaTeX comment lines.  That is, LaTex markup
        in comment tokens is not escaped so that LaTeX can render it (default:
        ``False``).  *New in Pygments 1.2.*

    `mathescape`
        If set to ``True``, enables LaTeX math mode escape in comments. That
        is, ``'$...$'`` inside a comment will trigger math mode (default:
        ``False``).  *New in Pygments 1.2.*
    """
    name = 'LaTeX'
    aliases = ['latex', 'tex']
    filenames = ['*.tex']

    def __init__(self, **options):
        Formatter.__init__(self, **options)
        self.docclass = options.get('docclass', 'article')
        self.preamble = options.get('preamble', '')
        self.linenos = get_bool_opt(options, 'linenos', False)
        self.linenostart = abs(get_int_opt(options, 'linenostart', 1))
        self.linenostep = abs(get_int_opt(options, 'linenostep', 1))
        self.verboptions = options.get('verboptions', '')
        self.nobackground = get_bool_opt(options, 'nobackground', False)
        self.commandprefix = options.get('commandprefix', 'PY')
        self.texcomments = get_bool_opt(options, 'texcomments', False)
        self.mathescape = get_bool_opt(options, 'mathescape', False)

        self._create_stylesheet()

    def _create_stylesheet(self):
        t2n = self.ttype2name = {Token: ''}
        c2d = self.cmd2def = {}
        cp = self.commandprefix

        def rgbcolor(col):
            if col:
                return ','.join(['%.2f' %(int(col[i] + col[i + 1], 16) / 255.0)
                                 for i in (0, 2, 4)])
            else:
                return '1,1,1'

        for ttype, ndef in self.style:
            name = _get_ttype_name(ttype)
            cmndef = ''
            if ndef['bold']:
                cmndef += r'\let\$$@bf=\textbf'
            if ndef['italic']:
                cmndef += r'\let\$$@it=\textit'
            if ndef['underline']:
                cmndef += r'\let\$$@ul=\underline'
            if ndef['roman']:
                cmndef += r'\let\$$@ff=\textrm'
            if ndef['sans']:
                cmndef += r'\let\$$@ff=\textsf'
            if ndef['mono']:
                cmndef += r'\let\$$@ff=\textsf'
            if ndef['color']:
                cmndef += (r'\def\$$@tc##1{\textcolor[rgb]{%s}{##1}}' %
                           rgbcolor(ndef['color']))
            if ndef['border']:
                cmndef += (r'\def\$$@bc##1{\setlength{\fboxsep}{0pt}'
                           r'\fcolorbox[rgb]{%s}{%s}{\strut ##1}}' %
                           (rgbcolor(ndef['border']),
                            rgbcolor(ndef['bgcolor'])))
            elif ndef['bgcolor']:
                cmndef += (r'\def\$$@bc##1{\setlength{\fboxsep}{0pt}'
                           r'\colorbox[rgb]{%s}{\strut ##1}}' %
                           rgbcolor(ndef['bgcolor']))
            if cmndef == '':
                continue
            cmndef = cmndef.replace('$$', cp)
            t2n[ttype] = name
            c2d[name] = cmndef

    def get_style_defs(self, arg=''):
        """
        Return the command sequences needed to define the commands
        used to format text in the verbatim environment. ``arg`` is ignored.
        """
        cp = self.commandprefix
        styles = []
        for name, definition in self.cmd2def.iteritems():
            styles.append(r'\expandafter\def\csname %s@tok@%s\endcsname{%s}' %
                          (cp, name, definition))
        return STYLE_TEMPLATE % {'cp': self.commandprefix,
                                 'styles': '\n'.join(styles)}

    def format_unencoded(self, tokensource, outfile):
        # TODO: add support for background colors
        t2n = self.ttype2name
        cp = self.commandprefix

        if self.full:
            realoutfile = outfile
            outfile = StringIO()

        outfile.write(r'\begin{Verbatim}[commandchars=\\\{\}')
        if self.linenos:
            start, step = self.linenostart, self.linenostep
            outfile.write(',numbers=left' +
                          (start and ',firstnumber=%d' % start or '') +
                          (step and ',stepnumber=%d' % step or ''))
        if self.mathescape or self.texcomments:
            outfile.write(r',codes={\catcode`\$=3\catcode`\^=7\catcode`\_=8}')
        if self.verboptions:
            outfile.write(',' + self.verboptions)
        outfile.write(']\n')

        for ttype, value in tokensource:
            if ttype in Token.Comment:
                if self.texcomments:
                    # Try to guess comment starting lexeme and escape it ...
                    start = value[0:1]
                    for i in xrange(1, len(value)):
                        if start[0] != value[i]:
                            break
                        start += value[i]

                    value = value[len(start):]
                    start = escape_tex(start, self.commandprefix)

                    # ... but do not escape inside comment.
                    value = start + value
                elif self.mathescape:
                    # Only escape parts not inside a math environment.
                    parts = value.split('$')
                    in_math = False
                    for i, part in enumerate(parts):
                        if not in_math:
                            parts[i] = escape_tex(part, self.commandprefix)
                        in_math = not in_math
                    value = '$'.join(parts)
                else:
                    value = escape_tex(value, self.commandprefix)
            else:
                value = escape_tex(value, self.commandprefix)
            styles = []
            while ttype is not Token:
                try:
                    styles.append(t2n[ttype])
                except KeyError:
                    # not in current style
                    styles.append(_get_ttype_name(ttype))
                ttype = ttype.parent
            styleval = '+'.join(reversed(styles))
            if styleval:
                spl = value.split('\n')
                for line in spl[:-1]:
                    if line:
                        outfile.write("\\%s{%s}{%s}" % (cp, styleval, line))
                    outfile.write('\n')
                if spl[-1]:
                    outfile.write("\\%s{%s}{%s}" % (cp, styleval, spl[-1]))
            else:
                outfile.write(value)

        outfile.write('\\end{Verbatim}\n')

        if self.full:
            realoutfile.write(DOC_TEMPLATE %
                dict(docclass  = self.docclass,
                     preamble  = self.preamble,
                     title     = self.title,
                     encoding  = self.encoding or 'latin1',
                     styledefs = self.get_style_defs(),
                     code      = outfile.getvalue()))
<commit_msg>Closes #785: make sure LateX formatter output is Unicode.<commit_after># -*- coding: utf-8 -*-
"""
    pygments.formatters.latex
    ~~~~~~~~~~~~~~~~~~~~~~~~~

    Formatter for LaTeX fancyvrb output.

    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

from pygments.formatter import Formatter
from pygments.token import Token, STANDARD_TYPES
from pygments.util import get_bool_opt, get_int_opt, StringIO

__all__ = ['LatexFormatter']

def escape_tex(text, commandprefix):
    return text.replace('\\', '\x00'). \
                replace('{', '\x01'). \
                replace('}', '\x02'). \
                replace('\x00', r'\%sZbs{}' % commandprefix). \
                replace('\x01', r'\%sZob{}' % commandprefix). \
                replace('\x02', r'\%sZcb{}' % commandprefix). \
                replace('^', r'\%sZca{}' % commandprefix). \
                replace('_', r'\%sZus{}' % commandprefix). \
                replace('&', r'\%sZam{}' % commandprefix). \
                replace('<', r'\%sZlt{}' % commandprefix). \
                replace('>', r'\%sZgt{}' % commandprefix). \
                replace('#', r'\%sZsh{}' % commandprefix). \
                replace('%', r'\%sZpc{}' % commandprefix). \
                replace('$', r'\%sZdl{}' % commandprefix). \
                replace('-', r'\%sZhy{}' % commandprefix). \
                replace("'", r'\%sZsq{}' % commandprefix). \
                replace('"', r'\%sZdq{}' % commandprefix). \
                replace('~', r'\%sZti{}' % commandprefix)

DOC_TEMPLATE = r'''
\documentclass{%(docclass)s}
\usepackage{fancyvrb}
\usepackage{color}
\usepackage[%(encoding)s]{inputenc}
%(preamble)s

%(styledefs)s

\begin{document}

\section*{%(title)s}

%(code)s
\end{document}
'''

## Small explanation of the mess below :)
#
# The previous version of the LaTeX formatter just assigned a command to
# each token type defined in the current style.  That obviously is
# problematic if the highlighted code is produced for a different style
# than the style commands themselves.
#
# This version works much like the HTML formatter which assigns multiple
# CSS classes to each <span> tag, from the most specific to the least
# specific token type, thus falling back to the parent token type if one
# is not defined.  Here, the classes are there too and use the same short
# forms given in token.STANDARD_TYPES.
#
# Highlighted code now only uses one custom command, which by default is
# \PY and selectable by the commandprefix option (and in addition the
# escapes \PYZat, \PYZlb and \PYZrb which haven't been renamed for
# backwards compatibility purposes).
#
# \PY has two arguments: the classes, separated by +, and the text to
# render in that style.  The classes are resolved into the respective
# style commands by magic, which serves to ignore unknown classes.
#
# The magic macros are:
# * \PY@it, \PY@bf, etc. are unconditionally wrapped around the text
#   to render in \PY@do.  Their definition determines the style.
# * \PY@reset resets \PY@it etc. to do nothing.
# * \PY@toks parses the list of classes, using magic inspired by the
#   keyval package (but modified to use plusses instead of commas
#   because fancyvrb redefines commas inside its environments).
# * \PY@tok processes one class, calling the \PY@tok@classname command
#   if it exists.
# * \PY@tok@classname sets the \PY@it etc. to reflect the chosen style
#   for its class.
# * \PY resets the style, parses the classnames and then calls \PY@do.
#
# Tip: to read this code, print it out in substituted form using e.g.
# >>> print STYLE_TEMPLATE % {'cp': 'PY'}

STYLE_TEMPLATE = r'''
\makeatletter
\def\%(cp)s@reset{\let\%(cp)s@it=\relax \let\%(cp)s@bf=\relax%%
    \let\%(cp)s@ul=\relax \let\%(cp)s@tc=\relax%%
    \let\%(cp)s@bc=\relax \let\%(cp)s@ff=\relax}
\def\%(cp)s@tok#1{\csname %(cp)s@tok@#1\endcsname}
\def\%(cp)s@toks#1+{\ifx\relax#1\empty\else%%
    \%(cp)s@tok{#1}\expandafter\%(cp)s@toks\fi}
\def\%(cp)s@do#1{\%(cp)s@bc{\%(cp)s@tc{\%(cp)s@ul{%%
    \%(cp)s@it{\%(cp)s@bf{\%(cp)s@ff{#1}}}}}}}
\def\%(cp)s#1#2{\%(cp)s@reset\%(cp)s@toks#1+\relax+\%(cp)s@do{#2}}

%(styles)s

\def\%(cp)sZbs{\char`\\}
\def\%(cp)sZus{\char`\_}
\def\%(cp)sZob{\char`\{}
\def\%(cp)sZcb{\char`\}}
\def\%(cp)sZca{\char`\^}
\def\%(cp)sZam{\char`\&}
\def\%(cp)sZlt{\char`\<}
\def\%(cp)sZgt{\char`\>}
\def\%(cp)sZsh{\char`\#}
\def\%(cp)sZpc{\char`\%%}
\def\%(cp)sZdl{\char`\$}
\def\%(cp)sZhy{\char`\-}
\def\%(cp)sZsq{\char`\'}
\def\%(cp)sZdq{\char`\"}
\def\%(cp)sZti{\char`\~}
%% for compatibility with earlier versions
\def\%(cp)sZat{@}
\def\%(cp)sZlb{[}
\def\%(cp)sZrb{]}
\makeatother
'''

def _get_ttype_name(ttype):
    fname = STANDARD_TYPES.get(ttype)
    if fname:
        return fname
    aname = ''
    while fname is None:
        aname = ttype[-1] + aname
        ttype = ttype.parent
        fname = STANDARD_TYPES.get(ttype)
    return fname + aname

class LatexFormatter(Formatter):
    r"""
    Format tokens as LaTeX code. This needs the `fancyvrb` and `color`
    standard packages.

    Without the `full` option, code is formatted as one ``Verbatim``
    environment, like this:

    .. sourcecode:: latex

        \begin{Verbatim}[commandchars=\\{\}]
        \PY{k}{def }\PY{n+nf}{foo}(\PY{n}{bar}):
            \PY{k}{pass}
        \end{Verbatim}

    The special command used here (``\PY``) and all the other macros it needs
    are output by the `get_style_defs` method.

    With the `full` option, a complete LaTeX document is output, including
    the command definitions in the preamble.

    The `get_style_defs()` method of a `LatexFormatter` returns a string
    containing ``\def`` commands defining the macros needed inside the
    ``Verbatim`` environments.

    Additional options accepted:

    `style`
        The style to use, can be a string or a Style subclass (default:
        ``'default'``).

    `full`
        Tells the formatter to output a "full" document, i.e. a complete
        self-contained document (default: ``False``).

    `title`
        If `full` is true, the title that should be used to caption the
        document (default: ``''``).

    `docclass`
        If the `full` option is enabled, this is the document class to use
        (default: ``'article'``).

    `preamble`
        If the `full` option is enabled, this can be further preamble commands,
        e.g. ``\usepackage`` (default: ``''``).

    `linenos`
        If set to ``True``, output line numbers (default: ``False``).

    `linenostart`
        The line number for the first line (default: ``1``).

    `linenostep`
        If set to a number n > 1, only every nth line number is printed.

    `verboptions`
        Additional options given to the Verbatim environment (see the *fancyvrb*
        docs for possible values) (default: ``''``).

    `commandprefix`
        The LaTeX commands used to produce colored output are constructed
        using this prefix and some letters (default: ``'PY'``).
        *New in Pygments 0.7.*

        *New in Pygments 0.10:* the default is now ``'PY'`` instead of ``'C'``.

    `texcomments`
        If set to ``True``, enables LaTeX comment lines.  That is, LaTex markup
        in comment tokens is not escaped so that LaTeX can render it (default:
        ``False``).  *New in Pygments 1.2.*

    `mathescape`
        If set to ``True``, enables LaTeX math mode escape in comments. That
        is, ``'$...$'`` inside a comment will trigger math mode (default:
        ``False``).  *New in Pygments 1.2.*
    """
    name = 'LaTeX'
    aliases = ['latex', 'tex']
    filenames = ['*.tex']

    def __init__(self, **options):
        Formatter.__init__(self, **options)
        self.docclass = options.get('docclass', 'article')
        self.preamble = options.get('preamble', '')
        self.linenos = get_bool_opt(options, 'linenos', False)
        self.linenostart = abs(get_int_opt(options, 'linenostart', 1))
        self.linenostep = abs(get_int_opt(options, 'linenostep', 1))
        self.verboptions = options.get('verboptions', '')
        self.nobackground = get_bool_opt(options, 'nobackground', False)
        self.commandprefix = options.get('commandprefix', 'PY')
        self.texcomments = get_bool_opt(options, 'texcomments', False)
        self.mathescape = get_bool_opt(options, 'mathescape', False)

        self._create_stylesheet()

    def _create_stylesheet(self):
        t2n = self.ttype2name = {Token: ''}
        c2d = self.cmd2def = {}
        cp = self.commandprefix

        def rgbcolor(col):
            if col:
                return ','.join(['%.2f' %(int(col[i] + col[i + 1], 16) / 255.0)
                                 for i in (0, 2, 4)])
            else:
                return '1,1,1'

        for ttype, ndef in self.style:
            name = _get_ttype_name(ttype)
            cmndef = ''
            if ndef['bold']:
                cmndef += r'\let\$$@bf=\textbf'
            if ndef['italic']:
                cmndef += r'\let\$$@it=\textit'
            if ndef['underline']:
                cmndef += r'\let\$$@ul=\underline'
            if ndef['roman']:
                cmndef += r'\let\$$@ff=\textrm'
            if ndef['sans']:
                cmndef += r'\let\$$@ff=\textsf'
            if ndef['mono']:
                cmndef += r'\let\$$@ff=\textsf'
            if ndef['color']:
                cmndef += (r'\def\$$@tc##1{\textcolor[rgb]{%s}{##1}}' %
                           rgbcolor(ndef['color']))
            if ndef['border']:
                cmndef += (r'\def\$$@bc##1{\setlength{\fboxsep}{0pt}'
                           r'\fcolorbox[rgb]{%s}{%s}{\strut ##1}}' %
                           (rgbcolor(ndef['border']),
                            rgbcolor(ndef['bgcolor'])))
            elif ndef['bgcolor']:
                cmndef += (r'\def\$$@bc##1{\setlength{\fboxsep}{0pt}'
                           r'\colorbox[rgb]{%s}{\strut ##1}}' %
                           rgbcolor(ndef['bgcolor']))
            if cmndef == '':
                continue
            cmndef = cmndef.replace('$$', cp)
            t2n[ttype] = name
            c2d[name] = cmndef

    def get_style_defs(self, arg=''):
        """
        Return the command sequences needed to define the commands
        used to format text in the verbatim environment. ``arg`` is ignored.
        """
        cp = self.commandprefix
        styles = []
        for name, definition in self.cmd2def.iteritems():
            styles.append(r'\expandafter\def\csname %s@tok@%s\endcsname{%s}' %
                          (cp, name, definition))
        return STYLE_TEMPLATE % {'cp': self.commandprefix,
                                 'styles': '\n'.join(styles)}

    def format_unencoded(self, tokensource, outfile):
        # TODO: add support for background colors
        t2n = self.ttype2name
        cp = self.commandprefix

        if self.full:
            realoutfile = outfile
            outfile = StringIO()

        outfile.write(ur'\begin{Verbatim}[commandchars=\\\{\}')
        if self.linenos:
            start, step = self.linenostart, self.linenostep
            outfile.write(u',numbers=left' +
                          (start and u',firstnumber=%d' % start or u'') +
                          (step and u',stepnumber=%d' % step or u''))
        if self.mathescape or self.texcomments:
            outfile.write(ur',codes={\catcode`\$=3\catcode`\^=7\catcode`\_=8}')
        if self.verboptions:
            outfile.write(u',' + self.verboptions)
        outfile.write(u']\n')

        for ttype, value in tokensource:
            if ttype in Token.Comment:
                if self.texcomments:
                    # Try to guess comment starting lexeme and escape it ...
                    start = value[0:1]
                    for i in xrange(1, len(value)):
                        if start[0] != value[i]:
                            break
                        start += value[i]

                    value = value[len(start):]
                    start = escape_tex(start, self.commandprefix)

                    # ... but do not escape inside comment.
                    value = start + value
                elif self.mathescape:
                    # Only escape parts not inside a math environment.
                    parts = value.split('$')
                    in_math = False
                    for i, part in enumerate(parts):
                        if not in_math:
                            parts[i] = escape_tex(part, self.commandprefix)
                        in_math = not in_math
                    value = '$'.join(parts)
                else:
                    value = escape_tex(value, self.commandprefix)
            else:
                value = escape_tex(value, self.commandprefix)
            styles = []
            while ttype is not Token:
                try:
                    styles.append(t2n[ttype])
                except KeyError:
                    # not in current style
                    styles.append(_get_ttype_name(ttype))
                ttype = ttype.parent
            styleval = '+'.join(reversed(styles))
            if styleval:
                spl = value.split('\n')
                for line in spl[:-1]:
                    if line:
                        outfile.write("\\%s{%s}{%s}" % (cp, styleval, line))
                    outfile.write('\n')
                if spl[-1]:
                    outfile.write("\\%s{%s}{%s}" % (cp, styleval, spl[-1]))
            else:
                outfile.write(value)

        outfile.write(u'\\end{Verbatim}\n')

        if self.full:
            realoutfile.write(DOC_TEMPLATE %
                dict(docclass  = self.docclass,
                     preamble  = self.preamble,
                     title     = self.title,
                     encoding  = self.encoding or 'latin1',
                     styledefs = self.get_style_defs(),
                     code      = outfile.getvalue()))
<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""
Downloads assets to the static directory.
This is used to allow easier updating without having to clog the git repository
with frequently updated minified JS/CSS
"""
import urllib.request
import os
import sys
from ansicolor import black, red
from collections import namedtuple

Asset = namedtuple("Asset", ["filename", "url"])
assets = [
    #Bootstrap
    Asset("css/bootstrap.min.css", "https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/css/bootstrap.min.css"),
    Asset("js/bootstrap.min.js", "https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/js/bootstrap.min.js"),
    Asset("fonts/glyphicons-halflings-regular.eot", "https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/fonts/glyphicons-halflings-regular.eot"),
    Asset("fonts/glyphicons-halflings-regular.svg", "https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/fonts/glyphicons-halflings-regular.svg"),
    Asset("fonts/glyphicons-halflings-regular.ttf", "https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/fonts/glyphicons-halflings-regular.ttf"),
    Asset("fonts/glyphicons-halflings-regular.woff", "https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/fonts/glyphicons-halflings-regular.woff"),
    Asset("fonts/glyphicons-halflings-regular.woff2", "https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/fonts/glyphicons-halflings-regular.woff2"),
    #Angular
    Asset("js/angular.min.js", "https://ajax.googleapis.com/ajax/libs/angularjs/1.3.13/angular.min.js"),
    #Angular Bootstrap directives
    Asset("js/angular-bootstrap.min.js", "https://angular-ui.github.io/bootstrap/ui-bootstrap-tpls-0.12.0.min.js"),
    #JQuery & plugins
    Asset("js/jquery.min.js", "https://ajax.googleapis.com/ajax/libs/jquery/2.1.3/jquery.min.js"),
    Asset("js/jquery.highlight.js", "https://raw.githubusercontent.com/bartaz/sandbox.js/master/jquery.highlight.js"),
]

def ensureFileIsPresent(asset, directory, forceDownload=False):
    (filename, url) = asset
    filepath = os.path.join(directory, filename)
    if url is None: # --> no need to download
        return
    if not os.path.isfile(filepath) or forceDownload:
        print (black("Downloading %s" % filename, bold=True))
        urllib.request.urlretrieve(url, filepath)

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("-f", "--force-download", action="store_true", help="Force downloading assets")
    parser.add_argument("-d", "--directory", default="./static", help="Directory to download (must exist)")
    args = parser.parse_args()
    #
    if not os.path.isdir(args.directory):
        print(red("%s is not a directory" % args.directory, bold=True))
        sys.exit(1)
    #Run download if file
    for asset in assets:
        ensureFileIsPresent(asset, args.directory, args.force_download)<commit_msg>Auto-create directories in asset downloader<commit_after>#!/usr/bin/env python3
"""
Downloads assets to the static directory.
This is used to allow easier updating without having to clog the git repository
with frequently updated minified JS/CSS
"""
import urllib.request
import os
import sys
from ansicolor import black, red
from collections import namedtuple

Asset = namedtuple("Asset", ["filename", "url"])
assets = [
    #Bootstrap
    Asset("css/bootstrap.min.css", "https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/css/bootstrap.min.css"),
    Asset("js/bootstrap.min.js", "https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/js/bootstrap.min.js"),
    Asset("fonts/glyphicons-halflings-regular.eot", "https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/fonts/glyphicons-halflings-regular.eot"),
    Asset("fonts/glyphicons-halflings-regular.svg", "https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/fonts/glyphicons-halflings-regular.svg"),
    Asset("fonts/glyphicons-halflings-regular.ttf", "https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/fonts/glyphicons-halflings-regular.ttf"),
    Asset("fonts/glyphicons-halflings-regular.woff", "https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/fonts/glyphicons-halflings-regular.woff"),
    Asset("fonts/glyphicons-halflings-regular.woff2", "https://maxcdn.bootstrapcdn.com/bootstrap/3.3.2/fonts/glyphicons-halflings-regular.woff2"),
    #Angular
    Asset("js/angular.min.js", "https://ajax.googleapis.com/ajax/libs/angularjs/1.3.13/angular.min.js"),
    #Angular Bootstrap directives
    Asset("js/angular-bootstrap.min.js", "https://angular-ui.github.io/bootstrap/ui-bootstrap-tpls-0.12.0.min.js"),
    #JQuery & plugins
    Asset("js/jquery.min.js", "https://ajax.googleapis.com/ajax/libs/jquery/2.1.3/jquery.min.js"),
    Asset("js/jquery.highlight.js", "https://raw.githubusercontent.com/bartaz/sandbox.js/master/jquery.highlight.js"),
]

def ensureFileIsPresent(asset, directory, forceDownload=False):
    (filename, url) = asset
    filepath = os.path.join(directory, filename)
    if url is None: # --> no need to download
        return
    if not os.path.isfile(filepath) or forceDownload:
        #Create directory if required
        dirname = os.path.dirname(filepath)
        if not os.path.exists(dirname):
            os.makedirs(dirname)
        #Perform download
        print (black("Downloading %s" % filename, bold=True))
        urllib.request.urlretrieve(url, filepath)

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("-f", "--force-download", action="store_true", help="Force downloading assets")
    parser.add_argument("-d", "--directory", default="./static", help="Directory to download (must exist)")
    args = parser.parse_args()
    #Check if the --directory arg is valid
    if not os.path.isdir(args.directory):
        print(red("%s is not a directory" % args.directory, bold=True))
        sys.exit(1)
    #Create subdirs if not already present

    #Run download if file
    for asset in assets:
        ensureFileIsPresent(asset, args.directory, args.force_download)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import absolute_import, division, print_function
from __future__ import unicode_literals

from pkg_resources import resource_filename

import nose
from nose.tools import *

import numpy as np

from sknano.core.crystallography import AlphaQuartz, DiamondStructure, \
    CaesiumChlorideStructure, RocksaltStructure, ZincblendeStructure, \
    Gold, Copper, MoS2
from sknano.generators import AlphaQuartzGenerator, \
    DiamondStructureGenerator, GoldGenerator, CopperGenerator, \
    CaesiumChlorideStructureGenerator, RocksaltStructureGenerator, \
    ZincblendeStructureGenerator, MoS2Generator
from sknano.io import XYZReader, DATAReader
from sknano.testing import GeneratorTestFixtures

class TestCase(GeneratorTestFixtures):

    def test1(self):
        quartz = AlphaQuartzGenerator()
        quartz.save()
        self.tmpdata.append(quartz.fname)
        assert_equals(quartz.atoms.Natoms, AlphaQuartz().basis.Natoms)

    def test2(self):
        quartz = AlphaQuartzGenerator(scaling_matrix=[3, 3, 3])
        quartz.save()
        self.tmpdata.append(quartz.fname)
        assert_equals(quartz.atoms.Natoms, AlphaQuartz().basis.Natoms * 3**3)

    def test3(self):
        gold = GoldGenerator()
        gold.save()
        self.tmpdata.append(gold.fname)
        assert_equals(gold.atoms.Natoms, Gold().basis.Natoms)

    def test4(self):
        gold = GoldGenerator(scaling_matrix=[5, 5, 5])
        gold.save()
        self.tmpdata.append(gold.fname)
        assert_equals(gold.atoms.Natoms, Gold().basis.Natoms * 5**3)

    def test5(self):
        copper = CopperGenerator()
        copper.save()
        self.tmpdata.append(copper.fname)
        assert_equals(copper.atoms.Natoms, Copper().basis.Natoms)

    def test6(self):
        copper = CopperGenerator(scaling_matrix=[5, 5, 5])
        copper.save()
        self.tmpdata.append(copper.fname)
        assert_equals(copper.atoms.Natoms, Copper().basis.Natoms * 5**3)

    def test7(self):
        diamond = DiamondStructureGenerator()
        diamond.save()
        self.tmpdata.append(diamond.fname)
        assert_equals(diamond.atoms.Natoms, DiamondStructure().basis.Natoms)

    def test8(self):
        diamond = DiamondStructureGenerator(scaling_matrix=[3, 3, 3])
        diamond.save()
        self.tmpdata.append(diamond.fname)
        assert_equals(diamond.atoms.Natoms,
                      DiamondStructure().basis.Natoms * 3**3)

    def test9(self):
        caesium_chloride = CaesiumChlorideStructureGenerator()
        caesium_chloride.save()
        self.tmpdata.append(caesium_chloride.fname)
        assert_equals(caesium_chloride.atoms.Natoms,
                      CaesiumChlorideStructure().basis.Natoms)

    def test10(self):
        caesium_chloride = \
            CaesiumChlorideStructureGenerator(scaling_matrix=[3, 3, 3])
        caesium_chloride.save()
        self.tmpdata.append(caesium_chloride.fname)
        assert_equals(caesium_chloride.atoms.Natoms,
                      CaesiumChlorideStructure().basis.Natoms * 3**3)

    def test11(self):
        molybdenum_disulphide = MoS2Generator()
        molybdenum_disulphide.save()
        self.tmpdata.append(molybdenum_disulphide.fname)
        assert_equals(molybdenum_disulphide.atoms.Natoms,
                      MoS2().basis.Natoms)

    def test12(self):
        molybdenum_disulphide = MoS2Generator(scaling_matrix=[3, 3, 3])
        molybdenum_disulphide.save()
        self.tmpdata.append(molybdenum_disulphide.fname)
        assert_equals(molybdenum_disulphide.atoms.Natoms,
                      MoS2().basis.Natoms * 3**3)

if __name__ == '__main__':
    nose.runmodule()
<commit_msg>updating bulk structure generator unit tests<commit_after># -*- coding: utf-8 -*-

from __future__ import absolute_import, division, print_function
from __future__ import unicode_literals

from pkg_resources import resource_filename

import nose
from nose.tools import *

import numpy as np

from sknano.core.crystallography import AlphaQuartz, DiamondStructure, \
    CaesiumChlorideStructure, RocksaltStructure, ZincblendeStructure, \
    Gold, Copper, MoS2
from sknano.generators import BCCStructureGenerator, FCCStructureGenerator, \
    AlphaQuartzGenerator, DiamondStructureGenerator, IronGenerator, \
    GoldGenerator, CopperGenerator, CaesiumChlorideStructureGenerator, \
    RocksaltStructureGenerator, ZincblendeStructureGenerator, MoS2Generator
from sknano.io import XYZReader, DATAReader
from sknano.testing import GeneratorTestFixtures

class TestCase(GeneratorTestFixtures):

    def test1(self):
        quartz = AlphaQuartzGenerator()
        quartz.save()
        self.tmpdata.append(quartz.fname)
        assert_equals(quartz.atoms.Natoms, AlphaQuartz().basis.Natoms)

    def test2(self):
        quartz = AlphaQuartzGenerator(scaling_matrix=[3, 3, 3])
        quartz.save()
        self.tmpdata.append(quartz.fname)
        assert_equals(quartz.atoms.Natoms, AlphaQuartz().basis.Natoms * 3**3)

    def test3(self):
        gold = GoldGenerator()
        gold.save()
        self.tmpdata.append(gold.fname)
        assert_equals(gold.atoms.Natoms, Gold().basis.Natoms)

    def test4(self):
        gold = GoldGenerator(scaling_matrix=[5, 5, 5])
        gold.save()
        self.tmpdata.append(gold.fname)
        assert_equals(gold.atoms.Natoms, Gold().basis.Natoms * 5**3)

    def test5(self):
        copper = CopperGenerator()
        copper.save()
        self.tmpdata.append(copper.fname)
        assert_equals(copper.atoms.Natoms, Copper().basis.Natoms)

    def test6(self):
        copper = CopperGenerator(scaling_matrix=[5, 5, 5])
        copper.save()
        self.tmpdata.append(copper.fname)
        assert_equals(copper.atoms.Natoms, Copper().basis.Natoms * 5**3)

    def test7(self):
        diamond = DiamondStructureGenerator()
        diamond.save()
        self.tmpdata.append(diamond.fname)
        assert_equals(diamond.atoms.Natoms, DiamondStructure().basis.Natoms)

    def test8(self):
        diamond = DiamondStructureGenerator(scaling_matrix=[3, 3, 3])
        diamond.save()
        self.tmpdata.append(diamond.fname)
        assert_equals(diamond.atoms.Natoms,
                      DiamondStructure().basis.Natoms * 3**3)

    def test9(self):
        caesium_chloride = CaesiumChlorideStructureGenerator()
        caesium_chloride.save()
        self.tmpdata.append(caesium_chloride.fname)
        assert_equals(caesium_chloride.atoms.Natoms,
                      CaesiumChlorideStructure().basis.Natoms)

    def test10(self):
        caesium_chloride = \
            CaesiumChlorideStructureGenerator(scaling_matrix=[3, 3, 3])
        caesium_chloride.save()
        self.tmpdata.append(caesium_chloride.fname)
        assert_equals(caesium_chloride.atoms.Natoms,
                      CaesiumChlorideStructure().basis.Natoms * 3**3)

    def test11(self):
        molybdenum_disulphide = MoS2Generator()
        molybdenum_disulphide.save()
        self.tmpdata.append(molybdenum_disulphide.fname)
        assert_equals(molybdenum_disulphide.atoms.Natoms,
                      MoS2().basis.Natoms)

    def test12(self):
        molybdenum_disulphide = MoS2Generator(scaling_matrix=[3, 3, 3])
        molybdenum_disulphide.save()
        self.tmpdata.append(molybdenum_disulphide.fname)
        assert_equals(molybdenum_disulphide.atoms.Natoms,
                      MoS2().basis.Natoms * 3**3)

    def test13(self):
        assert_equal(GoldGenerator().atoms, FCCStructureGenerator('Au').atoms)

    def test14(self):
        assert_equal(IronGenerator().atoms, BCCStructureGenerator('Fe').atoms)

if __name__ == '__main__':
    nose.runmodule()
<|endoftext|>


<commit_before>import functools
import random
import os.path
import time

from zstacklib.utils import shell
from zstacklib.utils import log
from zstacklib.utils import linux

logger = log.get_logger(__name__)
LV_RESERVED_SIZE = 1024*1024*4
LVM_CONFIG_PATH = "/etc/lvm"
SANLOCK_CONFIG_FILE_PATH = "/etc/sanlock/sanlock.conf"
LVM_CONFIG_BACKUP_PATH = "/etc/lvm/zstack-backup"

class LvmlockdLockType(object):
    NULL = 0
    SHARE = 1
    EXCLUSIVE = 2

    @staticmethod
    def from_abbr(abbr):
        if abbr == "sh":
            return LvmlockdLockType.SHARE
        elif abbr == "ex":
            return LvmlockdLockType.EXCLUSIVE
        elif abbr == "un":
            return LvmlockdLockType.NULL
        else:
            raise Exception("unknown lock type %s" % abbr)

    @staticmethod
    def from_str(string):
        if string == "NULL":
            return LvmlockdLockType.SHARE
        elif string == "SHARE":
            return LvmlockdLockType.EXCLUSIVE
        elif string == "EXCLUSIVE":
            return LvmlockdLockType.NULL
        else:
            raise Exception("unknown lock type %s" % string)

class RetryException(Exception):
    pass

def calcLvReservedSize(size):
    size = int(size) + (size/1024/1024%1024 + 1) * LV_RESERVED_SIZE
    return size

def check_lvm_config_is_default():
    cmd = shell.ShellCmd("lvmconfig --type diff")
    cmd(is_exception=True)
    if cmd.stdout != "":
        return False
    else:
        return True

def backup_lvm_config():
    if not os.path.exists(LVM_CONFIG_PATH):
        logger.warn("can not find lvm config path: %s, backup failed" % LVM_CONFIG_PATH)
        return

    if not os.path.exists(LVM_CONFIG_BACKUP_PATH):
        os.makedirs(LVM_CONFIG_BACKUP_PATH)

    current_time = time.time()
    cmd = shell.ShellCmd("cp %s/lvm.conf %s/lvm-%s.conf; "
                         "cp %s/lvmlocal.conf %s/lvmlocal-%s.conf" %
                         (LVM_CONFIG_PATH, LVM_CONFIG_BACKUP_PATH, current_time,
                          LVM_CONFIG_PATH, LVM_CONFIG_BACKUP_PATH, current_time))
    cmd(is_exception=False)
    logger.debug("backup lvm config file success")

def reset_lvm_conf_default():
    if not os.path.exists(LVM_CONFIG_PATH):
        raise Exception("can not find lvm config path: %s, reset lvm config failed" % LVM_CONFIG_PATH)

    cmd = shell.ShellCmd("lvmconfig --type default > %s/lvm.conf; "
                         "lvmconfig --type default > %s/lvmlocal.conf" %
                         (LVM_CONFIG_PATH, LVM_CONFIG_PATH))
    cmd(is_exception=False)

def config_lvm_by_sed(keyword, entry, files):
    if not os.path.exists(LVM_CONFIG_PATH):
        raise Exception("can not find lvm config path: %s, config lvm failed" % LVM_CONFIG_PATH)

    for file in files:
        cmd = shell.ShellCmd("sed -i 's/.*%s.*/%s/g' %s/%s" %
                             (keyword, entry, LVM_CONFIG_PATH, file))
        cmd(is_exception=False)

def config_sanlock_by_sed(keyword, entry):
    if not os.path.exists(SANLOCK_CONFIG_FILE_PATH):
        raise Exception("can not find sanlock config path: %s, config sanlock failed" % LVM_CONFIG_PATH)

    cmd = shell.ShellCmd("sed -i 's/.*%s.*/%s/g' %s" %
                         (keyword, entry, SANLOCK_CONFIG_FILE_PATH))
    cmd(is_exception=False)

def config_lvm_conf(node, value):
    cmd = shell.ShellCmd("lvmconfig --mergedconfig --config %s=%s -f /etc/lvm/lvm.conf" % (node, value))
    cmd(is_exception=True)

def config_lvmlocal_conf(node, value):
    cmd = shell.ShellCmd("lvmconfig --mergedconfig --config %s=%s -f /etc/lvm/lvmlocal.conf" % (node, value))
    cmd(is_exception=True)

def start_lvmlockd():
    for service in ["lvm2-lvmlockd", "wdmd", "sanlock"]:
        cmd = shell.ShellCmd("systemctl start %s" % service)
        cmd(is_exception=True)

def start_vg_lock(vgUuid):
    @linux.retry(times=20, sleep_time=random.uniform(1,10))
    def vg_lock_is_adding(vgUuid):
        # NOTE(weiw): this means vg locking is adding rather than complete
        cmd = shell.ShellCmd("sanlock client status | grep -E 's lvm_%s.*\\:0 ADD'" % vgUuid)
        cmd(is_exception=False)
        if cmd.return_code == 0:
            raise RetryException("vg %s lock space is starting" % vgUuid)
        return False

    @linux.retry(times=3, sleep_time=random.uniform(0.1, 1))
    def vg_lock_exists(vgUuid):
        cmd = shell.ShellCmd("lvmlockctl -i | grep %s" % vgUuid)
        cmd(is_exception=False)
        if cmd.return_code != 0:
            raise RetryException("can not find lock space for vg %s via lvmlockctl" % vgUuid)
        elif vg_lock_is_adding(vgUuid) is False:
            return True
        else:
            return False

    @linux.retry(times=15, sleep_time=random.uniform(0.1, 30))
    def start_lock(vgUuid):
        cmd = shell.ShellCmd("vgchange --lock-start %s" % vgUuid)
        cmd(is_exception=True)
        if cmd.return_code != 0:
            raise Exception("vgchange --lock-start failed")

        vg_lock_exists(vgUuid)

    try:
        vg_lock_exists(vgUuid)
    except RetryException:
        start_lock(vgUuid)
    except Exception as e:
        raise e

def get_vg_size(vgUuid):
    cmd = shell.ShellCmd("vgs --nolocking --readonly %s --noheadings --separator : --units b -o vg_size,vg_free" % vgUuid)
    cmd(is_exception=True)
    return cmd.stdout.strip().split(':')[0].strip("B"), cmd.stdout.strip().split(':')[1].strip("B")

def add_vg_tag(vgUuid, tag):
    cmd = shell.ShellCmd("vgchange --addtag %s %s" % (tag, vgUuid))
    cmd(is_exception=True)

def has_lv_tag(path, tag):
    o = shell.call("lvs -Stags={%s} %s --nolocking --noheadings --readonly 2>/dev/null | wc -l" % (tag, path))
    return o.strip() == '1'

def get_meta_lv_path(path):
    return path+"_meta"

def delete_image(path, tag):
    def activate_and_remove(f):
        active_lv(f, shared=False)
        backing = linux.qcow2_get_backing_file(f)
        shell.check_run("lvremove -y -Stags={%s} %s" % (tag, f))
        return f

    fpath = path
    while fpath:
        backing = activate_and_remove(fpath)
        activate_and_remove(get_meta_lv_path(fpath))
        fpath = backing

def clean_vg_exists_host_tags(vgUuid, hostUuid, tag):
    cmd = shell.ShellCmd("vgs %s -otags --nolocking --noheading | grep -Po '%s::%s::[\d.]*'" % (vgUuid, tag, hostUuid))
    cmd(is_exception=False)
    exists_tags = cmd.stdout.strip().split("\n")
    if len(exists_tags) == 0:
        return
    t = " --deltag " + " --deltag ".join(exists_tags)
    cmd = shell.ShellCmd("vgchange %s %s" % (t, vgUuid))
    cmd(is_exception=False)

@linux.retry(times=5, sleep_time=random.uniform(0.1, 3))
def create_lv_from_absolute_path(path, size, tag="zs::sharedblock::volume"):
    vgName = path.split("/")[2]
    lvName = path.split("/")[3]

    cmd = shell.ShellCmd("lvcreate -an --addtag %s --size %sb --name %s %s" %
                         (tag, calcLvReservedSize(size), lvName, vgName))
    cmd(is_exception=True)

def get_lv_size(path):
    cmd = shell.ShellCmd("lvs --nolocking --readonly --noheading -osize --units b %s" % path)
    cmd(is_exception=True)
    return cmd.stdout.strip().strip("B")

def resize_lv(path, size):
    cmd = shell.ShellCmd("lvresize --size %sb %s" % (calcLvReservedSize(size), path))
    cmd(is_exception=True)

@linux.retry(times=10, sleep_time=random.uniform(0.1, 3))
def active_lv(path, shared=False):
    flag = "-ay"
    if shared:
        flag = "-asy"

    cmd = shell.ShellCmd("lvchange %s %s" % (flag, path))
    cmd(is_exception=True)

def deactive_lv(path, raise_exception=True):
    if not lv_exists(path):
        return
    cmd = shell.ShellCmd("lvchange -an %s" % path)
    cmd(is_exception=raise_exception)

def delete_lv(path, raise_exception=True):
    # remove meta-lv if any
    if lv_exists(get_meta_lv_path(path)):
        shell.run("lvremove -y %s" % get_meta_lv_path(path))
    if not lv_exists(path):
        return
    cmd = shell.ShellCmd("lvremove -y %s" % path)
    cmd(is_exception=raise_exception)
    return cmd.return_code

def lv_exists(path):
    cmd = shell.ShellCmd("lvs --nolocking --readonly %s" % path)
    cmd(is_exception=False)
    return cmd.return_code == 0

def lv_uuid(path):
    cmd = shell.ShellCmd("lvs --nolocking --readonly --noheadings %s -ouuid" % path)
    cmd(is_exception=False)
    return cmd.stdout.strip()

def lv_is_active(path):
    cmd = shell.ShellCmd("lvs --nolocking --readonly --noheadings %s -oactive | grep active" % path)
    cmd(is_exception=False)
    return cmd.return_code == 0

def get_lv_locking_type(path):
    if not lv_is_active(path):
        return LvmlockdLockType.NULL
    cmd = shell.ShellCmd("lvmlockctl -i | grep %s | awk '{print $3}'" % lv_uuid(path))
    cmd(is_exception=True)
    return LvmlockdLockType.from_abbr(cmd.stdout.strip())

def lv_operate(abs_path, shared=False):
    def wrap(f):
        @functools.wraps(f)
        def inner(*args, **kwargs):
            with OperateLv(abs_path, shared):
                retval = f(*args, **kwargs)
            return retval
        return inner
    return wrap

def qcow2_lv_recursive_operate(abs_path, shared=False):
    def wrap(f):
        @functools.wraps(f)
        def inner(*args, **kwargs):
            with RecursiveOperateLv(abs_path, shared):
                retval = f(*args, **kwargs)
            return retval
        return inner
    return wrap

class OperateLv(object):
    def __init__(self, abs_path, shared=False):
        self.abs_path = abs_path
        self.shared = shared
        self.exists_lock = get_lv_locking_type(abs_path)
        self.target_lock = LvmlockdLockType.EXCLUSIVE if shared == False else LvmlockdLockType.SHARE

    def __enter__(self):
        if self.exists_lock < self.target_lock:
            active_lv(self.abs_path, self.shared)

    def __exit__(self, exc_type, exc_val, exc_tb):
        if self.exists_lock == LvmlockdLockType.NULL:
            deactive_lv(self.abs_path, raise_exception=False)
        else:
            active_lv(self.abs_path, self.exists_lock == LvmlockdLockType.SHARE)

class RecursiveOperateLv(object):
    def __init__(self, abs_path, shared=False):
        self.abs_path = abs_path
        self.shared = shared
        self.exists_lock = get_lv_locking_type(abs_path)
        self.target_lock = LvmlockdLockType.EXCLUSIVE if shared == False else LvmlockdLockType.SHARE
        self.backing = None

    def __enter__(self):
        if self.exists_lock < self.target_lock:
            active_lv(self.abs_path, self.shared)
        if linux.qcow2_get_backing_file(self.abs_path) != "":
            self.backing = RecursiveOperateLv(linux.qcow2_get_backing_file(self.abs_path), True)

        if self.backing is not None:
            self.backing.__enter__()

    def __exit__(self, exc_type, exc_val, exc_tb):
        if self.backing is not None:
            self.backing.__exit__(exc_type, exc_val, exc_tb)
        if self.exists_lock == LvmlockdLockType.NULL:
            deactive_lv(self.abs_path, raise_exception=False)
        else:
            active_lv(self.abs_path, self.exists_lock == LvmlockdLockType.SHARE)
<commit_msg>Fix lv size calculate<commit_after>import functools
import random
import os.path
import time

from zstacklib.utils import shell
from zstacklib.utils import log
from zstacklib.utils import linux

logger = log.get_logger(__name__)
LV_RESERVED_SIZE = 1024*1024*4
LVM_CONFIG_PATH = "/etc/lvm"
SANLOCK_CONFIG_FILE_PATH = "/etc/sanlock/sanlock.conf"
LVM_CONFIG_BACKUP_PATH = "/etc/lvm/zstack-backup"

class LvmlockdLockType(object):
    NULL = 0
    SHARE = 1
    EXCLUSIVE = 2

    @staticmethod
    def from_abbr(abbr):
        if abbr == "sh":
            return LvmlockdLockType.SHARE
        elif abbr == "ex":
            return LvmlockdLockType.EXCLUSIVE
        elif abbr == "un":
            return LvmlockdLockType.NULL
        else:
            raise Exception("unknown lock type %s" % abbr)

    @staticmethod
    def from_str(string):
        if string == "NULL":
            return LvmlockdLockType.SHARE
        elif string == "SHARE":
            return LvmlockdLockType.EXCLUSIVE
        elif string == "EXCLUSIVE":
            return LvmlockdLockType.NULL
        else:
            raise Exception("unknown lock type %s" % string)

class RetryException(Exception):
    pass

def calcLvReservedSize(size):
    if size <= 1024 * 1024 * 1024:
        size = int(size) + LV_RESERVED_SIZE
    size = int(size) + (size/1024/1024 / 1024 + 1) * LV_RESERVED_SIZE
    return size

def check_lvm_config_is_default():
    cmd = shell.ShellCmd("lvmconfig --type diff")
    cmd(is_exception=True)
    if cmd.stdout != "":
        return False
    else:
        return True

def backup_lvm_config():
    if not os.path.exists(LVM_CONFIG_PATH):
        logger.warn("can not find lvm config path: %s, backup failed" % LVM_CONFIG_PATH)
        return

    if not os.path.exists(LVM_CONFIG_BACKUP_PATH):
        os.makedirs(LVM_CONFIG_BACKUP_PATH)

    current_time = time.time()
    cmd = shell.ShellCmd("cp %s/lvm.conf %s/lvm-%s.conf; "
                         "cp %s/lvmlocal.conf %s/lvmlocal-%s.conf" %
                         (LVM_CONFIG_PATH, LVM_CONFIG_BACKUP_PATH, current_time,
                          LVM_CONFIG_PATH, LVM_CONFIG_BACKUP_PATH, current_time))
    cmd(is_exception=False)
    logger.debug("backup lvm config file success")

def reset_lvm_conf_default():
    if not os.path.exists(LVM_CONFIG_PATH):
        raise Exception("can not find lvm config path: %s, reset lvm config failed" % LVM_CONFIG_PATH)

    cmd = shell.ShellCmd("lvmconfig --type default > %s/lvm.conf; "
                         "lvmconfig --type default > %s/lvmlocal.conf" %
                         (LVM_CONFIG_PATH, LVM_CONFIG_PATH))
    cmd(is_exception=False)

def config_lvm_by_sed(keyword, entry, files):
    if not os.path.exists(LVM_CONFIG_PATH):
        raise Exception("can not find lvm config path: %s, config lvm failed" % LVM_CONFIG_PATH)

    for file in files:
        cmd = shell.ShellCmd("sed -i 's/.*%s.*/%s/g' %s/%s" %
                             (keyword, entry, LVM_CONFIG_PATH, file))
        cmd(is_exception=False)

def config_sanlock_by_sed(keyword, entry):
    if not os.path.exists(SANLOCK_CONFIG_FILE_PATH):
        raise Exception("can not find sanlock config path: %s, config sanlock failed" % LVM_CONFIG_PATH)

    cmd = shell.ShellCmd("sed -i 's/.*%s.*/%s/g' %s" %
                         (keyword, entry, SANLOCK_CONFIG_FILE_PATH))
    cmd(is_exception=False)

def config_lvm_conf(node, value):
    cmd = shell.ShellCmd("lvmconfig --mergedconfig --config %s=%s -f /etc/lvm/lvm.conf" % (node, value))
    cmd(is_exception=True)

def config_lvmlocal_conf(node, value):
    cmd = shell.ShellCmd("lvmconfig --mergedconfig --config %s=%s -f /etc/lvm/lvmlocal.conf" % (node, value))
    cmd(is_exception=True)

def start_lvmlockd():
    for service in ["lvm2-lvmlockd", "wdmd", "sanlock"]:
        cmd = shell.ShellCmd("systemctl start %s" % service)
        cmd(is_exception=True)

def start_vg_lock(vgUuid):
    @linux.retry(times=20, sleep_time=random.uniform(1,10))
    def vg_lock_is_adding(vgUuid):
        # NOTE(weiw): this means vg locking is adding rather than complete
        cmd = shell.ShellCmd("sanlock client status | grep -E 's lvm_%s.*\\:0 ADD'" % vgUuid)
        cmd(is_exception=False)
        if cmd.return_code == 0:
            raise RetryException("vg %s lock space is starting" % vgUuid)
        return False

    @linux.retry(times=3, sleep_time=random.uniform(0.1, 1))
    def vg_lock_exists(vgUuid):
        cmd = shell.ShellCmd("lvmlockctl -i | grep %s" % vgUuid)
        cmd(is_exception=False)
        if cmd.return_code != 0:
            raise RetryException("can not find lock space for vg %s via lvmlockctl" % vgUuid)
        elif vg_lock_is_adding(vgUuid) is False:
            return True
        else:
            return False

    @linux.retry(times=15, sleep_time=random.uniform(0.1, 30))
    def start_lock(vgUuid):
        cmd = shell.ShellCmd("vgchange --lock-start %s" % vgUuid)
        cmd(is_exception=True)
        if cmd.return_code != 0:
            raise Exception("vgchange --lock-start failed")

        vg_lock_exists(vgUuid)

    try:
        vg_lock_exists(vgUuid)
    except RetryException:
        start_lock(vgUuid)
    except Exception as e:
        raise e

def get_vg_size(vgUuid):
    cmd = shell.ShellCmd("vgs --nolocking --readonly %s --noheadings --separator : --units b -o vg_size,vg_free" % vgUuid)
    cmd(is_exception=True)
    return cmd.stdout.strip().split(':')[0].strip("B"), cmd.stdout.strip().split(':')[1].strip("B")

def add_vg_tag(vgUuid, tag):
    cmd = shell.ShellCmd("vgchange --addtag %s %s" % (tag, vgUuid))
    cmd(is_exception=True)

def has_lv_tag(path, tag):
    o = shell.call("lvs -Stags={%s} %s --nolocking --noheadings --readonly 2>/dev/null | wc -l" % (tag, path))
    return o.strip() == '1'

def get_meta_lv_path(path):
    return path+"_meta"

def delete_image(path, tag):
    def activate_and_remove(f):
        active_lv(f, shared=False)
        backing = linux.qcow2_get_backing_file(f)
        shell.check_run("lvremove -y -Stags={%s} %s" % (tag, f))
        return f

    fpath = path
    while fpath:
        backing = activate_and_remove(fpath)
        activate_and_remove(get_meta_lv_path(fpath))
        fpath = backing

def clean_vg_exists_host_tags(vgUuid, hostUuid, tag):
    cmd = shell.ShellCmd("vgs %s -otags --nolocking --noheading | grep -Po '%s::%s::[\d.]*'" % (vgUuid, tag, hostUuid))
    cmd(is_exception=False)
    exists_tags = cmd.stdout.strip().split("\n")
    if len(exists_tags) == 0:
        return
    t = " --deltag " + " --deltag ".join(exists_tags)
    cmd = shell.ShellCmd("vgchange %s %s" % (t, vgUuid))
    cmd(is_exception=False)

@linux.retry(times=5, sleep_time=random.uniform(0.1, 3))
def create_lv_from_absolute_path(path, size, tag="zs::sharedblock::volume"):
    vgName = path.split("/")[2]
    lvName = path.split("/")[3]

    cmd = shell.ShellCmd("lvcreate -an --addtag %s --size %sb --name %s %s" %
                         (tag, calcLvReservedSize(size), lvName, vgName))
    cmd(is_exception=True)

def get_lv_size(path):
    cmd = shell.ShellCmd("lvs --nolocking --readonly --noheading -osize --units b %s" % path)
    cmd(is_exception=True)
    return cmd.stdout.strip().strip("B")

def resize_lv(path, size):
    cmd = shell.ShellCmd("lvresize --size %sb %s" % (calcLvReservedSize(size), path))
    cmd(is_exception=True)

@linux.retry(times=10, sleep_time=random.uniform(0.1, 3))
def active_lv(path, shared=False):
    flag = "-ay"
    if shared:
        flag = "-asy"

    cmd = shell.ShellCmd("lvchange %s %s" % (flag, path))
    cmd(is_exception=True)

def deactive_lv(path, raise_exception=True):
    if not lv_exists(path):
        return
    cmd = shell.ShellCmd("lvchange -an %s" % path)
    cmd(is_exception=raise_exception)

def delete_lv(path, raise_exception=True):
    # remove meta-lv if any
    if lv_exists(get_meta_lv_path(path)):
        shell.run("lvremove -y %s" % get_meta_lv_path(path))
    if not lv_exists(path):
        return
    cmd = shell.ShellCmd("lvremove -y %s" % path)
    cmd(is_exception=raise_exception)
    return cmd.return_code

def lv_exists(path):
    cmd = shell.ShellCmd("lvs --nolocking --readonly %s" % path)
    cmd(is_exception=False)
    return cmd.return_code == 0

def lv_uuid(path):
    cmd = shell.ShellCmd("lvs --nolocking --readonly --noheadings %s -ouuid" % path)
    cmd(is_exception=False)
    return cmd.stdout.strip()

def lv_is_active(path):
    cmd = shell.ShellCmd("lvs --nolocking --readonly --noheadings %s -oactive | grep active" % path)
    cmd(is_exception=False)
    return cmd.return_code == 0

def get_lv_locking_type(path):
    if not lv_is_active(path):
        return LvmlockdLockType.NULL
    cmd = shell.ShellCmd("lvmlockctl -i | grep %s | awk '{print $3}'" % lv_uuid(path))
    cmd(is_exception=True)
    return LvmlockdLockType.from_abbr(cmd.stdout.strip())

def lv_operate(abs_path, shared=False):
    def wrap(f):
        @functools.wraps(f)
        def inner(*args, **kwargs):
            with OperateLv(abs_path, shared):
                retval = f(*args, **kwargs)
            return retval
        return inner
    return wrap

def qcow2_lv_recursive_operate(abs_path, shared=False):
    def wrap(f):
        @functools.wraps(f)
        def inner(*args, **kwargs):
            with RecursiveOperateLv(abs_path, shared):
                retval = f(*args, **kwargs)
            return retval
        return inner
    return wrap

class OperateLv(object):
    def __init__(self, abs_path, shared=False):
        self.abs_path = abs_path
        self.shared = shared
        self.exists_lock = get_lv_locking_type(abs_path)
        self.target_lock = LvmlockdLockType.EXCLUSIVE if shared == False else LvmlockdLockType.SHARE

    def __enter__(self):
        if self.exists_lock < self.target_lock:
            active_lv(self.abs_path, self.shared)

    def __exit__(self, exc_type, exc_val, exc_tb):
        if self.exists_lock == LvmlockdLockType.NULL:
            deactive_lv(self.abs_path, raise_exception=False)
        else:
            active_lv(self.abs_path, self.exists_lock == LvmlockdLockType.SHARE)

class RecursiveOperateLv(object):
    def __init__(self, abs_path, shared=False):
        self.abs_path = abs_path
        self.shared = shared
        self.exists_lock = get_lv_locking_type(abs_path)
        self.target_lock = LvmlockdLockType.EXCLUSIVE if shared == False else LvmlockdLockType.SHARE
        self.backing = None

    def __enter__(self):
        if self.exists_lock < self.target_lock:
            active_lv(self.abs_path, self.shared)
        if linux.qcow2_get_backing_file(self.abs_path) != "":
            self.backing = RecursiveOperateLv(linux.qcow2_get_backing_file(self.abs_path), True)

        if self.backing is not None:
            self.backing.__enter__()

    def __exit__(self, exc_type, exc_val, exc_tb):
        if self.backing is not None:
            self.backing.__exit__(exc_type, exc_val, exc_tb)
        if self.exists_lock == LvmlockdLockType.NULL:
            deactive_lv(self.abs_path, raise_exception=False)
        else:
            active_lv(self.abs_path, self.exists_lock == LvmlockdLockType.SHARE)
<|endoftext|>