<commit_before>'''
Copyright (c) 2014, Emmanuel Levijarvi
All rights reserved.
License BSD

iotrelay.py provides an application and a framework for passing data
between data sources and handlers.
'''
import argparse
import datetime
from collections import defaultdict
import pkg_resources
import os
import sys
import signal
import threading
import logging
try:
    import configparser
except ImportError:
    import ConfigParser as configparser

logger = logging.getLogger()

DEFAULT_CONFIG = os.path.join(os.path.expanduser("~"), '.iotrelay.cfg')
GROUP = 'iotrelay'
version = "1.0.1"

class ConfigParser(configparser.SafeConfigParser):
    '''ConfigParser is a subclass of the standard library's ConfigParser.

    It adds the ability to parse an option as a list. See the Python
    Standard Library Documentation for more.
    '''
    @staticmethod
    def _make_list(value, sep):
        return [v.strip() for v in value.split(sep)]

    def getlist(self, section, option, sep=','):
        '''Parse an option as a list, optionally specifying the separator
        by specifying the sep argument.
        '''
        return self._make_list(self.get(section, option), sep)

class Reading(object):
    '''Reading provides a container for passing a datum,or "Reading",
    between sources and handlers.

    Attributes:
        reading_type (str):
        value (str):
        timestamp (datetime.datetime):
        series_key (str):
    '''
    def __init__(self, reading_type, value, timestamp=None, series_key=None):
        self.reading_type = reading_type
        self.value = value
        self.series_key = series_key
        if series_key is None:
            self.series_key = reading_type
        if timestamp is None:
            self.timestamp = datetime.datetime.utcnow()
        else:
            self.timestamp = timestamp
        self.iteration = False

    def __iter__(self):
        return self

    def next(self):
        return self.__next__()

    def __next__(self):
        if self.iteration:
            raise StopIteration
        self.iteration = True
        return self

    def __str__(self):
        return "{0}: {1!s}, {2!s}".format(self.timestamp.isoformat(),
                                          self.series_key,
                                          self.value)

    def __repr__(self):
        return ("Reading({self.reading_type!r}, {self.value!r}, "
                "{self.timestamp!r}, {self.series_key!r})".format(self=self))

class Relay(object):
    def __init__(self, config):
        self.config = config
        self.stop_event = threading.Event()
        signal.signal(signal.SIGTERM, self.stop)
        signal.signal(signal.SIGINT, self.stop)
        self.sources = []
        self.handlers = defaultdict(list)

    def load_plugins(self):
        logger.debug('IoTrelay loading plugins')
        for entrypoint in pkg_resources.iter_entry_points(group=GROUP):
            plugin_name = entrypoint.dist.project_name
            try:
                plugin_config = dict(self.config.items(plugin_name))
            except configparser.NoSectionError:
                logging.warning('Plugin: {0} not loaded. It has not been '
                                'configured.'.format(plugin_name))
                continue
            plugin = entrypoint.load()(plugin_config)
            if entrypoint.name == 'handler':
                try:
                    reading_types = self.config.getlist(plugin_name,
                                                        'reading types')
                except configparser.NoSectionError:
                    msg = "No 'reading type' specified for handler {0}"
                    logger.warning(msg.format(plugin_name))
                else:
                    for reading_type in reading_types:
                        self.handlers[reading_type].append(plugin)
            elif entrypoint.name == 'source':
                self.sources.append(plugin)
            logger.debug("Plugin: {0}.{1}, loaded".format(plugin_name,
                                                          entrypoint.name))

    def stop(self, signum, stack):
        self.stop_event.set()

    def run(self):
        while not self.stop_event.is_set():
            for source in self.sources:
                readings = source.get_readings()
                if readings is None:
                    continue
                for reading in readings:
                    for handler in self.handlers.get(reading.reading_type, []):
                        if reading.value is None:
                            logger.warning('None value from {0}'.format(
                                reading.series_key))
                            continue
                        handler.set_reading(reading)
            self.stop_event.wait(1)

    def flush_handlers(self):
        flushed = set()
        for handler_list in self.handlers.values():
            for handler in handler_list:
                if handler in flushed:
                    continue
                try:
                    handler.flush()
                except AttributeError:
                    pass
                flushed.add(handler)

def main():
    parser = argparse.ArgumentParser(description="Internet of Things Relay")
    parser.add_argument('-c', '--config-file', help="Configuration Filename")
    parser.add_argument('--log-level', help="Log Level", default='info',
                        choices=('debug', 'info', 'warning', 'info'))
    args = parser.parse_args()
    logging.basicConfig(format='%(asctime)s %(message)s',
                        level=args.log_level.upper())
    config = ConfigParser()
    if args.config_file is None:
        config_file = DEFAULT_CONFIG
    else:
        config_file = args.config_file
    try:
        f = open(config_file, 'r')
    except IOError as e:
        logger.critical("Cannot open config file {0}. {1}.".format(e.filename,
                                                                   e.strerror))
        sys.exit(1)
    with f:
        config.readfp(f)
    r = Relay(config)
    r.load_plugins()
    r.run()
    r.flush_handlers()

if __name__ == "__main__":
    main()
<commit_msg>add Exception class for plugins to use<commit_after>'''
Copyright (c) 2014, Emmanuel Levijarvi
All rights reserved.
License BSD

iotrelay.py provides an application and a framework for passing data
between data sources and handlers.
'''
import argparse
import datetime
from collections import defaultdict
import pkg_resources
import os
import sys
import signal
import threading
import logging
try:
    import configparser
except ImportError:
    import ConfigParser as configparser

logger = logging.getLogger()

DEFAULT_CONFIG = os.path.join(os.path.expanduser("~"), '.iotrelay.cfg')
GROUP = 'iotrelay'
version = "1.0.1"

class Error(Exception):
    pass

class PluginError(Error):
    pass

class ConfigParser(configparser.SafeConfigParser):
    '''ConfigParser is a subclass of the standard library's ConfigParser.

    It adds the ability to parse an option as a list. See the Python
    Standard Library Documentation for more.
    '''
    @staticmethod
    def _make_list(value, sep):
        return [v.strip() for v in value.split(sep)]

    def getlist(self, section, option, sep=','):
        '''Parse an option as a list, optionally specifying the separator
        by specifying the sep argument.
        '''
        return self._make_list(self.get(section, option), sep)

class Reading(object):
    '''Reading provides a container for passing a datum,or "Reading",
    between sources and handlers.

    Attributes:
        reading_type (str):
        value (str):
        timestamp (datetime.datetime):
        series_key (str):
    '''
    def __init__(self, reading_type, value, timestamp=None, series_key=None):
        self.reading_type = reading_type
        self.value = value
        self.series_key = series_key
        if series_key is None:
            self.series_key = reading_type
        if timestamp is None:
            self.timestamp = datetime.datetime.utcnow()
        else:
            self.timestamp = timestamp
        self.iteration = False

    def __iter__(self):
        return self

    def next(self):
        return self.__next__()

    def __next__(self):
        if self.iteration:
            raise StopIteration
        self.iteration = True
        return self

    def __str__(self):
        return "{0}: {1!s}, {2!s}".format(self.timestamp.isoformat(),
                                          self.series_key,
                                          self.value)

    def __repr__(self):
        return ("Reading({self.reading_type!r}, {self.value!r}, "
                "{self.timestamp!r}, {self.series_key!r})".format(self=self))

class Relay(object):
    def __init__(self, config):
        self.config = config
        self.stop_event = threading.Event()
        signal.signal(signal.SIGTERM, self.stop)
        signal.signal(signal.SIGINT, self.stop)
        self.sources = []
        self.handlers = defaultdict(list)

    def load_plugins(self):
        logger.debug('IoTrelay loading plugins')
        for entrypoint in pkg_resources.iter_entry_points(group=GROUP):
            plugin_name = entrypoint.dist.project_name
            try:
                plugin_config = dict(self.config.items(plugin_name))
            except configparser.NoSectionError:
                logging.warning('Plugin: {0} not loaded. It has not been '
                                'configured.'.format(plugin_name))
                continue
            plugin = entrypoint.load()(plugin_config)
            if entrypoint.name == 'handler':
                try:
                    reading_types = self.config.getlist(plugin_name,
                                                        'reading types')
                except configparser.NoSectionError:
                    msg = "No 'reading type' specified for handler {0}"
                    logger.warning(msg.format(plugin_name))
                else:
                    for reading_type in reading_types:
                        self.handlers[reading_type].append(plugin)
            elif entrypoint.name == 'source':
                self.sources.append(plugin)
            logger.debug("Plugin: {0}.{1}, loaded".format(plugin_name,
                                                          entrypoint.name))

    def stop(self, signum, stack):
        self.stop_event.set()

    def run(self):
        while not self.stop_event.is_set():
            for source in self.sources:
                try:
                    readings = source.get_readings()
                except PluginError as e:
                    logger.error('Unable to read from source. {0}'.format(e))
                    continue
                if readings is None:
                    continue
                for reading in readings:
                    for handler in self.handlers.get(reading.reading_type, []):
                        if reading.value is None:
                            logger.warning('None value from {0}'.format(
                                reading.series_key))
                            continue
                        try:
                            handler.set_reading(reading)
                        except PluginError as e:
                            logger.error('Unable to send data {0}'.format(e))
            self.stop_event.wait(1)

    def flush_handlers(self):
        flushed = set()
        for handler_list in self.handlers.values():
            for handler in handler_list:
                if handler in flushed:
                    continue
                try:
                    handler.flush()
                except AttributeError:
                    pass
                flushed.add(handler)

def main():
    parser = argparse.ArgumentParser(description="Internet of Things Relay")
    parser.add_argument('-c', '--config-file', help="Configuration Filename")
    parser.add_argument('--log-level', help="Log Level", default='info',
                        choices=('debug', 'info', 'warning', 'info'))
    args = parser.parse_args()
    logging.basicConfig(format='%(asctime)s %(message)s',
                        level=args.log_level.upper())
    config = ConfigParser()
    if args.config_file is None:
        config_file = DEFAULT_CONFIG
    else:
        config_file = args.config_file
    try:
        f = open(config_file, 'r')
    except IOError as e:
        logger.critical("Cannot open config file {0}. {1}.".format(e.filename,
                                                                   e.strerror))
        sys.exit(1)
    with f:
        config.readfp(f)
    r = Relay(config)
    r.load_plugins()
    r.run()
    r.flush_handlers()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
This is an example showing how to copy a tree while overwriting one or more of
its branches with new values.
"""

from rootpy.tree import Tree, TreeModel
from rootpy.io import open as ropen
from rootpy.types import *
from random import gauss

# define the model
class Event(TreeModel):

    x = FloatCol()
    y = FloatCol()
    z = FloatCol()
    i = IntCol()

# first create a tree "test" in a file "test.root"
f = ropen("test.root", "recreate")

tree = Tree("test", model=Event)

# fill the tree
for i in xrange(10000):
    tree.x = gauss(.5, 1.)
    tree.y = gauss(.3, 2.)
    tree.z = gauss(13., 42.)
    tree.i = i
    tree.fill()
tree.write()

# Now we want to copy the tree above into a new file while overwriting a branch
f_copy = ropen("test_copy.root", "recreate")

# you may not know the entire model of the original tree but only the branches
# you intend to overwite, so I am not specifying the model=Event below as an
# example of how to deal with this.
tree_copy = Tree("test_copy")
# if the original tree was not handed to you through rootpy don't forget to:
# from rootpy.utils import asrootpy
# tree = asrootpy(tree)
# Here we specify the buffer for the new tree to use. We use the same buffer as
# the original tree. This creates all the same branches in the new tree but
# their addresses point to the same memory used by the original tree.
tree_copy.set_buffer(
        tree.buffer,
        create_branches=True)

# now loop over the original tree and fill the new tree
for entry in tree:
    # overwrite a branch value
    # this changes the value that will be written to the new tree but leaves the
    # value unchanged in the original tree on disk.
    entry.x = 3.141
    # "entry" is actually the buffer, which is shared between both trees.
    tree_copy.Fill()

# tree_copy is now a copy of tree where the "x" branch has been overwritten with
# a new values
tree_copy.Write()
f_copy.Close()
f.Close()
<commit_msg>spelling<commit_after>#!/usr/bin/env python

"""
This is an example showing how to copy a tree while overwriting one or more of
its branches with new values.
"""

from rootpy.tree import Tree, TreeModel
from rootpy.io import open as ropen
from rootpy.types import *
from random import gauss

# define the model
class Event(TreeModel):

    x = FloatCol()
    y = FloatCol()
    z = FloatCol()
    i = IntCol()

# first create a tree "test" in a file "test.root"
f = ropen("test.root", "recreate")

tree = Tree("test", model=Event)

# fill the tree
for i in xrange(10000):
    tree.x = gauss(.5, 1.)
    tree.y = gauss(.3, 2.)
    tree.z = gauss(13., 42.)
    tree.i = i
    tree.fill()
tree.write()

# Now we want to copy the tree above into a new file while overwriting a branch
f_copy = ropen("test_copy.root", "recreate")

# you may not know the entire model of the original tree but only the branches
# you intend to overwrite, so I am not specifying the model=Event below as an
# example of how to deal with this.
tree_copy = Tree("test_copy")
# if the original tree was not handed to you through rootpy don't forget to:
# from rootpy.utils import asrootpy
# tree = asrootpy(tree)
# Here we specify the buffer for the new tree to use. We use the same buffer as
# the original tree. This creates all the same branches in the new tree but
# their addresses point to the same memory used by the original tree.
tree_copy.set_buffer(
        tree.buffer,
        create_branches=True)

# now loop over the original tree and fill the new tree
for entry in tree:
    # overwrite a branch value
    # this changes the value that will be written to the new tree but leaves the
    # value unchanged in the original tree on disk.
    entry.x = 3.141
    # "entry" is actually the buffer, which is shared between both trees.
    tree_copy.Fill()

# tree_copy is now a copy of tree where the "x" branch has been overwritten with
# a new values
tree_copy.Write()
f_copy.Close()
f.Close()
<|endoftext|>


<commit_before>import subprocess, re
import helpers
from helpers import *

def chain_exists(name):
    argv = ['iptables', '-t', 'nat', '-nL']
    p = subprocess.Popen(argv, stdout = subprocess.PIPE)
    for line in p.stdout:
        if line.startswith('Chain %s ' % name):
            return True
    rv = p.wait()
    if rv:
        raise Fatal('%r returned %d' % (argv, rv))

def ipt(*args):
    argv = ['iptables', '-t', 'nat'] + list(args)
    debug1('>> %s\n' % ' '.join(argv))
    rv = subprocess.call(argv)
    if rv:
        raise Fatal('%r returned %d' % (argv, rv))

def do_it(port, subnets):
    chain = 'sshuttle-%s' % port

    # basic cleanup/setup of chains
    if chain_exists(chain):
        ipt('-D', 'OUTPUT', '-j', chain)
        ipt('-D', 'PREROUTING', '-j', chain)
        ipt('-F', chain)
        ipt('-X', chain)

    if subnets:
        ipt('-N', chain)
        ipt('-F', chain)
        ipt('-I', 'OUTPUT', '1', '-j', chain)
        ipt('-I', 'PREROUTING', '1', '-j', chain)

        # create new subnet entries
        for snet,swidth in subnets:
            ipt('-A', chain, '-j', 'REDIRECT',
                '--dest', '%s/%s' % (snet,swidth),
                '-p', 'tcp',
                '--to-ports', str(port),
                '-m', 'ttl', '!', '--ttl', '42'  # to prevent infinite loops
                )

# This is some iptables voodoo for setting up the Linux kernel's transparent
# proxying stuff.  If subnets is empty, we just delete our sshuttle chain;
# otherwise we delete it, then make it from scratch.
#
# We name the chain based on the transproxy port number so that it's possible
# to run multiple copies of sshuttle at the same time.  Of course, the
# multiple copies shouldn't have overlapping subnets, or only the most-
# recently-started one will win (because we use "-I OUTPUT 1" instead of
# "-A OUTPUT").
#
# This code is supposed to clean up after itself by deleting extra chains on
# exit.  In case that fails, it's not the end of the world; future runs will
# supercede it in the transproxy list, at least, so the leftover iptables
# chains are mostly harmless.
def main(port, subnets):
    assert(port > 0)
    assert(port <= 65535)

    if os.getuid() != 0:
        raise Fatal('you must be root (or enable su/sudo) to set up iptables')

    # because of limitations of the 'su' command, the *real* stdin/stdout
    # are both attached to stdout initially.  Clone stdout into stdin so we
    # can read from it.
    os.dup2(1, 0)

    debug1('iptables manager ready.\n')
    sys.stdout.write('READY\n')
    sys.stdout.flush()

    # ctrl-c shouldn't be passed along to me.  When the main sshuttle dies,
    # I'll die automatically.
    os.setsid()

    # we wait until we get some input before creating the rules.  That way,
    # sshuttle can launch us as early as possible (and get sudo password
    # authentication as early in the startup process as possible).
    sys.stdin.readline(128)
    try:
        debug1('iptables manager: starting transproxy.\n')
        do_it(port, subnets)

        sys.stdout.write('STARTED\n')

        try:
            sys.stdout.flush()

            # Now we wait until EOF or any other kind of exception.  We need
            # to stay running so that we don't need a *second* password
            # authentication at shutdown time - that cleanup is important!
            while sys.stdin.readline(128):
                pass
        except IOError:
            # the parent process died for some reason; he's surely been loud
            # enough, so no reason to report another error
            return

    finally:
        try:
            debug1('iptables manager: undoing changes.\n')
        except:
            pass
        do_it(port, [])
<commit_msg>iptables: if client dies before sending GO, just quit.<commit_after>import subprocess, re
import helpers
from helpers import *

def chain_exists(name):
    argv = ['iptables', '-t', 'nat', '-nL']
    p = subprocess.Popen(argv, stdout = subprocess.PIPE)
    for line in p.stdout:
        if line.startswith('Chain %s ' % name):
            return True
    rv = p.wait()
    if rv:
        raise Fatal('%r returned %d' % (argv, rv))

def ipt(*args):
    argv = ['iptables', '-t', 'nat'] + list(args)
    debug1('>> %s\n' % ' '.join(argv))
    rv = subprocess.call(argv)
    if rv:
        raise Fatal('%r returned %d' % (argv, rv))

def do_it(port, subnets):
    chain = 'sshuttle-%s' % port

    # basic cleanup/setup of chains
    if chain_exists(chain):
        ipt('-D', 'OUTPUT', '-j', chain)
        ipt('-D', 'PREROUTING', '-j', chain)
        ipt('-F', chain)
        ipt('-X', chain)

    if subnets:
        ipt('-N', chain)
        ipt('-F', chain)
        ipt('-I', 'OUTPUT', '1', '-j', chain)
        ipt('-I', 'PREROUTING', '1', '-j', chain)

        # create new subnet entries
        for snet,swidth in subnets:
            ipt('-A', chain, '-j', 'REDIRECT',
                '--dest', '%s/%s' % (snet,swidth),
                '-p', 'tcp',
                '--to-ports', str(port),
                '-m', 'ttl', '!', '--ttl', '42'  # to prevent infinite loops
                )

# This is some iptables voodoo for setting up the Linux kernel's transparent
# proxying stuff.  If subnets is empty, we just delete our sshuttle chain;
# otherwise we delete it, then make it from scratch.
#
# We name the chain based on the transproxy port number so that it's possible
# to run multiple copies of sshuttle at the same time.  Of course, the
# multiple copies shouldn't have overlapping subnets, or only the most-
# recently-started one will win (because we use "-I OUTPUT 1" instead of
# "-A OUTPUT").
#
# This code is supposed to clean up after itself by deleting extra chains on
# exit.  In case that fails, it's not the end of the world; future runs will
# supercede it in the transproxy list, at least, so the leftover iptables
# chains are mostly harmless.
def main(port, subnets):
    assert(port > 0)
    assert(port <= 65535)

    if os.getuid() != 0:
        raise Fatal('you must be root (or enable su/sudo) to set up iptables')

    # because of limitations of the 'su' command, the *real* stdin/stdout
    # are both attached to stdout initially.  Clone stdout into stdin so we
    # can read from it.
    os.dup2(1, 0)

    debug1('iptables manager ready.\n')
    sys.stdout.write('READY\n')
    sys.stdout.flush()

    # ctrl-c shouldn't be passed along to me.  When the main sshuttle dies,
    # I'll die automatically.
    os.setsid()

    # we wait until we get some input before creating the rules.  That way,
    # sshuttle can launch us as early as possible (and get sudo password
    # authentication as early in the startup process as possible).
    line = sys.stdin.readline(128)
    if not line:
        return  # parent died; nothing to do
    if line != 'GO\n':
        raise Fatal('iptables: expected GO but got %r' % line)
    try:
        if line:
            debug1('iptables manager: starting transproxy.\n')
            do_it(port, subnets)
            sys.stdout.write('STARTED\n')

        try:
            sys.stdout.flush()

            # Now we wait until EOF or any other kind of exception.  We need
            # to stay running so that we don't need a *second* password
            # authentication at shutdown time - that cleanup is important!
            while sys.stdin.readline(128):
                pass
        except IOError:
            # the parent process died for some reason; he's surely been loud
            # enough, so no reason to report another error
            return

    finally:
        try:
            debug1('iptables manager: undoing changes.\n')
        except:
            pass
        do_it(port, [])
<|endoftext|>


<commit_before>import gbdxtools
from gbdxtools.vectors import TermsAggDef, GeohashAggDef

gbdx = gbdxtools.Interface()
gbdx.vectors.aggregations_url = 'https://devvector.geobigdata.io/insight-vector/api/aggregation'

# Let's find all the Worldview 3 vector footprints in colorado
colorado_aoi = "POLYGON((-108.89 40.87,-102.19 40.87,-102.19 37.03,-108.89 37.03,-108.89 40.87))"

query = 'ingest_source:OSM'

# let's get 5 geohash buckets with the top 5 item_type values in each
child_agg = TermsAggDef('item_type')
agg = GeohashAggDef('3', children=child_agg)
result = gbdx.vectors.aggregate_query(colorado_aoi, agg, query, count=5)

# the result has a single-element list containing the top-level aggregation
for entry in result[0]['terms']:  # the 'terms' field contains our buckets
    geohash_str = entry['term']  # the 'term' entry contains our geohash
    print geohash_str
    child_aggs = entry[
        'aggregations']  # the 'aggregations' field contains the child aggregations for the 'item_type' values

    # since the child aggregations have the same structure, we can walk it the same way.
    # let's create a dict of item_types and their counts
    for child in child_aggs:
        types = {bucket['term']: bucket['count'] for bucket in child['terms']}

    # we could do something cool with these, but for now we'll just print them
    for item_type in types:
        print '\t%s:%s' % (item_type, types[item_type])
<commit_msg>Clean up vector aggregation example<commit_after>import gbdxtools
from gbdxtools.vectors import TermsAggDef, GeohashAggDef

gbdx = gbdxtools.Interface()

# Let's find all the Worldview 3 vector footprints in colorado
colorado_aoi = "POLYGON((-108.89 40.87,-102.19 40.87,-102.19 37.03,-108.89 37.03,-108.89 40.87))"

query = 'ingest_source:OSM'

# let's get 5 geohash buckets with the top 5 item_type values in each
child_agg = TermsAggDef('item_type')
agg = GeohashAggDef('3', children=child_agg)
result = gbdx.vectors.aggregate_query(colorado_aoi, agg, query, count=5)

# the result has a single-element list containing the top-level aggregation
for entry in result[0]['terms']:  # the 'terms' field contains our buckets
    geohash_str = entry['term']  # the 'term' entry contains our geohash
    print geohash_str
    child_aggs = entry[ 'aggregations']  # the 'aggregations' field contains the child aggregations for the 'item_type' values

    # since the child aggregations have the same structure, we can walk it the same way.
    # let's create a dict of item_types and their counts
    for child in child_aggs:
        types = {bucket['term']: bucket['count'] for bucket in child['terms']}

    # we could do something cool with these, but for now we'll just print them
    for item_type in types:
        print '\t%s:%s' % (item_type, types[item_type])
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import contextlib
import os
import sys

from pre_commit.envcontext import envcontext
from pre_commit.envcontext import Var
from pre_commit.languages import helpers
from pre_commit.util import clean_path_on_failure
from pre_commit.xargs import xargs

ENVIRONMENT_DIR = 'node_env'
get_default_version = helpers.basic_get_default_version
healthy = helpers.basic_healthy

def get_env_patch(venv):  # pragma: windows no cover
    config = os.path.join(venv, 'bin') if sys.platform == 'cygwin' else venv
    return (
        ('NODE_VIRTUAL_ENV', venv),
        ('NPM_CONFIG_PREFIX', config),
        ('npm_config_prefix', config),
        ('NODE_PATH', os.path.join(venv, 'lib', 'node_modules')),
        ('PATH', (os.path.join(venv, 'bin'), os.pathsep, Var('PATH'))),
    )

@contextlib.contextmanager
def in_env(repo_cmd_runner, language_version):  # pragma: windows no cover
    envdir = repo_cmd_runner.path(
        helpers.environment_dir(ENVIRONMENT_DIR, language_version),
    )
    with envcontext(get_env_patch(envdir)):
        yield

def install_environment(
        repo_cmd_runner, version, additional_dependencies,
):  # pragma: windows no cover
    additional_dependencies = tuple(additional_dependencies)
    assert repo_cmd_runner.exists('package.json')
    directory = helpers.environment_dir(ENVIRONMENT_DIR, version)

    env_dir = repo_cmd_runner.path(directory)
    with clean_path_on_failure(env_dir):
        cmd = [
            sys.executable, '-m', 'nodeenv', '--prebuilt',
            '{{prefix}}{}'.format(directory),
        ]

        if version != 'default':
            cmd.extend(['-n', version])

        repo_cmd_runner.run(cmd)

        with in_env(repo_cmd_runner, version):
            helpers.run_setup_cmd(
                repo_cmd_runner,
                ('npm', 'install', '-g', '.') + additional_dependencies,
            )

def run_hook(repo_cmd_runner, hook, file_args):  # pragma: windows no cover
    with in_env(repo_cmd_runner, hook['language_version']):
        return xargs(helpers.to_cmd(hook), file_args)
<commit_msg>NodeJS hooks compatibilty fix for Cygwin - take 2<commit_after>from __future__ import unicode_literals

import contextlib
import os
import sys

from pre_commit.envcontext import envcontext
from pre_commit.envcontext import Var
from pre_commit.languages import helpers
from pre_commit.util import clean_path_on_failure
from pre_commit.util import cmd_output
from pre_commit.xargs import xargs

ENVIRONMENT_DIR = 'node_env'
get_default_version = helpers.basic_get_default_version
healthy = helpers.basic_healthy

def get_env_patch(venv): # pragma: windows no cover
    if sys.platform == 'cygwin': # pragma: no cover
        _, win_venv, _ = cmd_output('cygpath', '-w', venv)
        install_prefix = r'{}\bin'.format(win_venv.strip())
    else:
        install_prefix = venv
    return (
        ('NODE_VIRTUAL_ENV', venv),
        ('NPM_CONFIG_PREFIX', install_prefix),
        ('npm_config_prefix', install_prefix),
        ('NODE_PATH', os.path.join(venv, 'lib', 'node_modules')),
        ('PATH', (os.path.join(venv, 'bin'), os.pathsep, Var('PATH'))),
    )

@contextlib.contextmanager
def in_env(repo_cmd_runner, language_version):  # pragma: windows no cover
    envdir = repo_cmd_runner.path(
        helpers.environment_dir(ENVIRONMENT_DIR, language_version),
    )
    with envcontext(get_env_patch(envdir)):
        yield

def install_environment(
        repo_cmd_runner, version, additional_dependencies,
):  # pragma: windows no cover
    additional_dependencies = tuple(additional_dependencies)
    assert repo_cmd_runner.exists('package.json')
    directory = helpers.environment_dir(ENVIRONMENT_DIR, version)

    env_dir = repo_cmd_runner.path(directory)
    with clean_path_on_failure(env_dir):
        cmd = [
            sys.executable, '-m', 'nodeenv', '--prebuilt',
            '{{prefix}}{}'.format(directory),
        ]

        if version != 'default':
            cmd.extend(['-n', version])

        repo_cmd_runner.run(cmd)

        with in_env(repo_cmd_runner, version):
            helpers.run_setup_cmd(
                repo_cmd_runner,
                ('npm', 'install', '-g', '.') + additional_dependencies,
            )

def run_hook(repo_cmd_runner, hook, file_args):  # pragma: windows no cover
    with in_env(repo_cmd_runner, hook['language_version']):
        return xargs(helpers.to_cmd(hook), file_args)
<|endoftext|>


<commit_before>from logging import getLogger

from django.db import models

from commands.utils import(
    send_cec_command,
    send_infrared_command,
    send_serial_command,
    send_tcp_command,
)

LOGGER = getLogger(__name__)

class RemoteButton(models.Model):

    lirc_code = models.CharField(
        unique=True,
        max_length=255,
    )

    def execute(self):
        effects = []
        LOGGER.warning('Executing macros: {}'.format(self.macros.all()))
        for command in self.macros.all():
            side_effects = command.execute()
            if side_effects is not None:
                effects += side_effects
        for effect in effects:
            effect.execute()

    def __repr__(self):
        return self.lirc_code

    def __str__(self):
        return self.__repr__()

class Command(models.Model):
    device = models.ForeignKey(
        'Device',
        related_name='commands'
    )
    trigger = models.ForeignKey(
        'CommandSet',
        related_name='commands'
    )
    command_type = models.CharField(
        max_length=255,
    )
    condition = models.ForeignKey(
        'Condition',
        related_name='+',
        null=True,
        blank=True,
    )
    data = models.CharField(
        null=True,
        blank=True,
        max_length=255,
    )

    def execute(self):
        handler = self.device.get_handler(self.command_type)
        if not self.condition or self.condition.met():
            handler(self.data)

    def __repr__(self):
        return '{} command {} triggered by {}'.format(self.command_type, self.data, self.trigger)

    def __str__(self):
        return self.__repr__()

class CommandSet(models.Model):

    name = models.CharField(max_length=255)

    trigger = models.ForeignKey(
        'RemoteButton',
        related_name='macros'
    )

    condition = models.ForeignKey(
        'Condition',
        related_name='+',
        null=True,
        blank=True,
    )

    def execute(self):
        if hasattr(self, 'condition') and not self.condition.met():
            LOGGER.warning('Conditions for {} were not met.'.format(self))
            return None
        for command in self.commands.all():
            LOGGER.warning('Conditions for {} were met. Executing command {}.'.format(self, command))
            command.execute()
        return list(self.side_effects.all())

    def __repr__(self):
        return self.name

    def __str__(self):
        return self.__repr__()

class Condition(models.Model):

    CONDITION_TYPES = [
        ('any', 'Any'),
        ('all', 'All'),
    ]

    condition_type = models.CharField(max_length=3, choices=CONDITION_TYPES)
    states = models.ManyToManyField('State', related_name='+', blank=True)
    nested_conditions = models.ManyToManyField('self',
        related_name='+',
        symmetrical=False,
        blank=True
    )

    def met(self):
        if self.condition_type == 'any':
            check_method = any
        else:
            check_method = all
        states_met = check_method(x.is_active() for x in self.states.all())
        nested_conditions_met = check_method(x.met() for x in self.nested_conditions.all())
        return check_method([states_met, nested_conditions_met])

class StateSet(models.Model):
    name = models.CharField(max_length=255)
    status = models.OneToOneField(
        'State',
        null=True,
        blank=True,
    )
    device = models.ForeignKey(
        'Device',
        related_name='state_sets'
    )

    def __repr__(self):
        return '{}: {}'.format(self.name, self.status)

    def __str__(self):
        return self.__repr__()

class State(models.Model):
    name = models.CharField(max_length=255)
    state_set = models.ForeignKey(
        'StateSet',
    )

    def is_active(self):
        return self == self.state_set.status

    def activate(self):
        self.state_set.status = self
        self.state_set.save()

    def __repr__(self):
        return self.name

    def __str__(self):
        return self.__repr__()

class StateSideEffect(models.Model):
    commands = models.ForeignKey(
        'CommandSet',
        related_name='side_effects',
    )
    states = models.ManyToManyField(
        'State',
        related_name='reachable_by'
    )

    def execute(self):
        for state in self.states.all():
            state.activate()

    def __repr__(self):
        return 'Set {} after {}'.format(', '.join(str(x) for x in self.states.all()), self.commands)

    def __str__(self):
        return self.__repr__()

class Device(models.Model):

    command_types = {
        'cec': 'cec_config',
        'tcp': 'tcp_config',
        'serial': 'serial_config',
        'infrared': 'irsend_config',
    }

    name = models.CharField(
        max_length=255
    )
    cec_config = models.OneToOneField(
        'CecConfig',
        null=True,
        blank=True,
    )
    tcp_config = models.OneToOneField(
        'TcpConfig',
        null=True,
        blank=True,
    )
    serial_config = models.OneToOneField(
        'SerialConfig',
        null=True,
        blank=True,
    )
    irsend_config = models.OneToOneField(
        'IrsendConfig',
        null=True,
        blank=True,
    )

    def get_handler(self, command_type):
        return getattr(self, self.command_types[command_type]).handler

    def __repr__(self):
        return self.name

    def __str__(self):
        return self.__repr__()

class CecConfig(models.Model):

    source_address = models.PositiveIntegerField()
    target_address = models.PositiveIntegerField()

    def handler(self, command):
        LOGGER.warning(
            'Sending CEC command {} from {} to {}.'.format(
                command,
                self.source_address,
                self.target_address,
            )
        )
        send_cec_command(
            source_address,
            target_address,
            command,
        )

    def __repr__(self):
        return 'CEC config' +  (' for {}'.format(self.device) if hasattr(self, 'device') else '')

    def __str__(self):
        return self.__repr__()

class SerialConfig(models.Model):

    port_name = models.CharField(max_length=255)
    baud_rate = models.PositiveIntegerField()
    byte_size = models.PositiveIntegerField()
    timeout = models.PositiveIntegerField()

    def handler(self, command):
        LOGGER.warning(
            'Sending serial command "{}" on port {} with byte size {}, baud rate {}, and timeout {}.'.format(
                command,
                self.port_name,
                self.byte_size,
                self.baud_rate,
                self.timeout,
            )
        )
        send_serial_command(
            self.port_name,
            self.baud_rate,
            self.byte_size,
            self.timeout,
            command,
        )

    def __repr__(self):
        return 'Serial config' +  (' for {}'.format(self.device) if hasattr(self, 'device') else '')

    def __str__(self):
        return self.__repr__()

class TcpConfig(models.Model):

    host = models.CharField(max_length=255)
    port = models.PositiveIntegerField()

    def handler(self, command):
        LOGGER.warning(
            'Sending TCP command "{}" to host {} on port number {}.'.format(
                command,
                self.host,
                self.port,
            )
        )
        send_tcp_command(host, port, command)

    def __repr__(self):
        return 'TCP config' +  (' for {}'.format(self.device) if hasattr(self, 'device') else '')

    def __str__(self):
        return self.__repr__()

class IrsendConfig(models.Model):

    remote_name = models.CharField(max_length=255)

    def handler(self, command):
        LOGGER.warning(
            'Sending IR command "{}" with remote "{}".'.format(command, self.remote_name)
        )
        send_infrared_command(self.remote_name, command)

    def __repr__(self):
        return 'Infrared config' +  (' for {}'.format(self.device) if hasattr(self, 'device') else '')

    def __str__(self):
        return self.__repr__()
<commit_msg>Bugfixes<commit_after>from logging import getLogger

from django.db import models

from commands.utils import(
    send_cec_command,
    send_infrared_command,
    send_serial_command,
    send_tcp_command,
)

LOGGER = getLogger(__name__)

class RemoteButton(models.Model):

    lirc_code = models.CharField(
        unique=True,
        max_length=255,
    )

    def execute(self):
        effects = []
        LOGGER.warning('Executing macros: {}'.format(self.macros.all()))
        for command in self.macros.all():
            side_effects = command.execute()
            if side_effects is not None:
                effects += side_effects
        for effect in effects:
            effect.execute()

    def __repr__(self):
        return self.lirc_code

    def __str__(self):
        return self.__repr__()

class Command(models.Model):
    device = models.ForeignKey(
        'Device',
        related_name='commands'
    )
    trigger = models.ForeignKey(
        'CommandSet',
        related_name='commands'
    )
    command_type = models.CharField(
        max_length=255,
    )
    condition = models.ForeignKey(
        'Condition',
        related_name='+',
        null=True,
        blank=True,
    )
    data = models.CharField(
        null=True,
        blank=True,
        max_length=255,
    )

    def execute(self):
        handler = self.device.get_handler(self.command_type)
        if not self.condition or self.condition.met():
            handler(self.data)

    def __repr__(self):
        return '{} command {} triggered by {}'.format(self.command_type, self.data, self.trigger)

    def __str__(self):
        return self.__repr__()

class CommandSet(models.Model):

    name = models.CharField(max_length=255)

    trigger = models.ForeignKey(
        'RemoteButton',
        related_name='macros'
    )

    condition = models.ForeignKey(
        'Condition',
        related_name='+',
        null=True,
        blank=True,
    )

    def execute(self):
        if hasattr(self, 'condition') and not self.condition.met():
            LOGGER.warning('Conditions for {} were not met.'.format(self))
            return None
        for command in self.commands.all():
            LOGGER.warning('Conditions for {} were met. Executing command {}.'.format(self, command))
            command.execute()
        return list(self.side_effects.all())

    def __repr__(self):
        return self.name

    def __str__(self):
        return self.__repr__()

class Condition(models.Model):

    CONDITION_TYPES = [
        ('any', 'Any'),
        ('all', 'All'),
    ]

    condition_type = models.CharField(max_length=3, choices=CONDITION_TYPES)
    states = models.ManyToManyField('State', related_name='+', blank=True)
    nested_conditions = models.ManyToManyField('self',
        related_name='+',
        symmetrical=False,
        blank=True
    )

    def met(self):
        if self.condition_type == 'any':
            check_method = any
        else:
            check_method = all
        states_met = check_method(x.is_active() for x in self.states.all())
        nested_conditions_met = check_method(x.met() for x in self.nested_conditions.all())
        return check_method([states_met, nested_conditions_met])

class StateSet(models.Model):
    name = models.CharField(max_length=255)
    status = models.OneToOneField(
        'State',
        null=True,
        blank=True,
    )
    device = models.ForeignKey(
        'Device',
        related_name='state_sets'
    )

    def __repr__(self):
        return '{}: {}'.format(self.name, self.status)

    def __str__(self):
        return self.__repr__()

class State(models.Model):
    name = models.CharField(max_length=255)
    state_set = models.ForeignKey(
        'StateSet',
    )

    def is_active(self):
        return self == self.state_set.status

    def activate(self):
        self.state_set.status = self
        self.state_set.save()

    def __repr__(self):
        return self.name

    def __str__(self):
        return self.__repr__()

class StateSideEffect(models.Model):
    commands = models.ForeignKey(
        'CommandSet',
        related_name='side_effects',
    )
    states = models.ManyToManyField(
        'State',
        related_name='reachable_by'
    )

    def execute(self):
        for state in self.states.all():
            state.activate()

    def __repr__(self):
        return 'Set {} after {}'.format(', '.join(str(x) for x in self.states.all()), self.commands)

    def __str__(self):
        return self.__repr__()

class Device(models.Model):

    command_types = {
        'cec': 'cec_config',
        'tcp': 'tcp_config',
        'serial': 'serial_config',
        'infrared': 'irsend_config',
    }

    name = models.CharField(
        max_length=255
    )
    cec_config = models.OneToOneField(
        'CecConfig',
        null=True,
        blank=True,
    )
    tcp_config = models.OneToOneField(
        'TcpConfig',
        null=True,
        blank=True,
    )
    serial_config = models.OneToOneField(
        'SerialConfig',
        null=True,
        blank=True,
    )
    irsend_config = models.OneToOneField(
        'IrsendConfig',
        null=True,
        blank=True,
    )

    def get_handler(self, command_type):
        return getattr(self, self.command_types[command_type]).handler

    def __repr__(self):
        return self.name

    def __str__(self):
        return self.__repr__()

class CecConfig(models.Model):

    source_address = models.PositiveIntegerField()
    target_address = models.PositiveIntegerField()

    def handler(self, command):
        LOGGER.warning(
            'Sending CEC command {} from {} to {}.'.format(
                command,
                self.source_address,
                self.target_address,
            )
        )
        send_cec_command(
            self.source_address,
            self.target_address,
            command,
        )

    def __repr__(self):
        return 'CEC config' +  (' for {}'.format(self.device) if hasattr(self, 'device') else '')

    def __str__(self):
        return self.__repr__()

class SerialConfig(models.Model):

    port_name = models.CharField(max_length=255)
    baud_rate = models.PositiveIntegerField()
    byte_size = models.PositiveIntegerField()
    timeout = models.PositiveIntegerField()

    def handler(self, command):
        LOGGER.warning(
            'Sending serial command "{}" on port {} with byte size {}, baud rate {}, and timeout {}.'.format(
                command,
                self.port_name,
                self.byte_size,
                self.baud_rate,
                self.timeout,
            )
        )
        send_serial_command(
            self.port_name,
            self.baud_rate,
            self.byte_size,
            self.timeout,
            command,
        )

    def __repr__(self):
        return 'Serial config' +  (' for {}'.format(self.device) if hasattr(self, 'device') else '')

    def __str__(self):
        return self.__repr__()

class TcpConfig(models.Model):

    host = models.CharField(max_length=255)
    port = models.PositiveIntegerField()

    def handler(self, command):
        LOGGER.warning(
            'Sending TCP command "{}" to host {} on port number {}.'.format(
                command,
                self.host,
                self.port,
            )
        )
        send_tcp_command(host, port, command)

    def __repr__(self):
        return 'TCP config' +  (' for {}'.format(self.device) if hasattr(self, 'device') else '')

    def __str__(self):
        return self.__repr__()

class IrsendConfig(models.Model):

    remote_name = models.CharField(max_length=255)

    def handler(self, command):
        LOGGER.warning(
            'Sending IR command "{}" with remote "{}".'.format(command, self.remote_name)
        )
        send_infrared_command(self.remote_name, command)

    def __repr__(self):
        return 'Infrared config' +  (' for {}'.format(self.device) if hasattr(self, 'device') else '')

    def __str__(self):
        return self.__repr__()
<|endoftext|>


<commit_before>import numpy as np
from climlab.process.diagnostic import DiagnosticProcess
from climlab.utils.legendre import P2
from climlab.domain.field import Field

class ConstantAlbedo(DiagnosticProcess):
    def __init__(self, albedo=0.33, **kwargs):
        '''Uniform prescribed albedo.'''
        super(ConstantAlbedo, self).__init__(**kwargs)
        self.param['albedo'] = albedo
        #lat = self.domains['default'].axes['lat'].points
        #albedo = np.ones_like(lat)
        # make sure that the diagnostic has the correct field dimensions.
        dom = self.domains['default']
        self.diagnostics['albedo'] = Field(albedo, domain=dom)

class P2Albedo(DiagnosticProcess):
    def __init__(self, a0=0.33, a2=0.25, **kwargs):
        '''Second order Legendre polynomial formula for surface albedo.'''
        super(P2Albedo, self).__init__(**kwargs)
        self.param['a0'] = a0
        self.param['a2'] = a2
        lat = self.domains['default'].axes['lat'].points
        phi = np.deg2rad(lat)
        albedo = a0 + a2 * P2(np.sin(phi))
        # make sure that the diagnostic has the correct field dimensions.
        dom = self.domains['default']
        self.diagnostics['albedo'] = Field(albedo, domain=dom)

class Iceline(DiagnosticProcess):
    def __init__(self, Tf=-10., **kwargs):
        super(DiagnosticProcess, self).__init__(**kwargs)
        self.param['Tf'] = Tf

    def find_icelines(self):
        Tf = self.param['Tf']
        Ts = self.state['Ts']
        iceline1 = 90.
        iceline2 = -90.
        lat_bounds = self.domains['Ts'].axes['lat'].bounds
        noice = np.where(Ts >= Tf, True, False)
        ice = np.where(Ts < Tf, True, False)
        self.diagnostics['noice'] = noice
        self.diagnostics['ice'] = ice
        if ice.all():
            # 100% ice cover
            icelat = np.array([-90., 90.])
        elif noice.all():
            # zero ice cover
            icelat = np.array([0., 0.])
        else:  # there is some ice edge
            # Taking np.diff of a boolean array gives True at the boundaries between True and False
            boundary_indices = np.where(np.diff(ice.squeeze()))[0] + 1
            icelat = lat_bounds[boundary_indices]  # an array of boundary latitudes
        self.diagnostics['icelat'] = icelat

    def compute(self):
        self.find_icelines()

class StepFunctionAlbedo(DiagnosticProcess):
    def __init__(self, Tf=-10., a0=0.33, a2=0.25, ai=0.6, **kwargs):
        super(DiagnosticProcess, self).__init__(**kwargs)
        self.param['Tf'] = Tf
        self.param['a0'] = a0
        self.param['a2'] = a2
        self.param['ai'] = ai
        sfc = self.domains['Ts']
        self.subprocess['iceline'] = Iceline(Tf=Tf, state=self.state)
        self.subprocess['warm albedo'] = P2Albedo(a0=a0, a2=a2, domains=sfc)
        self.subprocess['cold albedo'] = ConstantAlbedo(albedo=ai, domains=sfc)
        self.topdown = False  # call subprocess compute methods first

    def _get_current_albedo(self):
        '''Simple step-function albedo based on ice line at temperature Tf.'''
        ice = self.subprocess['iceline'].diagnostics['ice']
        # noice = self.subprocess['iceline'].diagnostics['noice']
        cold_albedo = self.subprocess['cold albedo'].diagnostics['albedo']
        warm_albedo = self.subprocess['warm albedo'].diagnostics['albedo']
        albedo = Field(np.where(ice, cold_albedo, warm_albedo), domain=self.domains['Ts'])
        return albedo

    def compute(self):
        self.diagnostics['albedo'] = self._get_current_albedo()
<commit_msg>Changed the default albedo parameters for StepFunctionAlbedo class -- now the same as I previously used in Snowball Earth notebook.<commit_after>import numpy as np
from climlab.process.diagnostic import DiagnosticProcess
from climlab.utils.legendre import P2
from climlab.domain.field import Field

class ConstantAlbedo(DiagnosticProcess):
    def __init__(self, albedo=0.33, **kwargs):
        '''Uniform prescribed albedo.'''
        super(ConstantAlbedo, self).__init__(**kwargs)
        self.param['albedo'] = albedo
        #lat = self.domains['default'].axes['lat'].points
        #albedo = np.ones_like(lat)
        # make sure that the diagnostic has the correct field dimensions.
        dom = self.domains['default']
        self.diagnostics['albedo'] = Field(albedo, domain=dom)

class P2Albedo(DiagnosticProcess):
    def __init__(self, a0=0.33, a2=0.25, **kwargs):
        '''Second order Legendre polynomial formula for surface albedo.'''
        super(P2Albedo, self).__init__(**kwargs)
        self.param['a0'] = a0
        self.param['a2'] = a2
        lat = self.domains['default'].axes['lat'].points
        phi = np.deg2rad(lat)
        albedo = a0 + a2 * P2(np.sin(phi))
        # make sure that the diagnostic has the correct field dimensions.
        dom = self.domains['default']
        self.diagnostics['albedo'] = Field(albedo, domain=dom)

class Iceline(DiagnosticProcess):
    def __init__(self, Tf=-10., **kwargs):
        super(DiagnosticProcess, self).__init__(**kwargs)
        self.param['Tf'] = Tf

    def find_icelines(self):
        Tf = self.param['Tf']
        Ts = self.state['Ts']
        iceline1 = 90.
        iceline2 = -90.
        lat_bounds = self.domains['Ts'].axes['lat'].bounds
        noice = np.where(Ts >= Tf, True, False)
        ice = np.where(Ts < Tf, True, False)
        self.diagnostics['noice'] = noice
        self.diagnostics['ice'] = ice
        if ice.all():
            # 100% ice cover
            icelat = np.array([-90., 90.])
        elif noice.all():
            # zero ice cover
            icelat = np.array([0., 0.])
        else:  # there is some ice edge
            # Taking np.diff of a boolean array gives True at the boundaries between True and False
            boundary_indices = np.where(np.diff(ice.squeeze()))[0] + 1
            icelat = lat_bounds[boundary_indices]  # an array of boundary latitudes
        self.diagnostics['icelat'] = icelat

    def compute(self):
        self.find_icelines()

class StepFunctionAlbedo(DiagnosticProcess):
    def __init__(self, Tf=-10., a0=0.3, a2=0.078, ai=0.62, **kwargs):
        super(DiagnosticProcess, self).__init__(**kwargs)
        self.param['Tf'] = Tf
        self.param['a0'] = a0
        self.param['a2'] = a2
        self.param['ai'] = ai
        sfc = self.domains['Ts']
        self.subprocess['iceline'] = Iceline(Tf=Tf, state=self.state)
        self.subprocess['warm albedo'] = P2Albedo(a0=a0, a2=a2, domains=sfc)
        self.subprocess['cold albedo'] = ConstantAlbedo(albedo=ai, domains=sfc)
        self.topdown = False  # call subprocess compute methods first

    def _get_current_albedo(self):
        '''Simple step-function albedo based on ice line at temperature Tf.'''
        ice = self.subprocess['iceline'].diagnostics['ice']
        # noice = self.subprocess['iceline'].diagnostics['noice']
        cold_albedo = self.subprocess['cold albedo'].diagnostics['albedo']
        warm_albedo = self.subprocess['warm albedo'].diagnostics['albedo']
        albedo = Field(np.where(ice, cold_albedo, warm_albedo), domain=self.domains['Ts'])
        return albedo

    def compute(self):
        self.diagnostics['albedo'] = self._get_current_albedo()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
'''bootstrap
Install build system and required tools to build nonlibc.

DISCLAIMER: this is a hack!
In my defense, package management is f'ing HARD and there just
	seems to be nothing better than hand-coding the nitpicky
	cross-platform hand-waving for. each. dependecy.

COROLLARY: please DON'T just add another tool to this project;
	try and keep to what's there already.

THE GOOD NEWS: at least this bootstrap is now a python script
	(as opposed to BASH), so tooling is python3-only.

As always, the quest for a
	[Universal Install Script](https://www.explainxkcd.com/wiki/index.php/1654:_Universal_Install_Script)
	continues!
'''

import subprocess
import sys
import re
from shutil import which
from os import environ

def run_cmd(args=[], shell=False, cwd=None, env={}):
	'''run_cmd()
	Run a command.
	Return stdout; raise an exception on failure.
	'''
	if verbose:
		print('EXEC: %s' % ' '.join(args))

	en = environ.copy()
	en.update(env)

	try:
		sub = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
							shell=shell, cwd=cwd, env=en, check=True);
	except subprocess.CalledProcessError as err:
		if verbose:
			print(err.stdout.decode('ascii'))
			print(err.stderr.decode('ascii'), file=sys.stderr)
		raise

	ret = sub.stdout.decode('ascii')
	if verbose:
		print(ret)
	return ret.strip()

def vers(version_string):
	'''vers()
	Takes a dodgy 'version_string' input

	Returns a list which can be compared (e.g. `>` `<` `==` etc.)
		with another list returned by this function.

	stolen from <https://stackoverflow.com/questions/1714027/version-number-comparison>
	'''
	try:
		return [int(x) for x in re.sub(r'(\.0+)*$','', version_string).split(".")]
	except:
		raise ValueError('''cannot parse version string '%s' ''' % version_string)

def pgm_vers(pgm_path, version_cmd='--version'):
	'''pgm_vers()
	Execute 'pgm_path' with 'version_cmd'; munge the output for a version string.

	Return a "listified" version string which can be compared with another
		"listified" version directly.
	'''
	re_vers = re.compile('\D*([\d.]+)')

	try:
		ret = run_cmd([ pgm_path, version_cmd ])
	except:
		return None

	m = re_vers.match(ret)
	if not m:
		raise ValueError('could not find a version ID: %s' % ret)

	return m.group(1)

#		Templates
#
# These are run through Python's str.format() function,
#+	against both the template itself and the target - which means
#+	they can reference any variable therein using standard Python formatting.
templates = {
			"apt-get" : {
				"platform" : [ "linux" ],
				"requires" : [ "apt-get" ],
				"cmd_list" : [ [ "sudo", "apt-get", "-y", "install", "{pkg_name}" ] ]
			},
			"brew" : {
				"platform" : [ "darwin" ],
				"requires" : [ "brew" ],
				"cmd_list" : [ [ "brew", "install", "{pkg_name}" ] ]
			},
			"gem" : {
				"platform" : [ "linux", "darwin" ],
				"requires" : [ "gem" ],
				"cmd_list" : [
					[ "sudo", "gem", "install", "rubygems-update" ],
					[ "sudo", "gem", "install", "{pkg_name}" ]
				]
			},
			"pip3" : {
				"platform" : [ "linux" ],
				"requires" : [ "pip3" ],
				"cmd_list" : [ [ "sudo", "-H", "pip3", "install", "{pkg_name}" ] ]
			},
			"port" : {
				"platform" : [ "darwin" ],
				"requires" : [ "port" ],
				"cmd_list" :  [ [ "sudo", "port", "install", "{pkg_name}" ] ]
			},
			"zip-install" : {
				"platform" : [ "linux", "darwin" ],
				"requires" : [ "mkdir", "wget", "unzip" ],
				"install_d" : "/usr/local/bin",
				"temp_dir" : "./toolchain",
				"cmd_list" :  [
								[ "mkdir", "-p", "{temp_dir}" ],
								[ "wget", "-O", "{temp_dir}/{bin_name}.zip", "{url}" ],
								[ "sudo", "unzip", "-o", "-d", "{install_d}/", "{temp_dir}/{bin_name}.zip" ],
								[ "sudo", "chmod", "go+rx", "{install_d}/{bin_name}" ]
							]
			}
		}

class template():
	'''A rule (and accompanying parameters) for the invocation of
		a specific command.
	'''

	def __init__(self, name='', template={}):
		self.name = name
		self.template = template

	def execute(self, recipe={}):
		for cmd in self.template['cmd_list']:
			# generate a command string, replacing any dictionary references
			cmd_str = [ tok.format(**recipe, **self.template) for tok in cmd ]
			# execute it
			run_cmd(cmd_str, recipe.get('shell', False), recipe.get('cwd', None))

	def is_valid(template):
		'''a template is valid if:

		-	our platform is included in its platform listing
		-	its binary exists on this system
		'''
		# one of the platform entries must match current platform
		if sys.platform not in template['platform']:
			return False

		# verify that required binaries are in place
		if 'requires' in template:
			for binary in template['requires']:
				if not which(binary):
					return False # TODO: try to install it?

		return True

#	Targets
#
# Each target is the name of a binary to be executed on the system.
#
# The Version object specifies 'minimum' or 'exact' version numbers to be
#+	achieved.
#
# Recipes are in a list to allow specific ordering (priority).
#
# If a 'cond' object is specified, each variable inside it will have its
#+	'eval' evaluated and a matching result selected from 'ret'
#
# All variables in a target are available for substitution inside recipes.
targets = {	"meson" : {
				"version" : { "minimum" : "0.41.2" },
				"recipes" : [
					{ "template" : "pip3", "recipe" : { "pkg_name" : "meson" } }
				]
			},
			"ninja" : {
				"version" : { "minimum" : "1.7.2" },
				"cond" : { "bin-type" : { "eval" : "sys.platform",
										"ret" : { "darwin" : "mac", "linux" : "linux" }
										}
				},
				"recipes" : [
					{ "template" : "apt-get", "recipe" : { "pkg_name" : "ninja-build" } },
					{ "template" : "port", "recipe" : { "pkg_name" : "ninja" } },
					{ "template" : "zip-install",
						"recipe" : { "pkg_name" : "ninja-build", "bin_name" : "ninja",
							"url" : "https://github.com/ninja-build/ninja/releases//download/v{version[minimum]}/ninja-{bin-type}.zip" }
					}
				]
			},
			"md2man-roff" : {
				"recipes" : [
					{ "template" : "gem", "recipe" : { "pkg_name" : "md2man" } }
				]
			}
		}

class target(dict):
	'''a required executable, and its recipes
	'''

	def __init__(self, name='', **kwargs):
		dict.__init__(self, **kwargs)
		self['name'] = name

		# resolve any conditional types, add them to own dictionary
		for label, test in self.get('cond', {}).items():
			res = eval(test['eval'])
			self[label] = test['ret'][res]

	def ok(self):
		# executable must exist
		path = which(self['name'])
		if not path:
			return False

		# always run a version check:
		#+	this verifies the binary will actually exec on this patform
		if 'version' in self:
			exist = pgm_vers(path, self['version'].get('version_cmd', '--version'))
			if 'minimum' in self['version'] and vers(exist) < vers(self['version']['minimum']):
				return False
			elif 'exactly' in self['version'] and vers(exist) != vers(self['version']['exactly']):
				return False

		return True

	def install(self, active_templates={}):
		'''try and install a target using provided 'active_templates'
		'''

		for obj in self['recipes']:
			# reject templates which don't apply to this platform
			if obj['template'] not in active_templates:
				continue

			# preprocess the recipe (variable replacement)
			recipe = { var : val.format(**self) for var, val in obj['recipe'].items() }
			# execute template
			try:
				if verbose:
					print('%s - try: %s' % (self['name'], obj['template']))
				active_templates[obj['template']].execute(recipe)
				if self.ok():
					break
			# if a recipe fails carry on
			except Exception as e:
				print('%s: template %s failed with:\n%s' % (self['name'], obj['template'], e.message),
						file=sys.stderr)
				continue
		# it is an error to have NO working recipes
		else:
			raise AssertionError('failed to install %s for %s' % (self['name'], sys.platform))

def irequire(templates, targets, quiet=False):
	'''entry point
	'''
	global verbose
	verbose = not quiet

	active_templates = { name : template(name, tmp)
							for name, tmp in templates.items()
							if template.is_valid(tmp)
						}

	targets = [ target(name, **tgt) for name, tgt in targets.items() ]

	for tgt in targets:
		if not tgt.ok():
			tgt.install(active_templates)

if __name__ == "__main__":
	# TODO: implement
	print('loading JSON from arguments or from stdin not yet supported', file=sys.stderr)
	exit(1)
<commit_msg>update apt; require minimum python version<commit_after>#!/usr/bin/env python3
'''bootstrap
Install build system and required tools to build nonlibc.

DISCLAIMER: this is a hack!
In my defense, package management is f'ing HARD and there just
	seems to be nothing better than hand-coding the nitpicky
	cross-platform hand-waving for. each. dependecy.

COROLLARY: please DON'T just add another tool to this project;
	try and keep to what's there already.

THE GOOD NEWS: at least this bootstrap is now a python script
	(as opposed to BASH), so tooling is python3-only.

As always, the quest for a
	[Universal Install Script](https://www.explainxkcd.com/wiki/index.php/1654:_Universal_Install_Script)
	continues!
'''

import subprocess
import sys
import re
from shutil import which
from os import environ

def run_cmd(args=[], shell=False, cwd=None, env={}):
	'''run_cmd()
	Run a command.
	Return stdout; raise an exception on failure.
	'''
	if verbose:
		print('EXEC: %s' % ' '.join(args))

	en = environ.copy()
	en.update(env)

	try:
		sub = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
							shell=shell, cwd=cwd, env=en, check=True);
	except subprocess.CalledProcessError as err:
		if verbose:
			print(err.stdout.decode('ascii'))
			print(err.stderr.decode('ascii'), file=sys.stderr)
		raise

	ret = sub.stdout.decode('ascii')
	if verbose:
		print(ret)
	return ret.strip()

def vers(version_string):
	'''vers()
	Takes a dodgy 'version_string' input

	Returns a list which can be compared (e.g. `>` `<` `==` etc.)
		with another list returned by this function.

	stolen from <https://stackoverflow.com/questions/1714027/version-number-comparison>
	'''
	try:
		return [int(x) for x in re.sub(r'(\.0+)*$','', version_string).split(".")]
	except:
		raise ValueError('''cannot parse version string '%s' ''' % version_string)

def pgm_vers(pgm_path, version_cmd='--version'):
	'''pgm_vers()
	Execute 'pgm_path' with 'version_cmd'; munge the output for a version string.

	Return a "listified" version string which can be compared with another
		"listified" version directly.
	'''
	re_vers = re.compile('\D*([\d.]+)')

	try:
		ret = run_cmd([ pgm_path, version_cmd ])
	except:
		return None

	m = re_vers.match(ret)
	if not m:
		raise ValueError('could not find a version ID: %s' % ret)

	return m.group(1)

#		Templates
#
# These are run through Python's str.format() function,
#+	against both the template itself and the target - which means
#+	they can reference any variable therein using standard Python formatting.
templates = {
			"apt-get" : {
				"platform" : [ "linux" ],
				"requires" : [ "apt-get" ],
				"cmd_list" : [
					[ "sudo", "apt-get", "update" ],
					[ "sudo", "apt-get", "-y", "install", "{pkg_name}" ]
				]
			},
			"brew" : {
				"platform" : [ "darwin" ],
				"requires" : [ "brew" ],
				"cmd_list" : [ [ "brew", "install", "{pkg_name}" ] ]
			},
			"gem" : {
				"platform" : [ "linux", "darwin" ],
				"requires" : [ "gem" ],
				"cmd_list" : [
					[ "sudo", "gem", "install", "rubygems-update" ],
					[ "sudo", "gem", "install", "{pkg_name}" ]
				]
			},
			"pip3" : {
				"platform" : [ "linux" ],
				"requires" : [ "pip3" ],
				"cmd_list" : [ [ "sudo", "-H", "pip3", "install", "{pkg_name}" ] ]
			},
			"port" : {
				"platform" : [ "darwin" ],
				"requires" : [ "port" ],
				"cmd_list" :  [ [ "sudo", "port", "install", "{pkg_name}" ] ]
			},
			"zip-install" : {
				"platform" : [ "linux", "darwin" ],
				"requires" : [ "mkdir", "wget", "unzip" ],
				"install_d" : "/usr/local/bin",
				"temp_dir" : "./toolchain",
				"cmd_list" :  [
								[ "mkdir", "-p", "{temp_dir}" ],
								[ "wget", "-O", "{temp_dir}/{bin_name}.zip", "{url}" ],
								[ "sudo", "unzip", "-o", "-d", "{install_d}/", "{temp_dir}/{bin_name}.zip" ],
								[ "sudo", "chmod", "go+rx", "{install_d}/{bin_name}" ]
							]
			}
		}

class template():
	'''A rule (and accompanying parameters) for the invocation of
		a specific command.
	'''

	def __init__(self, name='', template={}):
		self.name = name
		self.template = template

	def execute(self, recipe={}):
		for cmd in self.template['cmd_list']:
			# generate a command string, replacing any dictionary references
			cmd_str = [ tok.format(**recipe, **self.template) for tok in cmd ]
			# execute it
			run_cmd(cmd_str, recipe.get('shell', False), recipe.get('cwd', None))

	def is_valid(template):
		'''a template is valid if:

		-	our platform is included in its platform listing
		-	its binary exists on this system
		'''
		# one of the platform entries must match current platform
		if sys.platform not in template['platform']:
			return False

		# verify that required binaries are in place
		if 'requires' in template:
			for binary in template['requires']:
				if not which(binary):
					return False # TODO: try to install it?

		return True

#	Targets
#
# Each target is the name of a binary to be executed on the system.
#
# The Version object specifies 'minimum' or 'exact' version numbers to be
#+	achieved.
#
# Recipes are in a list to allow specific ordering (priority).
#
# If a 'cond' object is specified, each variable inside it will have its
#+	'eval' evaluated and a matching result selected from 'ret'
#
# All variables in a target are available for substitution inside recipes.
targets = {	"meson" : {
				"version" : { "minimum" : "0.41.2" },
				"recipes" : [
					{ "template" : "pip3", "recipe" : { "pkg_name" : "meson" } }
				]
			},
			"ninja" : {
				"version" : { "minimum" : "1.7.2" },
				"cond" : { "bin-type" : { "eval" : "sys.platform",
										"ret" : { "darwin" : "mac", "linux" : "linux" }
										}
				},
				"recipes" : [
					{ "template" : "apt-get", "recipe" : { "pkg_name" : "ninja-build" } },
					{ "template" : "port", "recipe" : { "pkg_name" : "ninja" } },
					{ "template" : "zip-install",
						"recipe" : { "pkg_name" : "ninja-build", "bin_name" : "ninja",
							"url" : "https://github.com/ninja-build/ninja/releases//download/v{version[minimum]}/ninja-{bin-type}.zip" }
					}
				]
			},
			"md2man-roff" : {
				"recipes" : [
					{ "template" : "gem", "recipe" : { "pkg_name" : "md2man" } }
				]
			}
		}

class target(dict):
	'''a required executable, and its recipes
	'''

	def __init__(self, name='', **kwargs):
		dict.__init__(self, **kwargs)
		self['name'] = name

		# resolve any conditional types, add them to own dictionary
		for label, test in self.get('cond', {}).items():
			res = eval(test['eval'])
			self[label] = test['ret'][res]

	def ok(self):
		# executable must exist
		path = which(self['name'])
		if not path:
			return False

		# always run a version check:
		#+	this verifies the binary will actually exec on this patform
		if 'version' in self:
			exist = pgm_vers(path, self['version'].get('version_cmd', '--version'))
			if 'minimum' in self['version'] and vers(exist) < vers(self['version']['minimum']):
				return False
			elif 'exactly' in self['version'] and vers(exist) != vers(self['version']['exactly']):
				return False

		return True

	def install(self, active_templates={}):
		'''try and install a target using provided 'active_templates'
		'''

		for obj in self['recipes']:
			# reject templates which don't apply to this platform
			if obj['template'] not in active_templates:
				continue

			# preprocess the recipe (variable replacement)
			recipe = { var : val.format(**self) for var, val in obj['recipe'].items() }
			# execute template
			try:
				if verbose:
					print('%s - try: %s' % (self['name'], obj['template']))
				active_templates[obj['template']].execute(recipe)
				if self.ok():
					break
			# if a recipe fails carry on
			except Exception as e:
				print('%s: template %s failed with:\n%s' % (self['name'], obj['template'], e.message),
						file=sys.stderr)
				continue
		# it is an error to have NO working recipes
		else:
			raise AssertionError('failed to install %s for %s' % (self['name'], sys.platform))

def irequire(templates, targets, quiet=False):
	'''entry point
	'''
	global verbose
	verbose = not quiet

	active_templates = { name : template(name, tmp)
							for name, tmp in templates.items()
							if template.is_valid(tmp)
						}

	targets = [ target(name, **tgt) for name, tgt in targets.items() ]

	for tgt in targets:
		if not tgt.ok():
			tgt.install(active_templates)

# must be at least python 3.5
if vers(sys.version) < vers('3.5'):
	print('python version %s < minimum 3.5 required' % sys.version, file=sys.stderr)
	exit(1)

if __name__ == "__main__":
	# TODO: implement
	print('loading JSON from arguments or from stdin not yet supported', file=sys.stderr)
	exit(1)
<|endoftext|>


<commit_before>from all_pairs_rankrls import AllPairsRankRLS
from cg_rankrls import CGRankRLS
from cg_rls import CGRLS
#from conditional_ranking import ConditionalRanking
#from cost_sensitive_greedy_rls import CostSensitiveGreedyRLS
#from dual_bundle_learner import DualBundleLearner
#from dual_cg_rankrls import DualCGRankRLS
#from dual_rankrls import DualRankRLS
#from floating_rls import FloatingRLS
from kron_rls import KronRLS
#from greedy_label_rankrls import GreedyLabelRankRLS
from greedy_rls import GreedyRLS
#from kernel_dependency import KernelDependency
from label_rankrls import LabelRankRLS
#from learner_interface import LearnerInterface
from mmc import MMC
#from multi_task_greedy_rls import MultiTaskGreedyRLS
from rls import RLS
#from space_efficient_greedy_rls import SpaceEfficientGreedyRLS
#from steepest_descent_mmc import SteepestDescentMMC

<commit_msg>Added some try - excepts for cython imports.<commit_after>from all_pairs_rankrls import AllPairsRankRLS
from cg_rankrls import CGRankRLS
from cg_rls import CGRLS
#from conditional_ranking import ConditionalRanking
#from cost_sensitive_greedy_rls import CostSensitiveGreedyRLS
#from dual_bundle_learner import DualBundleLearner
#from dual_cg_rankrls import DualCGRankRLS
#from dual_rankrls import DualRankRLS
#from floating_rls import FloatingRLS
try:
    from kron_rls import KronRLS
except ImportError, e:
    print e

#from greedy_label_rankrls import GreedyLabelRankRLS
try:
    from greedy_rls import GreedyRLS
except ImportError, e:
    print e
#from kernel_dependency import KernelDependency
from label_rankrls import LabelRankRLS
#from learner_interface import LearnerInterface
try:
    from mmc import MMC
except ImportError, e:
    print e
#from multi_task_greedy_rls import MultiTaskGreedyRLS
from rls import RLS
#from space_efficient_greedy_rls import SpaceEfficientGreedyRLS
#from steepest_descent_mmc import SteepestDescentMMC

<|endoftext|>


<commit_before># global x86_64, hostname_id, distro, network_type, virtualization, gui, develop, monitoring, roles

# pylama:ignore=E0602
def is_manjaro() -> bool:
    return distro == "manjaro"  # type: ignore

def is_x86_64() -> bool:
    return x86_64  # type: ignore

def is_role(name: str) -> bool:
    return name in roles.split(",")  # type: ignore

def is_monitoring(name: str) -> bool:
    return name in monitoring.split(",")  # type: ignore

def is_gui(name: str) -> bool:
    return name in gui.split(",")  # type: ignore

def is_develop(name: str) -> bool:
    return name in develop.split(",")  # type: ignore

def is_network_type(name: str) -> bool:
    return name in network_type.split(",")  # type: ignore

def get_hostname_id() -> bool:
    return hostname_id  # type: ignore

def get_virtualization() -> bool:
    return virtualization  # type: ignore

def system():
    system_pkgs = []
    if is_manjaro():
        system_pkgs += ["linux512"]
    else:
        system_pkgs += ["linux"]

    # utils
    system_pkgs += ["linux-firmware",
                    "base",
                    "polkit",
                    "gnupg",
                    "wget",
                    "curl",
                    "git",
                    "rsync",
                    "logrotate",
                    "nano",
                    "vim",
                    "pacutils",
                    "pkgfile",  # pkgfile makepkg (get package for makepkg)
                    "dialog",
                    "libnewt",  # external dialog
                    "mlocate",
                    "inetutils",  # for set hostname (crazy ansible code)
                    "man-db"]

    # ansible
    system_pkgs += ["ansible", "python", "python-lxml"]

    # terminal
    system_pkgs += ["urxvt-perls",
                    "fd",  # fast find alternative
                    "exa",  # ls alternative
                    "ncdu",  # disk usage analyzer
                    "bitwarden-cli-bin",
                    "zsh",
                    "fzf"]

    # archivers
    system_pkgs += ["p7zip", "unzip", "unrar"]

    if is_x86_64():
        system_pkgs += ["yay",  # AUR package manager
                        "refind",  # UEFI boot manager
                        "pkgcacheclean"]  # clean the pacman cache

    return system_pkgs

def driver():
    driver_pkgs = []

    if not is_gui("none"):
        driver_pkgs = ["mesa"]

    if get_hostname_id() == "archhost":
        driver_pkgs += ["xf86-video-intel"]
    # elif get_hostname_id() == "master":
    #    driver_pkgs += ["linux512-nvidia"]
    elif get_hostname_id() == "xnote":
        driver_pkgs += ["bbswitch",
                        # "bumblebee",
                        # "nvidia",
                        "xf86-video-intel",
                        "xf86-input-libinput"]  # touchpad
    elif get_hostname_id() == "worknote":
        driver_pkgs += ["bbswitch",
                        # "bumblebee",
                        # "nvidia",
                        "xf86-video-intel",
                        "xf86-input-libinput"]  # touchpad
    elif get_hostname_id() == "kvmtest":
        driver_pkgs += ["spice-vdagent", "xf86-video-qxl"]

    return driver_pkgs

def network():
    network_pkgs = ["net-tools",
                    "smbclient",
                    "httpie",
                    "openvpn",
                    "openssh"]  # ssh server

    if is_network_type("wireless"):
        network_pkgs += ["iwd"]

    return network_pkgs

def vm():
    if get_virtualization() == "kvm_qemu":
        return ["qemu",
                "virt-viewer",  # for SPICE
                "edk2-ovmf"]  # for UEFI
    elif get_virtualization() == "kvm_libvirt":
        return ["qemu",
                "libvirt",  # additional interface
                "virt-viewer",  # for SPICE
                "virt-manager",  # GUI for libvirt
                "edk2-ovmf",  # for UEFI
                "ebtables",  # for network
                "dnsmasq"]  # for network
    else:
        return []

def desktop_env():
    if is_gui("none"):
        return []

    gui_pkgs = ["xorg-server",
                "xorg-xinit",
                "xorg-xfontsel",  # font select
                "xorg-xprop",  # window info (xprop | grep WM_CLASS)
                "xorg-xev",  # keypress info
                "xorg-xwininfo",  # select window
                "arandr",  # screen position
                "ddcutil",  # brightness\monitor control
                "xcursor-ize-vision",  # a couple of X cursor that similar to Windows 7 cursor
                "perwindowlayoutd",  # daemon to make per window layout (also exists "kbdd-git")
                "scrot",  # for screenshots
                "flameshot",  # for screenshots
                "libnotify",  # create notifications message
                "wmctrl",  # windows manipulation
                "xclip",  # save data to clipboard
                "gsmartcontrol",  # UI for smartctl
                "xrectsel"]  # get select region

    # rofi
    gui_pkgs += ["rofi", "rofi-proxy", "python-googletrans", "python-psutil"]

    if is_gui("lightdm"):
        gui_pkgs += ["lightdm", "lightdm-gtk-greeter"]

    if is_gui("awesome"):
        gui_pkgs += ["awesome",
                     "redshift",  # brightness control
                     "mate-icon-theme",
                     "inter-font"]

    if is_gui("kde"):
        gui_pkgs += ["plasma-desktop",  # minimal install
                     "kdeconnect",  # connect to phone
                     "plasma-pa",  # pulse audio plugins
                     "kscreen",  # settings for screen configuration
                     "kinfocenter",  # show information about computer
                     "plasma-disks",  # monitoring disks (part of kinfocenter)
                     "plasma-systemmonitor",  # system monitor
                     "plasma5-applets-virtual-desktop-bar-git",  # tilling bar
                     "dolphin-plugins",
                     "print-manager"]
        if is_manjaro():
            gui_pkgs += ["manjaro-kde-settings",
                         "manjaro-settings-manager-knotifier",
                         "manjaro-settings-manager-kcm",
                         "mhwd",  # hardware detection library
                         "pamac-gtk"]

    return gui_pkgs

def development():
    if is_develop("none"):
        return []

    develop_pkgs = []
    if is_develop("std"):
        # "pycharm-community-edition", "pycharm-professional", "clion", "clion-cmake"
        develop_pkgs += ["git",
                         "icdiff",  # console diff
                         "meld",
                         "emacs",
                         "sublime-merge",
                         "sublime-text-dev",
                         "visual-studio-code-bin"]

    if is_develop("cpp"):
        develop_pkgs += ["clang",
                         "cmake",
                         "ninja",
                         "gdb",
                         "conan",
                         "protobuf",
                         "cpupower",  # for disable CPU powersafe mode in tests
                         "cpp-dependencies",
                         "vulkan-mesa-layers",  # show vulkan draw statistics
                         "python-dateutil",  # for include-what-you-use
                         "include-what-you-use"]

    if is_develop("python"):
        develop_pkgs += ["python-pip",
                         "python-nose",
                         "python-jedi",  # for vs-code ?
                         "pylama",  # linter
                         "mypy",  # linter
                         "python-pylint",  # linter
                         "python-pytest",
                         "python-termcolor",  # for ansible
                         "python-virtualenv",
                         "tk",
                         "swig",
                         "portaudio"]  # for pyaudio and my audio-lib

    if is_develop("go"):
        develop_pkgs += ["go", "protobuf"]

    if is_develop("rust"):
        develop_pkgs += ["rust", "cargo", "rust-src", "rust-racer"]

    if is_develop("rust3D"):
        develop_pkgs += ["sdl2", "sdl2_image"]

    if is_develop("sqlite"):
        develop_pkgs += ["sqlite-analyzer"]

    if is_develop("android"):
        develop_pkgs += ["adb"]

    return develop_pkgs

def monitoring_utils():
    if is_monitoring("none"):
        return []

    monitoring_pkgs = []
    if is_monitoring("std"):
        monitoring_pkgs += ["iftop",  # network monitor
                            "htop",  # process monitor
                            "iotop",  # disk monitor
                            "hwinfo"]  # info about hardware

        if is_x86_64():
            monitoring_pkgs += ["hw-probe"]  # check hardware and find drivers

    if is_monitoring("notebook"):
        monitoring_pkgs += ["powertop"]

    if is_monitoring("hddtemp"):
        monitoring_pkgs += ["hddtemp",  # disk temperature
                            "smartmontools"]

    if is_monitoring("ups"):
        monitoring_pkgs += ["apcupsd"]

    return monitoring_pkgs

def font():
    if not is_role("font"):
        return []

    return ["font-manager",  # viewer for fonts
            "ttf-ms-fonts",
            "ttf-tahoma",
            # "ttf-vista-fonts",
            "ttf-fixedsys-excelsior-linux",
            "ttf-droid",
            "ttf-dejavu",
            "ttf-ubuntu-font-family",
            "inter-font",  # for awesome
            "noto-fonts-emoji",  # emoji for chrome
            "adobe-source-code-pro-fonts"]

def docker():
    if not is_role("docker"):
        return []

    return ["docker", "docker-compose"]

def automount():
    if not is_role("automount"):
        return []

    return ["nfs-utils"]

def web():
    if not is_role("web"):
        return []

    return ["firefox", "firefox-i18n-ru", "google-chrome"]

def game():
    if not is_role("game"):
        return []

    game_pkgs = []
    if is_manjaro():
        game_pkgs += ["steam-manjaro"]
    else:
        game_pkgs += ["steam"]

    game_pkgs += ["playonlinux",
                  "lib32-libldap",  # for WOT ?
                  "minecraft-launcher"]

    return game_pkgs

def messengers():
    if not is_role("messengers"):
        return []

    return ["telegram-desktop", "slack-desktop"]

def audio():
    if not is_role("audio"):
        return []

    audio_pkgs = ["pulseaudio"]

    if not is_gui("none"):
        audio_pkgs += ["pavucontrol", "volumeicon", "pasystray", "spotify"]

    return audio_pkgs

def media():
    if not is_role("media"):
        return []

    return ["viewnior",  # image viewer
            "gimp",
            "krita",
            "inkscape",  # vector editor
            "blender",
            "smplayer",
            "deadbeef"]

def pdf():
    if not is_role("pdf"):
        return []

    return ["mupdf"]  # pdf viewer (analog: llpp-git)

def office():
    if not is_role("office"):
        return []

    return ["libreoffice-fresh-ru"]

def file_managers():
    if not is_role("file_managers"):
        return []

    return ["doublecmd-gtk2",
            "fsearch-git",
            "yandex-disk",  # yandex-disk setup/start
            "dropbox"]

def file_managers_keys():
    if not is_role("file_managers"):
        return []

    # dropbox
    return ["1C61A2656FB57B7E4DE0F4C1FC918B335044912E"]

def torrent():
    if not is_role("torrent"):
        return []

    return ["transmission-remote-gui"]  # transmission-remote-gui-bin - not work now

def spell_checkers():
    if not is_role("spell_checkers"):
        return []

    return ["enchant", "hunspell-en_us", "hunspell-ru-aot", "languagetool"]

def plex():
    if not is_role("plex"):
        return []

    return ["plex-media-server"]

def work():
    if not is_role("work"):
        return []

    return ["openconnect"]  # for vpn to work

groups = ["base-devel"]

packages = []
packages += system()
packages += driver()
packages += network()
packages += vm()
packages += desktop_env()
packages += development()
packages += monitoring_utils()
packages += font()
packages += docker()
packages += automount()
packages += web()
packages += game()
packages += messengers()
packages += audio()
packages += media()
packages += pdf()
packages += office()
packages += spell_checkers()
packages += file_managers()
packages += torrent()
packages += plex()
packages += work()

keys = []
keys += file_managers_keys()
<commit_msg>install nvidia<commit_after># global x86_64, hostname_id, distro, network_type, virtualization, gui, develop, monitoring, roles

# pylama:ignore=E0602
def is_manjaro() -> bool:
    return distro == "manjaro"  # type: ignore

def is_x86_64() -> bool:
    return x86_64  # type: ignore

def is_role(name: str) -> bool:
    return name in roles.split(",")  # type: ignore

def is_monitoring(name: str) -> bool:
    return name in monitoring.split(",")  # type: ignore

def is_gui(name: str) -> bool:
    return name in gui.split(",")  # type: ignore

def is_develop(name: str) -> bool:
    return name in develop.split(",")  # type: ignore

def is_network_type(name: str) -> bool:
    return name in network_type.split(",")  # type: ignore

def get_hostname_id() -> bool:
    return hostname_id  # type: ignore

def get_virtualization() -> bool:
    return virtualization  # type: ignore

def system():
    system_pkgs = []
    if is_manjaro():
        system_pkgs += ["linux512"]
    else:
        system_pkgs += ["linux"]

    # utils
    system_pkgs += ["linux-firmware",
                    "base",
                    "polkit",
                    "gnupg",
                    "wget",
                    "curl",
                    "git",
                    "rsync",
                    "logrotate",
                    "nano",
                    "vim",
                    "pacutils",
                    "pkgfile",  # pkgfile makepkg (get package for makepkg)
                    "dialog",
                    "libnewt",  # external dialog
                    "mlocate",
                    "inetutils",  # for set hostname (crazy ansible code)
                    "man-db"]

    # ansible
    system_pkgs += ["ansible", "python", "python-lxml"]

    # terminal
    system_pkgs += ["urxvt-perls",
                    "fd",  # fast find alternative
                    "exa",  # ls alternative
                    "ncdu",  # disk usage analyzer
                    "bitwarden-cli-bin",
                    "zsh",
                    "fzf"]

    # archivers
    system_pkgs += ["p7zip", "unzip", "unrar"]

    if is_x86_64():
        system_pkgs += ["yay",  # AUR package manager
                        "refind",  # UEFI boot manager
                        "pkgcacheclean"]  # clean the pacman cache

    return system_pkgs

def driver():
    driver_pkgs = []

    if not is_gui("none"):
        driver_pkgs = ["mesa"]

    if get_hostname_id() == "archhost":
        driver_pkgs += ["xf86-video-intel"]
    elif get_hostname_id() == "master":
        driver_pkgs += ["nvidia"]
    elif get_hostname_id() == "xnote":
        driver_pkgs += ["bbswitch",
                        # "bumblebee",
                        # "nvidia",
                        "xf86-video-intel",
                        "xf86-input-libinput"]  # touchpad
    elif get_hostname_id() == "worknote":
        driver_pkgs += ["bbswitch",
                        # "bumblebee",
                        # "nvidia",
                        "xf86-video-intel",
                        "xf86-input-libinput"]  # touchpad
    elif get_hostname_id() == "kvmtest":
        driver_pkgs += ["spice-vdagent", "xf86-video-qxl"]

    return driver_pkgs

def network():
    network_pkgs = ["net-tools",
                    "smbclient",
                    "httpie",
                    "openvpn",
                    "openssh"]  # ssh server

    if is_network_type("wireless"):
        network_pkgs += ["iwd"]

    return network_pkgs

def vm():
    if get_virtualization() == "kvm_qemu":
        return ["qemu",
                "virt-viewer",  # for SPICE
                "edk2-ovmf"]  # for UEFI
    elif get_virtualization() == "kvm_libvirt":
        return ["qemu",
                "libvirt",  # additional interface
                "virt-viewer",  # for SPICE
                "virt-manager",  # GUI for libvirt
                "edk2-ovmf",  # for UEFI
                "ebtables",  # for network
                "dnsmasq"]  # for network
    else:
        return []

def desktop_env():
    if is_gui("none"):
        return []

    gui_pkgs = ["xorg-server",
                "xorg-xinit",
                "xorg-xfontsel",  # font select
                "xorg-xprop",  # window info (xprop | grep WM_CLASS)
                "xorg-xev",  # keypress info
                "xorg-xwininfo",  # select window
                "arandr",  # screen position
                "ddcutil",  # brightness\monitor control
                "xcursor-ize-vision",  # a couple of X cursor that similar to Windows 7 cursor
                "perwindowlayoutd",  # daemon to make per window layout (also exists "kbdd-git")
                "scrot",  # for screenshots
                "flameshot",  # for screenshots
                "libnotify",  # create notifications message
                "wmctrl",  # windows manipulation
                "xclip",  # save data to clipboard
                "gsmartcontrol",  # UI for smartctl
                "xrectsel"]  # get select region

    # rofi
    gui_pkgs += ["rofi", "rofi-proxy", "python-googletrans", "python-psutil"]

    if is_gui("lightdm"):
        gui_pkgs += ["lightdm", "lightdm-gtk-greeter"]

    if is_gui("awesome"):
        gui_pkgs += ["awesome",
                     "redshift",  # brightness control
                     "mate-icon-theme",
                     "inter-font"]

    if is_gui("kde"):
        gui_pkgs += ["plasma-desktop",  # minimal install
                     "kdeconnect",  # connect to phone
                     "plasma-pa",  # pulse audio plugins
                     "kscreen",  # settings for screen configuration
                     "kinfocenter",  # show information about computer
                     "plasma-disks",  # monitoring disks (part of kinfocenter)
                     "plasma-systemmonitor",  # system monitor
                     "plasma5-applets-virtual-desktop-bar-git",  # tilling bar
                     "dolphin-plugins",
                     "print-manager"]
        if is_manjaro():
            gui_pkgs += ["manjaro-kde-settings",
                         "manjaro-settings-manager-knotifier",
                         "manjaro-settings-manager-kcm",
                         "mhwd",  # hardware detection library
                         "pamac-gtk"]

    return gui_pkgs

def development():
    if is_develop("none"):
        return []

    develop_pkgs = []
    if is_develop("std"):
        # "pycharm-community-edition", "pycharm-professional", "clion", "clion-cmake"
        develop_pkgs += ["git",
                         "icdiff",  # console diff
                         "meld",
                         "emacs",
                         "sublime-merge",
                         "sublime-text-dev",
                         "visual-studio-code-bin"]

    if is_develop("cpp"):
        develop_pkgs += ["clang",
                         "cmake",
                         "ninja",
                         "gdb",
                         "conan",
                         "protobuf",
                         "cpupower",  # for disable CPU powersafe mode in tests
                         "cpp-dependencies",
                         "vulkan-mesa-layers",  # show vulkan draw statistics
                         "python-dateutil",  # for include-what-you-use
                         "include-what-you-use"]

    if is_develop("python"):
        develop_pkgs += ["python-pip",
                         "python-nose",
                         "python-jedi",  # for vs-code ?
                         "pylama",  # linter
                         "mypy",  # linter
                         "python-pylint",  # linter
                         "python-pytest",
                         "python-termcolor",  # for ansible
                         "python-virtualenv",
                         "tk",
                         "swig",
                         "portaudio"]  # for pyaudio and my audio-lib

    if is_develop("go"):
        develop_pkgs += ["go", "protobuf"]

    if is_develop("rust"):
        develop_pkgs += ["rust", "cargo", "rust-src", "rust-racer"]

    if is_develop("rust3D"):
        develop_pkgs += ["sdl2", "sdl2_image"]

    if is_develop("sqlite"):
        develop_pkgs += ["sqlite-analyzer"]

    if is_develop("android"):
        develop_pkgs += ["adb"]

    return develop_pkgs

def monitoring_utils():
    if is_monitoring("none"):
        return []

    monitoring_pkgs = []
    if is_monitoring("std"):
        monitoring_pkgs += ["iftop",  # network monitor
                            "htop",  # process monitor
                            "iotop",  # disk monitor
                            "hwinfo"]  # info about hardware

        if is_x86_64():
            monitoring_pkgs += ["hw-probe"]  # check hardware and find drivers

    if is_monitoring("notebook"):
        monitoring_pkgs += ["powertop"]

    if is_monitoring("hddtemp"):
        monitoring_pkgs += ["hddtemp",  # disk temperature
                            "smartmontools"]

    if is_monitoring("ups"):
        monitoring_pkgs += ["apcupsd"]

    return monitoring_pkgs

def font():
    if not is_role("font"):
        return []

    return ["font-manager",  # viewer for fonts
            "ttf-ms-fonts",
            "ttf-tahoma",
            # "ttf-vista-fonts",
            "ttf-fixedsys-excelsior-linux",
            "ttf-droid",
            "ttf-dejavu",
            "ttf-ubuntu-font-family",
            "inter-font",  # for awesome
            "noto-fonts-emoji",  # emoji for chrome
            "adobe-source-code-pro-fonts"]

def docker():
    if not is_role("docker"):
        return []

    return ["docker", "docker-compose"]

def automount():
    if not is_role("automount"):
        return []

    return ["nfs-utils"]

def web():
    if not is_role("web"):
        return []

    return ["firefox", "firefox-i18n-ru", "google-chrome"]

def game():
    if not is_role("game"):
        return []

    game_pkgs = []
    if is_manjaro():
        game_pkgs += ["steam-manjaro"]
    else:
        game_pkgs += ["steam"]

    game_pkgs += ["playonlinux",
                  "lib32-libldap",  # for WOT ?
                  "minecraft-launcher"]

    return game_pkgs

def messengers():
    if not is_role("messengers"):
        return []

    return ["telegram-desktop", "slack-desktop"]

def audio():
    if not is_role("audio"):
        return []

    audio_pkgs = ["pulseaudio"]

    if not is_gui("none"):
        audio_pkgs += ["pavucontrol", "volumeicon", "pasystray", "spotify"]

    return audio_pkgs

def media():
    if not is_role("media"):
        return []

    return ["viewnior",  # image viewer
            "gimp",
            "krita",
            "inkscape",  # vector editor
            "blender",
            "smplayer",
            "deadbeef"]

def pdf():
    if not is_role("pdf"):
        return []

    return ["mupdf"]  # pdf viewer (analog: llpp-git)

def office():
    if not is_role("office"):
        return []

    return ["libreoffice-fresh-ru"]

def file_managers():
    if not is_role("file_managers"):
        return []

    return ["doublecmd-gtk2",
            "fsearch-git",
            "yandex-disk",  # yandex-disk setup/start
            "dropbox"]

def file_managers_keys():
    if not is_role("file_managers"):
        return []

    # dropbox
    return ["1C61A2656FB57B7E4DE0F4C1FC918B335044912E"]

def torrent():
    if not is_role("torrent"):
        return []

    return ["transmission-remote-gui"]  # transmission-remote-gui-bin - not work now

def spell_checkers():
    if not is_role("spell_checkers"):
        return []

    return ["enchant", "hunspell-en_us", "hunspell-ru-aot", "languagetool"]

def plex():
    if not is_role("plex"):
        return []

    return ["plex-media-server"]

def work():
    if not is_role("work"):
        return []

    return ["openconnect"]  # for vpn to work

groups = ["base-devel"]

packages = []
packages += system()
packages += driver()
packages += network()
packages += vm()
packages += desktop_env()
packages += development()
packages += monitoring_utils()
packages += font()
packages += docker()
packages += automount()
packages += web()
packages += game()
packages += messengers()
packages += audio()
packages += media()
packages += pdf()
packages += office()
packages += spell_checkers()
packages += file_managers()
packages += torrent()
packages += plex()
packages += work()

keys = []
keys += file_managers_keys()
<|endoftext|>