<commit_before>from __future__ import absolute_import, division, print_function

import os
import sys
from os.path import join, normpath, isabs
import subprocess
import multiprocessing
import warnings

import conda.config as cc

from conda_build.config import config

from conda_build import source
from conda_build.scripts import prepend_bin_path

def get_perl_ver():
    return str(config.CONDA_PERL)

def get_py_ver():
    return '.'.join(str(config.CONDA_PY))

def get_npy_ver():
    if config.CONDA_NPY:
        # Convert int -> string, e.g.
        #   17 -> '1.7'
        #   110 -> '1.10'
        conda_npy = str(config.CONDA_NPY)
        return conda_npy[0] + '.' + conda_npy[1:]
    return ''

def get_stdlib_dir():
    return join(config.build_prefix, 'Lib' if sys.platform == 'win32' else
                                'lib/python%s' % get_py_ver())

def get_sp_dir():
    return join(get_stdlib_dir(), 'site-packages')

def get_git_build_info(src_dir, git_url, expected_rev):
    expected_rev = expected_rev or 'master'
    env = os.environ.copy()
    d = {}
    git_dir = join(src_dir, '.git')
    if not os.path.exists(git_dir):
        return d

    env['GIT_DIR'] = git_dir
    try:
        # Verify current commit matches expected commit
        current_commit = subprocess.check_output(["git", "log", "-n1", "--format=%H"], env=env)
        current_commit = current_commit.decode('utf-8')
        expected_tag_commit = subprocess.check_output(["git", "log", "-n1", "--format=%H", expected_rev], env=env)
        expected_tag_commit = expected_tag_commit.decode('utf-8')

        # Verify correct remote url.
        # (Need to find the git cache directory, and check the remote from there.)
        cache_details = subprocess.check_output(["git", "remote", "-v"], env=env)
        cache_details = cache_details.decode('utf-8')
        cache_dir = cache_details.split('\n')[0].split()[1]
        assert "conda-bld/git_cache" in cache_dir

        env['GIT_DIR'] = cache_dir
        remote_details = subprocess.check_output(["git", "remote", "-v"], env=env)
        remote_details = remote_details.decode('utf-8')
        remote_url = remote_details.split('\n')[0].split()[1]
        if '://' not in remote_url:
            # Local filepaths are allowed, but make sure we normalize them
            remote_url = normpath(remote_url)

        # If the current source directory in conda-bld/work doesn't match the user's
        # metadata git_url or git_rev, then we aren't looking at the right source.
        if remote_url != git_url or current_commit != expected_tag_commit:
            return d
    except subprocess.CalledProcessError:
        return d

    env['GIT_DIR'] = git_dir

    # grab information from describe
    key_name = lambda a: "GIT_DESCRIBE_{}".format(a)
    keys = [key_name("TAG"), key_name("NUMBER"), key_name("HASH")]
    env = {str(key): str(value) for key, value in env.items()}
    process = subprocess.Popen(["git", "describe", "--tags", "--long", "HEAD"],
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                               env=env)
    output = process.communicate()[0].strip()
    output = output.decode('utf-8')
    parts = output.rsplit('-', 2)
    parts_length = len(parts)
    if parts_length == 3:
        d.update(dict(zip(keys, parts)))
    # get the _full_ hash of the current HEAD
    process = subprocess.Popen(["git", "rev-parse", "HEAD"],
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                               env=env)
    output = process.communicate()[0].strip()
    output = output.decode('utf-8')
    d['GIT_FULL_HASH'] = output
    # set up the build string
    if key_name('NUMBER') in d and key_name('HASH') in d:
        d['GIT_BUILD_STR'] = '{}_{}'.format(d[key_name('NUMBER')],
                                            d[key_name('HASH')])

    return d

def get_dict(m=None, prefix=None):
    if not prefix:
        prefix = config.build_prefix

    python = config.build_python
    d = {'CONDA_BUILD': '1', 'PYTHONNOUSERSITE': '1'}
    d['CONDA_DEFAULT_ENV'] = config.build_prefix
    d['ARCH'] = str(cc.bits)
    d['PREFIX'] = prefix
    d['PYTHON'] = python
    d['PY3K'] = str(config.PY3K)
    d['STDLIB_DIR'] = get_stdlib_dir()
    d['SP_DIR'] = get_sp_dir()
    d['SYS_PREFIX'] = sys.prefix
    d['SYS_PYTHON'] = sys.executable
    d['PERL_VER'] = get_perl_ver()
    d['PY_VER'] = get_py_ver()
    if get_npy_ver():
        d['NPY_VER'] = get_npy_ver()
    d['SRC_DIR'] = source.get_dir()
    if "LANG" in os.environ:
        d['LANG'] = os.environ['LANG']
    if "HTTPS_PROXY" in os.environ:
        d['HTTPS_PROXY'] = os.environ['HTTPS_PROXY']
    if "HTTP_PROXY" in os.environ:
        d['HTTP_PROXY'] = os.environ['HTTP_PROXY']

    if m:
        for var_name in m.get_value('build/script_env', []):
            value = os.getenv(var_name)
            if value is None:
                warnings.warn(
                    "The environment variable '%s' is undefined." % var_name,
                    UserWarning
                )
            else:
                d[var_name] = value

    if sys.platform == "darwin":
        # multiprocessing.cpu_count() is not reliable on OSX
        # See issue #645 on github.com/conda/conda-build
        out, err = subprocess.Popen('sysctl -n hw.logicalcpu', shell=True, stdout=subprocess.PIPE).communicate()
        d['CPU_COUNT'] = out.decode('utf-8').strip()
    else:
        try:
            d['CPU_COUNT'] = str(multiprocessing.cpu_count())
        except NotImplementedError:
            d['CPU_COUNT'] = "1"

    if m.get_value('source/git_url'):
        git_url = m.get_value('source/git_url')
        if '://' not in git_url:
            # If git_url is a relative path instead of a url, convert it to an abspath
            if not isabs(git_url):
                git_url = join(m.path, git_url)
            git_url = normpath(join(m.path, git_url))
        d.update(**get_git_build_info(d['SRC_DIR'],
                                      git_url,
                                      m.get_value('source/git_rev')))

    d['PATH'] = dict(os.environ)['PATH']
    d = prepend_bin_path(d, prefix)

    if sys.platform == 'win32':         # -------- Windows
        d['SCRIPTS'] = join(prefix, 'Scripts')
        d['LIBRARY_PREFIX'] = join(prefix, 'Library')
        d['LIBRARY_BIN'] = join(d['LIBRARY_PREFIX'], 'bin')
        d['LIBRARY_INC'] = join(d['LIBRARY_PREFIX'], 'include')
        d['LIBRARY_LIB'] = join(d['LIBRARY_PREFIX'], 'lib')
        # This probably should be done more generally
        d['CYGWIN_PREFIX'] = prefix.replace('\\', '/').replace('C:', '/cygdrive/c')

        d['R'] = join(prefix, 'Scripts', 'R.exe')
    else:                               # -------- Unix
        d['HOME'] = os.getenv('HOME', 'UNKNOWN')
        d['PKG_CONFIG_PATH'] = join(prefix, 'lib', 'pkgconfig')
        d['R'] = join(prefix, 'bin', 'R')

    if sys.platform == 'darwin':         # -------- OSX
        d['OSX_ARCH'] = 'i386' if cc.bits == 32 else 'x86_64'
        d['CFLAGS'] = '-arch %(OSX_ARCH)s' % d
        d['CXXFLAGS'] = d['CFLAGS']
        d['LDFLAGS'] = d['CFLAGS']
        d['MACOSX_DEPLOYMENT_TARGET'] = '10.6'

    elif sys.platform.startswith('linux'):      # -------- Linux
        d['LD_RUN_PATH'] = prefix + '/lib'

    if m:
        d['PKG_NAME'] = m.name()
        d['PKG_VERSION'] = m.version()
        d['PKG_BUILDNUM'] = str(m.build_number())
        d['PKG_BUILD_STRING'] = str(m.build_id())
        d['RECIPE_DIR'] = m.path

    return d

if __name__ == '__main__':
    e = get_dict()
    for k in sorted(e):
        assert isinstance(e[k], str), k
        print('%s=%s' % (k, e[k]))
<commit_msg>Add -m32 to CFLAGS and CXXFLAGS for multilib gcc<commit_after>from __future__ import absolute_import, division, print_function

import os
import sys
from os.path import join, normpath, isabs
import subprocess
import multiprocessing
import warnings

import conda.config as cc

from conda_build.config import config

from conda_build import source
from conda_build.scripts import prepend_bin_path

def get_perl_ver():
    return str(config.CONDA_PERL)

def get_py_ver():
    return '.'.join(str(config.CONDA_PY))

def get_npy_ver():
    if config.CONDA_NPY:
        # Convert int -> string, e.g.
        #   17 -> '1.7'
        #   110 -> '1.10'
        conda_npy = str(config.CONDA_NPY)
        return conda_npy[0] + '.' + conda_npy[1:]
    return ''

def get_stdlib_dir():
    return join(config.build_prefix, 'Lib' if sys.platform == 'win32' else
                                'lib/python%s' % get_py_ver())

def get_sp_dir():
    return join(get_stdlib_dir(), 'site-packages')

def get_git_build_info(src_dir, git_url, expected_rev):
    expected_rev = expected_rev or 'master'
    env = os.environ.copy()
    d = {}
    git_dir = join(src_dir, '.git')
    if not os.path.exists(git_dir):
        return d

    env['GIT_DIR'] = git_dir
    try:
        # Verify current commit matches expected commit
        current_commit = subprocess.check_output(["git", "log", "-n1", "--format=%H"], env=env)
        current_commit = current_commit.decode('utf-8')
        expected_tag_commit = subprocess.check_output(["git", "log", "-n1", "--format=%H", expected_rev], env=env)
        expected_tag_commit = expected_tag_commit.decode('utf-8')

        # Verify correct remote url.
        # (Need to find the git cache directory, and check the remote from there.)
        cache_details = subprocess.check_output(["git", "remote", "-v"], env=env)
        cache_details = cache_details.decode('utf-8')
        cache_dir = cache_details.split('\n')[0].split()[1]
        assert "conda-bld/git_cache" in cache_dir

        env['GIT_DIR'] = cache_dir
        remote_details = subprocess.check_output(["git", "remote", "-v"], env=env)
        remote_details = remote_details.decode('utf-8')
        remote_url = remote_details.split('\n')[0].split()[1]
        if '://' not in remote_url:
            # Local filepaths are allowed, but make sure we normalize them
            remote_url = normpath(remote_url)

        # If the current source directory in conda-bld/work doesn't match the user's
        # metadata git_url or git_rev, then we aren't looking at the right source.
        if remote_url != git_url or current_commit != expected_tag_commit:
            return d
    except subprocess.CalledProcessError:
        return d

    env['GIT_DIR'] = git_dir

    # grab information from describe
    key_name = lambda a: "GIT_DESCRIBE_{}".format(a)
    keys = [key_name("TAG"), key_name("NUMBER"), key_name("HASH")]
    env = {str(key): str(value) for key, value in env.items()}
    process = subprocess.Popen(["git", "describe", "--tags", "--long", "HEAD"],
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                               env=env)
    output = process.communicate()[0].strip()
    output = output.decode('utf-8')
    parts = output.rsplit('-', 2)
    parts_length = len(parts)
    if parts_length == 3:
        d.update(dict(zip(keys, parts)))
    # get the _full_ hash of the current HEAD
    process = subprocess.Popen(["git", "rev-parse", "HEAD"],
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                               env=env)
    output = process.communicate()[0].strip()
    output = output.decode('utf-8')
    d['GIT_FULL_HASH'] = output
    # set up the build string
    if key_name('NUMBER') in d and key_name('HASH') in d:
        d['GIT_BUILD_STR'] = '{}_{}'.format(d[key_name('NUMBER')],
                                            d[key_name('HASH')])

    return d

def get_dict(m=None, prefix=None):
    if not prefix:
        prefix = config.build_prefix

    python = config.build_python
    d = {'CONDA_BUILD': '1', 'PYTHONNOUSERSITE': '1'}
    d['CONDA_DEFAULT_ENV'] = config.build_prefix
    d['ARCH'] = str(cc.bits)
    d['PREFIX'] = prefix
    d['PYTHON'] = python
    d['PY3K'] = str(config.PY3K)
    d['STDLIB_DIR'] = get_stdlib_dir()
    d['SP_DIR'] = get_sp_dir()
    d['SYS_PREFIX'] = sys.prefix
    d['SYS_PYTHON'] = sys.executable
    d['PERL_VER'] = get_perl_ver()
    d['PY_VER'] = get_py_ver()
    if get_npy_ver():
        d['NPY_VER'] = get_npy_ver()
    d['SRC_DIR'] = source.get_dir()
    if "LANG" in os.environ:
        d['LANG'] = os.environ['LANG']
    if "HTTPS_PROXY" in os.environ:
        d['HTTPS_PROXY'] = os.environ['HTTPS_PROXY']
    if "HTTP_PROXY" in os.environ:
        d['HTTP_PROXY'] = os.environ['HTTP_PROXY']

    if m:
        for var_name in m.get_value('build/script_env', []):
            value = os.getenv(var_name)
            if value is None:
                warnings.warn(
                    "The environment variable '%s' is undefined." % var_name,
                    UserWarning
                )
            else:
                d[var_name] = value

    if sys.platform == "darwin":
        # multiprocessing.cpu_count() is not reliable on OSX
        # See issue #645 on github.com/conda/conda-build
        out, err = subprocess.Popen('sysctl -n hw.logicalcpu', shell=True, stdout=subprocess.PIPE).communicate()
        d['CPU_COUNT'] = out.decode('utf-8').strip()
    else:
        try:
            d['CPU_COUNT'] = str(multiprocessing.cpu_count())
        except NotImplementedError:
            d['CPU_COUNT'] = "1"

    if m.get_value('source/git_url'):
        git_url = m.get_value('source/git_url')
        if '://' not in git_url:
            # If git_url is a relative path instead of a url, convert it to an abspath
            if not isabs(git_url):
                git_url = join(m.path, git_url)
            git_url = normpath(join(m.path, git_url))
        d.update(**get_git_build_info(d['SRC_DIR'],
                                      git_url,
                                      m.get_value('source/git_rev')))

    d['PATH'] = dict(os.environ)['PATH']
    d = prepend_bin_path(d, prefix)

    if sys.platform == 'win32':         # -------- Windows
        d['SCRIPTS'] = join(prefix, 'Scripts')
        d['LIBRARY_PREFIX'] = join(prefix, 'Library')
        d['LIBRARY_BIN'] = join(d['LIBRARY_PREFIX'], 'bin')
        d['LIBRARY_INC'] = join(d['LIBRARY_PREFIX'], 'include')
        d['LIBRARY_LIB'] = join(d['LIBRARY_PREFIX'], 'lib')
        # This probably should be done more generally
        d['CYGWIN_PREFIX'] = prefix.replace('\\', '/').replace('C:', '/cygdrive/c')

        d['R'] = join(prefix, 'Scripts', 'R.exe')
    else:                               # -------- Unix
        d['HOME'] = os.getenv('HOME', 'UNKNOWN')
        d['PKG_CONFIG_PATH'] = join(prefix, 'lib', 'pkgconfig')
        d['R'] = join(prefix, 'bin', 'R')

    if sys.platform == 'darwin':         # -------- OSX
        d['OSX_ARCH'] = 'i386' if cc.bits == 32 else 'x86_64'
        d['CFLAGS'] = '-arch %(OSX_ARCH)s' % d
        d['CXXFLAGS'] = d['CFLAGS']
        d['LDFLAGS'] = d['CFLAGS']
        d['MACOSX_DEPLOYMENT_TARGET'] = '10.6'

    elif sys.platform.startswith('linux'):      # -------- Linux
        d['LD_RUN_PATH'] = prefix + '/lib'
        if cc.bits == 32:
            d['CFLAGS'] = '-m32'
            d['CXXFLAGS'] = d['CFLAGS']

    if m:
        d['PKG_NAME'] = m.name()
        d['PKG_VERSION'] = m.version()
        d['PKG_BUILDNUM'] = str(m.build_number())
        d['PKG_BUILD_STRING'] = str(m.build_id())
        d['RECIPE_DIR'] = m.path

    return d

if __name__ == '__main__':
    e = get_dict()
    for k in sorted(e):
        assert isinstance(e[k], str), k
        print('%s=%s' % (k, e[k]))
<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import print_function

import datetime
import fnmatch
from glob import glob
import os
import time

from git import Repo
import conda.api
import conda.fetch

from conda_gitenv.resolve import tempdir, create_tracking_branches
from conda_gitenv.lock import Locked
from conda_gitenv import manifest_branch_prefix

def tags_by_label(labels_directory):
    tags = {}
    if os.path.isdir(labels_directory):
        for label_fname in glob(os.path.join(labels_directory, '*.txt')):
            with open(label_fname, 'r') as fh:
                tag_name = fh.read().strip()
            label = os.path.splitext(os.path.basename(label_fname))[0]
            tags[label] = tag_name
    return tags

def tags_by_env(repo):
    tags = {}
    for tag in repo.tags:
        env_name = tag.name.split('-')[1]
        tags.setdefault(env_name, []).append(tag)
    return tags

def deploy_tag(repo, tag_name, target):
    tag = repo.tags[tag_name]
    # Checkout the tag in a detached head form.
    repo.head.reference = tag.commit
    repo.head.reset(working_tree=True)

    # Pull out the environment name from the form "env-<env_name>-<deployed_name>".
    env_name = tag_name.split('-')[1]
    deployed_name = tag_name.split('-', 2)[2]

    manifest_fname = os.path.join(repo.working_dir, 'env.manifest')
    if not os.path.exists(manifest_fname):
        raise ValueError("The tag '{}' doesn't have a manifested environment.".format(tag_name))
    with open(manifest_fname, 'r') as fh:
        manifest = sorted(line.strip().split('\t') for line in fh)
    create_env(manifest, os.path.join(target, env_name, deployed_name), os.path.join(target, '.pkg_cache'))

def create_env(pkgs, target, pkg_cache):
    # We lock the specific environment we are wanting to create. If other requests come in for the
    # exact same environment, they will have to wait for this to finish (good).
    with Locked(target):
        pkg_names = set(pkg for _, pkg in pkgs)
        if os.path.exists(target):
            # The environment we want to deploy already exists. We should just double check that
            # there aren't already packages in there which we need to remove before we install anything
            # new.
            linked = conda.install.linked(target)
            for pkg in linked:
                if pkg not in pkg_names:
                    conda.install.unlink(target, pkg)
        else:
            linked = []

        if set(linked) == pkg_names:
            # We don't need to re-link everything - it is already as expected.
            # The downside is that we are not verifying that each package is installed correctly.
            return

        for source, pkg in pkgs:
            index = conda.fetch.fetch_index([source], use_cache=False)
            # Deal with the fact that a recent conda includes the source in the index key.
            index = {pkg['fn']: pkg for pkg in index.values()}

            tar_name = pkg + '.tar.bz2'
            pkg_info = index.get(tar_name, None)
            if pkg_info is None:
                raise ValueError('Distribution {} is no longer available in the channel.'.format(tar_name))
            dist_name = pkg
            # We force a lock on retrieving anything which needs access to a distribution of this
            # name. If other requests come in to get the exact same package they will have to wait
            # for this to finish (good). If conda itself it fetching these pacakges then there is
            # the potential for a race condition (bad) - there is no solution to this unless
            # conda/conda is updated to be more precise with its locks.
            lock_name = os.path.join(pkg_cache, dist_name)
            with Locked(lock_name):
                schannel_dist_name = dist_name
                if pkg_info['schannel'] != 'defaults':
                    schannel_dist_name='{}::{}'.format(pkg_info['schannel'],
                                                       dist_name)
                if not conda.install.is_extracted(schannel_dist_name):
                    if not conda.install.is_fetched(schannel_dist_name):
                        print('Fetching {}'.format(dist_name))
                        conda.fetch.fetch_pkg(pkg_info)
                    conda.install.extract(schannel_dist_name)
                conda.install.link(target, schannel_dist_name)

def deploy_repo(repo, target, desired_env_labels=('*')):
    env_tags = tags_by_env(repo)
    for branch in repo.branches:
        # We only want environment branches, not manifest branches.
        if not branch.name.startswith(manifest_branch_prefix):
            manifest_branch_name = manifest_branch_prefix + branch.name
            # If there is no equivalent manifest branch, we need to
            # skip this environment.
            if manifest_branch_name not in repo.branches:
                continue
            manifest_branch = repo.branches[manifest_branch_name]
            branch.checkout()
            all_labelled_tags = tags_by_label(os.path.join(repo.working_dir, 'labels'))

            # Create a latest tag that points to the most recently tagged environment.
            if env_tags.get(branch.name):
                latest_tag = max(env_tags[branch.name],
                                 key=lambda t: t.commit.committed_date)
                all_labelled_tags['latest'] = latest_tag.name

            # Only deploy environments that match the given pattern.
            labelled_tags = {}
            for label, tag in all_labelled_tags.items():
                if any([fnmatch.fnmatch('{}/{}'.format(branch.name, label),
                                        env_label) for env_label in desired_env_labels]):
                    labelled_tags[label] = tag

            for tag in set(labelled_tags.values()):
                deploy_tag(repo, tag, target)
            for label, tag in labelled_tags.items():
                with Locked(os.path.join(target, label)):
                    deployed_name = tag.split('-', 2)[2]
                    label_target = deployed_name
                    label_location = os.path.join(target, branch.name, label)

                    if os.path.exists(label_location):
                        # Unix only:
                        if os.readlink(label_location) != label_target:
                            os.remove(label_location)

                    if not os.path.exists(label_location):
                        print('Linking {}/{} to {}'.format(branch.name, label, tag))
                        os.symlink(label_target, label_location)

def configure_parser(parser):
    parser.add_argument('repo_uri', help='Repo to deploy.')
    parser.add_argument('target', help='Location to deploy the environments to.')
    parser.add_argument('--env_labels', nargs='+',  default=['*'],
                        help='Pattern to match environment labels to. In the '
                             'form "{environment}/{label}".',)
    parser.set_defaults(function=handle_args)
    return parser

def handle_args(args):
    with tempdir() as repo_directory:
        repo = Repo.clone_from(args.repo_uri, repo_directory)
        create_tracking_branches(repo)
        deploy_repo(repo, args.target, args.env_labels)

def main():
    import argparse

    parser = argparse.ArgumentParser(description='Deploy the tracked environments.')
    configure_parser(parser)
    args = parser.parse_args()
    return args.function(args)

if __name__ == '__main__':
    main()
<commit_msg>Set pkgs_dir to be pkg_cache and not PREFIX/pkgs. (#27)<commit_after>#!/usr/bin/env python
from __future__ import print_function

import datetime
import fnmatch
from glob import glob
import os
import time

from git import Repo
import conda.api
import conda.fetch

from conda_gitenv.resolve import tempdir, create_tracking_branches
from conda_gitenv.lock import Locked
from conda_gitenv import manifest_branch_prefix

def tags_by_label(labels_directory):
    tags = {}
    if os.path.isdir(labels_directory):
        for label_fname in glob(os.path.join(labels_directory, '*.txt')):
            with open(label_fname, 'r') as fh:
                tag_name = fh.read().strip()
            label = os.path.splitext(os.path.basename(label_fname))[0]
            tags[label] = tag_name
    return tags

def tags_by_env(repo):
    tags = {}
    for tag in repo.tags:
        env_name = tag.name.split('-')[1]
        tags.setdefault(env_name, []).append(tag)
    return tags

def deploy_tag(repo, tag_name, target, pkg_cache):
    tag = repo.tags[tag_name]
    # Checkout the tag in a detached head form.
    repo.head.reference = tag.commit
    repo.head.reset(working_tree=True)

    # Pull out the environment name from the form "env-<env_name>-<deployed_name>".
    env_name = tag_name.split('-')[1]
    deployed_name = tag_name.split('-', 2)[2]

    manifest_fname = os.path.join(repo.working_dir, 'env.manifest')
    if not os.path.exists(manifest_fname):
        raise ValueError("The tag '{}' doesn't have a manifested environment.".format(tag_name))
    with open(manifest_fname, 'r') as fh:
        manifest = sorted(line.strip().split('\t') for line in fh)
    create_env(manifest, os.path.join(target, env_name, deployed_name), pkg_cache)

def create_env(pkgs, target, pkg_cache):
    # We lock the specific environment we are wanting to create. If other requests come in for the
    # exact same environment, they will have to wait for this to finish (good).
    with Locked(target):
        pkg_names = set(pkg for _, pkg in pkgs)
        if os.path.exists(target):
            # The environment we want to deploy already exists. We should just double check that
            # there aren't already packages in there which we need to remove before we install anything
            # new.
            linked = conda.install.linked(target)
            for pkg in linked:
                if pkg not in pkg_names:
                    conda.install.unlink(target, pkg)
        else:
            linked = []

        if set(linked) == pkg_names:
            # We don't need to re-link everything - it is already as expected.
            # The downside is that we are not verifying that each package is installed correctly.
            return

        for source, pkg in pkgs:
            index = conda.fetch.fetch_index([source], use_cache=False)
            # Deal with the fact that a recent conda includes the source in the index key.
            index = {pkg['fn']: pkg for pkg in index.values()}

            tar_name = pkg + '.tar.bz2'
            pkg_info = index.get(tar_name, None)
            if pkg_info is None:
                raise ValueError('Distribution {} is no longer available in the channel.'.format(tar_name))
            dist_name = pkg
            # We force a lock on retrieving anything which needs access to a distribution of this
            # name. If other requests come in to get the exact same package they will have to wait
            # for this to finish (good). If conda itself it fetching these pacakges then there is
            # the potential for a race condition (bad) - there is no solution to this unless
            # conda/conda is updated to be more precise with its locks.
            lock_name = os.path.join(pkg_cache, dist_name)
            with Locked(lock_name):
                schannel_dist_name = dist_name
                if pkg_info['schannel'] != 'defaults':
                    schannel_dist_name='{}::{}'.format(pkg_info['schannel'],
                                                       dist_name)
                if not conda.install.is_extracted(schannel_dist_name):
                    if not conda.install.is_fetched(schannel_dist_name):
                        print('Fetching {}'.format(dist_name))
                        conda.fetch.fetch_pkg(pkg_info)
                    conda.install.extract(schannel_dist_name)
                conda.install.link(target, schannel_dist_name)

def deploy_repo(repo, target, desired_env_labels=('*')):
    # Cache settings to be reinstated at the end.
    orig_package_cache_ = conda.install.package_cache_
    orig_pkgs_dirs = conda.install.pkgs_dirs

    # Empty package cache so that it will reinitialised.
    conda.install.package_cache_ = {}
    # Set pkgs_dirs location to be the specified pkg_cache.
    pkg_cache = os.path.join(target, '.pkg_cache')
    conda.install.pkgs_dirs = [pkg_cache]

    env_tags = tags_by_env(repo)
    for branch in repo.branches:
        # We only want environment branches, not manifest branches.
        if not branch.name.startswith(manifest_branch_prefix):
            manifest_branch_name = manifest_branch_prefix + branch.name
            # If there is no equivalent manifest branch, we need to
            # skip this environment.
            if manifest_branch_name not in repo.branches:
                continue
            manifest_branch = repo.branches[manifest_branch_name]
            branch.checkout()
            all_labelled_tags = tags_by_label(os.path.join(repo.working_dir, 'labels'))

            # Create a latest tag that points to the most recently tagged environment.
            if env_tags.get(branch.name):
                latest_tag = max(env_tags[branch.name],
                                 key=lambda t: t.commit.committed_date)
                all_labelled_tags['latest'] = latest_tag.name

            # Only deploy environments that match the given pattern.
            labelled_tags = {}
            for label, tag in all_labelled_tags.items():
                if any([fnmatch.fnmatch('{}/{}'.format(branch.name, label),
                                        env_label) for env_label in desired_env_labels]):
                    labelled_tags[label] = tag

            for tag in set(labelled_tags.values()):
                deploy_tag(repo, tag, target, pkg_cache)
            for label, tag in labelled_tags.items():
                with Locked(os.path.join(target, label)):
                    deployed_name = tag.split('-', 2)[2]
                    label_target = deployed_name
                    label_location = os.path.join(target, branch.name, label)

                    if os.path.exists(label_location):
                        # Unix only:
                        if os.readlink(label_location) != label_target:
                            os.remove(label_location)

                    if not os.path.exists(label_location):
                        print('Linking {}/{} to {}'.format(branch.name, label, tag))
                        os.symlink(label_target, label_location)

    conda.install.package_cache_ = orig_package_cache_
    conda.install.pkgs_dirs = orig_pkgs_dirs

def configure_parser(parser):
    parser.add_argument('repo_uri', help='Repo to deploy.')
    parser.add_argument('target', help='Location to deploy the environments to.')
    parser.add_argument('--env_labels', nargs='+',  default=['*'],
                        help='Pattern to match environment labels to. In the '
                             'form "{environment}/{label}".',)
    parser.set_defaults(function=handle_args)
    return parser

def handle_args(args):
    with tempdir() as repo_directory:
        repo = Repo.clone_from(args.repo_uri, repo_directory)
        create_tracking_branches(repo)
        deploy_repo(repo, args.target, args.env_labels)

def main():
    import argparse

    parser = argparse.ArgumentParser(description='Deploy the tracked environments.')
    configure_parser(parser)
    args = parser.parse_args()
    return args.function(args)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># coding=utf-8
'''
Helpers for data mining tasks.
'''
from collections import defaultdict

import csv
import json
import math

from rdflib import Graph, RDF, RDFS, Namespace

nsTaxMeOn = Namespace("http://www.yso.fi/onto/taxmeon/")
nsRanks = Namespace("http://www.yso.fi/onto/taxonomic-ranks/")
nsHaliasSchema = Namespace("http://ldf.fi/schema/halias/")

DATA_DIR = '../data/'

common_species = {  # Support 0.5 items
    'alli', 'haahka', 'haapana', 'haarapääsky', 'harmaalokki', 'hippiäinen', 'isokoskelo', 'kalalokki', 'korppi',
    'kottarainen', 'kyhmyjoutsen', 'merilokki', 'merimetso', 'mustarastas', 'naurulokki', 'niittykirvinen', 'peippo',
    'punarinta', 'räkättirastas', 'sinisorsa', 'sinitiainen', 'talitiainen', 'telkkä', 'tukkakoskelo', 'tukkasotka',
    'tylli', 'varis', 'varpushaukka', 'viherpeippo', 'vihervarpunen', 'västäräkki'}
#    {
#    # Support 0.75 items:
#    'haahka', 'harmaalokki', 'isokoskelo', 'kalalokki', 'korppi', 'kyhmyjoutsen', 'merilokki', 'naurulokki', 'peippo',
#    'sinisorsa', 'sinitiainen', 'talitiainen', 'telkkä', 'varis', 'viherpeippo', 'vihervarpunen'
#    }

# len(common_species) == 16

def read_observation_basket(filename):
    """
    Read observation itemsets from file.

    :param filename:
    """
    itemsets = []

    with open(filename) as csvfile:
        transaction_reader = csv.reader(csvfile, delimiter=',', skipinitialspace=True)
        for row in transaction_reader:
            itemsets.append(tuple(sorted(row)))

    return itemsets

def get_species(itemsets):
    return set([species for itemset in itemsets for species in itemset])

def read_observation_sequences(filename=DATA_DIR + 'observation.sequence'):
    """
    Read observation sequences from file.

    :param filename: sequence file (JSON)
    """
    with open(filename) as f:
        sequences = json.load(f)

    return sequences

def get_all_taxa():
    itemsets = read_observation_basket(DATA_DIR + 'observation.basket')
    return list(set([item for itemset in itemsets for item in itemset]))

def get_yearly_sequences(prune_common_species = False):
    '''
    Put each years' observations into separate sequence

    :param prune_common_species: Leave out the most commonly (year round) observed species
    :return:
    '''
    sequences = read_observation_sequences(DATA_DIR + 'observation.sequence')
    year_seqs = []

    pruned_species = common_species if prune_common_species else []

    for year in range(1979, 2009):
        good_keys, good_seqs = zip(*[(date, obs) for date, obs in sorted(sequences.items()) if date.startswith(str(year))])

        year_seqs.append([[species for species, _, _ in sorted(seq) if species not in pruned_species]
                          for seq in good_seqs])

    return year_seqs

def get_all_names(finnish_list):
    '''
    Get scientific name and english name from finnish name

    :param finnish_list:
    :return: list of tuples (finnish, scientific, english)

    >>> get_all_names(['peippo', 'mustavaris'])
    [('peippo', 'Fringilla coelebs', 'chaffinch'), ('mustavaris', 'Corvus frugilegus', 'rook')]
    '''

    taxon_ontology = Graph()
    taxon_ontology.parse(DATA_DIR + 'halias_taxon_ontology.ttl', format='turtle')

    name_list = []

    for finnish in finnish_list:
        taxa = taxon_ontology.subjects(RDF.type, nsTaxMeOn["TaxonInChecklist"])
        for taxon in taxa:
            labels = taxon_ontology.objects(taxon, RDFS.label)
        #    if nsRanks["Species"] in taxon_ontology.objects(taxon, RDF.type):
            save_this = False
            eng = None
            sci = None
            for label in labels:
                if label.language == 'fi' and str(label) == finnish:
                    save_this = True
                elif label.language == 'en':
                    eng = str(label)
                else:
                    # Assume sci
                    sci = str(label)

            if save_this:
                name_list.append((finnish, sci, eng))

    assert len(name_list) == len(finnish_list)

    return name_list

def local_name(uri):
    return uri.split('/')[-1]

def get_species_itemsets(use_all_species=False):
    '''
    Get data about single species as transactions

    :return:
    '''
    all_taxa = get_all_taxa()

    species_itemsets = []
    species_names = []

    taxon_ontology = Graph()
    taxon_ontology.parse(DATA_DIR + 'halias_taxon_ontology.ttl', format='turtle')

    species_nodes = taxon_ontology.subjects(RDF.type, nsRanks["Species"])

    for sp in species_nodes:
        this_species = []

        labels = taxon_ontology.objects(sp, RDFS.label)
    #    if nsRanks["Species"] in taxon_ontology.objects(taxon, RDF.type):

        for label in labels:
            if label.language == 'fi':
                finnish = str(label)

        if finnish not in all_taxa:
            continue

        this_species.append(finnish)

        # Get upper taxa
        ancestor_list = []
        for ancestor in taxon_ontology.transitive_objects(sp, nsTaxMeOn['isPartOfHigherTaxon']):
            ancestor_label = None
            for label in taxon_ontology.objects(ancestor, RDFS.label):
                if label.language == 'fi' and not ancestor_label:
                    ancestor_label = str(label)
                elif not label.language:
                    ancestor_label = str(label)

            if ancestor_label and ancestor_label not in ['Aves', 'Animalia']:
                ancestor_list.append(ancestor_label)

        this_species += ancestor_list[1:]

        conservation_status = next(taxon_ontology.objects(sp, nsHaliasSchema['hasConservationStatus2010']), False)
        if conservation_status:
            this_species.append(local_name(str(conservation_status)))
        rarity = next(taxon_ontology.objects(sp, nsHaliasSchema['rarity']), False)
        if rarity:
            this_species.append(local_name(str(rarity)))
        charas = taxon_ontology.objects(sp, nsHaliasSchema['hasCharacteristic'])
        if charas:
            this_species += ['tuntomerkki %s' % local_name(str(chara)) for chara in charas]

        if charas or use_all_species:
            # Take only species with characteristics
            species_itemsets.append(this_species)
            species_names.append(finnish)

    # if not use_all_species:
    #     # Prune species without characteristics
    #     species_itemsets = [itemset for itemset in species_itemsets if
    #                         [item for item in itemset if item.startswith('tuntomerkki')]]

    seqs = read_observation_sequences()

    amounts = defaultdict(int)
    sums_pres = defaultdict(int)
    sums_cover = defaultdict(int)
    sums_humi = defaultdict(int)
    sums_rain = defaultdict(int)
    sums_temp = defaultdict(int)

    sums_windsp = defaultdict(int)

    for date, obses in seqs.items():
        for obs in obses:
            species, count_mig, count_tot, month_num, \
                weather_pressure, weather_cover, weather_humidity, weather_rainfall, weather_temp_day, weather_wind_list, \
                weather_std_cover, weather_std_temp, weather_std_wind = obs
            if species in species_names:
                amounts[species] += int(count_tot)
                sums_pres[species] += int(weather_pressure) * int(count_tot)
                if weather_cover is not None:
                    sums_cover[species] += int(weather_cover) * int(count_tot)
                if weather_humidity is not None:
                    sums_humi[species] += int(weather_humidity) * int(count_tot)
                if weather_rainfall is not None:
                    sums_rain[species] += int(weather_rainfall) * int(count_tot)
                sums_temp[species] += int(weather_temp_day) * int(count_tot)
                for wind in weather_wind_list:
                    windspeed = wind[-2:] if wind[-2:].isnumeric() else wind[-1]
                    sums_windsp[species] += int(windspeed)

    pres_list = []
    cover_list = []
    humi_list = []
    rain_list = []
    temp_list = []
    windsp_list = []

    for sp, n in amounts.items():
        pres_list.append(sums_pres[sp] / float(n))
        cover_list.append(sums_cover[sp] / float(n))
        humi_list.append(sums_humi[sp] / float(n))
        rain_list.append(sums_rain[sp] / float(n))
        temp_list.append(sums_temp[sp] / float(n))
        windsp_list.append(sums_windsp[sp] / float(n))

    pres_limit_low = sorted(pres_list)[int(round(len(pres_list) / 3.0))]
    pres_limit_high = sorted(pres_list)[int(round(len(pres_list) / 3.0 * 2))]

    cover_limit_low = sorted(cover_list)[int(round(len(cover_list) / 3.0))]
    cover_limit_high = sorted(cover_list)[int(round(len(cover_list) / 3.0 * 2))]

    humi_limit_low = sorted(humi_list)[int(round(len(humi_list) / 3.0))]
    humi_limit_high = sorted(humi_list)[int(round(len(humi_list) / 3.0 * 2))]

    rain_limit_low = sorted(rain_list)[int(round(len(rain_list) / 3.0))]
    rain_limit_high = sorted(rain_list)[int(round(len(rain_list) / 3.0 * 2))]

    temp_limit_low = sorted(temp_list)[int(round(len(temp_list) / 3.0))]
    temp_limit_high = sorted(temp_list)[int(round(len(temp_list) / 3.0 * 2))]

    windsp_limit_low = sorted(windsp_list)[int(round(len(windsp_list) / 3.0))]
    windsp_limit_high = sorted(windsp_list)[int(round(len(windsp_list) / 3.0 * 2))]

    print('Pressure limits: '
          '\t low = [{min:.2f}, {low:.2f}] \t average = ({low:.2f}, {high:.2f}] \t high = ({high:.2f}, {max:.2f}]'
          .format(min=min(pres_list), low=pres_limit_low, high=pres_limit_high, max=max(pres_list)))
    print('Cloud cover limits: '
          '\t low = [{min:.2f}, {low:.2f}] \t average = ({low:.2f}, {high:.2f}] \t high = ({high:.2f}, {max:.2f}]'
          .format(min=min(cover_list), low=cover_limit_low, high=cover_limit_high, max=max(cover_list)))
    print('Humidity limits: '
          '\t low = [{min:.2f}, {low:.2f}] \t average = ({low:.2f}, {high:.2f}] \t high = ({high:.2f}, {max:.2f}]'
          .format(min=min(humi_list), low=humi_limit_low, high=humi_limit_high, max=max(humi_list)))
    print('Rainfall limits: '
          '\t low = [{min:.2f}, {low:.2f}] \t average = ({low:.2f}, {high:.2f}] \t high = ({high:.2f}, {max:.2f}]'
          .format(min=min(rain_list), low=rain_limit_low, high=rain_limit_high, max=max(rain_list)))
    print('Temperature limits: '
          '\t low = [{min:.2f}, {low:.2f}] \t average = ({low:.2f}, {high:.2f}] \t high = ({high:.2f}, {max:.2f}]'
          .format(min=min(temp_list), low=temp_limit_low, high=temp_limit_high, max=max(temp_list)))
    print('Wind speed limits: '
          '\t low = [{min:.2f}, {low:.2f}] \t average = ({low:.2f}, {high:.2f}] \t high = ({high:.2f}, {max:.2f}]'
          .format(min=min(windsp_list), low=windsp_limit_low, high=windsp_limit_high, max=max(windsp_list)))

    for sp, n in amounts.items():
        for sp_list in species_itemsets:
            if sp == sp_list[0]:
                pressure = sums_pres[sp] / float(n)
                pressure = 'low' if pressure <= pres_limit_low else 'average' if pressure <= pres_limit_high else 'high'
                sp_list.append('air pressure %s' % pressure)

                cover = sums_cover[sp] / float(n)
                cover = 'low' if cover <= cover_limit_low else 'average' if cover <= cover_limit_high else 'high'
                sp_list.append('cloud cover %s' % cover)

                humi = sums_humi[sp] / float(n)
                humi = 'low' if humi <= humi_limit_low else 'average' if humi <= humi_limit_high else 'high'
                sp_list.append('humidity %s' % humi)

                rain = sums_rain[sp] / float(n)
                rain = 'low' if rain <= rain_limit_low else 'average' if rain <= rain_limit_high else 'high'
                sp_list.append('rainfall %s' % rain)

                temp_day = sums_temp[sp] / float(n)
                temp_day = 'low' if temp_day <= temp_limit_low else 'average' if temp_day <= temp_limit_high else 'high'
                sp_list.append('day temperature %s' % temp_day)

                windsp = sums_temp[sp] / float(n)
                windsp = 'low' if windsp <= windsp_limit_low else 'average' if windsp <= windsp_limit_high else 'high'
                sp_list.append('day windspeed %s' % windsp)

    return species_itemsets

def rules_to_tuples(rules_dicts):
    """
    Transform generated rule dicts to tuples

    :param rules_dict:
    :return:
    >>> rulez = [{(('isokoskelo',), ('sinisorsa',)): (0.956, 0.894, 1.013, 0.952)}]
    >>> rules_to_tuples(rulez)
    [(('isokoskelo',), ('sinisorsa',), 0.956, 0.894, 1.013, 0.952)]
    """
    return [list(f.items())[0][0] + list(f.items())[0][1] for f in rules_dicts]<commit_msg>Fix characteristics pruning<commit_after># coding=utf-8
'''
Helpers for data mining tasks.
'''
from collections import defaultdict

import csv
import json
import math

from rdflib import Graph, RDF, RDFS, Namespace

nsTaxMeOn = Namespace("http://www.yso.fi/onto/taxmeon/")
nsRanks = Namespace("http://www.yso.fi/onto/taxonomic-ranks/")
nsHaliasSchema = Namespace("http://ldf.fi/schema/halias/")

DATA_DIR = '../data/'

common_species = {  # Support 0.5 items
    'alli', 'haahka', 'haapana', 'haarapääsky', 'harmaalokki', 'hippiäinen', 'isokoskelo', 'kalalokki', 'korppi',
    'kottarainen', 'kyhmyjoutsen', 'merilokki', 'merimetso', 'mustarastas', 'naurulokki', 'niittykirvinen', 'peippo',
    'punarinta', 'räkättirastas', 'sinisorsa', 'sinitiainen', 'talitiainen', 'telkkä', 'tukkakoskelo', 'tukkasotka',
    'tylli', 'varis', 'varpushaukka', 'viherpeippo', 'vihervarpunen', 'västäräkki'}
#    {
#    # Support 0.75 items:
#    'haahka', 'harmaalokki', 'isokoskelo', 'kalalokki', 'korppi', 'kyhmyjoutsen', 'merilokki', 'naurulokki', 'peippo',
#    'sinisorsa', 'sinitiainen', 'talitiainen', 'telkkä', 'varis', 'viherpeippo', 'vihervarpunen'
#    }

# len(common_species) == 16

def read_observation_basket(filename):
    """
    Read observation itemsets from file.

    :param filename:
    """
    itemsets = []

    with open(filename) as csvfile:
        transaction_reader = csv.reader(csvfile, delimiter=',', skipinitialspace=True)
        for row in transaction_reader:
            itemsets.append(tuple(sorted(row)))

    return itemsets

def get_species(itemsets):
    return set([species for itemset in itemsets for species in itemset])

def read_observation_sequences(filename=DATA_DIR + 'observation.sequence'):
    """
    Read observation sequences from file.

    :param filename: sequence file (JSON)
    """
    with open(filename) as f:
        sequences = json.load(f)

    return sequences

def get_all_taxa():
    itemsets = read_observation_basket(DATA_DIR + 'observation.basket')
    return list(set([item for itemset in itemsets for item in itemset]))

def get_yearly_sequences(prune_common_species = False):
    '''
    Put each years' observations into separate sequence

    :param prune_common_species: Leave out the most commonly (year round) observed species
    :return:
    '''
    sequences = read_observation_sequences(DATA_DIR + 'observation.sequence')
    year_seqs = []

    pruned_species = common_species if prune_common_species else []

    for year in range(1979, 2009):
        good_keys, good_seqs = zip(*[(date, obs) for date, obs in sorted(sequences.items()) if date.startswith(str(year))])

        year_seqs.append([[species for species, _, _ in sorted(seq) if species not in pruned_species]
                          for seq in good_seqs])

    return year_seqs

def get_all_names(finnish_list):
    '''
    Get scientific name and english name from finnish name

    :param finnish_list:
    :return: list of tuples (finnish, scientific, english)

    >>> get_all_names(['peippo', 'mustavaris'])
    [('peippo', 'Fringilla coelebs', 'chaffinch'), ('mustavaris', 'Corvus frugilegus', 'rook')]
    '''

    taxon_ontology = Graph()
    taxon_ontology.parse(DATA_DIR + 'halias_taxon_ontology.ttl', format='turtle')

    name_list = []

    for finnish in finnish_list:
        taxa = taxon_ontology.subjects(RDF.type, nsTaxMeOn["TaxonInChecklist"])
        for taxon in taxa:
            labels = taxon_ontology.objects(taxon, RDFS.label)
        #    if nsRanks["Species"] in taxon_ontology.objects(taxon, RDF.type):
            save_this = False
            eng = None
            sci = None
            for label in labels:
                if label.language == 'fi' and str(label) == finnish:
                    save_this = True
                elif label.language == 'en':
                    eng = str(label)
                else:
                    # Assume sci
                    sci = str(label)

            if save_this:
                name_list.append((finnish, sci, eng))

    assert len(name_list) == len(finnish_list)

    return name_list

def local_name(uri):
    return uri.split('/')[-1]

def get_species_itemsets(use_all_species=False):
    '''
    Get data about single species as transactions

    :return:
    '''
    all_taxa = get_all_taxa()

    species_itemsets = []
    species_names = []

    taxon_ontology = Graph()
    taxon_ontology.parse(DATA_DIR + 'halias_taxon_ontology.ttl', format='turtle')

    species_nodes = taxon_ontology.subjects(RDF.type, nsRanks["Species"])

    for sp in species_nodes:
        this_species = []

        labels = taxon_ontology.objects(sp, RDFS.label)
    #    if nsRanks["Species"] in taxon_ontology.objects(taxon, RDF.type):

        for label in labels:
            if label.language == 'fi':
                finnish = str(label)

        if finnish not in all_taxa:
            continue

        this_species.append(finnish)

        # Get upper taxa
        ancestor_list = []
        for ancestor in taxon_ontology.transitive_objects(sp, nsTaxMeOn['isPartOfHigherTaxon']):
            ancestor_label = None
            for label in taxon_ontology.objects(ancestor, RDFS.label):
                if label.language == 'fi' and not ancestor_label:
                    ancestor_label = str(label)
                elif not label.language:
                    ancestor_label = str(label)

            if ancestor_label and ancestor_label not in ['Aves', 'Animalia']:
                ancestor_list.append(ancestor_label)

        this_species += ancestor_list[1:]

        conservation_status = next(taxon_ontology.objects(sp, nsHaliasSchema['hasConservationStatus2010']), False)
        if conservation_status:
            this_species.append(local_name(str(conservation_status)))
        rarity = next(taxon_ontology.objects(sp, nsHaliasSchema['rarity']), False)
        if rarity:
            this_species.append(local_name(str(rarity)))
        charas = taxon_ontology.objects(sp, nsHaliasSchema['hasCharacteristic'])
        charas = ['tuntomerkki %s' % local_name(str(chara)) for chara in charas]

        if charas:
            this_species += charas

        if charas or use_all_species:
            # Take only species with characteristics
            species_itemsets.append(this_species)
            species_names.append(finnish)

    # if not use_all_species:
    #     # Prune species without characteristics
    #     species_itemsets = [itemset for itemset in species_itemsets if
    #                         [item for item in itemset if item.startswith('tuntomerkki')]]

    seqs = read_observation_sequences()

    amounts = defaultdict(int)
    sums_pres = defaultdict(int)
    sums_cover = defaultdict(int)
    sums_humi = defaultdict(int)
    sums_rain = defaultdict(int)
    sums_temp = defaultdict(int)

    sums_windsp = defaultdict(int)

    for date, obses in seqs.items():
        for obs in obses:
            species, count_mig, count_tot, month_num, \
                weather_pressure, weather_cover, weather_humidity, weather_rainfall, weather_temp_day, weather_wind_list, \
                weather_std_cover, weather_std_temp, weather_std_wind = obs
            if species in species_names:
                amounts[species] += int(count_tot)
                sums_pres[species] += int(weather_pressure) * int(count_tot)
                if weather_cover is not None:
                    sums_cover[species] += int(weather_cover) * int(count_tot)
                if weather_humidity is not None:
                    sums_humi[species] += int(weather_humidity) * int(count_tot)
                if weather_rainfall is not None:
                    sums_rain[species] += int(weather_rainfall) * int(count_tot)
                sums_temp[species] += int(weather_temp_day) * int(count_tot)
                for wind in weather_wind_list:
                    windspeed = wind[-2:] if wind[-2:].isnumeric() else wind[-1]
                    sums_windsp[species] += int(windspeed)

    pres_list = []
    cover_list = []
    humi_list = []
    rain_list = []
    temp_list = []
    windsp_list = []

    for sp, n in amounts.items():
        pres_list.append(sums_pres[sp] / float(n))
        cover_list.append(sums_cover[sp] / float(n))
        humi_list.append(sums_humi[sp] / float(n))
        rain_list.append(sums_rain[sp] / float(n))
        temp_list.append(sums_temp[sp] / float(n))
        windsp_list.append(sums_windsp[sp] / float(n))

    pres_limit_low = sorted(pres_list)[int(round(len(pres_list) / 3.0))]
    pres_limit_high = sorted(pres_list)[int(round(len(pres_list) / 3.0 * 2))]

    cover_limit_low = sorted(cover_list)[int(round(len(cover_list) / 3.0))]
    cover_limit_high = sorted(cover_list)[int(round(len(cover_list) / 3.0 * 2))]

    humi_limit_low = sorted(humi_list)[int(round(len(humi_list) / 3.0))]
    humi_limit_high = sorted(humi_list)[int(round(len(humi_list) / 3.0 * 2))]

    rain_limit_low = sorted(rain_list)[int(round(len(rain_list) / 3.0))]
    rain_limit_high = sorted(rain_list)[int(round(len(rain_list) / 3.0 * 2))]

    temp_limit_low = sorted(temp_list)[int(round(len(temp_list) / 3.0))]
    temp_limit_high = sorted(temp_list)[int(round(len(temp_list) / 3.0 * 2))]

    windsp_limit_low = sorted(windsp_list)[int(round(len(windsp_list) / 3.0))]
    windsp_limit_high = sorted(windsp_list)[int(round(len(windsp_list) / 3.0 * 2))]

    print('Pressure limits: '
          '\t low = [{min:.2f}, {low:.2f}] \t average = ({low:.2f}, {high:.2f}] \t high = ({high:.2f}, {max:.2f}]'
          .format(min=min(pres_list), low=pres_limit_low, high=pres_limit_high, max=max(pres_list)))
    print('Cloud cover limits: '
          '\t low = [{min:.2f}, {low:.2f}] \t average = ({low:.2f}, {high:.2f}] \t high = ({high:.2f}, {max:.2f}]'
          .format(min=min(cover_list), low=cover_limit_low, high=cover_limit_high, max=max(cover_list)))
    print('Humidity limits: '
          '\t low = [{min:.2f}, {low:.2f}] \t average = ({low:.2f}, {high:.2f}] \t high = ({high:.2f}, {max:.2f}]'
          .format(min=min(humi_list), low=humi_limit_low, high=humi_limit_high, max=max(humi_list)))
    print('Rainfall limits: '
          '\t low = [{min:.2f}, {low:.2f}] \t average = ({low:.2f}, {high:.2f}] \t high = ({high:.2f}, {max:.2f}]'
          .format(min=min(rain_list), low=rain_limit_low, high=rain_limit_high, max=max(rain_list)))
    print('Temperature limits: '
          '\t low = [{min:.2f}, {low:.2f}] \t average = ({low:.2f}, {high:.2f}] \t high = ({high:.2f}, {max:.2f}]'
          .format(min=min(temp_list), low=temp_limit_low, high=temp_limit_high, max=max(temp_list)))
    print('Wind speed limits: '
          '\t low = [{min:.2f}, {low:.2f}] \t average = ({low:.2f}, {high:.2f}] \t high = ({high:.2f}, {max:.2f}]'
          .format(min=min(windsp_list), low=windsp_limit_low, high=windsp_limit_high, max=max(windsp_list)))

    for sp, n in amounts.items():
        for sp_list in species_itemsets:
            if sp == sp_list[0]:
                pressure = sums_pres[sp] / float(n)
                pressure = 'low' if pressure <= pres_limit_low else 'average' if pressure <= pres_limit_high else 'high'
                sp_list.append('air pressure %s' % pressure)

                cover = sums_cover[sp] / float(n)
                cover = 'low' if cover <= cover_limit_low else 'average' if cover <= cover_limit_high else 'high'
                sp_list.append('cloud cover %s' % cover)

                humi = sums_humi[sp] / float(n)
                humi = 'low' if humi <= humi_limit_low else 'average' if humi <= humi_limit_high else 'high'
                sp_list.append('humidity %s' % humi)

                rain = sums_rain[sp] / float(n)
                rain = 'low' if rain <= rain_limit_low else 'average' if rain <= rain_limit_high else 'high'
                sp_list.append('rainfall %s' % rain)

                temp_day = sums_temp[sp] / float(n)
                temp_day = 'low' if temp_day <= temp_limit_low else 'average' if temp_day <= temp_limit_high else 'high'
                sp_list.append('day temperature %s' % temp_day)

                windsp = sums_temp[sp] / float(n)
                windsp = 'low' if windsp <= windsp_limit_low else 'average' if windsp <= windsp_limit_high else 'high'
                sp_list.append('day windspeed %s' % windsp)

    return species_itemsets

def rules_to_tuples(rules_dicts):
    """
    Transform generated rule dicts to tuples

    :param rules_dict:
    :return:
    >>> rulez = [{(('isokoskelo',), ('sinisorsa',)): (0.956, 0.894, 1.013, 0.952)}]
    >>> rules_to_tuples(rulez)
    [(('isokoskelo',), ('sinisorsa',), 0.956, 0.894, 1.013, 0.952)]
    """
    return [list(f.items())[0][0] + list(f.items())[0][1] for f in rules_dicts]<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import markdown, pystache, yaml;
import datetime, json, re, os, shutil, sys;

config = {};
inputDir = os.path.dirname(sys.argv[-1]);

# load config files except for the last argument
for i in range(1, len(sys.argv)-1):
    configFile = sys.argv[i];
    if configFile.endswith('.json'):
        with open(configFile) as data_file:
            jmap = json.load(data_file);
            for k in jmap:
                config[k] = jmap[k];
    if configFile.endswith('.yaml'):
        with open(configFile) as data_file:
            ymap = yaml.load(data_file);
            for k in ymap:
                config[k] = ymap[k];

# load last argument
with (open(sys.argv[-1], 'r')) as f:
    fileText = f.read();
yamlMatch = re.compile(r'^---\n(.*)\n---\n(.*)', re.DOTALL).match(fileText);
yamlText = yamlMatch.group(1);
mdText = yamlMatch.group(2);
ymap = yaml.load(yamlText);
for k in ymap:
    config[k] = ymap[k];

# load data file
if ('data-file' in config) and (config['data-file'].endswith('.json')):
    with (open(inputDir + '/' + config['data-file'], 'r')) as f:
        jmap = json.load(f);
        for k in jmap:
            config[k] = jmap[k];

# set draft flag if published ts is in the future
now = str(datetime.datetime.now()).replace(' ', 'T')[0:-3];
if ('published' in config):
    published = str(config['published']);
    if (published > now):
        config['draft'] = True;

# create a subset of properties for the archive DB
datamap = {};
for key in ['url', 'lang', 'title', 'draft', 'description', 'summary', 'published', 'updated', 'author', 'tag', 'category']:
    if (key in config):
        datamap[key] = str(config[key]);

# update properties in the archive DB
noarchive = ('no-archive' in config) and (config['no-archive'] == True);
if (not noarchive) and ('archive-json' in config) and (os.path.exists(config['archive-json'])):
    with (open(config['archive-json'], 'r')) as f:
        jmap = json.load(f);
        same = False;
        if config['url'] in jmap:
            dbmap = jmap[config['url']];
            same = len(set(dbmap.items()) ^ set(datamap.items())) == 0;
        if not same:
            jmap[config['url']] = datamap;
            with open(config['archive-json'], 'w') as outfile:
                json.dump(jmap, outfile, sort_keys=True, indent=2);

# no work with draft posts
if ('draft' in config) and (config['draft'] == True):
    exit(0);

# author extended data -> authorz, works on single author
# TODO(isoos): make it available for multiple author
if ('author' in config) and ('author-data' in config) and ('authorz' not in config):
    authorz = [];
    adata = config['author-data'];
    if config['author'] in adata:
        authorz.append(adata[config['author']]);
    config['authorz'] = authorz;

# convert dates to short format
if ('published' in config) and ('published-date' not in config):
    config['published-date'] = str(config['published'])[0:10];
if ('updated' in config) and ('updated-date' not in config):
    config['updated-date'] = str(config['updated'])[0:10];

# transform the list of lang:url maps into a better suited map
if 'translation' in config:
    newhreflang = [];
    for m in config['translation']:
        for k in m:
            v = m[k];
            newhreflang.append({'lang': k, 'href': v});
    config['translation'] = newhreflang;

# run markdown
md = markdown.Markdown(extensions = [
    'markdown.extensions.extra',
    'markdown.extensions.meta',
    'markdown.extensions.sane_lists',
    'markdown.extensions.toc(baselevel=2,marker="")'])

# create context for mustache
config['content'] = md.convert(mdText);
config['md-meta'] = md.Meta;
if ('toc' in config) and (config['toc'] == True):
    config['md-toc'] = md.toc;

if (not 'output-file' in config):
    if config['url'].endswith('.html'):
        config['output-file'] = config['url'];
    else:
        if config['url'].endswith('/'):
            config['output-file'] = config['url'] + 'index.html';

with (open(config['page-template'], 'r')) as f:
    mustacheText = f.read();
newHtmlText = pystache.render(mustacheText, config);

htmlFileName = config['output-dir'] + config['output-file'];
outputDir = os.path.dirname(htmlFileName);
if not os.path.exists(outputDir):
    print('Creating: ' + outputDir);
    os.makedirs(outputDir)

# Writing output HTML
oldHtmlText = '';
if os.path.exists(htmlFileName):
    with (open(htmlFileName, 'rb')) as f:
        oldHtmlText = f.read().decode('utf-8');
if newHtmlText != oldHtmlText:
    print('Writing: ' + htmlFileName);
    with open(htmlFileName, 'wb') as f:
        f.write(newHtmlText.encode('utf-8'));

# copy referenced assets
assets = [];
if 'asset' in config:
    for a in config['asset']:
        assets.append(a);
for m in re.compile(r'src=\"(.*?)\"').findall(config['content']):
    assets.append(m);
if 'cover-image' in config:
    assets.append(config['cover-image']);

for src in assets:
    fullSrc = inputDir + '/' + src;
    if '/' in src:
        continue;
    if not os.path.isfile(fullSrc):
        continue;
    dest = outputDir + '/' + src;
    doCopy = False;
    if os.path.isfile(dest):
        doCopy = os.path.getmtime(fullSrc) > os.path.getmtime(dest);
    else:
        doCopy = True;
    if doCopy:
        print('Copy: ' + fullSrc + ' -> ' + dest);
        shutil.copy2(fullSrc, dest);
<commit_msg>has-authorz support<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import markdown, pystache, yaml;
import datetime, json, re, os, shutil, sys;

config = {};
inputDir = os.path.dirname(sys.argv[-1]);

# load config files except for the last argument
for i in range(1, len(sys.argv)-1):
    configFile = sys.argv[i];
    if configFile.endswith('.json'):
        with open(configFile) as data_file:
            jmap = json.load(data_file);
            for k in jmap:
                config[k] = jmap[k];
    if configFile.endswith('.yaml'):
        with open(configFile) as data_file:
            ymap = yaml.load(data_file);
            for k in ymap:
                config[k] = ymap[k];

# load last argument
with (open(sys.argv[-1], 'r')) as f:
    fileText = f.read();
yamlMatch = re.compile(r'^---\n(.*)\n---\n(.*)', re.DOTALL).match(fileText);
yamlText = yamlMatch.group(1);
mdText = yamlMatch.group(2);
ymap = yaml.load(yamlText);
for k in ymap:
    config[k] = ymap[k];

# load data file
if ('data-file' in config) and (config['data-file'].endswith('.json')):
    with (open(inputDir + '/' + config['data-file'], 'r')) as f:
        jmap = json.load(f);
        for k in jmap:
            config[k] = jmap[k];

# set draft flag if published ts is in the future
now = str(datetime.datetime.now()).replace(' ', 'T')[0:-3];
if ('published' in config):
    published = str(config['published']);
    if (published > now):
        config['draft'] = True;

# create a subset of properties for the archive DB
datamap = {};
for key in ['url', 'lang', 'title', 'draft', 'description', 'summary', 'published', 'updated', 'author', 'tag', 'category']:
    if (key in config):
        datamap[key] = str(config[key]);

# update properties in the archive DB
noarchive = ('no-archive' in config) and (config['no-archive'] == True);
if (not noarchive) and ('archive-json' in config) and (os.path.exists(config['archive-json'])):
    with (open(config['archive-json'], 'r')) as f:
        jmap = json.load(f);
        same = False;
        if config['url'] in jmap:
            dbmap = jmap[config['url']];
            same = len(set(dbmap.items()) ^ set(datamap.items())) == 0;
        if not same:
            jmap[config['url']] = datamap;
            with open(config['archive-json'], 'w') as outfile:
                json.dump(jmap, outfile, sort_keys=True, indent=2);

# no work with draft posts
if ('draft' in config) and (config['draft'] == True):
    exit(0);

# author extended data -> authorz, works on single author
# TODO(isoos): make it available for multiple author
if ('author' in config) and ('author-data' in config) and ('authorz' not in config):
    authorz = [];
    adata = config['author-data'];
    if config['author'] in adata:
        authorz.append(adata[config['author']]);
    config['authorz'] = authorz;

# separate flag to indicate the presence of authorz info
if 'authorz' in config:
    config['has-authorz'] = True;

# convert dates to short format
if ('published' in config) and ('published-date' not in config):
    config['published-date'] = str(config['published'])[0:10];
if ('updated' in config) and ('updated-date' not in config):
    config['updated-date'] = str(config['updated'])[0:10];

# transform the list of lang:url maps into a better suited map
if 'translation' in config:
    newhreflang = [];
    for m in config['translation']:
        for k in m:
            v = m[k];
            newhreflang.append({'lang': k, 'href': v});
    config['translation'] = newhreflang;

# run markdown
md = markdown.Markdown(extensions = [
    'markdown.extensions.extra',
    'markdown.extensions.meta',
    'markdown.extensions.sane_lists',
    'markdown.extensions.toc(baselevel=2,marker="")'])

# create context for mustache
config['content'] = md.convert(mdText);
config['md-meta'] = md.Meta;
if ('toc' in config) and (config['toc'] == True):
    config['md-toc'] = md.toc;

if (not 'output-file' in config):
    if config['url'].endswith('.html'):
        config['output-file'] = config['url'];
    else:
        if config['url'].endswith('/'):
            config['output-file'] = config['url'] + 'index.html';

with (open(config['page-template'], 'r')) as f:
    mustacheText = f.read();
newHtmlText = pystache.render(mustacheText, config);

htmlFileName = config['output-dir'] + config['output-file'];
outputDir = os.path.dirname(htmlFileName);
if not os.path.exists(outputDir):
    print('Creating: ' + outputDir);
    os.makedirs(outputDir)

# Writing output HTML
oldHtmlText = '';
if os.path.exists(htmlFileName):
    with (open(htmlFileName, 'rb')) as f:
        oldHtmlText = f.read().decode('utf-8');
if newHtmlText != oldHtmlText:
    print('Writing: ' + htmlFileName);
    with open(htmlFileName, 'wb') as f:
        f.write(newHtmlText.encode('utf-8'));

# copy referenced assets
assets = [];
if 'asset' in config:
    for a in config['asset']:
        assets.append(a);
for m in re.compile(r'src=\"(.*?)\"').findall(config['content']):
    assets.append(m);
if 'cover-image' in config:
    assets.append(config['cover-image']);

for src in assets:
    fullSrc = inputDir + '/' + src;
    if '/' in src:
        continue;
    if not os.path.isfile(fullSrc):
        continue;
    dest = outputDir + '/' + src;
    doCopy = False;
    if os.path.isfile(dest):
        doCopy = os.path.getmtime(fullSrc) > os.path.getmtime(dest);
    else:
        doCopy = True;
    if doCopy:
        print('Copy: ' + fullSrc + ' -> ' + dest);
        shutil.copy2(fullSrc, dest);
<|endoftext|>


<commit_before># monitor.py  27/09/2015  D.J.Whale
#
# Monitor settings of Energine MiHome plugs

import time

from energenie import OpenHEMS, Devices
from energenie import radio
from Timer import Timer

def trace(msg):
    print(str(msg))

#----- TEST APPLICATION -------------------------------------------------------

directory = {}

def allkeys(d):
    result = ""
    for k in d:
        if len(result) != 0:
            result += ','
        result += str(k)
    return result

def updateDirectory(message):
    """Update the local directory with information about this device"""
    now      = time.time()
    header   = message["header"]
    sensorId = header["sensorid"]

    if not directory.has_key(sensorId):
        # new device discovered
        desc = Devices.getDescription(header["mfrid"], header["productid"])
        print("ADD device:%s %s" % (hex(sensorId), desc))
        directory[sensorId] = {"header": message["header"]}
        print(allkeys(directory))

    directory[sensorId]["time"] = now
    #TODO would be good to keep recs, but need to iterate through all and key by paramid,
    #not as a list index, else merging will be hard.

SWITCH_MESSAGE = {
    "header": {
        "mfrid":       Devices.MFRID,
        "productid":   Devices.PRODUCTID_R1_MONITOR_AND_CONTROL,
        "encryptPIP":  Devices.CRYPT_PIP,
        "sensorid":    0 # FILL IN
    },
    "recs": [
        {
            "wr":      True,
            "paramid": OpenHEMS.PARAM_SWITCH_STATE,
            "typeid":  OpenHEMS.Value.UINT,
            "length":  1,
            "value":   0 # FILL IN
        }
    ]
}

JOIN_ACK_MESSAGE = {
    "header": {
        "mfrid":       0, # FILL IN
        "productid":   0, # FILL IN
        "encryptPIP":  Devices.CRYPT_PIP,
        "sensorid":    0 # FILL IN
    },
    "recs": [
        {
            "wr":      False,
            "paramid": OpenHEMS.PARAM_JOIN,
            "typeid":  OpenHEMS.Value.UINT,
            "length":  0
        }
    ]
}

def monitor():
    """Send discovery and monitor messages, and capture any responses"""

    # Define the schedule of message polling
    sendSwitchTimer    = Timer(5, 1)   # every 5 seconds offset by initial 1
    switch_state       = 0             # OFF
    radio.receiver()
    decoded            = None

    while True:
        # See if there is a payload, and if there is, process it
        if radio.isReceiveWaiting():
            trace("receiving payload")
            payload = radio.receive()
            try:
                decoded = OpenHEMS.decode(payload)
            except OpenHEMS.OpenHEMSException as e:
                print("Can't decode payload:" + str(e))
                continue

            OpenHEMS.showMessage(decoded)
            updateDirectory(decoded)
            #TODO: Should remember report time of each device,
            #and reschedule command messages to avoid their transmit slot
            #making it less likely to miss an incoming message due to
            #the radio being in transmit mode

            # assume only 1 rec in a join, for now
            if decoded["recs"][0]["paramid"] == OpenHEMS.PARAM_JOIN:
                #TODO: write OpenHEMS.getFromMessage("header_mfrid")
                response = OpenHEMS.alterMessage(JOIN_ACK_MESSAGE,
                    header_mfrid=decoded["header"]["mfrid"],
                    header_productid=decoded["header"]["productid"],
                    header_sensorid=decoded["header"]["sensorid"])
                p = OpenHEMS.encode(response)
                radio.transmitter()
                radio.transmit(p)
                radio.receiver()

        if sendSwitchTimer.check() and decoded != None:
            request = OpenHEMS.alterMessage(SWITCH_MESSAGE,
                header_sensorid=decoded["header"]["sensorid"],
                recs_0_value=switch_state)
            p = OpenHEMS.encode(request)
            radio.transmitter()
            radio.transmit(p)
            radio.receiver()
            switch_state = (switch_state+1) % 2 # toggle

if __name__ == "__main__":

    radio.init()
    OpenHEMS.init(Devices.CRYPT_PID)

    try:
        monitor()

    finally:
        radio.finished()

# END
<commit_msg>Added a simple data logging to CSV capability<commit_after># monitor.py  27/09/2015  D.J.Whale
#
# Monitor settings of Energine MiHome plugs

import time

from energenie import OpenHEMS, Devices
from energenie import radio
from Timer import Timer
import os

LOG_FILENAME = "energenie.csv"

def trace(msg):
    print(str(msg))

log_file = None

def logMessage (msg):
    HEADINGS = 'timestamp,mfrid,prodid,sensorid,flags,switch,voltage,freq,reactive,real'

    global log_file
    if log_file == None:
        if not os.path.isfile(LOG_FILENAME):
            log_file = open(LOG_FILENAME, 'w')
            log_file.write(HEADINGS + '\n')
        else:
            log_file = open(LOG_FILENAME, 'a') # append

    # get the header
    header    = msg['header']
    timestamp = time.time()
    mfrid     = header['mfrid']
    productid = header['productid']
    sensorid  = header['sensorid']

    # set defaults for any data that doesn't appear in this message
    # but build flags so we know which ones this contains
    flags = [0,0,0,0,0]
    switch = None
    voltage = None
    freq = None
    reactive = None
    real = None

    # capture any data that we want
    for rec in msg['recs']:
        paramid = rec['paramid']
        try:
            value = rec['value']
        except:
            value = None

        if   paramid == OpenHEMS.PARAM_SWITCH_STATE:
            switch = value
            flags[0] = 1
        elif paramid == OpenHEMS.PARAM_VOLTAGE:
            flags[1] = 1
            voltage = value
        elif paramid == OpenHEMS.PARAM_FREQUENCY:
            flags[2] = 1
            freq = value
        elif paramid == OpenHEMS.PARAM_REACTIVE_POWER:
            flags[3] = 1
            reactive = value
        elif paramid == OpenHEMS.PARAM_REAL_POWER:
            flags[4] = 1
            real = value

    # generate a line of CSV
    flags = "".join([str(a) for a in flags])
    csv = "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s" % (timestamp, mfrid, productid, sensorid, flags, switch, voltage, freq, reactive, real)
    log_file.write(csv + '\n')
    log_file.flush()
    print(csv) # testing

#----- TEST APPLICATION -------------------------------------------------------

directory = {}

def allkeys(d):
    result = ""
    for k in d:
        if len(result) != 0:
            result += ','
        result += str(k)
    return result

def updateDirectory(message):
    """Update the local directory with information about this device"""
    now      = time.time()
    header   = message["header"]
    sensorId = header["sensorid"]

    if not directory.has_key(sensorId):
        # new device discovered
        desc = Devices.getDescription(header["mfrid"], header["productid"])
        print("ADD device:%s %s" % (hex(sensorId), desc))
        directory[sensorId] = {"header": message["header"]}
        print(allkeys(directory))

    directory[sensorId]["time"] = now
    #TODO would be good to keep recs, but need to iterate through all and key by paramid,
    #not as a list index, else merging will be hard.

SWITCH_MESSAGE = {
    "header": {
        "mfrid":       Devices.MFRID,
        "productid":   Devices.PRODUCTID_R1_MONITOR_AND_CONTROL,
        "encryptPIP":  Devices.CRYPT_PIP,
        "sensorid":    0 # FILL IN
    },
    "recs": [
        {
            "wr":      True,
            "paramid": OpenHEMS.PARAM_SWITCH_STATE,
            "typeid":  OpenHEMS.Value.UINT,
            "length":  1,
            "value":   0 # FILL IN
        }
    ]
}

JOIN_ACK_MESSAGE = {
    "header": {
        "mfrid":       0, # FILL IN
        "productid":   0, # FILL IN
        "encryptPIP":  Devices.CRYPT_PIP,
        "sensorid":    0 # FILL IN
    },
    "recs": [
        {
            "wr":      False,
            "paramid": OpenHEMS.PARAM_JOIN,
            "typeid":  OpenHEMS.Value.UINT,
            "length":  0
        }
    ]
}

def monitor():
    """Send discovery and monitor messages, and capture any responses"""

    # Define the schedule of message polling
    sendSwitchTimer    = Timer(60, 1)   # every n seconds offset by initial 1
    switch_state       = 0             # OFF
    radio.receiver()
    decoded            = None

    while True:
        # See if there is a payload, and if there is, process it
        if radio.isReceiveWaiting():
            trace("receiving payload")
            payload = radio.receive()
            try:
                decoded = OpenHEMS.decode(payload)
            except OpenHEMS.OpenHEMSException as e:
                print("Can't decode payload:" + str(e))
                continue

            OpenHEMS.showMessage(decoded)
            updateDirectory(decoded)
            logMessage(decoded)

            #TODO: Should remember report time of each device,
            #and reschedule command messages to avoid their transmit slot
            #making it less likely to miss an incoming message due to
            #the radio being in transmit mode

            # assume only 1 rec in a join, for now
            if decoded["recs"][0]["paramid"] == OpenHEMS.PARAM_JOIN:
                #TODO: write OpenHEMS.getFromMessage("header_mfrid")
                response = OpenHEMS.alterMessage(JOIN_ACK_MESSAGE,
                    header_mfrid=decoded["header"]["mfrid"],
                    header_productid=decoded["header"]["productid"],
                    header_sensorid=decoded["header"]["sensorid"])
                p = OpenHEMS.encode(response)
                radio.transmitter()
                radio.transmit(p)
                radio.receiver()

        if sendSwitchTimer.check() and decoded != None:
            request = OpenHEMS.alterMessage(SWITCH_MESSAGE,
                header_sensorid=decoded["header"]["sensorid"],
                recs_0_value=switch_state)
            p = OpenHEMS.encode(request)
            radio.transmitter()
            radio.transmit(p)
            radio.receiver()
            switch_state = (switch_state+1) % 2 # toggle

if __name__ == "__main__":

    radio.init()
    OpenHEMS.init(Devices.CRYPT_PID)

    try:
        monitor()

    finally:
        radio.finished()

# END
<|endoftext|>


<commit_before>import numpy as np
import time
import os

import theano
import theano.tensor as T
from theano.tensor.shared_randomstreams import RandomStreams

import parameters
import log
import vectors
import validation
import kit

floatX = theano.config.floatX
empty = lambda *shape: np.empty(shape, dtype='int32')
rnd1 = lambda d0: np.random.rand(d0).astype(dtype=floatX)
rnd2 = lambda d0, d1: np.random.rand(d0, d1).astype(dtype=floatX)

def train(fileIndexMap, wordIndexMap, wordEmbeddings, contexts, metricsPath, negative):
    contextsCount = len(contexts)

    filesCount = len(fileIndexMap)
    fileEmbeddingSize = 200
    wordsCount = len(wordIndexMap)
    wordEmbeddingSize = wordEmbeddings.shape[1]
    windowSize = contexts.shape[1] - 1
    contextSize = windowSize - 1

    fileEmbeddings = theano.shared(rnd2(filesCount, fileEmbeddingSize).astype(float), 'fileEmbeddings', borrow=True)
    wordEmbeddings = theano.shared(wordEmbeddings.astype(floatX), 'wordEmbeddings', borrow=True)
    weights = theano.shared(rnd2(fileEmbeddingSize + contextSize * wordEmbeddingSize, wordsCount).astype(float), 'weights', borrow=True)

    ctxs = T.imatrix('ctxs')
    fileIndices = ctxs[:,0:1]
    wordIndices = ctxs[:,1:-1]
    targetIndices = T.reshape(ctxs[:,-1:], (ctxs.shape[0],1))

    random = RandomStreams(0)
    negativeSampleIndices = random.random_integers((targetIndices.shape[0], negative), low=0, high=wordsCount-1)
    indices = T.concatenate([targetIndices, negativeSampleIndices], axis=1)

    files = fileEmbeddings[fileIndices]
    fileFeatures = T.flatten(files, outdim=2)
    words = wordEmbeddings[wordIndices]
    wordFeatures = T.flatten(words, outdim=2)
    features = T.concatenate([fileFeatures, wordFeatures], axis=1)

    subWeights = weights[:,indices].dimshuffle(1,0,2)

    probabilities = T.batched_dot(features, subWeights)

    parameters = [fileEmbeddings, weights]

    l1Coefficient = T.scalar('l1Coefficient', dtype=floatX)
    l2Coefficient = T.scalar('l2Coefficient', dtype=floatX)

    l1Regularization = l1Coefficient * sum([abs(p).sum() for p in parameters])
    l2Regularization = l2Coefficient * sum([(p ** 2).sum() for p in parameters])

    cost = -T.mean(T.log(T.exp(probabilities[:,0])) + T.sum(T.log(T.exp(-probabilities[:,1:])), axis=1)) + \
           l1Regularization + l2Regularization

    learningRate = T.scalar('learningRate', dtype=floatX)

    parameters = [fileEmbeddings]
    subParameters = [files]
    gradients = [T.grad(cost, wrt=subP) for subP in subParameters]
    updates = [(p, T.inc_subtensor(subP, -learningRate * g)) for p, subP, g in zip(parameters, subParameters, gradients)]

    parameters = [weights]
    gradients = [T.grad(cost, wrt=p) for p in parameters]
    updates = updates + [(p, p - learningRate * g) for p, g in zip(parameters, gradients)]

    batchIndex = T.iscalar('batchIndex')
    batchSize = T.iscalar('batchSize')
    trainingContexts = theano.shared(contexts, 'trainingContexts', borrow=True)

    trainModel = theano.function(
        inputs=[batchIndex, batchSize, learningRate, l1Coefficient, l2Coefficient],
        outputs=cost,
        updates=updates,
        givens={
            ctxs: trainingContexts[batchIndex * batchSize : (batchIndex + 1) * batchSize]
        }
    )

    startTime = time.time()
    epochs = 30
    bs = 50 # bs for batchSize
    for epoch in xrange(0, epochs):
        contextsCount = contexts.shape[0]
        batchesCount = contextsCount / bs + int(contextsCount % bs > 0)

        for bi in xrange(0, batchesCount): # bi for batchIndex
            error = trainModel(bi, bs, 0.01, 0.02, 0.005)
            if error <= 0:
                break

        fe = fileEmbeddings.get_value()
        we = lambda key: fe[fileIndexMap['../data/Cockatoo/Prepared/cockatoo/{0}.txt'.format(key)]]
        distance = lambda left, right: vectors.euclideanDistance(we(left), we(right))

        metrics = {
            'tanks': distance('tank_0', 'tank_1'),
            'viruses': distance('virus_0', 'virus_1'),
            'tankVirus': distance('tank_0', 'virus_0'),
            'error': error
        }

        validation.dump(metricsPath, epoch, metrics)

        elapsed = time.time() - startTime
        secondsPerEpoch = elapsed / (epoch + 1)

        log.progress('Training model: {0:.3f}%. {1:.3f} sec per epoch. Error: {2:.3f}. Tanks={3:.3f}. Viruses={4:.3f}. Tank/Virus={5:.3f}.',
                     epoch + 1, epochs,
                     secondsPerEpoch,
                     float(error),
                     metrics['tanks'],
                     metrics['viruses'],
                     metrics['tankVirus'])

        if error <= 0:
            break

    validation.compareEmbeddings(fileIndexMap, fileEmbeddings.get_value())
    validation.plotEmbeddings(fileIndexMap, fileEmbeddings.get_value())

def main():
    pathTo = kit.PathTo('Cockatoo')

    fileIndexMap = parameters.loadIndexMap(pathTo.fileIndexMap)
    wordIndexMap = parameters.loadIndexMap(pathTo.wordIndexMap)
    indexWordMap = parameters.loadIndexMap(pathTo.wordIndexMap, inverse=True)
    wordEmbeddings = parameters.loadEmbeddings(pathTo.wordEmbeddings)
    metricsPath = pathTo.metrics('history.csv')

    if os.path.exists(metricsPath):
        os.remove(metricsPath)

    contextProvider = parameters.IndexContextProvider(pathTo.contexts)
    contexts = contextProvider[:]

    log.info('Contexts loading complete. {0} contexts loaded {1} words each.', len(contexts), contextProvider.contextSize)

    train(fileIndexMap, wordIndexMap, wordEmbeddings, contexts, metricsPath, negative=10)

if __name__ == '__main__':
    main()
<commit_msg>Refactoring. Moved some arguments to train interface. Added model stub.<commit_after>import numpy as np
import time
import os

import theano
import theano.tensor as T
from theano.tensor.shared_randomstreams import RandomStreams

import parameters
import log
import vectors
import validation
import kit

floatX = theano.config.floatX
rnd2 = lambda d0, d1: np.random.rand(d0, d1).astype(dtype=floatX)

class Model():
    def __init__(self, fileIndexMap, fileEmbeddingSize, wordIndexMap, wordEmbeddings, contexts, metricsPath,
                 epochs, batchSize, learningRate, negative, l1, l2):
        filesCount = len(fileIndexMap)
        wordsCount = len(wordIndexMap)
        wordEmbeddingSize = wordEmbeddings.shape[1]
        windowSize = contexts.shape[1] - 1
        contextSize = windowSize - 1

        fileEmbeddings = theano.shared(rnd2(filesCount, fileEmbeddingSize), 'fileEmbeddings', borrow=True)
        wordEmbeddings = theano.shared(wordEmbeddings, 'wordEmbeddings', borrow=True)
        weights = theano.shared(rnd2(fileEmbeddingSize + contextSize * wordEmbeddingSize, wordsCount), 'weights', borrow=True)

        ctxs = T.imatrix('ctxs')
        fileIndices = ctxs[:,0:1]
        wordIndices = ctxs[:,1:-1]
        targetIndices = T.reshape(ctxs[:,-1:], (ctxs.shape[0],1))

        random = RandomStreams(0)
        negativeSampleIndices = random.random_integers((targetIndices.shape[0], negative), low=0, high=wordsCount-1)
        indices = T.concatenate([targetIndices, negativeSampleIndices], axis=1)

        files = fileEmbeddings[fileIndices]
        fileFeatures = T.flatten(files, outdim=2)
        words = wordEmbeddings[wordIndices]
        wordFeatures = T.flatten(words, outdim=2)
        features = T.concatenate([fileFeatures, wordFeatures], axis=1)

        subWeights = weights[:,indices].dimshuffle(1,0,2)

        probabilities = T.batched_dot(features, subWeights)

        parameters = [fileEmbeddings, weights]

        l1Coefficient = T.scalar('l1Coefficient', dtype=floatX)
        l2Coefficient = T.scalar('l2Coefficient', dtype=floatX)

        l1Regularization = l1Coefficient * sum([abs(p).sum() for p in parameters])
        l2Regularization = l2Coefficient * sum([(p ** 2).sum() for p in parameters])

        cost = -T.mean(T.log(T.exp(probabilities[:,0])) + T.sum(T.log(T.exp(-probabilities[:,1:])), axis=1)) + \
               l1Regularization + l2Regularization

        lr = T.scalar('learningRate', dtype=floatX)

        parameters = [fileEmbeddings, weights]
        subParameters = [files]
        gradients = [T.grad(cost, wrt=subP) for subP in subParameters]
        updates = [(p, T.inc_subtensor(subP, -lr * g)) for p, subP, g in zip(parameters, subParameters, gradients)]

        parameters = [weights]
        gradients = [T.grad(cost, wrt=p) for p in parameters]
        updates = updates + [(p, p - lr * g) for p, g in zip(parameters, gradients)]

        batchIndex = T.iscalar('batchIndex')
        bs = T.iscalar('batchSize')
        trainingContexts = theano.shared(contexts, 'trainingContexts', borrow=True)

        self.trainModel = theano.function(
            inputs=[batchIndex, bs, lr, l1Coefficient, l2Coefficient],
            outputs=cost,
            updates=updates,
            givens={
                ctxs: trainingContexts[batchIndex * bs : (batchIndex + 1) * bs]
            }
        )

def train(fileIndexMap, fileEmbeddingSize, wordIndexMap, wordEmbeddings, contexts, metricsPath,
          epochs, batchSize, learningRate, negative, l1, l2):
    filesCount = len(fileIndexMap)
    wordsCount = len(wordIndexMap)
    wordEmbeddingSize = wordEmbeddings.shape[1]
    windowSize = contexts.shape[1] - 1
    contextSize = windowSize - 1

    fileEmbeddings = theano.shared(rnd2(filesCount, fileEmbeddingSize), 'fileEmbeddings', borrow=True)
    wordEmbeddings = theano.shared(wordEmbeddings, 'wordEmbeddings', borrow=True)
    weights = theano.shared(rnd2(fileEmbeddingSize + contextSize * wordEmbeddingSize, wordsCount), 'weights', borrow=True)

    ctxs = T.imatrix('ctxs')
    fileIndices = ctxs[:,0:1]
    wordIndices = ctxs[:,1:-1]
    targetIndices = T.reshape(ctxs[:,-1:], (ctxs.shape[0],1))

    random = RandomStreams(0)
    negativeSampleIndices = random.random_integers((targetIndices.shape[0], negative), low=0, high=wordsCount-1)
    indices = T.concatenate([targetIndices, negativeSampleIndices], axis=1)

    files = fileEmbeddings[fileIndices]
    fileFeatures = T.flatten(files, outdim=2)
    words = wordEmbeddings[wordIndices]
    wordFeatures = T.flatten(words, outdim=2)
    features = T.concatenate([fileFeatures, wordFeatures], axis=1)

    subWeights = weights[:,indices].dimshuffle(1,0,2)

    probabilities = T.batched_dot(features, subWeights)

    parameters = [fileEmbeddings, weights]

    l1Coefficient = T.scalar('l1Coefficient', dtype=floatX)
    l2Coefficient = T.scalar('l2Coefficient', dtype=floatX)

    l1Regularization = l1Coefficient * sum([abs(p).sum() for p in parameters])
    l2Regularization = l2Coefficient * sum([(p ** 2).sum() for p in parameters])

    cost = -T.mean(T.log(T.exp(probabilities[:,0])) + T.sum(T.log(T.exp(-probabilities[:,1:])), axis=1)) + \
           l1Regularization + l2Regularization

    lr = T.scalar('learningRate', dtype=floatX)

    parameters = [fileEmbeddings, weights]
    subParameters = [files]
    gradients = [T.grad(cost, wrt=subP) for subP in subParameters]
    updates = [(p, T.inc_subtensor(subP, -lr * g)) for p, subP, g in zip(parameters, subParameters, gradients)]

    parameters = [weights]
    gradients = [T.grad(cost, wrt=p) for p in parameters]
    updates = updates + [(p, p - lr * g) for p, g in zip(parameters, gradients)]

    batchIndex = T.iscalar('batchIndex')
    bs = T.iscalar('batchSize')
    trainingContexts = theano.shared(contexts, 'trainingContexts', borrow=True)

    trainModel = theano.function(
        inputs=[batchIndex, bs, lr, l1Coefficient, l2Coefficient],
        outputs=cost,
        updates=updates,
        givens={
            ctxs: trainingContexts[batchIndex * bs : (batchIndex + 1) * bs]
        }
    )

    startTime = time.time()
    for epoch in xrange(0, epochs):
        contextsCount = contexts.shape[0]
        batchesCount = contextsCount / batchSize + int(contextsCount % batchSize > 0)

        for bi in xrange(0, batchesCount):
            error = trainModel(bi, batchSize, learningRate, l1, l2)
            if error <= 0:
                break

        fe = fileEmbeddings.get_value()
        we = lambda key: fe[fileIndexMap['../data/Duplicates/Prepared/duplicates/{0}.txt'.format(key)]]
        distance = lambda left, right: vectors.euclideanDistance(we(left), we(right))

        metrics = {
            'tanks': distance('tank_0', 'tank_1'),
            'viruses': distance('virus_0', 'virus_1'),
            'tankVirus': distance('tank_0', 'virus_0'),
            'error': error
        }

        validation.dump(metricsPath, epoch, metrics)

        elapsed = time.time() - startTime
        secondsPerEpoch = elapsed / (epoch + 1)

        log.progress('Training model: {0:.3f}%. {1:.3f} sec per epoch. Error: {2:.3f}. Tanks={3:.3f}. Viruses={4:.3f}. Tank/Virus={5:.3f}.',
                     epoch + 1, epochs,
                     secondsPerEpoch,
                     float(error),
                     metrics['tanks'],
                     metrics['viruses'],
                     metrics['tankVirus'])

        if error <= 0:
            break

    validation.compareEmbeddings(fileIndexMap, fileEmbeddings.get_value())
    # validation.plotEmbeddings(fileIndexMap, fileEmbeddings.get_value())

def main():
    pathTo = kit.PathTo('Duplicates')

    fileIndexMap = parameters.loadIndexMap(pathTo.fileIndexMap)
    wordIndexMap = parameters.loadIndexMap(pathTo.wordIndexMap)
    indexWordMap = parameters.loadIndexMap(pathTo.wordIndexMap, inverse=True)
    wordEmbeddings = parameters.loadEmbeddings(pathTo.wordEmbeddings)
    metricsPath = pathTo.metrics('history.csv')

    if os.path.exists(metricsPath):
        os.remove(metricsPath)

    contextProvider = parameters.IndexContextProvider(pathTo.contexts)
    contexts = contextProvider[:]

    log.info('Contexts loading complete. {0} contexts loaded {1} words each.', len(contexts), contextProvider.contextSize)

    train(fileIndexMap, 200, wordIndexMap, wordEmbeddings, contexts, metricsPath,
          epochs=5,
          batchSize=50,
          learningRate=0.01,
          negative=10,
          l1=0.02,
          l2=0.005)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>
#--------------------------------------------------------------------------------
# isoplot - A lightweight data plotting and analysis application
#
# Author: Scott Nietfeld
# Initial Commit: 2016-10-12
#
# Compiling the GUI:  pyuic4 isoplot_gui.ui -o isoplot_gui.py
#--------------------------------------------------------------------------------

import os

def load_module(path):
    import imp

    print("Loading module from %s" % path)
    full_path = os.path.abspath(path)
    base_name = os.path.basename(full_path)
    mod_name= base_name.split('.')[0]

    print("Full path: %s" % full_path)
    print("Base name: %s" % base_name)
    print("Module name: %s" % mod_name)

    mod = imp.load_source(base_name, full_path)
    return mod

def get_functions(mod):
    import inspect

    all_functions = inspect.getmembers(mod, inspect.isfunction)
    return all_functions

def unload_module(path):
    pass

def load_data_file(path):
    pass

def run_usr_function(path):
    pass

if __name__=="__main__":
    print("Hello.")

    mod = load_module("./default_plots.py")
    f_list = get_functions(mod)

    print f_list

<commit_msg>get_functions() now returns functions organized in a dictionary instead of a list of tuples.<commit_after>
#--------------------------------------------------------------------------------
# isoplot - A lightweight data plotting and analysis application
#
# Author: Scott Nietfeld
# Initial Commit: 2016-10-12
#
# Compiling the GUI:  pyuic4 isoplot_gui.ui -o isoplot_gui.py
#--------------------------------------------------------------------------------

import os

def load_module(path):
    import imp

    print("Loading module from %s" % path)
    full_path = os.path.abspath(path)
    base_name = os.path.basename(full_path)
    mod_name= base_name.split('.')[0]

    print("Full path: %s" % full_path)
    print("Base name: %s" % base_name)
    print("Module name: %s" % mod_name)

    mod = imp.load_source(base_name, full_path)
    return mod

def get_functions(mod):
    import inspect

    all_functions = inspect.getmembers(mod, inspect.isfunction)

    f_dict = {}
    for (fname, f) in all_functions:
        f_dict[fname] = f

    return f_dict

def unload_module(path):
    pass

def load_data_file(path):
    pass

def run_usr_function(path):
    pass

if __name__=="__main__":
    print("Hello.")

    mod = load_module("./default_plots.py")
    f_dict = get_functions(mod)

    print f_dict
    f_dict['scatter'](None, None)
    f_dict['line'](None, None)

<|endoftext|>


<commit_before>
from common.options import define

# Main

define("host",
       default="http://social-dev.anthill.local",
       help="Public hostname of this service",
       type=str)

define("listen",
       default="port:10300",
       help="Socket to listen. Could be a port number (port:N), or a unix domain socket (unix:PATH)",
       type=str)

define("name",
       default="social",
       help="Service short name. User to discover by discovery service.",
       type=str)

# MySQL database

define("db_host",
       default="127.0.0.1",
       type=str,
       help="MySQL database location")

define("db_username",
       default="anthill",
       type=str,
       help="MySQL account username")

define("db_password",
       default="",
       type=str,
       help="MySQL account password")

define("db_name",
       default="social",
       type=str,
       help="MySQL database name")

# Regular cache

define("cache_host",
       default="127.0.0.1",
       help="Location of a regular cache (redis).",
       group="cache",
       type=str)

define("cache_port",
       default=6379,
       help="Port of regular cache (redis).",
       group="cache",
       type=int)

define("cache_db",
       default=4,
       help="Database of regular cache (redis).",
       group="cache",
       type=int)

define("cache_max_connections",
       default=500,
       help="Maximum connections to the regular cache (connection pool).",
       group="cache",
       type=int)<commit_msg>Refused the *.anthill.local domain for local testing, using *.anthill instead<commit_after>
from common.options import define

# Main

define("host",
       default="http://social-dev.anthill",
       help="Public hostname of this service",
       type=str)

define("listen",
       default="port:10300",
       help="Socket to listen. Could be a port number (port:N), or a unix domain socket (unix:PATH)",
       type=str)

define("name",
       default="social",
       help="Service short name. User to discover by discovery service.",
       type=str)

# MySQL database

define("db_host",
       default="127.0.0.1",
       type=str,
       help="MySQL database location")

define("db_username",
       default="anthill",
       type=str,
       help="MySQL account username")

define("db_password",
       default="",
       type=str,
       help="MySQL account password")

define("db_name",
       default="social",
       type=str,
       help="MySQL database name")

# Regular cache

define("cache_host",
       default="127.0.0.1",
       help="Location of a regular cache (redis).",
       group="cache",
       type=str)

define("cache_port",
       default=6379,
       help="Port of regular cache (redis).",
       group="cache",
       type=int)

define("cache_db",
       default=4,
       help="Database of regular cache (redis).",
       group="cache",
       type=int)

define("cache_max_connections",
       default=500,
       help="Maximum connections to the regular cache (connection pool).",
       group="cache",
       type=int)<|endoftext|>


<commit_before>#!/usr/bin/env python
# leds.py

import RPi.GPIO as GPIO
import time

GPIO.setmode(GPIO.BOARD)
GPIO.setwarnings(False)
GPIO.cleanup()
GPIO.setup(7, GPIO.OUT)
GPIO.setup(11, GPIO.OUT)
GPIO.setup(12, GPIO.OUT)

while True:
	GPIO.output(7, GPIO.HIGH)
	time.sleep(1000)
	GPIO.output(11, GPIO.HIGH)
	time.sleep(1000)
	GPIO.output(12, GPIO.HIGH)
	time.sleep(1000)

GPIO.cleanup()<commit_msg>fix @ Sat Sep 27 21:28:20 EDT 2014<commit_after>#!/usr/bin/env python
# leds.py

import RPi.GPIO as GPIO
import time

GPIO.setmode(GPIO.BOARD)
GPIO.setwarnings(False)
GPIO.cleanup()
GPIO.setup(7, GPIO.OUT)
GPIO.setup(11, GPIO.OUT)
GPIO.setup(12, GPIO.OUT)

while True:
	GPIO.output(7, GPIO.HIGH)
	time.sleep(1)
	GPIO.output(11, GPIO.HIGH)
	time.sleep(1)
	GPIO.output(12, GPIO.HIGH)
	time.sleep(1)

GPIO.cleanup()<|endoftext|>