<commit_before>#!/usr/bin/env python
#
# Copyright (c) Vicent Marti. All rights reserved.
#
# This file is part of clar, distributed under the ISC license.
# For full terms see the included COPYING file.
#

from __future__ import with_statement
from string import Template
import re, fnmatch, os, codecs, pickle

class Module(object):
    class Template(object):
        def __init__(self, module):
            self.module = module

        def _render_callback(self, cb):
            if not cb:
                return '    { NULL, NULL }'
            return '    { "%s", &%s }' % (cb['short_name'], cb['symbol'])

    class DeclarationTemplate(Template):
        def render(self):
            out = "\n".join("extern %s;" % cb['declaration'] for cb in self.module.callbacks) + "\n"

            if self.module.initialize:
                out += "extern %s;\n" % self.module.initialize['declaration']

            if self.module.cleanup:
                out += "extern %s;\n" % self.module.cleanup['declaration']

            return out

    class CallbacksTemplate(Template):
        def render(self):
            out = "static const struct clar_func _clar_cb_%s[] = {\n" % self.module.name
            out += ",\n".join(self._render_callback(cb) for cb in self.module.callbacks)
            out += "\n};\n"
            return out

    class InfoTemplate(Template):
        def render(self):
            return Template(
            r"""
    {
        "${clean_name}",
    ${initialize},
    ${cleanup},
        ${cb_ptr}, ${cb_count}, ${enabled}
    }"""
            ).substitute(
                clean_name = self.module.clean_name(),
                initialize = self._render_callback(self.module.initialize),
                cleanup = self._render_callback(self.module.cleanup),
                cb_ptr = "_clar_cb_%s" % self.module.name,
                cb_count = len(self.module.callbacks),
                enabled = int(self.module.enabled)
            )

    def __init__(self, name):
        self.name = name

        self.mtime = 0
        self.enabled = True
        self.modified = False

    def clean_name(self):
        return self.name.replace("_", "::")

    def _skip_comments(self, text):
        SKIP_COMMENTS_REGEX = re.compile(
            r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
            re.DOTALL | re.MULTILINE)

        def _replacer(match):
            s = match.group(0)
            return "" if s.startswith('/') else s

        return re.sub(SKIP_COMMENTS_REGEX, _replacer, text)

    def parse(self, contents):
        TEST_FUNC_REGEX = r"^(void\s+(test_%s__(\w+))\(\s*void\s*\))\s*\{"

        contents = self._skip_comments(contents)
        regex = re.compile(TEST_FUNC_REGEX % self.name, re.MULTILINE)

        self.callbacks = []
        self.initialize = None
        self.cleanup = None

        for (declaration, symbol, short_name) in regex.findall(contents):
            data = {
                "short_name" : short_name,
                "declaration" : declaration,
                "symbol" : symbol
            }

            if short_name == 'initialize':
                self.initialize = data
            elif short_name == 'cleanup':
                self.cleanup = data
            else:
                self.callbacks.append(data)

        return self.callbacks != []

    def refresh(self, path):
        self.modified = False

        try:
            st = os.stat(path)

            # Not modified
            if st.st_mtime == self.mtime:
                return True

            self.modified = True
            self.mtime = st.st_mtime

            with codecs.open(path, encoding='utf-8') as fp:
                raw_content = fp.read()

        except IOError:
            return False

        return self.parse(raw_content)

class TestSuite(object):

    def __init__(self, path):
        self.path = path

    def should_generate(self, path):
        if not os.path.isfile(path):
            return True

        if any(module.modified for module in self.modules.values()):
            return True

        return False

    def find_modules(self):
        modules = []
        for root, _, files in os.walk(self.path):
            module_root = root[len(self.path):]
            module_root = [c for c in module_root.split(os.sep) if c]

            tests_in_module = fnmatch.filter(files, "*.c")

            for test_file in tests_in_module:
                full_path = os.path.join(root, test_file)
                module_name = "_".join(module_root + [test_file[:-2]])

                modules.append((full_path, module_name))

        return modules

    def load_cache(self):
        path = os.path.join(self.path, '.clarcache')
        cache = {}

        try:
            fp = open(path, 'rb')
            cache = pickle.load(fp)
            fp.close()
        except (IOError, ValueError):
            pass

        return cache

    def save_cache(self):
        path = os.path.join(self.path, '.clarcache')
        with open(path, 'wb') as cache:
            pickle.dump(self.modules, cache)

    def load(self, force = False):
        module_data = self.find_modules()
        self.modules = {} if force else self.load_cache()

        for path, name in module_data:
            if name not in self.modules:
                self.modules[name] = Module(name)

            if not self.modules[name].refresh(path):
                del self.modules[name]

    def disable(self, excluded):
        for exclude in excluded:
            for module in self.modules.values():
                name = module.clean_name()
                if name.startswith(exclude):
                    module.enabled = False
                    module.modified = True

    def suite_count(self):
        return len(self.modules)

    def callback_count(self):
        return sum(len(module.callbacks) for module in self.modules.values())

    def write(self):
        output = os.path.join(self.path, 'clar.suite')

        if not self.should_generate(output):
            return False

        with open(output, 'w') as data:
            for module in self.modules.values():
                t = Module.DeclarationTemplate(module)
                data.write(t.render())

            for module in self.modules.values():
                t = Module.CallbacksTemplate(module)
                data.write(t.render())

            suites = "static struct clar_suite _clar_suites[] = {" + ','.join(
                Module.InfoTemplate(module).render() for module in sorted(self.modules.values(), key=lambda module: module.name)
            ) + "\n};\n"

            data.write(suites)

            data.write("static const size_t _clar_suite_count = %d;\n" % self.suite_count())
            data.write("static const size_t _clar_callback_count = %d;\n" % self.callback_count())

        self.save_cache()
        return True

if __name__ == '__main__':
    from optparse import OptionParser

    parser = OptionParser()
    parser.add_option('-f', '--force', action="store_true", dest='force', default=False)
    parser.add_option('-x', '--exclude', dest='excluded', action='append', default=[])

    options, args = parser.parse_args()

    for path in args or ['.']:
        suite = TestSuite(path)
        suite.load(options.force)
        suite.disable(options.excluded)
        if suite.write():
            print("Written `clar.suite` (%d tests in %d suites)" % (suite.callback_count(), suite.suite_count()))

<commit_msg>Allow more whites in function declaration<commit_after>#!/usr/bin/env python
#
# Copyright (c) Vicent Marti. All rights reserved.
#
# This file is part of clar, distributed under the ISC license.
# For full terms see the included COPYING file.
#

from __future__ import with_statement
from string import Template
import re, fnmatch, os, codecs, pickle

class Module(object):
    class Template(object):
        def __init__(self, module):
            self.module = module

        def _render_callback(self, cb):
            if not cb:
                return '    { NULL, NULL }'
            return '    { "%s", &%s }' % (cb['short_name'], cb['symbol'])

    class DeclarationTemplate(Template):
        def render(self):
            out = "\n".join("extern %s;" % cb['declaration'] for cb in self.module.callbacks) + "\n"

            if self.module.initialize:
                out += "extern %s;\n" % self.module.initialize['declaration']

            if self.module.cleanup:
                out += "extern %s;\n" % self.module.cleanup['declaration']

            return out

    class CallbacksTemplate(Template):
        def render(self):
            out = "static const struct clar_func _clar_cb_%s[] = {\n" % self.module.name
            out += ",\n".join(self._render_callback(cb) for cb in self.module.callbacks)
            out += "\n};\n"
            return out

    class InfoTemplate(Template):
        def render(self):
            return Template(
            r"""
    {
        "${clean_name}",
    ${initialize},
    ${cleanup},
        ${cb_ptr}, ${cb_count}, ${enabled}
    }"""
            ).substitute(
                clean_name = self.module.clean_name(),
                initialize = self._render_callback(self.module.initialize),
                cleanup = self._render_callback(self.module.cleanup),
                cb_ptr = "_clar_cb_%s" % self.module.name,
                cb_count = len(self.module.callbacks),
                enabled = int(self.module.enabled)
            )

    def __init__(self, name):
        self.name = name

        self.mtime = 0
        self.enabled = True
        self.modified = False

    def clean_name(self):
        return self.name.replace("_", "::")

    def _skip_comments(self, text):
        SKIP_COMMENTS_REGEX = re.compile(
            r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
            re.DOTALL | re.MULTILINE)

        def _replacer(match):
            s = match.group(0)
            return "" if s.startswith('/') else s

        return re.sub(SKIP_COMMENTS_REGEX, _replacer, text)

    def parse(self, contents):
        TEST_FUNC_REGEX = r"^(void\s+(test_%s__(\w+))\s*\(\s*void\s*\))\s*\{"

        contents = self._skip_comments(contents)
        regex = re.compile(TEST_FUNC_REGEX % self.name, re.MULTILINE)

        self.callbacks = []
        self.initialize = None
        self.cleanup = None

        for (declaration, symbol, short_name) in regex.findall(contents):
            data = {
                "short_name" : short_name,
                "declaration" : declaration,
                "symbol" : symbol
            }

            if short_name == 'initialize':
                self.initialize = data
            elif short_name == 'cleanup':
                self.cleanup = data
            else:
                self.callbacks.append(data)

        return self.callbacks != []

    def refresh(self, path):
        self.modified = False

        try:
            st = os.stat(path)

            # Not modified
            if st.st_mtime == self.mtime:
                return True

            self.modified = True
            self.mtime = st.st_mtime

            with codecs.open(path, encoding='utf-8') as fp:
                raw_content = fp.read()

        except IOError:
            return False

        return self.parse(raw_content)

class TestSuite(object):

    def __init__(self, path):
        self.path = path

    def should_generate(self, path):
        if not os.path.isfile(path):
            return True

        if any(module.modified for module in self.modules.values()):
            return True

        return False

    def find_modules(self):
        modules = []
        for root, _, files in os.walk(self.path):
            module_root = root[len(self.path):]
            module_root = [c for c in module_root.split(os.sep) if c]

            tests_in_module = fnmatch.filter(files, "*.c")

            for test_file in tests_in_module:
                full_path = os.path.join(root, test_file)
                module_name = "_".join(module_root + [test_file[:-2]])

                modules.append((full_path, module_name))

        return modules

    def load_cache(self):
        path = os.path.join(self.path, '.clarcache')
        cache = {}

        try:
            fp = open(path, 'rb')
            cache = pickle.load(fp)
            fp.close()
        except (IOError, ValueError):
            pass

        return cache

    def save_cache(self):
        path = os.path.join(self.path, '.clarcache')
        with open(path, 'wb') as cache:
            pickle.dump(self.modules, cache)

    def load(self, force = False):
        module_data = self.find_modules()
        self.modules = {} if force else self.load_cache()

        for path, name in module_data:
            if name not in self.modules:
                self.modules[name] = Module(name)

            if not self.modules[name].refresh(path):
                del self.modules[name]

    def disable(self, excluded):
        for exclude in excluded:
            for module in self.modules.values():
                name = module.clean_name()
                if name.startswith(exclude):
                    module.enabled = False
                    module.modified = True

    def suite_count(self):
        return len(self.modules)

    def callback_count(self):
        return sum(len(module.callbacks) for module in self.modules.values())

    def write(self):
        output = os.path.join(self.path, 'clar.suite')

        if not self.should_generate(output):
            return False

        with open(output, 'w') as data:
            for module in self.modules.values():
                t = Module.DeclarationTemplate(module)
                data.write(t.render())

            for module in self.modules.values():
                t = Module.CallbacksTemplate(module)
                data.write(t.render())

            suites = "static struct clar_suite _clar_suites[] = {" + ','.join(
                Module.InfoTemplate(module).render() for module in sorted(self.modules.values(), key=lambda module: module.name)
            ) + "\n};\n"

            data.write(suites)

            data.write("static const size_t _clar_suite_count = %d;\n" % self.suite_count())
            data.write("static const size_t _clar_callback_count = %d;\n" % self.callback_count())

        self.save_cache()
        return True

if __name__ == '__main__':
    from optparse import OptionParser

    parser = OptionParser()
    parser.add_option('-f', '--force', action="store_true", dest='force', default=False)
    parser.add_option('-x', '--exclude', dest='excluded', action='append', default=[])

    options, args = parser.parse_args()

    for path in args or ['.']:
        suite = TestSuite(path)
        suite.load(options.force)
        suite.disable(options.excluded)
        if suite.write():
            print("Written `clar.suite` (%d tests in %d suites)" % (suite.callback_count(), suite.suite_count()))

<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
##
# Copyright (c) 2010-2022 WALLIX, SARL. All rights reserved.
# Licensed computer software. Property of WALLIX.
# Product name: Packager
# Author(s): Christophe Grosjean, Jonathan Poelen, Raphael Zhou, Meng Tan
# Module description: Create package and git tag
##

import os
import re
import shutil
import argparse
import datetime
from typing import Dict, Tuple, List, Iterable
from .io import writeall, readall
from .version import less_version
from .shell import shell_cmd, git_uncommited_changes, git_tag_exists, git_last_tag

var_ident = '[A-Z][A-Z0-9_]*'

class PackagerError(Exception):
    pass

def replace_dict_all(text: str, dico: Dict[str, str]) -> str:
    rgx_replace_var = re.compile(f'%{var_ident}%')
    text_parts = []
    pos = 0
    for m in rgx_replace_var.finditer(text):
        text_parts.append(text[pos:m.start()])
        var = m.group()
        text_parts.append(dico.get(var[1:-1], var))
        pos = m.end()
    text_parts.append(text[pos:])
    return ''.join(text_parts)

def read_and_update_config(filename: str,
                           configs: Dict[str, str] = None,
                           encoding: str = 'utf-8') -> Dict[str, str]:
    """ Parse target Config files """
    parse_config_rgx = re.compile(rf'^({var_ident})\s*=(.*)')
    configs = {} if configs is None else configs

    with open(filename, encoding=encoding) as f:
        for line in f:
            if line.startswith('include '):
                directory = os.path.dirname(filename)
                included_path = os.path.join(directory, line[8:].strip())
                read_and_update_config(included_path, configs)
            else:
                m = re.search(parse_config_rgx, line)
                if m is not None:
                    configs[m.group(1).strip()] = m.group(2).strip()

    if 'PKG_DISTRIBUTION' not in configs:
        if configs['DIST_NAME'].lower() == 'ubuntu':
            configs['PKG_DISTRIBUTION'] = 'unstable'
        else:
            configs['PKG_DISTRIBUTION'] = configs['DIST_ID']

    if 'TARGET_NAME' not in configs:
        if configs['DIST_NAME'].lower() == 'ubuntu':
            configs['TARGET_NAME'] = '+' + configs['DIST_ID']

    return configs

def update_config_variables(config: Dict[str, str], variables: Iterable[str]) -> List[str]:
    """Return unparsed variable"""
    rgx_var = re.compile(rf'^({var_ident})(\+?=)(.*)')
    variable_errors = []
    for var in variables:
        m = rgx_var.match(var)
        if m is None:
            variable_errors.append(var)
            continue
        k = m.group(1)
        newvalue = m.group(3) if m.group(2) == '=' else config.get(k, '') + m.group(3)
        config[k] = newvalue
    return variable_errors

def print_configs(configs: Dict[str, str]) -> None:
    variables = []
    for k, v in configs.items():
        variables.append(f'{k} = {v}')
    print('\n'.join(variables))

def get_changelog_entry(project_name: str,
                        version: str,
                        maintainer: str,
                        urgency: str,
                        utc: str,
                        encoding: str = 'utf-8') -> str:
    editor = os.environ.get('EDITOR') or 'nano'

    changelog = []

    tmp_changelog = f'/tmp/{version}-changelog.tmp'
    cmd = f'{editor} {tmp_changelog}'
    if os.system(cmd):
        raise PackagerError(f'Error in `{cmd}`')

    with open(tmp_changelog, encoding=encoding) as f:
        for line in f:
            if line and line != '\n':
                changelog.append('  * ')
                changelog.append(line)

    os.remove(tmp_changelog)

    if not changelog:
        raise PackagerError('Change log is empty')

    now = datetime.datetime.today().strftime(f'%a, %d %b %Y %H:%M:%S +{utc}')
    return (f'{project_name or "%PROJECT_NAME%"} ({version}%TARGET_NAME%) %PKG_DISTRIBUTION%; '
            f'urgency={urgency}\n\n{"".join(changelog)}\n\n -- {maintainer}  {now}\n\n')

def update_changelog(changelog_path: str, changelog: str) -> None:
    changelog += readall(changelog_path)
    writeall(changelog_path, changelog)

def create_build_directory(package_template: str, output_build: str, configs: Dict[str, str]) -> None:
    rgx_tempfile = re.compile('^#.*#$|~$')

    try:
        os.mkdir(output_build, 0o766)
    except FileExistsError:
        pass

    filenames = filter(lambda fname: rgx_tempfile.search(fname) is None,
                       os.listdir(package_template))

    for filename in filenames:
        out = readall(f'{package_template}/{filename}')
        out = replace_dict_all(out, configs)
        writeall(f'{output_build}/{filename}', out)

def check_version(old_version: str, new_version: str) -> None:
    if not less_version(old_version, new_version):
        raise PackagerError(f'New version ({new_version}) is less than'
                            f' old version ({old_version})')

def check_pattern_version(old_version: str, new_version: str, pattern: str) -> None:
    if pattern:
        m = re.match(pattern, old_version)
        if m is None:
            raise PackagerError('Unable to retrieve the version number'
                                f'from {old_version}')
        if not new_version.startswith(m.group(0)):
            raise PackagerError(f'The last tag number ({old_version}) of '
                                f'the repository is not compatible with'
                                f'the new tag number {new_version}. '
                                f'Pattern is "{pattern}"')

def regex_version_or_die(pattern: str) -> re.Pattern:
    try:
        return re.compile(pattern)
    except re.error as e:
        raise PackagerError(f'Invalid error on regex version: {pattern}') from e

def read_version_or_die(pattern: str, file_version: str) -> Tuple[str, Tuple[int, int], str]:
    if not file_version:
        raise PackagerError('File version is empty')

    rgx_version = regex_version_or_die(pattern)
    content = readall(file_version)

    m = rgx_version.match(content)

    if m is None:
        raise PackagerError(
            f'Version not found: file={file_version}  pattern={pattern}')

    return m.group(1), m.span(1), content

def argument_parser(description: str = 'Packager for proxies repositories') -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description=description)

    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument('-b', '--build', action='store_true',
                       help='create a package directory')
    group.add_argument('-g', '--get-version', action='store_true')
    group.add_argument('-n', '--new-version', metavar='VERSION',
                       help='update version and create a tag')
    group.add_argument('--show-config', action='store_true',
                       help='show configs')

    parser.add_argument('--no-check-uncommited', action='store_true')
    parser.add_argument('--check-uncommited',
                        dest='no_check_uncommited', action='store_false')

    group = parser.add_argument_group('Version options')
    group.add_argument('--file-version', metavar='PATH')
    group.add_argument('--pattern-version', metavar='REGEX',
                       default=r'\s*(?:[a-zA-Z_][a-zA-Z0-9_]*)?VERSION\s*=\s*[\'"]([^\'"]*)[\'"]',
                       help='pattern for version')

    group = parser.add_argument_group('Update version and tag')
    group.add_argument('--force-version', action='store_true')
    group.add_argument('--start-pattern-version', default=r'^\d+\.\d+\.',
                       help='new version and current version must be the same')
    group.add_argument('--no-commit-and-tag', action='store_true')
    group.add_argument('--commit-and-tag',
                       dest='no_commit_and_tag', action='store_false')
    group.add_argument('--no-push', action='store_true')
    group.add_argument('--push', dest='no_push', action='store_false')
    group.add_argument('--commit-message', metavar='TEMPLATE', default='Version %s',
                       help='commit message template, %%s is replaced with new version')

    group = parser.add_argument_group('Build package options')
    group.add_argument('--target', metavar='NAME', help='target file path')
    group.add_argument('--output-build', metavar='DIRNAME',
                       default='debian', help='build directory')
    group.add_argument('--clean-build', action='store_true',
                       help='remove existing build directory')
    group.add_argument('--package-template', metavar='DIRNAME',
                       default='packaging/template/debian', help='package template directory')
    group.add_argument('--build-package', action='store_true',
                       help='run dpkg-buildpackage')
    group.add_argument('--project-name', metavar='NAME')
    group.add_argument('--project-version', metavar='VERSION')
    group.add_argument('--update-changelog', action='store_true')
    group.add_argument('--urgency', default='low')
    group.add_argument('--utc', default='0200')
    group.add_argument('--maintainer',
                       default='Proxies Team <R&D-Project-Bastion-Proxies@wallix.com>')
    group.add_argument('--distribution-name')
    group.add_argument('--distribution-version')
    group.add_argument('--distribution-id')
    group.add_argument('--package-distribution')
    group.add_argument('--no-check-version', action='store_true')
    group.add_argument('--check-version',
                       dest='no_check_version', action='store_false')
    group.add_argument('-s', '--variable', metavar='VARIABLE=VALUE', action='append', default=[],
                       help='support of VARIABLE=VALUE and VARIABLE+=VALUE')

    return parser

class Hook:
    def normalize_version(self, version: str) -> str:
        """normalized version from file"""
        return version

    def sanitize_version(self, old_version: str, new_version: str) -> str:
        """convert to a writable version"""
        return new_version

    def post_updated_version(self, old_version: str, new_version: str) -> None:
        pass

def run_packager(args: argparse.Namespace, hook: Hook = Hook()) -> None:
    if not args.show_config and not args.get_version and not args.no_check_uncommited:
        changes = git_uncommited_changes()
        if changes:
            raise PackagerError(f'Your repository has uncommited changes:\n{changes}\n'
                                'Please commit before packaging or use --no-check-uncommited')

    # get version
    new_version = args.new_version
    if args.get_version or new_version is not None:
        results = read_version_or_die(args.pattern_version, args.file_version)
        print(hook.normalize_version(results[0]))
        return

    # set version
    if new_version is not None:
        if not new_version:
            raise PackagerError('New version is empty')

        version, pos, content = read_version_or_die(
            args.pattern_version, args.file_version)
        version = hook.normalize_version(version)

        if not args.force_version:
            check_version(version, new_version)
            check_pattern_version(version, new_version,
                                  args.start_pattern_version)

        new_version = hook.sanitize_version(version, new_version)

        tag_exists, tag_cat = git_tag_exists(new_version)
        if tag_exists:
            raise PackagerError(
                f'Tag {new_version} already exists ({tag_cat})')

        writeall(args.file_version,
                 f'{content[:pos[0]]}{new_version}{content[pos[1]:]}')
        hook.post_updated_version(version, new_version)

        if not args.no_commit_and_tag:
            shell_cmd(['git', 'commit', '-am',
                       args.commit_message % (new_version,)])
            shell_cmd(['git', 'tag', new_version])
            if not args.no_push:
                shell_cmd(['git', 'push'])
                shell_cmd(['git', 'push', '--tags'])
        return

    if not args.target:
        raise PackagerError('--target is missing')

    # Build / Show config
    if not args.distribution_name or not args.distribution_version or not args.distribution_id:
        import distro
        args.distribution_id = args.distribution_id or distro.name()
        args.distribution_name = args.distribution_name or distro.id()
        args.distribution_version = args.distribution_version or distro.version()

    configs = {
        'PKG_DISTRIBUTION': args.package_distribution,
        'DIST_ID': args.distribution_id,
        'DIST_NAME': args.distribution_name,
        'DIST_VERSION': args.distribution_version,
        'PROJECT_VERSION': args.project_version,
        'PROJECT_NAME': args.project_name,
        'URGENCY': args.urgency,
        'MAINTAINER': args.maintainer,
        'ARCH': 'any',
        'TARGET_NAME': '',
        'UTC': args.utc or '0200',
    }

    read_and_update_config(args.target, configs)

    variable_errors = update_config_variables(configs, args.variable)
    if variable_errors:
        errors_s = '", "'.join(variable_errors)
        raise PackagerError(f'Parse error on -s / --variable: "{errors_s}"')

    if args.show_config:
        print_configs(configs)
        return

    project_version = configs['PROJECT_VERSION']
    project_name = configs['PROJECT_NAME']

    if not args.no_check_version or not project_version:
        results = read_version_or_die(args.pattern_version, args.file_version)
        version = hook.normalize_version(results[0])
        if not project_version:
            project_version = version
        elif not args.no_check_version and (project_version != version or project_version != git_last_tag()):
            raise PackagerError('Repository head mismatch current version. '
                                'Ignored with --no-check-version')
    elif project_version:
        project_version = hook.normalize_version(project_version)

    if not project_version or not project_name:
        errors = []
        if not project_version:
            errors.append(
                '--project-version or -s PROJECT_VERSION=... is missing')
        if not project_name:
            errors.append('--project-name or -s PROJECT_NAME=... is missing')
        raise PackagerError(', '.join(errors))

    if args.update_changelog:
        changelog = get_changelog_entry(project_name,
                                        project_version,
                                        configs['MAINTAINER'],
                                        configs['URGENCY'],
                                        configs['UTC'])
        update_changelog(f'{args.package_template}/changelog', changelog)

    if args.clean_build:
        try:
            shutil.rmtree(args.output_build)
        except OSError:
            pass

    create_build_directory(args.package_template, args.output_build, configs)

    if args.build_package:
        status = os.system('dpkg-buildpackage -b -tc -us -uc -r')
        if status:
            raise PackagerError('Build failed: dpkg-buildpackage error')
<commit_msg>packager.py: remove --new-version<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
##
# Copyright (c) 2010-2022 WALLIX, SARL. All rights reserved.
# Licensed computer software. Property of WALLIX.
# Product name: Packager
# Author(s): Christophe Grosjean, Jonathan Poelen, Raphael Zhou, Meng Tan
# Module description: Create package and git tag
##

import os
import re
import shutil
import argparse
import datetime
from typing import Dict, Tuple, List, Iterable
from .io import writeall, readall
from .version import less_version
from .shell import git_uncommited_changes, git_last_tag

var_ident = '[A-Z][A-Z0-9_]*'

class PackagerError(Exception):
    pass

def replace_dict_all(text: str, dico: Dict[str, str]) -> str:
    rgx_replace_var = re.compile(f'%{var_ident}%')
    text_parts = []
    pos = 0
    for m in rgx_replace_var.finditer(text):
        text_parts.append(text[pos:m.start()])
        var = m.group()
        text_parts.append(dico.get(var[1:-1], var))
        pos = m.end()
    text_parts.append(text[pos:])
    return ''.join(text_parts)

def read_and_update_config(filename: str,
                           configs: Dict[str, str] = None,
                           encoding: str = 'utf-8') -> Dict[str, str]:
    """ Parse target Config files """
    parse_config_rgx = re.compile(rf'^({var_ident})\s*=(.*)')
    configs = {} if configs is None else configs

    with open(filename, encoding=encoding) as f:
        for line in f:
            if line.startswith('include '):
                directory = os.path.dirname(filename)
                included_path = os.path.join(directory, line[8:].strip())
                read_and_update_config(included_path, configs)
            else:
                m = re.search(parse_config_rgx, line)
                if m is not None:
                    configs[m.group(1).strip()] = m.group(2).strip()

    if 'PKG_DISTRIBUTION' not in configs:
        if configs['DIST_NAME'].lower() == 'ubuntu':
            configs['PKG_DISTRIBUTION'] = 'unstable'
        else:
            configs['PKG_DISTRIBUTION'] = configs['DIST_ID']

    if 'TARGET_NAME' not in configs:
        if configs['DIST_NAME'].lower() == 'ubuntu':
            configs['TARGET_NAME'] = '+' + configs['DIST_ID']

    return configs

def update_config_variables(config: Dict[str, str], variables: Iterable[str]) -> List[str]:
    """Return unparsed variable"""
    rgx_var = re.compile(rf'^({var_ident})(\+?=)(.*)')
    variable_errors = []
    for var in variables:
        m = rgx_var.match(var)
        if m is None:
            variable_errors.append(var)
            continue
        k = m.group(1)
        newvalue = m.group(3) if m.group(2) == '=' else config.get(k, '') + m.group(3)
        config[k] = newvalue
    return variable_errors

def print_configs(configs: Dict[str, str]) -> None:
    variables = []
    for k, v in configs.items():
        variables.append(f'{k} = {v}')
    print('\n'.join(variables))

def get_changelog_entry(project_name: str,
                        version: str,
                        maintainer: str,
                        urgency: str,
                        utc: str,
                        encoding: str = 'utf-8') -> str:
    editor = os.environ.get('EDITOR') or 'nano'

    changelog = []

    tmp_changelog = f'/tmp/{version}-changelog.tmp'
    cmd = f'{editor} {tmp_changelog}'
    if os.system(cmd):
        raise PackagerError(f'Error in `{cmd}`')

    with open(tmp_changelog, encoding=encoding) as f:
        for line in f:
            if line and line != '\n':
                changelog.append('  * ')
                changelog.append(line)

    os.remove(tmp_changelog)

    if not changelog:
        raise PackagerError('Change log is empty')

    now = datetime.datetime.today().strftime(f'%a, %d %b %Y %H:%M:%S +{utc}')
    return (f'{project_name or "%PROJECT_NAME%"} ({version}%TARGET_NAME%) %PKG_DISTRIBUTION%; '
            f'urgency={urgency}\n\n{"".join(changelog)}\n\n -- {maintainer}  {now}\n\n')

def update_changelog(changelog_path: str, changelog: str) -> None:
    changelog += readall(changelog_path)
    writeall(changelog_path, changelog)

def create_build_directory(package_template: str, output_build: str, configs: Dict[str, str]) -> None:
    rgx_tempfile = re.compile('^#.*#$|~$')

    try:
        os.mkdir(output_build, 0o766)
    except FileExistsError:
        pass

    filenames = filter(lambda fname: rgx_tempfile.search(fname) is None,
                       os.listdir(package_template))

    for filename in filenames:
        out = readall(f'{package_template}/{filename}')
        out = replace_dict_all(out, configs)
        writeall(f'{output_build}/{filename}', out)

def regex_version_or_die(pattern: str) -> re.Pattern:
    try:
        return re.compile(pattern)
    except re.error as e:
        raise PackagerError(f'Invalid error on regex version: {pattern}') from e

def read_version_or_die(pattern: str, file_version: str) -> Tuple[str, Tuple[int, int], str]:
    if not file_version:
        raise PackagerError('File version is empty')

    rgx_version = regex_version_or_die(pattern)
    content = readall(file_version)

    m = rgx_version.match(content)

    if m is None:
        raise PackagerError(
            f'Version not found: file={file_version}  pattern={pattern}')

    return m.group(1), m.span(1), content

def argument_parser(description: str = 'Packager for proxies repositories') -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description=description)

    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument('-b', '--build', action='store_true',
                       help='create a package directory')
    group.add_argument('-g', '--get-version', action='store_true')
    group.add_argument('--show-config', action='store_true',
                       help='show configs')

    parser.add_argument('--no-check-uncommited', action='store_true')
    parser.add_argument('--check-uncommited',
                        dest='no_check_uncommited', action='store_false')

    group = parser.add_argument_group('Version options')
    group.add_argument('--file-version', metavar='PATH')
    group.add_argument('--pattern-version', metavar='REGEX',
                       default=r'\s*(?:[a-zA-Z_][a-zA-Z0-9_]*)?VERSION\s*=\s*[\'"]([^\'"]*)[\'"]',
                       help='pattern for version')

    group = parser.add_argument_group('Build package options')
    group.add_argument('--target', metavar='NAME', help='target file path')
    group.add_argument('--output-build', metavar='DIRNAME',
                       default='debian', help='build directory')
    group.add_argument('--clean-build', action='store_true',
                       help='remove existing build directory')
    group.add_argument('--package-template', metavar='DIRNAME',
                       default='packaging/template/debian', help='package template directory')
    group.add_argument('--build-package', action='store_true',
                       help='run dpkg-buildpackage')
    group.add_argument('--project-name', metavar='NAME')
    group.add_argument('--project-version', metavar='VERSION')
    group.add_argument('--update-changelog', action='store_true')
    group.add_argument('--urgency', default='low')
    group.add_argument('--utc', default='0200')
    group.add_argument('--maintainer',
                       default='Proxies Team <R&D-Project-Bastion-Proxies@wallix.com>')
    group.add_argument('--distribution-name')
    group.add_argument('--distribution-version')
    group.add_argument('--distribution-id')
    group.add_argument('--package-distribution')
    group.add_argument('--no-check-version', action='store_true')
    group.add_argument('--check-version',
                       dest='no_check_version', action='store_false')
    group.add_argument('-s', '--variable', metavar='VARIABLE=VALUE', action='append', default=[],
                       help='support of VARIABLE=VALUE and VARIABLE+=VALUE')

    return parser

class Hook:
    def normalize_version(self, version: str) -> str:
        """normalized version from file"""
        return version

def run_packager(args: argparse.Namespace, hook: Hook = Hook()) -> None:
    if not args.show_config and not args.get_version and not args.no_check_uncommited:
        changes = git_uncommited_changes()
        if changes:
            raise PackagerError(f'Your repository has uncommited changes:\n{changes}\n'
                                'Please commit before packaging or use --no-check-uncommited')

    # get version
    new_version = args.new_version
    if args.get_version or new_version is not None:
        results = read_version_or_die(args.pattern_version, args.file_version)
        print(hook.normalize_version(results[0]))
        return

    if not args.target:
        raise PackagerError('--target is missing')

    # Build / Show config
    if not args.distribution_name or not args.distribution_version or not args.distribution_id:
        import distro
        args.distribution_id = args.distribution_id or distro.name()
        args.distribution_name = args.distribution_name or distro.id()
        args.distribution_version = args.distribution_version or distro.version()

    configs = {
        'PKG_DISTRIBUTION': args.package_distribution,
        'DIST_ID': args.distribution_id,
        'DIST_NAME': args.distribution_name,
        'DIST_VERSION': args.distribution_version,
        'PROJECT_VERSION': args.project_version,
        'PROJECT_NAME': args.project_name,
        'URGENCY': args.urgency,
        'MAINTAINER': args.maintainer,
        'ARCH': 'any',
        'TARGET_NAME': '',
        'UTC': args.utc or '0200',
    }

    read_and_update_config(args.target, configs)

    variable_errors = update_config_variables(configs, args.variable)
    if variable_errors:
        errors_s = '", "'.join(variable_errors)
        raise PackagerError(f'Parse error on -s / --variable: "{errors_s}"')

    if args.show_config:
        print_configs(configs)
        return

    project_version = configs['PROJECT_VERSION']
    project_name = configs['PROJECT_NAME']

    if not args.no_check_version or not project_version:
        results = read_version_or_die(args.pattern_version, args.file_version)
        version = hook.normalize_version(results[0])
        if not project_version:
            project_version = version
        elif not args.no_check_version and (project_version != version or project_version != git_last_tag()):
            raise PackagerError('Repository head mismatch current version. '
                                'Ignored with --no-check-version')
    elif project_version:
        project_version = hook.normalize_version(project_version)

    if not project_version or not project_name:
        errors = []
        if not project_version:
            errors.append(
                '--project-version or -s PROJECT_VERSION=... is missing')
        if not project_name:
            errors.append('--project-name or -s PROJECT_NAME=... is missing')
        raise PackagerError(', '.join(errors))

    if args.update_changelog:
        changelog = get_changelog_entry(project_name,
                                        project_version,
                                        configs['MAINTAINER'],
                                        configs['URGENCY'],
                                        configs['UTC'])
        update_changelog(f'{args.package_template}/changelog', changelog)

    if args.clean_build:
        try:
            shutil.rmtree(args.output_build)
        except OSError:
            pass

    create_build_directory(args.package_template, args.output_build, configs)

    if args.build_package:
        status = os.system('dpkg-buildpackage -b -tc -us -uc -r')
        if status:
            raise PackagerError('Build failed: dpkg-buildpackage error')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Runs the server in uwsgi or http modes.

Also supports starting nginx proxy.

:copyright: Copyright (c) 2015 RadiaSoft LLC.  All Rights Reserved.
:license: http://www.apache.org/licenses/LICENSE-2.0.html
"""
from __future__ import absolute_import, division, print_function
from pykern import pkcli
from pykern import pkcollections
from pykern import pkconfig
from pykern import pkinspect
from pykern import pkio
from pykern import pkjinja
from pykern import pksubprocess
from pykern.pkdebug import pkdc, pkdexc, pkdp
import json
import os
import py
import socket
import sys

def celery():
    """Start celery"""
    assert pkconfig.channel_in('dev')
    import celery.bin.celery
    import sirepo.celery_tasks
    run_dir = _run_dir().join('celery').ensure(dir=True)
    with pkio.save_chdir(run_dir):
        celery.bin.celery.main(argv=[
            'celery',
            'worker',
            '--app=sirepo.celery_tasks',
            '--no-color',
            '-Ofair',
            '--queue=' + ','.join(sirepo.celery_tasks.QUEUE_NAMES),
        ])

def flower():
    """Start flower"""
    assert pkconfig.channel_in('dev')
    run_dir = _run_dir().join('flower').ensure(dir=True)
    with pkio.save_chdir(run_dir):
        from flower.command import FlowerCommand
        FlowerCommand().execute_from_commandline([
            'flower',
            '--address=' + cfg.ip,
            '--app=sirepo.celery_tasks',
            '--no-color',
            '--persistent',
        ])

def http():
    """Starts Flask server in http mode.

    Used for development only.
    """
    from sirepo import server

    with pkio.save_chdir(_run_dir()):
        app = server.init()
        app.run(
            host=cfg.ip,
            port=cfg.port,
            threaded=True,
            use_reloader=pkconfig.channel_in('dev'),
        )

def nginx_proxy():
    """Starts nginx in container.

    Used for development only.
    """
    assert pkconfig.channel_in('dev')
    run_dir = _run_dir().join('nginx_proxy').ensure(dir=True)
    with pkio.save_chdir(run_dir):
        f = run_dir.join('default.conf')
        values = dict(pkcollections.map_items(cfg))
        pkjinja.render_resource('nginx_proxy.conf', values, output=f)
        cmd = [
            'docker',
            'run',
            '--net=host',
            '--rm',
            '--volume={}:/etc/nginx/conf.d/default.conf'.format(f),
            'nginx',
        ]
        pksubprocess.check_call_with_signals(cmd)

def rabbitmq():
    assert pkconfig.channel_in('dev')
    run_dir = _run_dir().join('rabbitmq').ensure(dir=True)
    with pkio.save_chdir(run_dir):
        cmd = [
            'docker',
            'run',
            '--env=RABBITMQ_NODE_IP_ADDRESS=' + cfg.ip,
            '--net=host',
            '--rm',
            '--volume={}:/var/lib/rabbitmq'.format(run_dir),
            'rabbitmq:management',
        ]
        pksubprocess.check_call_with_signals(cmd)

def uwsgi():
    """Starts UWSGI server"""
    run_dir = _run_dir()
    with pkio.save_chdir(run_dir):
        values = dict(pkcollections.map_items(cfg))
        values['logto'] = None if pkconfig.channel_in('dev') else str(run_dir.join('uwsgi.log'))
        # uwsgi.py must be first, because values['uwsgi_py'] referenced by uwsgi.yml
        for f in ('uwsgi.py', 'uwsgi.yml'):
            output = run_dir.join(f)
            values[f.replace('.', '_')] = str(output)
            pkjinja.render_resource(f, values, output=output)
        cmd = ['uwsgi', '--yaml=' + values['uwsgi_yml']]
        pksubprocess.check_call_with_signals(cmd)

def _cfg_emails(value):
    """Parse a list of emails separated by comma, colons, semicolons or spaces.

    Args:
        value (object): if list or tuple, use verbatim; else split
    Returns:
        list: validated emails
    """
    import pyisemail
    try:
        if not isinstance(value, (list, tuple)):
            value = re.split(r'[,;:\s]+', value)
    except Exception:
        pkcli.command_error('{}: invalid email list', value)
    for v in value:
        if not pyisemail.is_email(value):
            pkcli.command_error('{}: invalid email', v)

def _cfg_int(lower, upper):
    def wrapper(value):
        v = int(value)
        assert lower <= v <= upper, \
            'value must be from {} to {}'.format(lower, upper)
        return v
    return wrapper

def _cfg_ip(value):
    try:
        socket.inet_aton(value)
        return value
    except socket.error:
        pkcli.command_error('{}: ip is not a valid IPv4 address', value)

def _run_dir():
    from sirepo import server

    if not isinstance(cfg.run_dir, type(py.path.local())):
        cfg.run_dir = pkio.mkdir_parent(cfg.run_dir) if cfg.run_dir else server.cfg.db_dir.new()
    return cfg.run_dir

cfg = pkconfig.init(
    ip=('0.0.0.0', _cfg_ip, 'what IP address to open'),
    nginx_proxy_port=(8080, _cfg_int(5001, 32767), 'port on which nginx_proxy listens'),
    port=(8000, _cfg_int(5001, 32767), 'port on which uwsgi or http listens'),
    processes=(1, _cfg_int(1, 16), 'how many uwsgi processes to start'),
    run_dir=(None, str, 'where to run the program (defaults db_dir)'),
    # uwsgi got hung up with 1024 threads on a 4 core VM with 4GB
    # so limit to 128, which is probably more than enough with
    # this application.
    threads=(10, _cfg_int(1, 128), 'how many uwsgi threads in each process'),
)
<commit_msg>raabbitmq: give a better hint that docker is not installed<commit_after># -*- coding: utf-8 -*-
"""Runs the server in uwsgi or http modes.

Also supports starting nginx proxy.

:copyright: Copyright (c) 2015 RadiaSoft LLC.  All Rights Reserved.
:license: http://www.apache.org/licenses/LICENSE-2.0.html
"""
from __future__ import absolute_import, division, print_function
from pykern import pkcli
from pykern import pkcollections
from pykern import pkconfig
from pykern import pkinspect
from pykern import pkio
from pykern import pkjinja
from pykern import pksubprocess
from pykern.pkdebug import pkdc, pkdexc, pkdp
import json
import os
import py
import socket
import sys

def celery():
    """Start celery"""
    assert pkconfig.channel_in('dev')
    import celery.bin.celery
    import sirepo.celery_tasks
    run_dir = _run_dir().join('celery').ensure(dir=True)
    with pkio.save_chdir(run_dir):
        celery.bin.celery.main(argv=[
            'celery',
            'worker',
            '--app=sirepo.celery_tasks',
            '--no-color',
            '-Ofair',
            '--queue=' + ','.join(sirepo.celery_tasks.QUEUE_NAMES),
        ])

def flower():
    """Start flower"""
    assert pkconfig.channel_in('dev')
    run_dir = _run_dir().join('flower').ensure(dir=True)
    with pkio.save_chdir(run_dir):
        from flower.command import FlowerCommand
        FlowerCommand().execute_from_commandline([
            'flower',
            '--address=' + cfg.ip,
            '--app=sirepo.celery_tasks',
            '--no-color',
            '--persistent',
        ])

def http():
    """Starts Flask server in http mode.

    Used for development only.
    """
    from sirepo import server

    with pkio.save_chdir(_run_dir()):
        app = server.init()
        app.run(
            host=cfg.ip,
            port=cfg.port,
            threaded=True,
            use_reloader=pkconfig.channel_in('dev'),
        )

def nginx_proxy():
    """Starts nginx in container.

    Used for development only.
    """
    assert pkconfig.channel_in('dev')
    run_dir = _run_dir().join('nginx_proxy').ensure(dir=True)
    with pkio.save_chdir(run_dir):
        f = run_dir.join('default.conf')
        values = dict(pkcollections.map_items(cfg))
        pkjinja.render_resource('nginx_proxy.conf', values, output=f)
        cmd = [
            'docker',
            'run',
            '--net=host',
            '--rm',
            '--volume={}:/etc/nginx/conf.d/default.conf'.format(f),
            'nginx',
        ]
        pksubprocess.check_call_with_signals(cmd)

def rabbitmq():
    assert pkconfig.channel_in('dev')
    run_dir = _run_dir().join('rabbitmq').ensure(dir=True)
    with pkio.save_chdir(run_dir):
        cmd = [
            'docker',
            'run',
            '--env=RABBITMQ_NODE_IP_ADDRESS=' + cfg.ip,
            '--net=host',
            '--rm',
            '--volume={}:/var/lib/rabbitmq'.format(run_dir),
            'rabbitmq:management',
        ]
        try:
            pksubprocess.check_call_with_signals(cmd)
        except OSError as e:
            if e.errno == errno.ENOENT:
                pkcli.command_error('docker is not installed')

def uwsgi():
    """Starts UWSGI server"""
    run_dir = _run_dir()
    with pkio.save_chdir(run_dir):
        values = dict(pkcollections.map_items(cfg))
        values['logto'] = None if pkconfig.channel_in('dev') else str(run_dir.join('uwsgi.log'))
        # uwsgi.py must be first, because values['uwsgi_py'] referenced by uwsgi.yml
        for f in ('uwsgi.py', 'uwsgi.yml'):
            output = run_dir.join(f)
            values[f.replace('.', '_')] = str(output)
            pkjinja.render_resource(f, values, output=output)
        cmd = ['uwsgi', '--yaml=' + values['uwsgi_yml']]
        pksubprocess.check_call_with_signals(cmd)

def _cfg_emails(value):
    """Parse a list of emails separated by comma, colons, semicolons or spaces.

    Args:
        value (object): if list or tuple, use verbatim; else split
    Returns:
        list: validated emails
    """
    import pyisemail
    try:
        if not isinstance(value, (list, tuple)):
            value = re.split(r'[,;:\s]+', value)
    except Exception:
        pkcli.command_error('{}: invalid email list', value)
    for v in value:
        if not pyisemail.is_email(value):
            pkcli.command_error('{}: invalid email', v)

def _cfg_int(lower, upper):
    def wrapper(value):
        v = int(value)
        assert lower <= v <= upper, \
            'value must be from {} to {}'.format(lower, upper)
        return v
    return wrapper

def _cfg_ip(value):
    try:
        socket.inet_aton(value)
        return value
    except socket.error:
        pkcli.command_error('{}: ip is not a valid IPv4 address', value)

def _run_dir():
    from sirepo import server

    if not isinstance(cfg.run_dir, type(py.path.local())):
        cfg.run_dir = pkio.mkdir_parent(cfg.run_dir) if cfg.run_dir else server.cfg.db_dir.new()
    return cfg.run_dir

cfg = pkconfig.init(
    ip=('0.0.0.0', _cfg_ip, 'what IP address to open'),
    nginx_proxy_port=(8080, _cfg_int(5001, 32767), 'port on which nginx_proxy listens'),
    port=(8000, _cfg_int(5001, 32767), 'port on which uwsgi or http listens'),
    processes=(1, _cfg_int(1, 16), 'how many uwsgi processes to start'),
    run_dir=(None, str, 'where to run the program (defaults db_dir)'),
    # uwsgi got hung up with 1024 threads on a 4 core VM with 4GB
    # so limit to 128, which is probably more than enough with
    # this application.
    threads=(10, _cfg_int(1, 128), 'how many uwsgi threads in each process'),
)
<|endoftext|>


<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst

# Packages may add whatever they like to this file, but
# should keep this content at the top.
# ----------------------------------------------------------------------------
from ._astropy_init import *   # noqa
# ----------------------------------------------------------------------------
try:
    import faulthandler

    faulthandler.enable()
except ImportError:
    pass

import warnings
import numpy as np
# warnings.filterwarnings("error", category=np.VisibleDeprecationWarning)
# warnings.filterwarnings("error", ".*")
warnings.filterwarnings("once", category=UserWarning)
warnings.filterwarnings("once", category=DeprecationWarning)
warnings.filterwarnings("once", category=np.VisibleDeprecationWarning)

__all__ = []
<commit_msg>Silence hdf5 path= warnings<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst

# Packages may add whatever they like to this file, but
# should keep this content at the top.
# ----------------------------------------------------------------------------
from ._astropy_init import *   # noqa
# ----------------------------------------------------------------------------
try:
    import faulthandler

    faulthandler.enable()
except ImportError:
    pass

import warnings
import numpy as np
# warnings.filterwarnings("error", category=np.VisibleDeprecationWarning)
# warnings.filterwarnings("error", ".*")
warnings.filterwarnings("once", category=UserWarning)
warnings.filterwarnings("once", category=DeprecationWarning)
warnings.filterwarnings("once", category=np.VisibleDeprecationWarning)
warnings.filterwarnings("ignore", "table path was not set via the path= ")

__all__ = []
<|endoftext|>


<commit_before>from __future__ import division
import time
import lcm
import os
import sys
import drc
import argparse
from collections import deque

"""
A very simple fall detector for the Atlas. Requires that the walking controller be running in matlab (use_mex=false). Listens to CONTROLLER_STATUS and attempts to set the robot to its brace position if a fall is detected.

Usage:
    python fall_detector.py
"""

DEFAULTS = {'vdot_threshold': 0.5,
            'vdot_count_threshold': 5,
            'vdot_memory': 10,
            'retransmit_delay_s': 5}

m = lcm.LCM()

class FallDetector:
    def __init__(self, opts):
        self.opts = opts
        self.vdot_errror_count = 0
        self.vdot_history = deque()
        self.last_t = None
        self.last_publish_time = 0

    def handle(self, channel, data):
        msg = drc.controller_status_t.decode(data)
        t = msg.utime / 1e6
        if self.last_t is None or t - self.last_t > 1 or t < 0.001:
            self.last_t = t
            self.vdot_errror_count = 0
            self.vdot_history = deque()
        else:
            if len(self.vdot_history) >= self.opts.memory:
                self.vdot_errror_count -= int(self.vdot_history.popleft())
            err = msg.Vdot > self.opts.vdot_threshold
            # print msg.Vdot
            if err:
                print "Positive Vdot ({:.5f}) at time: {:.3f}".format(msg.Vdot, t)
            self.vdot_history.append(err)
            self.vdot_errror_count += int(err)
            if self.vdot_errror_count > self.opts.count_threshold:
                if time.time() - self.last_publish_time > self.opts.retransmit_delay:
                    print "WARNING: Fall detected at time: {:.3f}".format(t)
                    print "Bracing NOW"
                    brace_msg = drc.utime_t()
                    brace_msg.utime = 0
                    m.publish("BRACE_FOR_FALL", brace_msg.encode())
                    status_msg = drc.system_status_t()
                    status_msg.utime = msg.utime
                    status_msg.system = 8
                    status_msg.importance = 0
                    status_msg.frequency = 0
                    status_msg.value = 'Possible fall detected. Bracing now!'
                    m.publish('SYSTEM_STATUS', status_msg.encode())
                    self.last_publish_time = time.time()
            self.last_t = t

def main():
    parser = argparse.ArgumentParser(description='Try to detect falls from controller status and trigger switch to bracing controller. Fall detection will be triggered if more than C values of vdot are above V for the last M controller cycles.')
    parser.add_argument('--vdot_threshold', '-v', metavar='V', action='store', type=float, default=DEFAULTS['vdot_threshold'])
    parser.add_argument('--count_threshold', '-c', metavar='C', action='store', type=float, default=DEFAULTS['vdot_count_threshold'], help='')
    parser.add_argument('--memory', '-m', metavar='M', action='store', type=int, default=DEFAULTS['vdot_memory'])
    parser.add_argument('--retransmit_delay', '-r', metavar='R', action='store', type=float, default=DEFAULTS['retransmit_delay_s'], help='Time (s) before sending another BRACE_FOR_FALL message')
    opts = parser.parse_args()

    f = FallDetector(opts)

    m.subscribe("CONTROLLER_STATUS", f.handle)
    print "Watching for falls..."
    while True:
        m.handle()

if __name__ == '__main__':
    main()
<commit_msg>doubled vdot threshold in fall classifier because it was getting tripped while rough terrain walking<commit_after>from __future__ import division
import time
import lcm
import os
import sys
import drc
import argparse
from collections import deque

"""
A very simple fall detector for the Atlas. Requires that the walking controller be running in matlab (use_mex=false). Listens to CONTROLLER_STATUS and attempts to set the robot to its brace position if a fall is detected.

Usage:
    python fall_detector.py
"""

DEFAULTS = {'vdot_threshold': 1.0,
            'vdot_count_threshold': 5,
            'vdot_memory': 10,
            'retransmit_delay_s': 5}

m = lcm.LCM()

class FallDetector:
    def __init__(self, opts):
        self.opts = opts
        self.vdot_errror_count = 0
        self.vdot_history = deque()
        self.last_t = None
        self.last_publish_time = 0

    def handle(self, channel, data):
        msg = drc.controller_status_t.decode(data)
        t = msg.utime / 1e6
        if self.last_t is None or t - self.last_t > 1 or t < 0.001:
            self.last_t = t
            self.vdot_errror_count = 0
            self.vdot_history = deque()
        else:
            if len(self.vdot_history) >= self.opts.memory:
                self.vdot_errror_count -= int(self.vdot_history.popleft())
            err = msg.Vdot > self.opts.vdot_threshold
            # print msg.Vdot
            if err:
                print "Positive Vdot ({:.5f}) at time: {:.3f}".format(msg.Vdot, t)
            self.vdot_history.append(err)
            self.vdot_errror_count += int(err)
            if self.vdot_errror_count > self.opts.count_threshold:
                if time.time() - self.last_publish_time > self.opts.retransmit_delay:
                    print "WARNING: Fall detected at time: {:.3f}".format(t)
                    print "Bracing NOW"
                    brace_msg = drc.utime_t()
                    brace_msg.utime = 0
                    m.publish("BRACE_FOR_FALL", brace_msg.encode())
                    status_msg = drc.system_status_t()
                    status_msg.utime = msg.utime
                    status_msg.system = 8
                    status_msg.importance = 0
                    status_msg.frequency = 0
                    status_msg.value = 'Possible fall detected. Bracing now!'
                    m.publish('SYSTEM_STATUS', status_msg.encode())
                    self.last_publish_time = time.time()
            self.last_t = t

def main():
    parser = argparse.ArgumentParser(description='Try to detect falls from controller status and trigger switch to bracing controller. Fall detection will be triggered if more than C values of vdot are above V for the last M controller cycles.')
    parser.add_argument('--vdot_threshold', '-v', metavar='V', action='store', type=float, default=DEFAULTS['vdot_threshold'])
    parser.add_argument('--count_threshold', '-c', metavar='C', action='store', type=float, default=DEFAULTS['vdot_count_threshold'], help='')
    parser.add_argument('--memory', '-m', metavar='M', action='store', type=int, default=DEFAULTS['vdot_memory'])
    parser.add_argument('--retransmit_delay', '-r', metavar='R', action='store', type=float, default=DEFAULTS['retransmit_delay_s'], help='Time (s) before sending another BRACE_FOR_FALL message')
    opts = parser.parse_args()

    f = FallDetector(opts)

    m.subscribe("CONTROLLER_STATUS", f.handle)
    print "Watching for falls..."
    while True:
        m.handle()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""
Generate T2* map from multi-echo BOLD images
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

.. autofunction:: init_bold_t2s_wf

"""
from nipype import logging
from nipype.pipeline import engine as pe
from nipype.interfaces import utility as niu

from ...engine import Workflow
from ...interfaces import T2SMap

from .util import init_skullstrip_bold_wf

DEFAULT_MEMORY_MIN_GB = 0.01
LOGGER = logging.getLogger('nipype.workflow')

# pylint: disable=R0914
def init_bold_t2s_wf(echo_times, mem_gb, omp_nthreads,
                     t2s_coreg=False, name='bold_t2s_wf'):
    """
    This workflow wraps the `tedana`_ `T2* workflow`_ to optimally
    combine multiple echos and derive a T2* map for optional use as a
    coregistration target.

    The following steps are performed:

    #. :abbr:`HMC (head motion correction)` on individual echo files.
    #. Compute the T2* map
    #. Create an optimally combined ME-EPI time series

    **Parameters**

        echo_times
            list of TEs associated with each echo
        mem_gb : float
            Size of BOLD file in GB
        omp_nthreads : int
            Maximum number of threads an individual process may use
        t2s_coreg : bool
            Use the calculated T2*-map for T2*-driven coregistration
        name : str
            Name of workflow (default: ``bold_t2s_wf``)

    **Inputs**

        bold_file
            list of individual echo files

    **Outputs**

        bold
            the optimally combined time series for all supplied echos
        bold_mask
            the binarized, skull-stripped adaptive T2* map
        bold_ref_brain
            the adaptive T2* map

    .. _tedana: https://github.com/me-ica/tedana
    .. _`T2* workflow`: https://tedana.readthedocs.io/en/latest/generated/tedana.workflows.t2smap_workflow.html#tedana.workflows.t2smap_workflow  # noqa

    """
    workflow = Workflow(name=name)
    workflow.__desc__ = """\
A T2* map was estimated from the preprocessed BOLD by fitting to a monoexponential signal
decay model with log-linear regression.
For each voxel, the maximal number of echoes with reliable signal in that voxel were
used to fit the model.
The calculated T2* map was then used to optimally combine preprocessed BOLD across
echoes following the method described in @posse_t2s.
The optimally combined time series was carried forward as the *preprocessed BOLD*{}.
""".format('' if not t2s_coreg else ', and the T2* map was also retained as the BOLD reference')

    inputnode = pe.Node(niu.IdentityInterface(fields=['bold_file']), name='inputnode')

    outputnode = pe.Node(niu.IdentityInterface(fields=['bold', 'bold_mask', 'bold_ref_brain']),
                         name='outputnode')

    LOGGER.log(25, 'Generating T2* map and optimally combined ME-EPI time series.')

    t2smap_node = pe.Node(T2SMap(echo_times=echo_times), name='t2smap_node')
    skullstrip_t2smap_wf = init_skullstrip_bold_wf(name='skullstrip_t2smap_wf')

    workflow.connect([
        (inputnode, t2smap_node, [('bold_file', 'in_files')]),
        (t2smap_node, outputnode, [('optimal_comb', 'bold')]),
        (t2smap_node, skullstrip_t2smap_wf, [('t2star_map', 'inputnode.in_file')]),
        (skullstrip_t2smap_wf, outputnode, [
            ('outputnode.mask_file', 'bold_mask'),
            ('outputnode.skull_stripped_file', 'bold_ref_brain')]),
    ])

    return workflow
<commit_msg>[skip ci] Adress oesteban review comments<commit_after># -*- coding: utf-8 -*-
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""
Generate T2* map from multi-echo BOLD images
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

.. autofunction:: init_bold_t2s_wf

"""
from nipype import logging
from nipype.pipeline import engine as pe
from nipype.interfaces import utility as niu

from ...engine import Workflow
from ...interfaces import T2SMap

from .util import init_skullstrip_bold_wf

DEFAULT_MEMORY_MIN_GB = 0.01
LOGGER = logging.getLogger('nipype.workflow')

# pylint: disable=R0914
def init_bold_t2s_wf(echo_times, mem_gb, omp_nthreads,
                     t2s_coreg=False, name='bold_t2s_wf'):
    """
    This workflow wraps the `tedana`_ `T2* workflow`_ to optimally
    combine multiple echos and derive a T2* map for optional use as a
    coregistration target.

    The following steps are performed:

    #. :abbr:`HMC (head motion correction)` on individual echo files.
    #. Compute the T2* map
    #. Create an optimally combined ME-EPI time series

    **Parameters**

        echo_times
            list of TEs associated with each echo
        mem_gb : float
            Size of BOLD file in GB
        omp_nthreads : int
            Maximum number of threads an individual process may use
        t2s_coreg : bool
            Use the calculated T2*-map for T2*-driven coregistration
        name : str
            Name of workflow (default: ``bold_t2s_wf``)

    **Inputs**

        bold_file
            list of individual echo files

    **Outputs**

        bold
            the optimally combined time series for all supplied echos
        bold_mask
            the binarized, skull-stripped adaptive T2* map
        bold_ref_brain
            the adaptive T2* map

    .. _tedana: https://github.com/me-ica/tedana
    .. _`T2* workflow`: https://tedana.readthedocs.io/en/latest/generated/tedana.workflows.t2smap_workflow.html#tedana.workflows.t2smap_workflow  # noqa

    """
    workflow = Workflow(name=name)
    workflow.__desc__ = """\
A T2* map was estimated from the preprocessed BOLD by fitting to a monoexponential signal
decay model with log-linear regression.
For each voxel, the maximal number of echoes with reliable signal in that voxel were
used to fit the model.
The calculated T2* map was then used to optimally combine preprocessed BOLD across
echoes following the method described in [@posse_t2s].
The optimally combined time series was carried forward as the *preprocessed BOLD*{}.
""".format('' if not t2s_coreg else ', and the T2* map was also retained as the BOLD reference')

    inputnode = pe.Node(niu.IdentityInterface(fields=['bold_file']), name='inputnode')

    outputnode = pe.Node(niu.IdentityInterface(fields=['bold', 'bold_mask', 'bold_ref_brain']),
                         name='outputnode')

    LOGGER.log(25, 'Generating T2* map and optimally combined ME-EPI time series.')

    t2smap_node = pe.Node(T2SMap(echo_times=echo_times), name='t2smap_node')
    skullstrip_t2smap_wf = init_skullstrip_bold_wf(name='skullstrip_t2smap_wf')

    workflow.connect([
        (inputnode, t2smap_node, [('bold_file', 'in_files')]),
        (t2smap_node, outputnode, [('optimal_comb', 'bold')]),
        (t2smap_node, skullstrip_t2smap_wf, [('t2star_map', 'inputnode.in_file')]),
        (skullstrip_t2smap_wf, outputnode, [
            ('outputnode.mask_file', 'bold_mask'),
            ('outputnode.skull_stripped_file', 'bold_ref_brain')]),
    ])

    return workflow
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    wapiti.operations.models
    ~~~~~~~~~~~~~~~~~~~~~~~~

    This module provides structures and abstractions for creating consistent
    Operation interfaces, regardless of underlying Mediawiki API response
    types.

    For example the ``prop=revisions`` and ``list=usercontribs`` APIs
    both return lists of revision information, however not all of the
    attributes afforded by ``prop=revisions`` are available from
    ``list=usercontribs``. Wapiti models and operations strive to
    resolve and abstract this fact away from the user as sanely as
    possible.
"""
from __future__ import unicode_literals

from datetime import datetime
from collections import namedtuple, OrderedDict

def parse_timestamp(timestamp):
    return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%SZ')

LanguageLink = namedtuple('LanguageLink', 'url language origin_page')
InterwikiLink = namedtuple('InterwikiLink', 'url prefix origin_page')
ExternalLink = namedtuple('ExternalLink', 'url origin_page')

NamespaceDescriptor = namedtuple('NamespaceDescriptor', 'id title canonical')
InterwikiDescriptor = namedtuple('InterwikiDescriptor', 'alias url language')

_MISSING = object()

class WapitiModelAttr(object):
    def __init__(self, name, **kw):
        self.name = name
        self.mw_name = kw.pop('mw_name', name)
        self.display = kw.pop('display', False)
        try:
            self.default = kw.pop('default')
        except KeyError:
            self.default = _MISSING
        if kw:
            raise ValueError('got unexpected keyword arguments: %r'
                             % kw.keys())

    def __repr__(self):
        ret = [self.__class__.__name__, '(', repr(self.name)]
        if self.mw_name != self.name:
            ret.extend([', mw_name=', repr(self.mw_name)])
        if self.default is not _MISSING:
            ret.extend([', default=', repr(self.default)])
        if self.display:
            ret.extend([', display=', repr(self.display)])
        ret.append(')')
        return ''.join(ret)

    def __iter__(self):
        return iter((self.name, self.mw_name, self.default, self.display))

WMA = WapitiModelAttr  # Windows Media Audio

def title_talk2subject(title):
    talk_pref, _, title_suf = title.partition(':')
    subj_pref, _, _ = talk_pref.rpartition('talk')
    subj_pref = subj_pref.strip()
    new_title = subj_pref + ':' + title_suf
    new_title = new_title.lstrip(':')
    return new_title

def title_subject2talk(title):
    subj_pref, _, title_suf = title.partition(':')
    subj_pref = subj_pref.strip()
    if not subj_pref:
        talk_pref = 'Talk'
    elif subj_pref.endswith('talk'):
        talk_pref = subj_pref
    else:
        talk_pref = subj_pref + ' talk'
    new_title = talk_pref + ':' + title_suf
    return new_title

class WapitiModelMeta(type):
    """
    The foundation of Wapiti's data models, which attempt to add
    consistency and order to the wide variety of return types used
    across different Mediawiki APIs. This metaclass enables certain
    inheritance-like usage patterns in models. See WapitiModelBase's
    docstring for more information.

    The `attributes` dictionary is a mapping of Python class attribute
    names to Mediawiki API result keys (e.g., `pageid` becomes
    `page_id` on the Python object).

    The `defaults` dictionary is a mapping of Python attribute name to
    default value, if allowed. If an attribute does not have a default
    value, and is missing upon instantiation of a model, an exception
    will be raised.
    """
    attributes = []

    def __new__(cls, name, bases, attrs):
        all_attributes = OrderedDict()
        for base in bases:
            base_attr_list = getattr(base, 'attributes', [])
            base_attr_dict = OrderedDict([(a.name, a) for a in base_attr_list])
            all_attributes.update(base_attr_dict)
        attr_dict = OrderedDict([(a.name, a) for a
                                 in attrs.get('attributes', [])])
        all_attributes.update(attr_dict)
        attrs['attributes'] = all_attributes.values()
        ret = super(WapitiModelMeta, cls).__new__(cls, name, bases, attrs)
        return ret

class WapitiModelBase(object):
    """
    The more-concrete counterpart of WapitiModelMeta, which primarily
    provides generic initialization and display logic.

    There are two methods for instantiation, the standard
    ``__init__()`` (e.g., ``CategoryInfo()``), which takes attributes
    as keyword arguments, and ``from_query()``, which usually takes a
    dictionary deserialized from JSON, as returned by Mediawiki
    API. For information on `attributes` and `defaults`, see
    WapitiModelMeta.

    ``__repr__()`` and ``__str__()`` are powered by
    ``get_display_str()``.
    """

    __metaclass__ = WapitiModelMeta
    attributes = []

    def __init__(self, **kw):
        missing = []
        for attr in self.attributes:
            try:
                val = kw.pop(attr.name)
            except KeyError:
                if attr.default is _MISSING:
                    missing.append(attr.name)
                    continue
                val = attr.default
            setattr(self, attr.name, val)
        if missing:
            raise ValueError('missing expected keyword arguments: %r'
                             % missing)
        # TODO: raise on unexpected keyword arguments?
        return

    @classmethod
    def from_query(cls, q_dict, **kw):
        kwargs = {}
        all_q_dict = dict(kw)
        all_q_dict.update(q_dict)
        for name, mw_name, _, _ in cls.attributes:
            if mw_name is None:
                continue
            try:
                kwargs[name] = all_q_dict[mw_name]
            except KeyError:
                pass
        return cls(**kwargs)

    def get_display_str(self, raise_exc=True):
        attr_list = []
        try:
            for (name, _, _, do_disp) in self.attributes:
                if not do_disp:
                    continue
                # TODO: don't display values if equal to default?
                val = getattr(self, name)
                attr_list.append('%s=%r' % (name, val))
        except:
            if raise_exc:
                raise
            return super(WapitiModelBase, self).__str__()
        attr_str = ', '.join(attr_list)
        return ''.join([self.__class__.__name__, '(', attr_str, ')'])

    __str__ = get_display_str

    def __repr__(self):
        try:
            return self.get_display_str()
        except:
            return super(PageIdentifier, self).__repr__()

class PageIdentifier(WapitiModelBase):
    attributes = [WMA('title', display=True),
                  WMA('page_id', mw_name='pageid', display=True),
                  WMA('ns', display=True),
                  WMA('source')]

    @property
    def is_subject_page(self):
        return (self.ns >= 0 and self.ns % 2 == 0)

    @property
    def is_talk_page(self):
        return (self.ns >= 0 and self.ns % 2 == 1)

class PageInfo(PageIdentifier):
    attributes = [WMA('subject_id', mw_name='subjectid', default=None),
                  WMA('talk_id', mw_name='talkid', default=None)]

    def __init__(self, **kw):
        req_title = kw.pop('req_title', None)
        super(PageInfo, self).__init__(**kw)
        self.req_title = req_title or self.title

        if self.is_subject_page:
            self.subject_id = self.page_id
        elif self.is_talk_page:
            self.talk_id = self.page_id
        else:
            raise ValueError('special or nonexistent namespace: %r' % self.ns)

    def get_subject_info(self):
        if self.is_subject_page:
            return self
        if self.subject_id is None:
            raise ValueError('subject_id not set')
        subj_title = title_talk2subject(self.title)
        subj_ns = self.ns - 1
        kwargs = dict(self.__dict__)
        kwargs['title'] = subj_title
        kwargs['ns'] = subj_ns
        return PageInfo(**kwargs)

    def get_talk_info(self):
        if self.is_talk_page:
            return self
        if self.talk_id is None:
            raise ValueError('talk_id not set')
        talk_title = title_subject2talk(self.title)
        talk_ns = self.ns + 1
        kwargs = dict(self.__dict__)
        kwargs['title'] = talk_title
        kwargs['ns'] = talk_ns
        return PageInfo(**kwargs)

class CategoryInfo(PageInfo):
    attributes = [WMA('total_count', mw_name='size', default=0, display=True),
                  WMA('page_count', mw_name='pages', default=0),
                  WMA('file_count', mw_name='files', default=0),
                  WMA('subcat_count', mw_name='subcats', default=0, display=True)]

class RevisionInfo(PageInfo):
    attributes = [WMA('rev_id', mw_name='revid', display=True),
                  WMA('size'),
                  WMA('user_text', mw_name='user', default='!userhidden'),
                  WMA('user_id', mw_name='userid', default=-1),
                  WMA('timestamp', display=True),
                  WMA('comment', default=''),
                  WMA('parsed_comment', mw_name='parsedcomment', default=''),
                  WMA('tags')]

    # note that certain revisions may have hidden the fields
    # user_id, user_text, and comment for administrative reasons,
    # aka "oversighting"
    # TODO: is oversighting better handled in operation?

    def __init__(self, *a, **kw):
        super(RevisionInfo, self).__init__(*a, **kw)
        self.timestamp = parse_timestamp(self.timestamp)

class Revision(RevisionInfo):
    attributes = [WMA('parent_rev_id', mw_name='parentid', display=True),
                  WMA('content', mw_name='*', default=''),  # default=''?
                  WMA('is_parsed')]

class ImageInfo(PageIdentifier):
    attributes = [WMA('image_repo', mw_name='imagerepository'),
                  WMA('missing', default=False),
                  WMA('url', default=''),  # will only exist if non-local repo
                  WMA('dimensions', default=''),
                  WMA('mime', default=''),
                  WMA('thumbmime', default=''),
                  WMA('media_type', mw_name='mediatype', default=''),
                  WMA('metadata', default=''),
                  WMA('archive_name', mw_name='archivename', default=''),
                  WMA('bitdepth', default='')]

#
# Protections
#
NEW = 'NEW'
AUTOCONFIRMED = 'AUTOCONFIRMED'
SYSOP = 'SYSOP'
PROTECTION_ACTIONS = ('create', 'edit', 'move', 'upload')

Protection = namedtuple('Protection', 'level, expiry')

class ProtectionInfo(object):
    # TODO: turn into mixin, add to PageIdentifier
    """
    For more info on protection,
    see https://en.wikipedia.org/wiki/Wikipedia:Protection_policy
    """
    levels = {
        'new': NEW,
        'autoconfirmed': AUTOCONFIRMED,
        'sysop': SYSOP,
    }

    def __init__(self, protections, page_ident=None):
        self.page_ident = page_ident

        protections = protections or {}
        self.protections = {}
        for p in protections:
            if not p['expiry'] == 'infinity':
                expiry = parse_timestamp(p['expiry'])
            else:
                expiry = 'infinity'
            level = self.levels[p['level']]
            self.protections[p['type']] = Protection(level, expiry)

    @property
    def has_protection(self):
        return any([x.level != NEW for x in self.protections.values()])

    @property
    def has_indef(self):
        return any([x.expiry == 'infinity' for x in self.protections.values()])

    @property
    def is_full_prot(self):
        try:
            if self.protections['edit'].level == SYSOP and \
                    self.protections['move'].level == SYSOP:
                return True
            else:
                return False
        except (KeyError, AttributeError):
            return False

    @property
    def is_semi_prot(self):
        try:
            if self.protections['edit'].level == AUTOCONFIRMED:
                return True
            else:
                return False
        except (KeyError, AttributeError):
            return False

    def __repr__(self):
        return u'ProtectionInfo(%r)' % self.protections

class CoordinateIndentifier(object):
    def __init__(self, coord, page_ident=None):
        self.page_ident = page_ident
        self.lat = coord.get('lat')
        self.lon = coord.get('lon')
        self.type = coord.get('type')
        self.name = coord.get('name')
        self.dim = coord.get('dim')
        self.country = coord.get('country')
        self.region = coord.get('region')
        if coord.get('primary', False):
            self.primary = True
        else:
            self.primary = False
        return
<commit_msg>unique_on added to models<commit_after># -*- coding: utf-8 -*-
"""
    wapiti.operations.models
    ~~~~~~~~~~~~~~~~~~~~~~~~

    This module provides structures and abstractions for creating consistent
    Operation interfaces, regardless of underlying Mediawiki API response
    types.

    For example the ``prop=revisions`` and ``list=usercontribs`` APIs
    both return lists of revision information, however not all of the
    attributes afforded by ``prop=revisions`` are available from
    ``list=usercontribs``. Wapiti models and operations strive to
    resolve and abstract this fact away from the user as sanely as
    possible.
"""
from __future__ import unicode_literals

from datetime import datetime
from collections import namedtuple, OrderedDict

def parse_timestamp(timestamp):
    return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%SZ')

LanguageLink = namedtuple('LanguageLink', 'url language origin_page')
InterwikiLink = namedtuple('InterwikiLink', 'url prefix origin_page')
ExternalLink = namedtuple('ExternalLink', 'url origin_page')

NamespaceDescriptor = namedtuple('NamespaceDescriptor', 'id title canonical')
InterwikiDescriptor = namedtuple('InterwikiDescriptor', 'alias url language')

_MISSING = object()

class WapitiModelAttr(object):
    def __init__(self, name, **kw):
        self.name = name
        self.mw_name = kw.pop('mw_name', name)
        self.display = kw.pop('display', False)
        try:
            self.default = kw.pop('default')
        except KeyError:
            self.default = _MISSING
        if kw:
            raise ValueError('got unexpected keyword arguments: %r'
                             % kw.keys())

    def __repr__(self):
        ret = [self.__class__.__name__, '(', repr(self.name)]
        if self.mw_name != self.name:
            ret.extend([', mw_name=', repr(self.mw_name)])
        if self.default is not _MISSING:
            ret.extend([', default=', repr(self.default)])
        if self.display:
            ret.extend([', display=', repr(self.display)])
        ret.append(')')
        return ''.join(ret)

    def __iter__(self):
        return iter((self.name, self.mw_name, self.default, self.display))

WMA = WapitiModelAttr  # Windows Media Audio

def title_talk2subject(title):
    talk_pref, _, title_suf = title.partition(':')
    subj_pref, _, _ = talk_pref.rpartition('talk')
    subj_pref = subj_pref.strip()
    new_title = subj_pref + ':' + title_suf
    new_title = new_title.lstrip(':')
    return new_title

def title_subject2talk(title):
    subj_pref, _, title_suf = title.partition(':')
    subj_pref = subj_pref.strip()
    if not subj_pref:
        talk_pref = 'Talk'
    elif subj_pref.endswith('talk'):
        talk_pref = subj_pref
    else:
        talk_pref = subj_pref + ' talk'
    new_title = talk_pref + ':' + title_suf
    return new_title

def get_unique_func(val):
    if callable(val):
        return val
    elif isinstance(val, basestring):
        return lambda obj: getattr(obj, val, obj)
    try:
        if all([isinstance(v, basestring) for v in val]):
            return lambda obj: tuple([getattr(obj, v, obj) for v in val])
    except TypeError:
        pass
    raise TypeError('could not derive uniqueification function from %r' % val)

class WapitiModelMeta(type):
    """
    The foundation of Wapiti's data models, which attempt to add
    consistency and order to the wide variety of return types used
    across different Mediawiki APIs. This metaclass enables certain
    inheritance-like usage patterns in models. See WapitiModelBase's
    docstring for more information.

    The `attributes` dictionary is a mapping of Python class attribute
    names to Mediawiki API result keys (e.g., `pageid` becomes
    `page_id` on the Python object).

    The `defaults` dictionary is a mapping of Python attribute name to
    default value, if allowed. If an attribute does not have a default
    value, and is missing upon instantiation of a model, an exception
    will be raised.
    """
    attributes = []

    def __new__(cls, name, bases, attrs):
        all_attributes = OrderedDict()
        for base in bases:
            base_attr_list = getattr(base, 'attributes', [])
            base_attr_dict = OrderedDict([(a.name, a) for a in base_attr_list])
            all_attributes.update(base_attr_dict)
        attr_dict = OrderedDict([(a.name, a) for a
                                 in attrs.get('attributes', [])])
        all_attributes.update(attr_dict)
        attrs['attributes'] = all_attributes.values()
        if 'unique_on' in attrs:
            attrs['unique_key'] = property(get_unique_func(attrs['unique_on']))
        ret = super(WapitiModelMeta, cls).__new__(cls, name, bases, attrs)
        return ret

class WapitiModelBase(object):
    """
    The more-concrete counterpart of WapitiModelMeta, which primarily
    provides generic initialization and display logic.

    There are two methods for instantiation, the standard
    ``__init__()`` (e.g., ``CategoryInfo()``), which takes attributes
    as keyword arguments, and ``from_query()``, which usually takes a
    dictionary deserialized from JSON, as returned by Mediawiki
    API. For information on `attributes` and `defaults`, see
    WapitiModelMeta.

    ``__repr__()`` and ``__str__()`` are powered by
    ``get_display_str()``.
    """

    __metaclass__ = WapitiModelMeta
    attributes = []
    unique_on = lambda self: self

    def __init__(self, **kw):
        missing = []
        for attr in self.attributes:
            try:
                val = kw.pop(attr.name)
            except KeyError:
                if attr.default is _MISSING:
                    missing.append(attr.name)
                    continue
                val = attr.default
            setattr(self, attr.name, val)
        if missing:
            raise ValueError('missing expected keyword arguments: %r'
                             % missing)
        # TODO: raise on unexpected keyword arguments?
        return

    @classmethod
    def from_query(cls, q_dict, **kw):
        kwargs = {}
        all_q_dict = dict(kw)
        all_q_dict.update(q_dict)
        for name, mw_name, _, _ in cls.attributes:
            if mw_name is None:
                continue
            try:
                kwargs[name] = all_q_dict[mw_name]
            except KeyError:
                pass
        return cls(**kwargs)

    def get_display_str(self, raise_exc=True):
        attr_list = []
        try:
            for (name, _, _, do_disp) in self.attributes:
                if not do_disp:
                    continue
                # TODO: don't display values if equal to default?
                val = getattr(self, name)
                attr_list.append('%s=%r' % (name, val))
        except:
            if raise_exc:
                raise
            return super(WapitiModelBase, self).__str__()
        attr_str = ', '.join(attr_list)
        return ''.join([self.__class__.__name__, '(', attr_str, ')'])

    __str__ = get_display_str

    def __repr__(self):
        try:
            return self.get_display_str()
        except:
            return super(PageIdentifier, self).__repr__()

class PageIdentifier(WapitiModelBase):
    attributes = [WMA('title', display=True),
                  WMA('page_id', mw_name='pageid', display=True),
                  WMA('ns', display=True),
                  WMA('source')]

    unique_on = 'title'

    @property
    def is_subject_page(self):
        return (self.ns >= 0 and self.ns % 2 == 0)

    @property
    def is_talk_page(self):
        return (self.ns >= 0 and self.ns % 2 == 1)

class PageInfo(PageIdentifier):
    attributes = [WMA('subject_id', mw_name='subjectid', default=None),
                  WMA('talk_id', mw_name='talkid', default=None)]

    def __init__(self, **kw):
        req_title = kw.pop('req_title', None)
        super(PageInfo, self).__init__(**kw)
        self.req_title = req_title or self.title

        if self.is_subject_page:
            self.subject_id = self.page_id
        elif self.is_talk_page:
            self.talk_id = self.page_id
        else:
            raise ValueError('special or nonexistent namespace: %r' % self.ns)

    def get_subject_info(self):
        if self.is_subject_page:
            return self
        if self.subject_id is None:
            raise ValueError('subject_id not set')
        subj_title = title_talk2subject(self.title)
        subj_ns = self.ns - 1
        kwargs = dict(self.__dict__)
        kwargs['title'] = subj_title
        kwargs['ns'] = subj_ns
        return PageInfo(**kwargs)

    def get_talk_info(self):
        if self.is_talk_page:
            return self
        if self.talk_id is None:
            raise ValueError('talk_id not set')
        talk_title = title_subject2talk(self.title)
        talk_ns = self.ns + 1
        kwargs = dict(self.__dict__)
        kwargs['title'] = talk_title
        kwargs['ns'] = talk_ns
        return PageInfo(**kwargs)

class CategoryInfo(PageInfo):
    attributes = [WMA('total_count', mw_name='size', default=0, display=True),
                  WMA('page_count', mw_name='pages', default=0),
                  WMA('file_count', mw_name='files', default=0),
                  WMA('subcat_count', mw_name='subcats', default=0, display=True)]

class RevisionInfo(PageInfo):
    attributes = [WMA('rev_id', mw_name='revid', display=True),
                  WMA('size'),
                  WMA('user_text', mw_name='user', default='!userhidden'),
                  WMA('user_id', mw_name='userid', default=-1),
                  WMA('timestamp', display=True),
                  WMA('comment', default=''),
                  WMA('parsed_comment', mw_name='parsedcomment', default=''),
                  WMA('tags')]

    unique_on = 'rev_id'

    # note that certain revisions may have hidden the fields
    # user_id, user_text, and comment for administrative reasons,
    # aka "oversighting"
    # TODO: is oversighting better handled in operation?

    def __init__(self, *a, **kw):
        super(RevisionInfo, self).__init__(*a, **kw)
        self.timestamp = parse_timestamp(self.timestamp)

class Revision(RevisionInfo):
    attributes = [WMA('parent_rev_id', mw_name='parentid', display=True),
                  WMA('content', mw_name='*', default=''),  # default=''?
                  WMA('is_parsed')]

class ImageInfo(PageIdentifier):
    attributes = [WMA('image_repo', mw_name='imagerepository'),
                  WMA('missing', default=False),
                  WMA('url', default=''),  # will only exist if non-local repo
                  WMA('dimensions', default=''),
                  WMA('mime', default=''),
                  WMA('thumbmime', default=''),
                  WMA('media_type', mw_name='mediatype', default=''),
                  WMA('metadata', default=''),
                  WMA('archive_name', mw_name='archivename', default=''),
                  WMA('bitdepth', default='')]

#
# Protections
#
NEW = 'NEW'
AUTOCONFIRMED = 'AUTOCONFIRMED'
SYSOP = 'SYSOP'
PROTECTION_ACTIONS = ('create', 'edit', 'move', 'upload')

Protection = namedtuple('Protection', 'level, expiry')

class ProtectionInfo(object):
    # TODO: turn into mixin, add to PageIdentifier
    """
    For more info on protection,
    see https://en.wikipedia.org/wiki/Wikipedia:Protection_policy
    """
    levels = {
        'new': NEW,
        'autoconfirmed': AUTOCONFIRMED,
        'sysop': SYSOP,
    }

    def __init__(self, protections, page_ident=None):
        self.page_ident = page_ident

        protections = protections or {}
        self.protections = {}
        for p in protections:
            if not p['expiry'] == 'infinity':
                expiry = parse_timestamp(p['expiry'])
            else:
                expiry = 'infinity'
            level = self.levels[p['level']]
            self.protections[p['type']] = Protection(level, expiry)

    @property
    def has_protection(self):
        return any([x.level != NEW for x in self.protections.values()])

    @property
    def has_indef(self):
        return any([x.expiry == 'infinity' for x in self.protections.values()])

    @property
    def is_full_prot(self):
        try:
            if self.protections['edit'].level == SYSOP and \
                    self.protections['move'].level == SYSOP:
                return True
            else:
                return False
        except (KeyError, AttributeError):
            return False

    @property
    def is_semi_prot(self):
        try:
            if self.protections['edit'].level == AUTOCONFIRMED:
                return True
            else:
                return False
        except (KeyError, AttributeError):
            return False

    def __repr__(self):
        return u'ProtectionInfo(%r)' % self.protections

class CoordinateIndentifier(object):
    def __init__(self, coord, page_ident=None):
        self.page_ident = page_ident
        self.lat = coord.get('lat')
        self.lon = coord.get('lon')
        self.type = coord.get('type')
        self.name = coord.get('name')
        self.dim = coord.get('dim')
        self.country = coord.get('country')
        self.region = coord.get('region')
        if coord.get('primary', False):
            self.primary = True
        else:
            self.primary = False
        return
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Wrapper to run the code from the command line.

:copyright: Copyright (c) 2015 RadiaSoft LLC.  All Rights Reserved.
:license: http://www.apache.org/licenses/LICENSE-2.0.html
"""
from __future__ import absolute_import, division, print_function
from pykern import pkinspect
from pykern import pkio
from pykern.pkcollections import PKDict
from pykern.pkdebug import pkdp
from sirepo import mpi
from sirepo import simulation_db
from sirepo.template import template_common
import sirepo.template

def run(cfg_dir):
    """Run code in ``cfg_dir``

    Args:
        cfg_dir (str): directory to run code in
    """
    template = sirepo.template.import_module(pkinspect.module_basename(run))
    _run_code()
    a = PKDict(
        # see template.warppba.open_data_file (opens last frame)
        frameIndex=None,
        run_dir=pkio.py_path(cfg_dir),
        sim_in=simulation_db.read_json(template_common.INPUT_BASE_NAME),
    )
    a.frameReport = a.sim_in.report
    a.update(a.sim_in.models[a.frameReport])
    if a.frameReport == 'laserPreviewReport':
        res = template.sim_frame_fieldAnimation(a)
    elif a.frameReport == 'beamPreviewReport':
        res = template.sim_frame_beamAnimation(a)
    else:
        raise AssertionError('invalid report: {}'.format(a.frameReport))
    simulation_db.write_result(res)

def run_background(cfg_dir, sbatch=False):
    """Run code in ``cfg_dir`` with mpi

    Args:
        cfg_dir (str): directory to run code in
    """
    template_common.exec_parameters_with_mpi()
    simulation_db.write_result({})

def _run_code():
    """Run code program with isolated locals()
    """
    r = template_common.exec_parameters()
    # advance the window until zmin is >= 0 (avoids mirroring in output)
    i = r.inc_steps
    if r.USE_BEAM:
        for x in range(0, r.diag_period, i):
            r.step(i)
    else:
        while r.w3d.zmmin + r.top.zgrid < 0:
            r.step(i)
<commit_msg>warppba<commit_after># -*- coding: utf-8 -*-
"""Wrapper to run the code from the command line.

:copyright: Copyright (c) 2015 RadiaSoft LLC.  All Rights Reserved.
:license: http://www.apache.org/licenses/LICENSE-2.0.html
"""
from __future__ import absolute_import, division, print_function
from pykern import pkinspect
from pykern import pkio
from pykern.pkcollections import PKDict
from pykern.pkdebug import pkdp
from sirepo import mpi
from sirepo import simulation_db
from sirepo.template import template_common
import sirepo.template

def run(cfg_dir):
    """Run code in ``cfg_dir``

    Args:
        cfg_dir (str): directory to run code in
    """
    template = sirepo.template.import_module(pkinspect.module_basename(run))
    _run_code()
    a = PKDict(
        # see template.warppba.open_data_file (opens last frame)
        frameIndex=None,
        run_dir=pkio.py_path(cfg_dir),
        sim_in=simulation_db.read_json(template_common.INPUT_BASE_NAME),
    )
    a.frameReport = a.sim_in.report
    a.update(a.sim_in.models[a.frameReport])
    if a.frameReport == 'laserPreviewReport':
        res = template.sim_frame_fieldAnimation(a)
    elif a.frameReport == 'beamPreviewReport':
        res = template.sim_frame_beamAnimation(a)
    else:
        raise AssertionError('invalid report: {}'.format(a.frameReport))
    simulation_db.write_result(res)

def run_background(cfg_dir, sbatch=False):
    """Run code in ``cfg_dir`` with mpi

    Args:
        cfg_dir (str): directory to run code in
    """
    template_common.exec_parameters_with_mpi()

def _run_code():
    """Run code program with isolated locals()
    """
    r = template_common.exec_parameters()
    # advance the window until zmin is >= 0 (avoids mirroring in output)
    i = r.inc_steps
    if r.USE_BEAM:
        for x in range(0, r.diag_period, i):
            r.step(i)
    else:
        while r.w3d.zmmin + r.top.zgrid < 0:
            r.step(i)
<|endoftext|>