<commit_before>#!/usr/bin/env python

from argparse import ArgumentParser
from collections import Mapping, Sequence
import ctypes
from distutils.spawn import find_executable # pylint:disable=no-name-in-module,import-error
import errno
from importlib import import_module
from json import load as json_loadf
import inspect
import os
import signal
from subprocess import Popen
import sys

try:
    input = raw_input # pylint:disable=redefined-builtin
except NameError:
    pass

def flatten_dict(prefix, data):
    # see http://stackoverflow.com/a/6036037
    for key in data:
        item = data[key]
        if prefix:
            new_prefix = prefix + '_' + key.upper()
        else:
            new_prefix = key.upper()

        try:
            is_str = isinstance(item, basestring)
        except NameError:
            is_str = isinstance(item, str)

        if isinstance(item, Mapping):
            for child_item in flatten_dict(new_prefix, item):
                yield child_item
        elif isinstance(item, Sequence) and not is_str:
            yield (new_prefix, ';'.join(item))
        else:
            yield (new_prefix, item)

class Loader(object):

    config = None

    _modules = None
    _python_bin = None
    _CSL = None

    def __init__(self, config):
        self.config = config
        self._modules = []

        if os.name == 'nt':
            os.execvp = self._win_execvp

            if not hasattr(os, 'symlink'):
                os.symlink = self._win_symlink

    def setup_project_env(self, project=None, variant=None):
        project = self.config['project'] = project or\
                os.environ.get('PROJECT_NAME',
                self.config.get('default_project'))

        variant = self.config['variant'] = variant or\
                os.environ.get('PROJECT_VARIANT',
                self.config.get('default_variant'))

        os.environ['PROJECT_NAME'] = project
        os.environ['PROJECT_VARIANT'] = variant
        return project, variant

    def setup_shell_env(self, data):
        for key, value in flatten_dict('', data):
            os.environ[key] = value

    def setup_virtualenv(self):
        venv_dir = self.get_virtualenv_dir()
        venv_bin_dir = os.path.join(venv_dir, 'bin')

        os.environ['PYTHONUSERBASE'] = venv_dir

        paths = os.environ['PATH'].split(os.pathsep)
        if paths[0] != venv_bin_dir:
            paths.insert(0, venv_bin_dir)
            os.environ['PATH'] = os.pathsep.join(paths)

    @staticmethod
    def fix_dir_separator(slash_delim_path):
        """ Text replace directory separator.

        Replace slash character (/) with the OS' directory separator.
        """
        return slash_delim_path.replace('/', os.path.sep)

    def expand_path(self, original_path):
        """ Get real path relative to project root dir.
        """
        path = self.fix_dir_separator(original_path)
        path = os.path.expanduser(path)
        return os.path.join(self.config['work_dir'], path)

    def register(self, argparser, module):
        self._modules.append(module)

        for function in module.commands:
            name = function.__name__.replace('_', '-')

            command = argparser.add_parser(name, help=inspect.getdoc(function))

            args = list(self.get_function_signature(function))
            if args[0][0] == 'self':
                args.pop(0)
            if args[0][0] == 'loader':
                args.pop(0)

            for arg in args:
                if len(arg) == 1:
                    command.add_argument(arg[0], nargs='*')
                elif len(arg) == 2:
                    name = arg[0].replace('_', '-')
                    command.add_argument('--' + name, dest=arg[0],
                            required=True, type=arg[1])
                else:
                    name = arg[0].replace('_', '-')
                    command.add_argument('--' + name, dest=arg[0],
                            default=arg[2], type=arg[1])

    def execute(self, arguments):
        for module in self._modules:
            for function in module.commands:
                name = function.__name__.replace('_', '-')

                if name != arguments._command_:
                    continue

                args = list(self.get_function_signature(function))
                if args[0][0] == 'self':
                    args.pop(0)
                if args[0][0] == 'loader':
                    args.pop(0)

                params = []
                for arg in args:
                    if len(arg) == 1:
                        params.extend(getattr(arguments, arg[0]))
                    else:
                        params.append(getattr(arguments, arg[0]))

                return function(self, *params)

    @staticmethod
    def force_symlink(target, link_name):
        try:
            os.symlink(target, link_name)
        except OSError as e:
            if e.errno == errno.EEXIST:
                os.remove(link_name)
                os.symlink(target, link_name)
            else:
                pass

    @staticmethod
    def load_module(module):
        """Load a python module by its name."""
        try:
            return import_module(module)
        except ImportError:
            sys.stderr.write('Unable to load the module: %s.\n' % module)
            exit(-1)

    def get_virtualenv_dir(self):
        return os.path.join(self.config['work_dir'],
                self.config['virtualenv_dir'])

    def get_binargs(self, script, *args):
        venv_dir = self.get_virtualenv_dir()
        script_path = os.path.join(venv_dir, 'bin', script)
        if os.path.exists(script_path):
            python_bin = self.get_python_bin()
            executable = [python_bin, script_path]
        else:
            raise RuntimeError('Cannot find executable for ' + script)

        return executable + list(args)

    def get_python_bin(self):
        python_bin = getattr(self, '_python_bin', None)
        if python_bin is not None:
            os.environ['PYTHON_BIN'] = python_bin
            return python_bin

        python_bin = 'python' + self.config['python_version']
        self._python_bin = find_executable(python_bin)
        if self._python_bin is None:
            shellenv_key = python_bin.upper() + '_BIN'
            if shellenv_key in os.environ:
                self._python_bin = os.environ[shellenv_key]
            else:
                self._python_bin = input(python_bin + ' executable location: ')

        os.environ['PYTHON_BIN'] = self._python_bin
        return self._python_bin

    def _win_symlink(self, target, link_name):
        # see http://stackoverflow.com/a/8464306
        if not hasattr(self, '_CSL'):
            csl = ctypes.windll.kernel32.CreateSymbolicLinkW
            csl.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32)
            csl.restype = ctypes.c_ubyte
            self._CSL = csl

        if target is not None and os.path.isdir(target):
            flags = 1
        else:
            flags = 0

        if self._CSL(link_name, target, flags) == 0:
            raise ctypes.WinError()

    def _win_execvp(self, cmd, argv):
        # see:
        # https://github.com/jupyter/jupyter_core/pull/54
        # https://github.com/jupyter/jupyter_core/pull/88
        process = Popen([cmd] + argv[1:], shell=True)
        # Don't raise KeyboardInterrupt in the parent process.
        # Set  this after spawning, to avoid subprocess inheriting handler.
        signal.signal(signal.SIGINT, signal.SIG_IGN)
        process.wait()
        sys.exit(process.returncode)

    @staticmethod
    def get_function_signature(func):
        try:
            signature = inspect.signature(func)
            for name, param in signature.parameters.items():
                if param.kind == param.VAR_POSITIONAL:
                    yield (name,)
                elif param.kind == param.VAR_KEYWORD:
                    continue
                else:

                    if param.annotation is param.empty:
                        type_ = None
                    else:
                        type_ = param.annotation

                    if param.default is param.empty:
                        yield (name, type_)
                    else:
                        yield (name, type_, param.default)

        except AttributeError:
            args, varargs, _, defaults = inspect.getargspec(func) # pylint:disable=deprecated-method

            if defaults is not None:
                n_non_default = len(args) - len(defaults)
                for arg in args[0:n_non_default]:
                    yield (arg, None)

                for index, arg in enumerate(args[n_non_default:]):
                    default = defaults[index + 1 - n_non_default]
                    yield (arg, None, default)

            else:
                for arg in args:
                    yield (arg, None)

            if varargs is not None:
                yield (varargs,)

    @staticmethod
    def fix_pathname(dos_path):
        fixed_path = dos_path.replace('\\', os.path.sep)
        fixed_path = fixed_path.replace('/', os.path.sep)

        if sys.platform == 'msys':
            return os.path.join('/', fixed_path.replace(':', ''))
        elif sys.platform == 'cygwin':
            return os.path.join('/cygdrive',
                    fixed_path.replace(':', ''))

        return fixed_path

root_dir = os.environ.get('ROOT_DIR', os.path.dirname(
        os.path.abspath(__file__)))

root_dir = Loader.fix_pathname(root_dir)

runner_dir = os.path.dirname(os.path.realpath(__file__))
sys.path[0] = root_dir

try:
    runner_config_file = os.environ.get('RUNNER_CONFIG_FILE',
            os.path.join(root_dir, 'etc', 'runner.json'))

    with open(runner_config_file) as f:
        runner_config = json_loadf(f)
except Exception as e: # pylint:disable=broad-except
    sys.stderr.write('Unable read configuration file etc/runner.json:\n' +\
            repr(e) + '\n')

    exit(-1)

runner_config['work_dir'] = root_dir
os.environ['ROOT_DIR'] = root_dir

argparse = ArgumentParser()
subparsers = argparse.add_subparsers(dest='_command_')
loader = Loader(runner_config)

for module_name in runner_config.get('modules', []):
    mod = loader.load_module(module_name)
    loader.register(subparsers, mod)

os.chdir(root_dir)
exit(loader.execute(argparse.parse_args()))
<commit_msg>export RUNNER_CONFIG_FILE<commit_after>#!/usr/bin/env python

from argparse import ArgumentParser
from collections import Mapping, Sequence
import ctypes
from distutils.spawn import find_executable # pylint:disable=no-name-in-module,import-error
import errno
from importlib import import_module
from json import load as json_loadf
import inspect
import os
import signal
from subprocess import Popen
import sys

try:
    input = raw_input # pylint:disable=redefined-builtin
except NameError:
    pass

def flatten_dict(prefix, data):
    # see http://stackoverflow.com/a/6036037
    for key in data:
        item = data[key]
        if prefix:
            new_prefix = prefix + '_' + key.upper()
        else:
            new_prefix = key.upper()

        try:
            is_str = isinstance(item, basestring)
        except NameError:
            is_str = isinstance(item, str)

        if isinstance(item, Mapping):
            for child_item in flatten_dict(new_prefix, item):
                yield child_item
        elif isinstance(item, Sequence) and not is_str:
            yield (new_prefix, ';'.join(item))
        else:
            yield (new_prefix, item)

class Loader(object):

    config = None

    _modules = None
    _python_bin = None
    _CSL = None

    def __init__(self, config):
        self.config = config
        self._modules = []

        if os.name == 'nt':
            os.execvp = self._win_execvp

            if not hasattr(os, 'symlink'):
                os.symlink = self._win_symlink

    def setup_project_env(self, project=None, variant=None):
        project = self.config['project'] = project or\
                os.environ.get('PROJECT_NAME',
                self.config.get('default_project'))

        variant = self.config['variant'] = variant or\
                os.environ.get('PROJECT_VARIANT',
                self.config.get('default_variant'))

        os.environ['PROJECT_NAME'] = project
        os.environ['PROJECT_VARIANT'] = variant
        return project, variant

    def setup_shell_env(self, data):
        for key, value in flatten_dict('', data):
            os.environ[key] = value

    def setup_virtualenv(self):
        venv_dir = self.get_virtualenv_dir()
        venv_bin_dir = os.path.join(venv_dir, 'bin')

        os.environ['PYTHONUSERBASE'] = venv_dir

        paths = os.environ['PATH'].split(os.pathsep)
        if paths[0] != venv_bin_dir:
            paths.insert(0, venv_bin_dir)
            os.environ['PATH'] = os.pathsep.join(paths)

    @staticmethod
    def fix_dir_separator(slash_delim_path):
        """ Text replace directory separator.

        Replace slash character (/) with the OS' directory separator.
        """
        return slash_delim_path.replace('/', os.path.sep)

    def expand_path(self, original_path):
        """ Get real path relative to project root dir.
        """
        path = self.fix_dir_separator(original_path)
        path = os.path.expanduser(path)
        return os.path.join(self.config['work_dir'], path)

    def register(self, argparser, module):
        self._modules.append(module)

        for function in module.commands:
            name = function.__name__.replace('_', '-')

            command = argparser.add_parser(name, help=inspect.getdoc(function))

            args = list(self.get_function_signature(function))
            if args[0][0] == 'self':
                args.pop(0)
            if args[0][0] == 'loader':
                args.pop(0)

            for arg in args:
                if len(arg) == 1:
                    command.add_argument(arg[0], nargs='*')
                elif len(arg) == 2:
                    name = arg[0].replace('_', '-')
                    command.add_argument('--' + name, dest=arg[0],
                            required=True, type=arg[1])
                else:
                    name = arg[0].replace('_', '-')
                    command.add_argument('--' + name, dest=arg[0],
                            default=arg[2], type=arg[1])

    def execute(self, arguments):
        for module in self._modules:
            for function in module.commands:
                name = function.__name__.replace('_', '-')

                if name != arguments._command_:
                    continue

                args = list(self.get_function_signature(function))
                if args[0][0] == 'self':
                    args.pop(0)
                if args[0][0] == 'loader':
                    args.pop(0)

                params = []
                for arg in args:
                    if len(arg) == 1:
                        params.extend(getattr(arguments, arg[0]))
                    else:
                        params.append(getattr(arguments, arg[0]))

                return function(self, *params)

    @staticmethod
    def force_symlink(target, link_name):
        try:
            os.symlink(target, link_name)
        except OSError as e:
            if e.errno == errno.EEXIST:
                os.remove(link_name)
                os.symlink(target, link_name)
            else:
                pass

    @staticmethod
    def load_module(module):
        """Load a python module by its name."""
        try:
            return import_module(module)
        except ImportError:
            sys.stderr.write('Unable to load the module: %s.\n' % module)
            exit(-1)

    def get_virtualenv_dir(self):
        return os.path.join(self.config['work_dir'],
                self.config['virtualenv_dir'])

    def get_binargs(self, script, *args):
        venv_dir = self.get_virtualenv_dir()
        script_path = os.path.join(venv_dir, 'bin', script)
        if os.path.exists(script_path):
            python_bin = self.get_python_bin()
            executable = [python_bin, script_path]
        else:
            raise RuntimeError('Cannot find executable for ' + script)

        return executable + list(args)

    def get_python_bin(self):
        python_bin = getattr(self, '_python_bin', None)
        if python_bin is not None:
            os.environ['PYTHON_BIN'] = python_bin
            return python_bin

        python_bin = 'python' + self.config['python_version']
        self._python_bin = find_executable(python_bin)
        if self._python_bin is None:
            shellenv_key = python_bin.upper() + '_BIN'
            if shellenv_key in os.environ:
                self._python_bin = os.environ[shellenv_key]
            else:
                self._python_bin = input(python_bin + ' executable location: ')

        os.environ['PYTHON_BIN'] = self._python_bin
        return self._python_bin

    def _win_symlink(self, target, link_name):
        # see http://stackoverflow.com/a/8464306
        if not hasattr(self, '_CSL'):
            csl = ctypes.windll.kernel32.CreateSymbolicLinkW
            csl.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32)
            csl.restype = ctypes.c_ubyte
            self._CSL = csl

        if target is not None and os.path.isdir(target):
            flags = 1
        else:
            flags = 0

        if self._CSL(link_name, target, flags) == 0:
            raise ctypes.WinError()

    def _win_execvp(self, cmd, argv):
        # see:
        # https://github.com/jupyter/jupyter_core/pull/54
        # https://github.com/jupyter/jupyter_core/pull/88
        process = Popen([cmd] + argv[1:], shell=True)
        # Don't raise KeyboardInterrupt in the parent process.
        # Set  this after spawning, to avoid subprocess inheriting handler.
        signal.signal(signal.SIGINT, signal.SIG_IGN)
        process.wait()
        sys.exit(process.returncode)

    @staticmethod
    def get_function_signature(func):
        try:
            signature = inspect.signature(func)
            for name, param in signature.parameters.items():
                if param.kind == param.VAR_POSITIONAL:
                    yield (name,)
                elif param.kind == param.VAR_KEYWORD:
                    continue
                else:

                    if param.annotation is param.empty:
                        type_ = None
                    else:
                        type_ = param.annotation

                    if param.default is param.empty:
                        yield (name, type_)
                    else:
                        yield (name, type_, param.default)

        except AttributeError:
            args, varargs, _, defaults = inspect.getargspec(func) # pylint:disable=deprecated-method

            if defaults is not None:
                n_non_default = len(args) - len(defaults)
                for arg in args[0:n_non_default]:
                    yield (arg, None)

                for index, arg in enumerate(args[n_non_default:]):
                    default = defaults[index + 1 - n_non_default]
                    yield (arg, None, default)

            else:
                for arg in args:
                    yield (arg, None)

            if varargs is not None:
                yield (varargs,)

    @staticmethod
    def fix_pathname(dos_path):
        fixed_path = dos_path.replace('\\', os.path.sep)
        fixed_path = fixed_path.replace('/', os.path.sep)

        if sys.platform == 'msys':
            return os.path.join('/', fixed_path.replace(':', ''))
        elif sys.platform == 'cygwin':
            return os.path.join('/cygdrive',
                    fixed_path.replace(':', ''))

        return fixed_path

if __name__ == '__main__':
    root_dir = os.environ.get('ROOT_DIR', os.path.dirname(
            os.path.abspath(__file__)))

    root_dir = Loader.fix_pathname(root_dir)

    runner_dir = os.path.dirname(os.path.realpath(__file__))
    sys.path[0] = root_dir

    try:
        if 'RUNNER_CONFIG_FILE' in os.environ:
            runner_config_file = os.environ['RUNNER_CONFIG_FILE']
        else:
            runner_config_file = os.path.join(root_dir, 'etc', 'runner.json')
            os.environ['RUNNER_CONFIG_FILE'] = runner_config_file

        with open(runner_config_file) as f:
            runner_config = json_loadf(f)
    except Exception as e: # pylint:disable=broad-except
        sys.stderr.write('Unable read configuration file etc/runner.json:\n' +\
                repr(e) + '\n')

        exit(-1)

    runner_config['work_dir'] = root_dir
    os.environ['ROOT_DIR'] = root_dir

    argparse = ArgumentParser()
    subparsers = argparse.add_subparsers(dest='_command_')
    loader = Loader(runner_config)

    for module_name in runner_config.get('modules', []):
        mod = loader.load_module(module_name)
        loader.register(subparsers, mod)

    os.chdir(root_dir)
    exit(loader.execute(argparse.parse_args()))
<|endoftext|>


<commit_before>from itertools import chain

from django.db import transaction, IntegrityError

__all__ = ["merge_persons"]

text_attrs = [
    "first_name",
    "last_name",
    "gender",
    "contact_phone",
    "location_address1",
    "location_address2",
    "location_city",
    "location_citycode",
    "location_zip",
    "location_country",
]

boolean_attrs = [
    "is_insoumise",
    "subscribed",
    "membre_reseau_elus",
    "subscribed_sms",
    "draw_participation",
]

def merge_attr_or(e1, e2, attrs, default=None):
    if isinstance(attrs, str):
        attrs = [attrs]
    for attr in attrs:
        setattr(e1, attr, getattr(e1, attr) or getattr(e2, attr, default))

def merge_persons(p1, p2):
    """Merge the two persons together, keeping p1 and deleting p2, associating all information linked to p2 to p1"""

    with transaction.atomic():
        # check if it's not the same user
        if p1.pk == p2.pk:
            raise ValueError

        # simply copy any text attribute that is missing on p1
        merge_attr_or(p1, p2, text_attrs, default="")

        # simply take the OR of boolean fields
        merge_attr_or(p1, p2, boolean_attrs, default=False)

        # for commentaries, we juxtapose them
        p1.commentaires = f"{p1.commentaires}\n\n{p2.commentaires}".strip()

        # for la date de naissance :
        merge_attr_or(p1, p2, "date_of_birth", default=None)

        # for memberships, we want to carefully copy the properties in case there is a duplicate
        current_memberships = p1.memberships.select_for_update()
        groups = {m.supportgroup_id: m for m in current_memberships}

        # copying person tags
        missing_tags = p2.tags.all().difference(p1.tags.all())
        p1.tags.add(*missing_tags)

        for m2 in p2.memberships.all():
            if m2.supportgroup_id in groups:
                m1 = groups[m2.supportgroup_id]
                m1.created = min(m1.created, m2.created)
                m1.membership_type = max(m1.membership_type, m2.membership_type)
                merge_attr_or(m1, m2, "notifications_enabled", default=False)

                m1.save()
                m2.delete()
            else:
                m2.person = p1
                m2.save()

        # for organizer configs, we are as careful as we were for memberships
        current_organizer_configs = p1.organizer_configs.select_for_update()
        organized_events = {o.event_id: o for o in current_organizer_configs}

        for o2 in p2.organizer_configs.select_for_update():
            if o2.event_id in organized_events:
                o1 = organized_events[o2.event_id]
                merge_attr_or(o1, o2, "is_creator", default=False)
                merge_attr_or(o1, o2, "notifications_enabled", default=False)
                merge_attr_or(o1, o2, "as_group", default=False)

                o1.save()
                o2.delete()
            else:
                o2.person = p1
                o2.save()

        # we don't care as much for rsvps: reassign or delete is enough
        current_rsvps = p1.rsvps.select_for_update()
        rsvped_events = {r.event_id: r for r in current_rsvps}

        for r2 in p2.rsvps.all():
            if r2.event_id in rsvped_events:
                r2.delete()
            else:
                r2.person = p1
                r2.save()

        # we reassign email addresses as well
        # use list to make sure the querysets are evaluated now, before modifying the addresses
        email_order_1 = list(p1.get_personemail_order())
        email_order_2 = list(p2.get_personemail_order())
        for e2 in p2.emails.all():
            e2.person = p1
            e2.save()
        # and set back the order
        p1.set_personemail_order(email_order_1 + email_order_2)

        # We reassign simply for these categories
        p2.form_submissions.update(person=p1)
        p2.payments.update(person=p1)
        p2.subscriptions.update(person=p2)
        p2.event_images.update(author=p1)

        # we reassign poll choices only if p1 has not voted yet
        p1_votes = set(pc.poll_id for pc in p1.poll_choices.all())
        p2.poll_choices.exclude(poll_id__in=p1_votes).update(person=p1)

        # we reassign mandates as long as we don't get any integrity error:
        # in that case it most likely means the mandate is already registered
        # the first object
        all_mandates = chain(
            p2.mandats_municipaux.all(),
            p2.mandats_departementaux.all(),
            p2.mandats_regionaux.all(),
        )
        for m in all_mandates:
            try:
                with transaction.atomic():
                    m.person = p1
                    m.save()
            except IntegrityError:
                pass

        # finally we need to delete the second person
        p2.delete()
<commit_msg>feat: rend la fonction de merge de personnes plus sûre<commit_after>from itertools import chain

from django.db import transaction, IntegrityError

__all__ = ["merge_persons"]

from agir.people.models import Person

def _set_or_fields(o1, o2, attr, default=None):
    return setattr(o1, attr, getattr(o1, attr, None) or getattr(o2, attr, default))

# GENERIC MERGING STRATEGIES
def merge_nullable(p1, p2, field):
    # gère le cas particulier des valeurs « falsy » mais non `None`
    if getattr(p1, field.name, None) is not None:
        return
    setattr(p1, field.name, getattr(p2, field.name, None))

def merge_text_fields(p1, p2, field):
    _set_or_fields(p1, p2, field.name, default="")

def merge_boolean_field(p1, p2, field):
    _set_or_fields(p1, p2, field.name, default=False)

def merge_dicts(p1, p2, field):
    setattr(
        p1, field.name, {**getattr(p2, field.name, {}), **getattr(p1, field.name, {})}
    )

def merge_reassign_related(p1, p2, field):
    for rel_obj in getattr(p2, field.get_accessor_name()).all():
        try:
            setattr(rel_obj, field.remote_field.name, p1)
            rel_obj.save(update_fields=[field.remote_field.name])
        except IntegrityError:
            # It means the related object exists, in a sense, on p1 already
            # we don't need to do anything for this related object
            continue

def merge_with_priority_ordering(order):
    order = list(order)

    def merge_func(p1, p2, field):
        setattr(
            p1,
            field.name,
            min([getattr(p1, field.name), getattr(p2, field.name)], key=order.index),
        )

    return merge_func

# SPECIFIC MERGING STRATEGIES
def merge_newsletters(p1, p2, _field):
    p1.newsletters = list(set(p1.newsletters).union(p2.newsletters))

def merge_tags(p1, p2, _field):
    # copying person tags
    missing_tags = p2.tags.all().difference(p1.tags.all())
    p1.tags.add(*missing_tags)

def merge_email_addresses(p1, p2, _field):
    # we reassign email addresses as well
    # use list to make sure the querysets are evaluated now, before modifying the addresses
    email_order_1 = list(p1.get_personemail_order())
    email_order_2 = list(p2.get_personemail_order())
    for e2 in p2.emails.all():
        e2.person = p1
        e2.save()
    # and set back the order
    p1.set_personemail_order(email_order_1 + email_order_2)

def merge_memberships(p1, p2, _field):
    # for memberships, we want to carefully copy the properties in case there is a duplicate
    current_memberships = p1.memberships.select_for_update()
    groups = {m.supportgroup_id: m for m in current_memberships}

    for m2 in p2.memberships.all():
        if m2.supportgroup_id in groups:
            m1 = groups[m2.supportgroup_id]
            m1.created = min(m1.created, m2.created)
            m1.membership_type = max(m1.membership_type, m2.membership_type)
            _set_or_fields(m1, m2, "notifications_enabled", default=False)

            m1.save()
            m2.delete()
        else:
            m2.person = p1
            m2.save()

def merge_rsvps(p1, p2, _field):
    # we don't care as much for rsvps: reassign or delete is enough
    current_rsvps = p1.rsvps.select_for_update()
    rsvped_events = {r.event_id: r for r in current_rsvps}

    for r2 in p2.rsvps.all():
        if r2.event_id in rsvped_events:
            r2.delete()
        else:
            r2.person = p1
            r2.save()

def merge_organizer_configs(p1, p2, _field):
    # for organizer configs, we are as careful as we were for memberships
    current_organizer_configs = p1.organizer_configs.select_for_update()
    organized_events = {o.event_id: o for o in current_organizer_configs}

    for o2 in p2.organizer_configs.select_for_update():
        if o2.event_id in organized_events:
            o1 = organized_events[o2.event_id]
            _set_or_fields(o1, o2, "is_creator", default=False)
            _set_or_fields(o1, o2, "notifications_enabled", default=False)
            _set_or_fields(o1, o2, "as_group", default=False)

            o1.save()
            o2.delete()
        else:
            o2.person = p1
            o2.save()

def merge_poll_choices(p1, p2, _field):
    # we reassign poll choices only if p1 has not voted yet
    p1_votes = set(pc.poll_id for pc in p1.poll_choices.all())
    p2.poll_choices.exclude(poll_id__in=p1_votes).update(person=p1)

def merge_comments(p1, p2, _field):
    p1.commentaires = f"{p1.commentaires}\n\n{p2.commentaires}".strip()

MERGE_STRATEGIES = {
    # Many2one
    "form_submissions": merge_reassign_related,
    "emails": merge_email_addresses,
    "personvalidationsms": None,
    "events": None,
    "organized_events": None,
    "rsvps": merge_rsvps,
    "organizer_configs": merge_organizer_configs,
    "event_images": merge_reassign_related,
    "supportgroups": None,
    "memberships": merge_memberships,
    "poll_choices": merge_reassign_related,
    "payments": merge_reassign_related,
    "subscriptions": merge_reassign_related,
    "received_notification": merge_reassign_related,
    "notification": None,
    "municipales2020_commune": None,
    "mandat_municipal": merge_reassign_related,
    "mandat_departemental": merge_reassign_related,
    "mandat_regional": merge_reassign_related,
    "campaignsentevent": None,
    # Simple fields
    "created": None,
    "modified": None,
    "id": None,
    "nb_id": None,
    "coordinates": merge_nullable,
    "coordinates_type": merge_text_fields,
    "location_name": merge_text_fields,
    "location_address1": merge_text_fields,
    "location_address2": merge_text_fields,
    "location_citycode": merge_nullable,
    "location_city": merge_text_fields,
    "location_zip": merge_text_fields,
    "location_state": merge_text_fields,
    "location_country": merge_text_fields,
    "location_address": None,
    "role": None,
    "auto_login_salt": None,
    "is_insoumise": merge_boolean_field,
    "is_2022": merge_boolean_field,
    "membre_reseau_elus": merge_with_priority_ordering(
        [
            Person.MEMBRE_RESEAU_EXCLUS,
            Person.MEMBRE_RESEAU_OUI,
            Person.MEMBRE_RESEAU_NON,
            Person.MEMBRE_RESEAU_SOUHAITE,
            Person.MEMBRE_RESEAU_INCONNU,
        ]
    ),
    "newsletters": merge_newsletters,
    "subscribed_sms": merge_boolean_field,
    "event_notifications": merge_boolean_field,
    "group_notifications": merge_boolean_field,
    "draw_participation": merge_boolean_field,
    "first_name": merge_text_fields,
    "last_name": merge_text_fields,
    "contact_phone": merge_text_fields,
    "contact_phone_status": merge_with_priority_ordering(
        [
            Person.CONTACT_PHONE_VERIFIED,
            Person.CONTACT_PHONE_PENDING,
            Person.CONTACT_PHONE_UNVERIFIED,
        ]
    ),
    "gender": merge_text_fields,
    "date_of_birth": merge_nullable,
    "mandates": None,
    "meta": merge_dicts,
    "commentaires": merge_comments,
    "search": None,
    "tags": merge_tags,
}

def merge_persons(p1, p2):
    """Merge the two persons together, keeping p1 and deleting p2, associating all information linked to p2 to p1"""
    fields = Person._meta.get_fields()
    missing_fields = {f.name for f in fields}.difference(MERGE_STRATEGIES)
    unknown_fields = set(MERGE_STRATEGIES).difference({f.name for f in fields})

    if missing_fields or unknown_fields:
        message = []
        if missing_fields:
            message.append(f"Fields without strategy: {missing_fields!r}")
        if unknown_fields:
            message.append(f"Unknown fields' name: {unknown_fields!r}")

        raise AssertionError("\n".join(message))

    # check if it's not the same user
    if p1.pk == p2.pk:
        raise ValueError("Cannot merge a person instance with itself")

    with transaction.atomic():
        for f in fields:
            func = MERGE_STRATEGIES.get(f.name)
            if func is not None:
                func(p1, p2, f)

        p1.save()

        # finally we need to delete the second person
        p2.delete()
<|endoftext|>


<commit_before># pylint: disable=C0103
# pylint: disable=C0111
# pylint: disable=C0301
import os
import re
import json
import sys
import logging

from uuid import uuid4
from pprint import pformat
from subprocess import Popen, PIPE
from time import sleep

from lib.util import print_debug
from lib.util import print_message
from lib.util import check_slurm_job_submission
from lib.util import push_event
from lib.util import cmd_exists
from lib.util import get_climo_output_files
from JobStatus import JobStatus

class Climo(object):
    """
    A wrapper around ncclimo, used to compute the climotologies from raw output data
    """
    def __init__(self, config, event_list):
        self.event_list = event_list
        self.config = {}
        self.status = JobStatus.INVALID
        self.type = 'climo'
        self.uuid = uuid4().hex
        self.yearset = config.get('yearset', 0)
        self.job_id = 0
        self.depends_on = []
        self.outputs = {
            'status': self.status,
            'climos': '',
            'regrid': '',
            'console_output': ''
        }
        self.inputs = {
            'start_year': '',
            'end_year': '',
            'caseId': '',
            'annual_mode': 'sdd',
            'input_directory': '',
            'climo_output_directory': '',
            'regrid_output_directory': '',
            'regrid_map_path': '',
            'year_set': '',
            'ncclimo_path': ''
        }
        self.proc = None
        self.slurm_args = {
            'num_cores': '-n 16', # 16 cores
            'run_time': '-t 0-02:00', # 2 hours run time
            'num_machines': '-N 1', # run on one machine
            'oversubscribe': '--oversubscribe'
        }
        self.prevalidate(config)

    def get_type(self):
        """
        Returns job type
        """
        return self.type

    def execute(self, batch=False):
        """
        Calls ncclimo in a subprocess
        """
        ncclimo = os.path.join(self.config['ncclimo_path'], 'ncclimo')
        cmd = [
            ncclimo,
            '-c', self.config['caseId'],
            '-a', self.config['annual_mode'],
            '-s', str(self.config['start_year']),
            '-e', str(self.config['end_year']),
            '-i', self.config['input_directory'],
            '-r', self.config['regrid_map_path'],
            '-o', self.config['climo_output_directory'],
            '-O', self.config['regrid_output_directory'],
        ]
        if not batch:
            # Not running in batch mode
            while True:
                try:
                    self.proc = Popen(
                        cmd,
                        stdout=PIPE,
                        stderr=PIPE,
                        shell=False)
                    break
                except:
                    sleep(1)
            self.status = JobStatus.RUNNING
            done = 2
            console_output = ''
            while done != 0:
                done = self.proc.poll()
                lines = self.proc.stdout.readlines()
                for line in lines:
                    console_output += line
                lines = self.proc.stderr.readlines()
                for line in lines:
                    console_output += line
                print console_output
                if done < 0:
                    break
                sleep(1)
                self.outputs['console_output'] = console_output
                print console_output

            self.status = JobStatus.COMPLETED
            return 0
        else:
            # Submitting the job to SLURM
            expected_name = 'ncclimo_set_{year_set}_{start}_{end}_{uuid}'.format(
                year_set=self.config.get('year_set'),
                start=self.config.get('start_year'),
                end=self.config.get('end_year'),
                uuid=self.uuid[:5])
            run_scripts_path = os.path.join(os.getcwd(), 'run_scripts')
            run_script = os.path.join(run_scripts_path, expected_name)

            if not os.path.exists(run_scripts_path):
                os.makedirs(run_scripts_path)

            self.slurm_args['error_file'] = '-e {error_file}'.format(error_file=run_script + '.err')
            self.slurm_args['output_file'] = '-o {output_file}'.format(output_file=run_script + '.out')

            with open(run_script, 'w') as batchfile:
                batchfile.write('#!/bin/bash\n')
                slurm_prefix = '\n'.join(['#SBATCH ' + self.slurm_args[s] for s in self.slurm_args]) + '\n'
                batchfile.write(slurm_prefix)
                slurm_command = ' '.join(cmd)
                batchfile.write(slurm_command)

            slurm_cmd = ['sbatch', run_script]
            started = False
            retry_count = 0
            while not started and retry_count < 5:
                while True:
                    try:
                        self.proc = Popen(slurm_cmd, stdout=PIPE, stderr=PIPE)
                        break
                    except:
                        sleep(1)
                _, _ = self.proc.communicate()
                started, job_id = check_slurm_job_submission(expected_name)
                if started:
                    self.status = JobStatus.SUBMITTED
                    self.job_id = job_id
                    message = '## {type} id: {id} changed state to {state}'.format(
                        type=self.get_type(),
                        id=self.job_id,
                        state=self.status)
                    logging.info(message)
                    message = '{type} id: {id} changed state to {state}'.format(
                        type=self.get_type(),
                        id=self.job_id,
                        state=self.status)
                    self.event_list = push_event(self.event_list, message)

                else:
                    logging.warning('Error starting climo job, trying again attempt %s', str(retry_count))
                    retry_count += 1

            if retry_count >= 5:
                self.status = JobStatus.FAILED
                message = '## {type} id: {id} changed state to {state}'.format(
                    type=self.get_type(),
                    id=self.job_id,
                    state=self.status)
                logging.info(message)
                message = '{type} id: {id} changed state to {state}'.format(
                    type=self.get_type(),
                    id=self.job_id,
                    state=self.status)
                self.event_list = push_event(self.event_list, message)
                self.job_id = 0
            return self.job_id

    def set_status(self, status):
        self.status = status

    def save(self, conf_path):
        """
        Saves job configuration to a json file at conf_path
        """
        try:
            with open(conf_path, 'r') as infile:
                config = json.load(infile)
            with open(conf_path, 'w') as outfile:
                config[self.uuid]['inputs'] = self.config
                config[self.uuid]['outputs'] = self.outputs
                config[self.uuid]['type'] = self.type
                json.dump(config, outfile, indent=4)
        except Exception as e:
            print_message('Error saving configuration file')
            print_debug(e)
            raise

    def __str__(self):
        return pformat({
            'type': self.type,
            'config': self.config,
            'status': self.status,
            'depends_on': self.depends_on,
            'uuid': self.uuid,
            'job_id': self.job_id
        }, indent=4)

    def prevalidate(self, config):
        """
        Prerun validation for inputs
        """
        if self.status == JobStatus.VALID:
            return 0
        for i in config:
            if i not in self.inputs:
                # print_message("Unexpected arguement: {}, {}".format(i, config[i]))
                pass
            else:
                self.config[i] = config.get(i)
        self.status = JobStatus.VALID

        # after checking that the job is valid to run,
        # check if the output already exists and the job actually needs to run
        if os.path.exists(self.config.get('climo_output_directory')):
            set_start_year = self.config.get('start_year')
            set_end_year = self.config.get('end_year')
            contents = os.listdir(self.config.get('climo_output_directory'))

            file_list = get_climo_output_files(
                input_path=self.config.get('climo_output_directory'),
                set_start_year=self.config.get('start_year'),
                set_end_year=self.config.get('end_year'))

            if len(file_list) >= 17:
                self.status = JobStatus.COMPLETED
                # print_message('Ncclimo job already computed, skipping', 'ok')
                message = 'Ncclimo job already computed, skipping'
                self.event_list = push_event(self.event_list, message)
            return 0

    def postvalidate(self):
        """
        Post execution validation
        """
        print "post validation"
<commit_msg>Increases the maximum run time for ncclimo<commit_after># pylint: disable=C0103
# pylint: disable=C0111
# pylint: disable=C0301
import os
import re
import json
import sys
import logging

from uuid import uuid4
from pprint import pformat
from subprocess import Popen, PIPE
from time import sleep

from lib.util import print_debug
from lib.util import print_message
from lib.util import check_slurm_job_submission
from lib.util import push_event
from lib.util import cmd_exists
from lib.util import get_climo_output_files
from JobStatus import JobStatus

class Climo(object):
    """
    A wrapper around ncclimo, used to compute the climotologies from raw output data
    """
    def __init__(self, config, event_list):
        self.event_list = event_list
        self.config = {}
        self.status = JobStatus.INVALID
        self.type = 'climo'
        self.uuid = uuid4().hex
        self.yearset = config.get('yearset', 0)
        self.job_id = 0
        self.depends_on = []
        self.outputs = {
            'status': self.status,
            'climos': '',
            'regrid': '',
            'console_output': ''
        }
        self.inputs = {
            'start_year': '',
            'end_year': '',
            'caseId': '',
            'annual_mode': 'sdd',
            'input_directory': '',
            'climo_output_directory': '',
            'regrid_output_directory': '',
            'regrid_map_path': '',
            'year_set': '',
            'ncclimo_path': ''
        }
        self.proc = None
        self.slurm_args = {
            'num_cores': '-n 16', # 16 cores
            'run_time': '-t 0-05:00', # 2 hours run time
            'num_machines': '-N 1', # run on one machine
            'oversubscribe': '--oversubscribe'
        }
        self.prevalidate(config)

    def get_type(self):
        """
        Returns job type
        """
        return self.type

    def execute(self, batch=False):
        """
        Calls ncclimo in a subprocess
        """
        ncclimo = os.path.join(self.config['ncclimo_path'], 'ncclimo')
        # ncclimo = 'ncclimo'
        cmd = [
            ncclimo,
            '-c', self.config['caseId'],
            '-a', self.config['annual_mode'],
            '-s', str(self.config['start_year']),
            '-e', str(self.config['end_year']),
            '-i', self.config['input_directory'],
            '-r', self.config['regrid_map_path'],
            '-o', self.config['climo_output_directory'],
            '-O', self.config['regrid_output_directory'],
        ]
        if not batch:
            # Not running in batch mode
            while True:
                try:
                    self.proc = Popen(
                        cmd,
                        stdout=PIPE,
                        stderr=PIPE,
                        shell=False)
                    break
                except:
                    sleep(1)
            self.status = JobStatus.RUNNING
            done = 2
            console_output = ''
            while done != 0:
                done = self.proc.poll()
                lines = self.proc.stdout.readlines()
                for line in lines:
                    console_output += line
                lines = self.proc.stderr.readlines()
                for line in lines:
                    console_output += line
                print console_output
                if done < 0:
                    break
                sleep(1)
                self.outputs['console_output'] = console_output
                print console_output

            self.status = JobStatus.COMPLETED
            return 0
        else:
            # Submitting the job to SLURM
            expected_name = 'ncclimo_set_{year_set}_{start}_{end}_{uuid}'.format(
                year_set=self.config.get('year_set'),
                start=self.config.get('start_year'),
                end=self.config.get('end_year'),
                uuid=self.uuid[:5])
            run_scripts_path = os.path.join(os.getcwd(), 'run_scripts')
            run_script = os.path.join(run_scripts_path, expected_name)

            if not os.path.exists(run_scripts_path):
                os.makedirs(run_scripts_path)

            self.slurm_args['error_file'] = '-e {error_file}'.format(error_file=run_script + '.err')
            self.slurm_args['output_file'] = '-o {output_file}'.format(output_file=run_script + '.out')

            with open(run_script, 'w') as batchfile:
                batchfile.write('#!/bin/bash\n')
                slurm_prefix = '\n'.join(['#SBATCH ' + self.slurm_args[s] for s in self.slurm_args]) + '\n'
                batchfile.write(slurm_prefix)
                slurm_command = ' '.join(cmd)
                batchfile.write(slurm_command)

            slurm_cmd = ['sbatch', run_script]
            started = False
            retry_count = 0
            while not started and retry_count < 5:
                while True:
                    try:
                        self.proc = Popen(slurm_cmd, stdout=PIPE, stderr=PIPE)
                        break
                    except:
                        sleep(1)
                _, _ = self.proc.communicate()
                started, job_id = check_slurm_job_submission(expected_name)
                if started:
                    self.status = JobStatus.SUBMITTED
                    self.job_id = job_id
                    message = '## {type} id: {id} changed state to {state}'.format(
                        type=self.get_type(),
                        id=self.job_id,
                        state=self.status)
                    logging.info(message)
                    message = '{type} id: {id} changed state to {state}'.format(
                        type=self.get_type(),
                        id=self.job_id,
                        state=self.status)
                    self.event_list = push_event(self.event_list, message)

                else:
                    logging.warning('Error starting climo job, trying again attempt %s', str(retry_count))
                    retry_count += 1

            if retry_count >= 5:
                self.status = JobStatus.FAILED
                message = '## {type} id: {id} changed state to {state}'.format(
                    type=self.get_type(),
                    id=self.job_id,
                    state=self.status)
                logging.info(message)
                message = '{type} id: {id} changed state to {state}'.format(
                    type=self.get_type(),
                    id=self.job_id,
                    state=self.status)
                self.event_list = push_event(self.event_list, message)
                self.job_id = 0
            return self.job_id

    def set_status(self, status):
        self.status = status

    def save(self, conf_path):
        """
        Saves job configuration to a json file at conf_path
        """
        try:
            with open(conf_path, 'r') as infile:
                config = json.load(infile)
            with open(conf_path, 'w') as outfile:
                config[self.uuid]['inputs'] = self.config
                config[self.uuid]['outputs'] = self.outputs
                config[self.uuid]['type'] = self.type
                json.dump(config, outfile, indent=4)
        except Exception as e:
            print_message('Error saving configuration file')
            print_debug(e)
            raise

    def __str__(self):
        return pformat({
            'type': self.type,
            'config': self.config,
            'status': self.status,
            'depends_on': self.depends_on,
            'uuid': self.uuid,
            'job_id': self.job_id
        }, indent=4)

    def prevalidate(self, config):
        """
        Prerun validation for inputs
        """
        if self.status == JobStatus.VALID:
            return 0
        for i in config:
            if i not in self.inputs:
                # print_message("Unexpected arguement: {}, {}".format(i, config[i]))
                pass
            else:
                self.config[i] = config.get(i)
        self.status = JobStatus.VALID

        # after checking that the job is valid to run,
        # check if the output already exists and the job actually needs to run
        if os.path.exists(self.config.get('climo_output_directory')):
            set_start_year = self.config.get('start_year')
            set_end_year = self.config.get('end_year')
            contents = os.listdir(self.config.get('climo_output_directory'))

            file_list = get_climo_output_files(
                input_path=self.config.get('climo_output_directory'),
                set_start_year=self.config.get('start_year'),
                set_end_year=self.config.get('end_year'))

            if len(file_list) >= 17:
                self.status = JobStatus.COMPLETED
                # print_message('Ncclimo job already computed, skipping', 'ok')
                message = 'Ncclimo job already computed, skipping'
                self.event_list = push_event(self.event_list, message)
            return 0

    def postvalidate(self):
        """
        Post execution validation
        """
        print "post validation"
<|endoftext|>


<commit_before>"""
Infrastructure for dealing with plugins.
"""

import pkg_resources
import warnings

from collections.abc import Mapping

from .exceptions import PluginExistsError, PluginNotFoundError, InvalidPluginError, verify_type

__author__ = 'Aaron Hosford'
__all__ = [
    'PluginGroup',
    'CONFIG_LOADERS',
    'URL_SCHEMES',
    'load_plugins',
    'config_loader',
    'url_scheme',
]

class PluginGroup(Mapping):
    """
    A PluginGroup is a collection of plugins registered under the same entry point group name. It
    supports both install-time and run-time registration of plugins, case-insensitive lookup by
    name, and plugin type-checking.
    """

    def __init__(self, name, value_type=None):
        verify_type(name, str, non_empty=True)
        if value_type is not None:
            verify_type(value_type, type)

        self._name = name
        self._value_type = value_type
        self._original_names = {}
        self._registry = {}

    def load(self, warn=True):
        """
        Load any pre-registered plugins.

        :param warn: Whether to warn if a registered plugin could not be loaded.
        :return: None
        """
        for entry_point in pkg_resources.iter_entry_points(group=self._name):
            try:
                self.register(entry_point.name, entry_point.load())
            except Exception as exc:
                if warn:
                    warnings.warn(str(exc))

    def register(self, name, value):
        """
        Register a new plugin.

        :param name: The name of the plugin.
        :param value: The plugin.
        :return: None
        """
        verify_type(name, str, non_empty=True)
        if name.lower() in self._registry and self._registry[name.lower()] != value:
            raise PluginExistsError("Another plugin by this name has already been registered: %s" % name)
        if self._value_type is not None and not isinstance(value, self._value_type):
            raise InvalidPluginError("Plugin %s is not a/an %s." % (name, self._value_type.__name__))
        self._original_names[name] = name
        self._registry[name.lower()] = value

    def __getitem__(self, name):
        if not isinstance(name, str):
            return NotImplemented
        if name.lower() not in self._registry:
            raise PluginNotFoundError(name)
        return self._registry[name.lower()]

    def __setitem__(self, name, value):
        if not isinstance(name, str):
            return NotImplemented
        self.register(name, value)

    def __iter__(self):
        return iter(self._original_names)

    def __contains__(self, name):
        if not isinstance(name, str):
            return NotImplemented
        return name.lower() in self._registry

    def __len__(self):
        return len(self._registry)

    def get(self, name, default=None):
        """
        Get the plugin by name.

        :param name: The name of the plugin.
        :param default: The default value if the plugin does not exist.
        :return: The plugin, or the default.
        """
        verify_type(name, str, non_empty=True)
        return self._registry.get(name.lower(), default)

    def plugin(self, name=NotImplemented, value=NotImplemented):
        """
        A decorator for in-line registration of plugins.

        Registering a plugin function under its own name to group PLUGIN_GROUP::

            @PLUGIN_GROUP.plugin
            def aptly_named_plugin_function(arg1, arg2):
                ...

        Registering a plugin function to a different name to group PLUGIN_GROUP::

            @PLUGIN_GROUP.plugin('BetterPluginName')
            def less_aptly_named_plugin_function(arg1, arg2):
                ...

        :param value: The value to be registered as a plugin.
        :param name: The name to register the plugin under.
        :return: The value, unchanged, after registration, or a parameter-free plugin decorator.
        """

        assert name is not NotImplemented or value is not NotImplemented

        if value is NotImplemented and not isinstance(name, str):
            value = name
            name = NotImplemented

        if name is NotImplemented:
            name = value.__name__

        verify_type(name, str, non_empty=True)

        if value is NotImplemented:
            def registrar(obj):
                """
                A parameter-free decorator for in-line registration of plugins.

                :param obj: The value to be registered as a plugin.
                :return: The value, unchanged, after registration.
                """
                self.register(name, obj)
                return obj

            return registrar

        self.register(name, value)

        return value

CONFIG_LOADERS = PluginGroup('attila.config_loader')
URL_SCHEMES = PluginGroup('attila.url_scheme')

def load_plugins(warn=True):
    """
    Load all registered Attila plugins.

    Another package can register a plugin for use by attila by setting the entry_points parameter in
    the other package's setup.py script. See http://stackoverflow.com/a/9615473/4683578 and/or
    https://pythonhosted.org/setuptools/setuptools.use_html#dynamic-discovery-of-services-and-plugins
    for an explanation of how plugins work in Python.

    There are two distinct types of plugins that attila itself recognizes:
      * Config Loaders: These are loaded from the plugin group 'attila.config_loaders', and must be
        either subclasses of the attila.abc.configurations.Configurable class,
        class method which accepts a configparser.ConfigParser and a section name as its arguments.
      * Configured Objects: These are loaded from the plugin group 'attila.configured_object', and
        must be *instances* of the attila.abc.configurations.Configurable class.

    Each of these plugins is registered using the entry point name specified in the registering
    package's setup.py. The registered plugins can then be accessed via the attila.plugins.iter_*()
    and attila.plugins.get_*() methods.

    The Configurable Types are all loaded before any Configured Objects are loaded, allowing the
    Configurable Objects to be loaded, via the load_object() function, from a config section with a
    Type parameter that refers to a Configurable Type.
    """
    CONFIG_LOADERS.load(warn)
    URL_SCHEMES.load(warn)

def config_loader(name=NotImplemented, value=NotImplemented):
    """
    A decorator for in-line registration of config loaders.

    Registering a config loader function under its own name::

        @config_loader
        def aptly_named_config_loader(string):
            ...

    Registering a config loader function under a different name::

        @config_loader('BetterConfigLoaderName')
        def less_aptly_named_config_loader(string):
            ...

    Registering a config loader class under its own name::

        @config_loader
        class AptlyNamedConfigLoader(attila.abc.configurations.Configurable):
            ...

    Registering a config loader class under a different name::

        @config_loader('BetterConfigLoaderName')
        class LessAptlyNamedConfigLoader(attila.abc.configurations.Configurable):
            ...

    IMPORTANT NOTE: If you use this decorator, but your configuration loader
    isn't available, check to make sure the module you used it in has been
    imported; the decorator has to be executed as a statement before it takes
    effect.

    :param name: The name to register the plugin under.
    :param value: The value to register as a plugin.
    :return: The value, unchanged, after registration, or a parameter-free plugin decorator.
    """

    return CONFIG_LOADERS.plugin(name, value)

def url_scheme(name=NotImplemented, value=NotImplemented):
    """
    A decorator for in-line registration of URL schemes.

    Registering a URL scheme function under its own name::

        @config_loader
        def aptly_named_url_scheme(string):
            ...

    Registering a URL scheme function under a different name::

        @config_loader('BetterConfigLoaderName')
        def less_aptly_named_url_scheme(string):
            ...

    :param name: The name to register the plugin under.
    :param value: The value to register as a plugin.
    :return: The value, unchanged, after registration, or a parameter-free plugin decorator.
    """

    return URL_SCHEMES.plugin(name, value)
<commit_msg>Fixed documentation string of url_scheme<commit_after>"""
Infrastructure for dealing with plugins.
"""

import pkg_resources
import warnings

from collections.abc import Mapping

from .exceptions import PluginExistsError, PluginNotFoundError, InvalidPluginError, verify_type

__author__ = 'Aaron Hosford'
__all__ = [
    'PluginGroup',
    'CONFIG_LOADERS',
    'URL_SCHEMES',
    'load_plugins',
    'config_loader',
    'url_scheme',
]

class PluginGroup(Mapping):
    """
    A PluginGroup is a collection of plugins registered under the same entry point group name. It
    supports both install-time and run-time registration of plugins, case-insensitive lookup by
    name, and plugin type-checking.
    """

    def __init__(self, name, value_type=None):
        verify_type(name, str, non_empty=True)
        if value_type is not None:
            verify_type(value_type, type)

        self._name = name
        self._value_type = value_type
        self._original_names = {}
        self._registry = {}

    def load(self, warn=True):
        """
        Load any pre-registered plugins.

        :param warn: Whether to warn if a registered plugin could not be loaded.
        :return: None
        """
        for entry_point in pkg_resources.iter_entry_points(group=self._name):
            try:
                self.register(entry_point.name, entry_point.load())
            except Exception as exc:
                if warn:
                    warnings.warn(str(exc))

    def register(self, name, value):
        """
        Register a new plugin.

        :param name: The name of the plugin.
        :param value: The plugin.
        :return: None
        """
        verify_type(name, str, non_empty=True)
        if name.lower() in self._registry and self._registry[name.lower()] != value:
            raise PluginExistsError("Another plugin by this name has already been registered: %s" % name)
        if self._value_type is not None and not isinstance(value, self._value_type):
            raise InvalidPluginError("Plugin %s is not a/an %s." % (name, self._value_type.__name__))
        self._original_names[name] = name
        self._registry[name.lower()] = value

    def __getitem__(self, name):
        if not isinstance(name, str):
            return NotImplemented
        if name.lower() not in self._registry:
            raise PluginNotFoundError(name)
        return self._registry[name.lower()]

    def __setitem__(self, name, value):
        if not isinstance(name, str):
            return NotImplemented
        self.register(name, value)

    def __iter__(self):
        return iter(self._original_names)

    def __contains__(self, name):
        if not isinstance(name, str):
            return NotImplemented
        return name.lower() in self._registry

    def __len__(self):
        return len(self._registry)

    def get(self, name, default=None):
        """
        Get the plugin by name.

        :param name: The name of the plugin.
        :param default: The default value if the plugin does not exist.
        :return: The plugin, or the default.
        """
        verify_type(name, str, non_empty=True)
        return self._registry.get(name.lower(), default)

    def plugin(self, name=NotImplemented, value=NotImplemented):
        """
        A decorator for in-line registration of plugins.

        Registering a plugin function under its own name to group PLUGIN_GROUP::

            @PLUGIN_GROUP.plugin
            def aptly_named_plugin_function(arg1, arg2):
                ...

        Registering a plugin function to a different name to group PLUGIN_GROUP::

            @PLUGIN_GROUP.plugin('BetterPluginName')
            def less_aptly_named_plugin_function(arg1, arg2):
                ...

        :param value: The value to be registered as a plugin.
        :param name: The name to register the plugin under.
        :return: The value, unchanged, after registration, or a parameter-free plugin decorator.
        """

        assert name is not NotImplemented or value is not NotImplemented

        if value is NotImplemented and not isinstance(name, str):
            value = name
            name = NotImplemented

        if name is NotImplemented:
            name = value.__name__

        verify_type(name, str, non_empty=True)

        if value is NotImplemented:
            def registrar(obj):
                """
                A parameter-free decorator for in-line registration of plugins.

                :param obj: The value to be registered as a plugin.
                :return: The value, unchanged, after registration.
                """
                self.register(name, obj)
                return obj

            return registrar

        self.register(name, value)

        return value

CONFIG_LOADERS = PluginGroup('attila.config_loader')
URL_SCHEMES = PluginGroup('attila.url_scheme')

def load_plugins(warn=True):
    """
    Load all registered Attila plugins.

    Another package can register a plugin for use by attila by setting the entry_points parameter in
    the other package's setup.py script. See http://stackoverflow.com/a/9615473/4683578 and/or
    https://pythonhosted.org/setuptools/setuptools.use_html#dynamic-discovery-of-services-and-plugins
    for an explanation of how plugins work in Python.

    There are two distinct types of plugins that attila itself recognizes:
      * Config Loaders: These are loaded from the plugin group 'attila.config_loaders', and must be
        either subclasses of the attila.abc.configurations.Configurable class,
        class method which accepts a configparser.ConfigParser and a section name as its arguments.
      * Configured Objects: These are loaded from the plugin group 'attila.configured_object', and
        must be *instances* of the attila.abc.configurations.Configurable class.

    Each of these plugins is registered using the entry point name specified in the registering
    package's setup.py. The registered plugins can then be accessed via the attila.plugins.iter_*()
    and attila.plugins.get_*() methods.

    The Configurable Types are all loaded before any Configured Objects are loaded, allowing the
    Configurable Objects to be loaded, via the load_object() function, from a config section with a
    Type parameter that refers to a Configurable Type.
    """
    CONFIG_LOADERS.load(warn)
    URL_SCHEMES.load(warn)

def config_loader(name=NotImplemented, value=NotImplemented):
    """
    A decorator for in-line registration of config loaders.

    Registering a config loader function under its own name::

        @config_loader
        def aptly_named_config_loader(string):
            ...

    Registering a config loader function under a different name::

        @config_loader('BetterConfigLoaderName')
        def less_aptly_named_config_loader(string):
            ...

    Registering a config loader class under its own name::

        @config_loader
        class AptlyNamedConfigLoader(attila.abc.configurations.Configurable):
            ...

    Registering a config loader class under a different name::

        @config_loader('BetterConfigLoaderName')
        class LessAptlyNamedConfigLoader(attila.abc.configurations.Configurable):
            ...

    IMPORTANT NOTE: If you use this decorator, but your configuration loader
    isn't available, check to make sure the module you used it in has been
    imported; the decorator has to be executed as a statement before it takes
    effect.

    :param name: The name to register the plugin under.
    :param value: The value to register as a plugin.
    :return: The value, unchanged, after registration, or a parameter-free plugin decorator.
    """

    return CONFIG_LOADERS.plugin(name, value)

def url_scheme(name=NotImplemented, value=NotImplemented):
    """
    A decorator for in-line registration of URL schemes.

    Registering a URL scheme function under its own name::

        @url_scheme
        def aptly_named_url_scheme(string):
            ...

    Registering a URL scheme function under a different name::

        @url_scheme('BetterURLSchemeName')
        def less_aptly_named_url_scheme(string):
            ...

    :param name: The name to register the plugin under.
    :param value: The value to register as a plugin.
    :return: The value, unchanged, after registration, or a parameter-free plugin decorator.
    """

    return URL_SCHEMES.plugin(name, value)
<|endoftext|>


<commit_before>from flask import current_app
from whoosh.index import create_in, open_dir
from whoosh.fields import Schema
from whoosh.writing import AsyncWriter

import os
import Queue
# Find the stack on which we want to store the database connection.
# Starting with Flask 0.9, the _app_ctx_stack is the correct one,
# before that we need to use the _request_ctx_stack.
try:
    from flask import _app_ctx_stack as stack
except ImportError:
    from flask import _request_ctx_stack as stack

class DirectoryAlreadyExists(Exception):
    pass

class Whoosh(object):
    def __init__(self, app=None):
        self.app = app
        if app is not None:
            self.init_app(app)

    def init_app(self, app):
        app.config.setdefault('WHOOSH_INDEX_ROOT', '/tmp')
        app.config.setdefault('WHOOSH_INDEX_NAME', '')
        app.config.setdefault('WHOOSH_SEARCHER_MIN', 1)
        app.config.setdefault('WHOOSH_SEARCHER_MAX', 10)
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)

    def init_index(self, fields):
        index_root = current_app.config['WHOOSH_INDEX_ROOT']
        name = current_app.config['WHOOSH_INDEX_NAME'] or None

        if os.path.isdir(index_root) and os.listdir(index_root):
            raise DirectoryAlreadyExists()

        if os.path.exists(index_root) and not os.path.isdir(index_root):
            raise DirectoryAlreadyExists()

        if not os.path.exists(index_root):
            os.makedirs(index_root)

        schema = Schema(**fields)
        return create_in(index_root, schema = schema, indexname = name)

    def setup_whoosh(self):
        if not hasattr(current_app, 'extensions'):
            current_app.extensions = {}
        if 'whoosh' not in current_app.extensions:
            wm = WhooshManager(current_app.config, self.open_index())
            current_app.extensions['whoosh'] = wm

    def open_index(self):
        index_root = current_app.config['WHOOSH_INDEX_ROOT']
        name = current_app.config['WHOOSH_INDEX_NAME'] or None
        return open_dir(index_root, indexname = name)

    @property
    def searcher(self):
        self.setup_whoosh()
        ctx = stack.top
        if ctx is not None:
            if not hasattr(ctx, 'whoosh_search_accessor'):
                whoosh_manager = current_app.extensions['whoosh']
                ctx.whoosh_search_accessor = whoosh_manager.search_pool.get()
            searcher = ctx.whoosh_search_accessor.searcher
            searcher.refresh()
            return searcher

    @property
    def writer(self):
        self.setup_whoosh()
        ctx = stack.top
        if ctx is not None:
            if not hasattr(ctx, 'whoosh_writer'):
                whoosh_manager = current_app.extensions['whoosh']
                ctx.whoosh_writer = AsyncWriter(whoosh_manager.index)
            return ctx.whoosh_writer

    def teardown(self, exception):
        ctx = stack.top
        if hasattr(ctx, 'whoosh_search_accessor'):
            whoosh_manager = current_app.extensions['whoosh']
            whoosh_manager.search_pool.put(ctx.whoosh_search_accessor)

class WhooshManager(object):
    """
    The application level stuff for whoosh.
    """
    def __init__(self, config, index):
        self.minimum = config['WHOOSH_SEARCHER_MIN']
        self.maximum = config['WHOOSH_SEARCHER_MAX']
        self.index = index
        self.search_pool = self.initialize_searcher_queue()

    def initialize_searcher_queue(self):
        queue = Queue.LifoQueue(self.maximum)
        for i in range(self.minimum):
            queue.put(SearchAccessor(self.index))
        for i in range(self.maximum - self.minimum):
            queue.put(SearchAccessor(self.index, init=True))
        return queue

class SearchAccessor(object):
    def __init__(self, index, init=False):
        self.index = index
        self._searcher = None
        if init:
            self._searcher = self.index.searcher()

    @property
    def searcher(self):
        if self._searcher is None:
            self._searcher = self.index.searcher()
        return self._searcher
<commit_msg>add folder parameter to DirectoryAlreadyExists exception, and allow init_index to optionally clear the index<commit_after>from flask import current_app
from whoosh.index import create_in, open_dir, exists_in
from whoosh.fields import Schema
from whoosh.writing import AsyncWriter

import os
import Queue
# Find the stack on which we want to store the database connection.
# Starting with Flask 0.9, the _app_ctx_stack is the correct one,
# before that we need to use the _request_ctx_stack.
try:
    from flask import _app_ctx_stack as stack
except ImportError:
    from flask import _request_ctx_stack as stack

class DirectoryAlreadyExists(Exception):
    def __init__(self, folder):
        super(DirectoryAlreadyExists, self).__init__()
        self.folder = folder
    def __str__(self):
        return repr(self.folder)

class Whoosh(object):
    def __init__(self, app=None):
        self.app = app
        if app is not None:
            self.init_app(app)

    def init_app(self, app):
        app.config.setdefault('WHOOSH_INDEX_ROOT', '/tmp')
        app.config.setdefault('WHOOSH_INDEX_NAME', '')
        app.config.setdefault('WHOOSH_SEARCHER_MIN', 1)
        app.config.setdefault('WHOOSH_SEARCHER_MAX', 10)
        if hasattr(app, 'teardown_appcontext'):
            app.teardown_appcontext(self.teardown)
        else:
            app.teardown_request(self.teardown)

    def init_index(self, fields, clear=False):
        index_root = current_app.config['WHOOSH_INDEX_ROOT']
        name = current_app.config['WHOOSH_INDEX_NAME'] or None

        if os.path.exists(index_root) and not os.path.isdir(index_root):
            # index root exists and is not a directory
            raise DirectoryAlreadyExists(index_root)

        if os.path.isdir(index_root) and not exists_in(index_root, indexname=name) \
           and os.listdir(index_root):
            # index root is a directory and is non-empty and non-index
            raise DirectoryAlreadyExists(index_root)

        if os.path.isdir(index_root) and exists_in(index_root, indexname=name) \
           and not clear:
            # index root is an existing index and we don't have permission to clear it
            raise DirectoryAlreadyExists(index_root)

        # either the directory doesn't exist, or it does but it's empty, or
        # it does, and has an index, but we have permission to clear it.

        if not os.path.exists(index_root):
            os.makedirs(index_root)

        schema = Schema(**fields)
        return create_in(index_root, schema = schema, indexname = name)

    def setup_whoosh(self):
        if not hasattr(current_app, 'extensions'):
            current_app.extensions = {}
        if 'whoosh' not in current_app.extensions:
            wm = WhooshManager(current_app.config, self.open_index())
            current_app.extensions['whoosh'] = wm

    def open_index(self):
        index_root = current_app.config['WHOOSH_INDEX_ROOT']
        name = current_app.config['WHOOSH_INDEX_NAME'] or None
        return open_dir(index_root, indexname = name)

    @property
    def searcher(self):
        self.setup_whoosh()
        ctx = stack.top
        if ctx is not None:
            if not hasattr(ctx, 'whoosh_search_accessor'):
                whoosh_manager = current_app.extensions['whoosh']
                ctx.whoosh_search_accessor = whoosh_manager.search_pool.get()
            searcher = ctx.whoosh_search_accessor.searcher
            searcher.refresh()
            return searcher

    @property
    def writer(self):
        self.setup_whoosh()
        ctx = stack.top
        if ctx is not None:
            if not hasattr(ctx, 'whoosh_writer'):
                whoosh_manager = current_app.extensions['whoosh']
                ctx.whoosh_writer = AsyncWriter(whoosh_manager.index)
            return ctx.whoosh_writer

    def teardown(self, exception):
        ctx = stack.top
        if hasattr(ctx, 'whoosh_search_accessor'):
            whoosh_manager = current_app.extensions['whoosh']
            whoosh_manager.search_pool.put(ctx.whoosh_search_accessor)

class WhooshManager(object):
    """
    The application level stuff for whoosh.
    """
    def __init__(self, config, index):
        self.minimum = config['WHOOSH_SEARCHER_MIN']
        self.maximum = config['WHOOSH_SEARCHER_MAX']
        self.index = index
        self.search_pool = self.initialize_searcher_queue()

    def initialize_searcher_queue(self):
        queue = Queue.LifoQueue(self.maximum)
        for i in range(self.minimum):
            queue.put(SearchAccessor(self.index))
        for i in range(self.maximum - self.minimum):
            queue.put(SearchAccessor(self.index, init=True))
        return queue

class SearchAccessor(object):
    def __init__(self, index, init=False):
        self.index = index
        self._searcher = None
        if init:
            self._searcher = self.index.searcher()

    @property
    def searcher(self):
        if self._searcher is None:
            self._searcher = self.index.searcher()
        return self._searcher
<|endoftext|>


<commit_before>from mmd.molecule import *
from mmd.scf import *
from mmd.postscf import *
import numpy as np

np.set_printoptions(suppress=True,precision=14)

# read in geometry
geometry = './geoms/h2o.dat'

# init molecule and build integrals
mol = Molecule(filename=geometry,basis='sto-3g',gauge=[0.0,0.0,0.0])
mol.build()

# do the SCF
scf = SCF(mol)
scf.RHF()

scf.forces()

for atom in mol.atoms:
    print atom.forces

# do MP2
#mp2 = PostSCF(mol)
#mp2.MP2()

<commit_msg>add simple BOMD<commit_after>from mmd.molecule import *
from mmd.scf import *
from mmd.postscf import *
import numpy as np

# read in geometry
geometry = './geoms/h2o.dat'

# init molecule and build integrals
mol = Molecule(filename=geometry,basis='sto-3g')
mol.build()

# do the SCF
scf = SCF(mol)
scf.RHF()

# compute and print nuclear forces
scf.forces()
for atom in mol.atoms:
    print atom.forces

# do MP2
mp2 = PostSCF(mol)
mp2.MP2()

<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding:utf-8 -*-

"""Simple caching functionality using decorated functions.

Cached values are stored in a dicitonary. The keys are generated by a callable
given to the decorator. This can e.g. be used to cache expensive calculations
that depend on an object's state. In that case, the key generating-function
needs to be made aware of all relevant state attributes.
"""

# TODO: testing concept?

from functools import wraps

cache_registry = []

class NotInCacheError(Exception):
    """Exception to be thrown when a key that is not present in cache is read.
    """

    pass

class Cache(object):
    """Base class for caches.
    """

    def get(self, key):
        """Get a cached value from key.

        Parameters
        ----------
        key : string

        Raises
        ------
        NotInCacheError
            In case key does not exist in cache.
        """
        raise NotImplementedError()

    def set(self, key, value):
        """Set a key/value pair in cache.

        Parameters
        ----------
        key : string
        value : object
        """
        raise NotImplementedError()

    def clear(self):
        """Clean out all cached items.
        """

        raise NotImplementedError()

    # TODO: is it reasonable to add a keys member, a delete() function?

class FiniteCache(Cache):
    """Implements a cache that can only hold a given number of elements.

    Uses a dict, but watches out for the number of keys in dict. If the maximum
    size is reached, the oldest key/value pair needs to go. This can be used
    for computations that return large objects.
    """

    def __init__(self, max_size):
        """Create a cacher that behaves much like a dictionary with a finite
        number of entries.

        Parameters
        ----------
        max_size : int
            Maximum number of entries.
        """

        self.max_size = max_size
        self.cache = {}  # dicts are not ordered, so...
        self.keys_in_order = []  # lists are ordered!

    def get(self, key):
        """Get a cached value from key.

        Parameters
        ----------
        key : string

        Raises
        ------
        NotInCacheError
            In case key does not exist in cache.
        """

        try:
            cached_val = self.cache[key]
            return cached_val
        except KeyError:
            raise NotInCacheError

    def set(self, key, value):
        """Set a key/value pair in cache.

        Parameters
        ----------
        key : string
        value : object
        """

        self.keys_in_order.append(key)
        self.cache[key] = value

        # if cache has grown too big (i.e. has too many cached items): clean
        # out oldest cached item
        if len(self.cache) > self.max_size:
            oldest_key = self.keys_in_order[0]  # first item is oldest key
            self.cache.pop(oldest_key)
            self.keys_in_order.pop(0)

    def clear(self):
        """Clean out all cached items.
        """

        self.cache.clear()
        for key in self.keys_in_order:
            del key
        self.keys_in_order = []

def cacher(key_template=None, get_cacher=lambda: FiniteCache(5)):
    """Decorator that wraps a class member function with caching capabilities.

    Parameters
    ----------
    key_template : string
        A string that describes the class' state at runtime. The string needs
        to be such that .format() could be called on it.
    get_cache : callable, optional
        A callable to return an object with get(), set() and clear() methods.
        Default to `lambda: FiniteCache(5)`. The callable approach is used to
        ensure caches will be local to each decorated function.
    """

    if key_template is None:
        raise ValueError("A key_template must be given")

    cache = get_cacher()
    cache_registry.append(cache)
    # TODO: can we always do that? what a the requirements for this to work?
    # is a common interface enough?

    def decorate(func):
        func_name = func.__repr__()

        @wraps(func)
        def wrapper(*args, **kwargs):
            # bare-metal convert all positional arguments to keyword arguments:
            kwargs.update(dict(zip(func.func_code.co_varnames, args)))

            key = (func_name + "_"  # make cacher aware of function name
                   + key_template.format(**kwargs)  # evaluate key
                  )
            try:
                ret = cache.get(key)
            except NotInCacheError:
                ret = func(**kwargs)
                # TODO: call like that or use previously generated copy of kwargs together with args?
                cache.set(key, ret)

            return ret

        return wrapper

    return decorate

def clear_all_registered_caches():
    """Clear all caches registered.
    """

    for cache in cache_registry:
        cache.clear()
<commit_msg>Add keys to caches<commit_after>#!/usr/bin/env python
#-*- coding:utf-8 -*-

"""Simple caching functionality using decorated functions.

Cached values are stored in a dicitonary. The keys are generated by a callable
given to the decorator. This can e.g. be used to cache expensive calculations
that depend on an object's state. In that case, the key generating-function
needs to be made aware of all relevant state attributes.
"""

# TODO: testing concept?

from functools import wraps

cache_registry = []

class NotInCacheError(Exception):
    """Exception to be thrown when a key that is not present in cache is read.
    """

    pass

class Cache(object):
    """Base class for caches.
    """

    def get(self, key):
        """Get a cached value from key.

        Parameters
        ----------
        key : string

        Raises
        ------
        NotInCacheError
            In case key does not exist in cache.
        """
        raise NotImplementedError()

    def set(self, key, value):
        """Set a key/value pair in cache.

        Parameters
        ----------
        key : string
        value : object
        """
        raise NotImplementedError()

    def clear(self):
        """Clean out all cached items.
        """

        raise NotImplementedError()

    @property
    def keys(self):
        """Keys stored.
        """

        raise NotImplementedError()

    # TODO: is it reasonable to add a keys member, a delete() function?

class FiniteCache(Cache):
    """Implements a cache that can only hold a given number of elements.

    Uses a dict, but watches out for the number of keys in dict. If the maximum
    size is reached, the oldest key/value pair needs to go. This can be used
    for computations that return large objects.
    """

    def __init__(self, max_size):
        """Create a cacher that behaves much like a dictionary with a finite
        number of entries.

        Parameters
        ----------
        max_size : int
            Maximum number of entries.
        """

        self.max_size = max_size
        self.cache = {}  # dicts are not ordered, so...
        self.keys_in_order = []  # lists are ordered!

    def get(self, key):
        """Get a cached value from key.

        Parameters
        ----------
        key : string

        Raises
        ------
        NotInCacheError
            In case key does not exist in cache.
        """

        try:
            cached_val = self.cache[key]
            return cached_val
        except KeyError:
            raise NotInCacheError

    def set(self, key, value):
        """Set a key/value pair in cache.

        Parameters
        ----------
        key : string
        value : object
        """

        self.keys_in_order.append(key)
        self.cache[key] = value

        # if cache has grown too big (i.e. has too many cached items): clean
        # out oldest cached item
        if len(self.cache) > self.max_size:
            oldest_key = self.keys_in_order[0]  # first item is oldest key
            self.cache.pop(oldest_key)
            self.keys_in_order.pop(0)

    def clear(self):
        """Clean out all cached items.
        """

        self.cache.clear()
        for key in self.keys_in_order:
            del key
        self.keys_in_order = []

    @property
    def keys(self):
        return self.cache.keys()

def cacher(key_template=None, get_cacher=lambda: FiniteCache(5)):
    """Decorator that wraps a class member function with caching capabilities.

    Parameters
    ----------
    key_template : string
        A string that describes the class' state at runtime. The string needs
        to be such that .format() could be called on it.
    get_cache : callable, optional
        A callable to return an object with get(), set() and clear() methods.
        Default to `lambda: FiniteCache(5)`. The callable approach is used to
        ensure caches will be local to each decorated function.
    """

    if key_template is None:
        raise ValueError("A key_template must be given")

    cache = get_cacher()
    cache_registry.append(cache)
    # TODO: can we always do that? what a the requirements for this to work?
    # is a common interface enough?

    def decorate(func):
        func_name = func.__repr__()

        @wraps(func)
        def wrapper(*args, **kwargs):
            # bare-metal convert all positional arguments to keyword arguments:
            kwargs.update(dict(zip(func.func_code.co_varnames, args)))

            key = (func_name + "_"  # make cacher aware of function name
                   + key_template.format(**kwargs)  # evaluate key
                  )
            try:
                ret = cache.get(key)
            except NotInCacheError:
                ret = func(**kwargs)
                # TODO: call like that or use previously generated copy of kwargs together with args?
                cache.set(key, ret)

            return ret

        return wrapper

    return decorate

def clear_all_registered_caches():
    """Clear all caches registered.
    """

    for cache in cache_registry:
        cache.clear()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import BaseHTTPServer

ServerClass = BaseHTTPServer.HTTPServer
RequestHandlerClass = BaseHTTPServer.BaseHTTPRequestHandler

SERVER_NAME = ''
SERVER_PORT = 9000

class JsonPostResponder(RequestHandlerClass):
    def _get_requested_filename(self, path):
        return '.' + path

    def _get_content_from_stream(self, length, stream):
        return stream.read(length)

    def _get_content_from_file(self, filename):
        content = ''
        with open(filename, 'r') as content_file:
            content = content_file.read()
        return content

    def _send_response(self, status, content):
        self.send_response(status)
        self.end_headers()
        self.wfile.write(content)

    def _print_request(self, command, path, headers, content):
        print('\n%s --> %s\n%s%s\n' % (command, path, headers, content))

    def _print_response(self, status, path, content):
        print('%s <-- %s\n%s' % (status, path, content))

    def do_POST(self):
        content_length = int(self.headers['Content-Length'])
        content = self._get_content_from_stream(content_length, self.rfile)
        self._print_request(self.command, self.path, self.headers, content)

        try:
            status = 200
            filename = self._get_requested_filename(self.path)
            content = self._get_content_from_file(filename)
        except:
            status = 404
            content = 'Not found\n'
        self._print_response(status, self.path, content)
        self._send_response(status, content)

server_address = (SERVER_NAME, SERVER_PORT)
httpd = ServerClass(server_address, JsonPostResponder)
httpd.serve_forever()
<commit_msg>Dummy server request handler is not JSON-specific<commit_after>#!/usr/bin/env python

import BaseHTTPServer

ServerClass = BaseHTTPServer.HTTPServer
RequestHandlerClass = BaseHTTPServer.BaseHTTPRequestHandler

SERVER_NAME = ''
SERVER_PORT = 9000

class PostResponder(RequestHandlerClass):
    def _get_requested_filename(self, path):
        return '.' + path

    def _get_content_from_stream(self, length, stream):
        return stream.read(length)

    def _get_content_from_file(self, filename):
        content = ''
        with open(filename, 'r') as content_file:
            content = content_file.read()
        return content

    def _send_response(self, status, content):
        self.send_response(status)
        self.end_headers()
        self.wfile.write(content)

    def _print_request(self, command, path, headers, content):
        print('\n%s --> %s\n%s%s\n' % (command, path, headers, content))

    def _print_response(self, status, path, content):
        print('%s <-- %s\n%s' % (status, path, content))

    def do_POST(self):
        content_length = int(self.headers['Content-Length'])
        content = self._get_content_from_stream(content_length, self.rfile)
        self._print_request(self.command, self.path, self.headers, content)

        try:
            status = 200
            filename = self._get_requested_filename(self.path)
            content = self._get_content_from_file(filename)
        except:
            status = 404
            content = 'Not found\n'
        self._print_response(status, self.path, content)
        self._send_response(status, content)

server_address = (SERVER_NAME, SERVER_PORT)
httpd = ServerClass(server_address, PostResponder)
httpd.serve_forever()
<|endoftext|>