<commit_before># -*- encoding: utf-8 -*-
#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2017 Marek Marczykowski-Górecki
#                               <marmarek@invisiblethingslab.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program; if not, see <http://www.gnu.org/licenses/>.

'''Qubes VM objects.'''

import logging
import shlex

import subprocess
import warnings

import qubesadmin.base
import qubesadmin.exc
import qubesadmin.storage
import qubesadmin.features
import qubesadmin.devices
import qubesadmin.firewall
import qubesadmin.tags

if not hasattr(shlex, 'quote'):
    # python2 compat
    import pipes
    shlex.quote = pipes.quote

class QubesVM(qubesadmin.base.PropertyHolder):
    '''Qubes domain.'''

    log = None

    tags = None

    features = None

    devices = None

    firewall = None

    def __init__(self, app, name, klass=None):
        super(QubesVM, self).__init__(app, 'admin.vm.property.', name)
        self._volumes = None
        self._klass = klass
        self.log = logging.getLogger(name)
        self.tags = qubesadmin.tags.Tags(self)
        self.features = qubesadmin.features.Features(self)
        self.devices = qubesadmin.devices.DeviceManager(self)
        self.firewall = qubesadmin.firewall.Firewall(self)

    @property
    def name(self):
        '''Domain name'''
        return self._method_dest

    @name.setter
    def name(self, new_value):
        self.qubesd_call(
            self._method_dest,
            self._method_prefix + 'Set',
            'name',
            str(new_value).encode('utf-8'))
        self._method_dest = new_value
        self._volumes = None
        self.app.domains.clear_cache()

    def __str__(self):
        return self._method_dest

    def __lt__(self, other):
        if isinstance(other, QubesVM):
            return self.name < other.name
        return NotImplemented

    def __eq__(self, other):
        if isinstance(other, QubesVM):
            return self.name == other.name
        if isinstance(other, str):
            return self.name == other
        return NotImplemented

    def __hash__(self):
        return hash(self.name)

    def start(self):
        '''
        Start domain.

        :return:
        '''
        self.qubesd_call(self._method_dest, 'admin.vm.Start')

    def shutdown(self, force=False):
        '''
        Shutdown domain.

        :return:
        '''
        # TODO: force parameter
        # TODO: wait parameter (using event?)
        if force:
            raise NotImplementedError
        self.qubesd_call(self._method_dest, 'admin.vm.Shutdown')

    def kill(self):
        '''
        Kill domain (forcefuly shutdown).

        :return:
        '''
        self.qubesd_call(self._method_dest, 'admin.vm.Kill')

    def force_shutdown(self):
        '''Deprecated alias for :py:meth:`kill`'''
        warnings.warn(
            'Call to deprecated function force_shutdown(), use kill() instead',
            DeprecationWarning, stacklevel=2)
        return self.kill()

    def pause(self):
        '''
        Pause domain.

        Pause its execution without any prior notification.

        :return:
        '''
        self.qubesd_call(self._method_dest, 'admin.vm.Pause')

    def unpause(self):
        '''
        Unpause domain.

        Opposite to :py:meth:`pause`.

        :return:
        '''
        self.qubesd_call(self._method_dest, 'admin.vm.Unpause')

    def get_power_state(self):
        '''Return power state description string.

        Return value may be one of those:

        =============== ========================================================
        return value    meaning
        =============== ========================================================
        ``'Halted'``    Machine is not active.
        ``'Transient'`` Machine is running, but does not have :program:`guid`
                        or :program:`qrexec` available.
        ``'Running'``   Machine is ready and running.
        ``'Paused'``    Machine is paused.
        ``'Suspended'`` Machine is S3-suspended.
        ``'Halting'``   Machine is in process of shutting down (OS shutdown).
        ``'Dying'``     Machine is in process of shutting down (cleanup).
        ``'Crashed'``   Machine crashed and is unusable.
        ``'NA'``        Machine is in unknown state.
        =============== ========================================================

        .. seealso::

            http://wiki.libvirt.org/page/VM_lifecycle
                Description of VM life cycle from the point of view of libvirt.

            https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainState
                Libvirt's enum describing precise state of a domain.

        '''

        try:
            return self._get_current_state()['power_state']
        except qubesadmin.exc.QubesDaemonNoResponseError:
            return 'NA'

    def get_mem(self):
        return int(self._get_current_state()['mem'])

    def _get_current_state(self):
        state = {}
        response = self.qubesd_call(self._method_dest, 'admin.vm.CurrentState')
        for part in response.decode('ascii').split():
            name, value = part.split('=', 1)
            state[name] = value
        return state

    def is_halted(self):
        ''' Check whether this domain's state is 'Halted'
            :returns: :py:obj:`True` if this domain is halted, \
                :py:obj:`False` otherwise.
            :rtype: bool
        '''
        return self.get_power_state() == 'Halted'

    def is_paused(self):
        '''Check whether this domain is paused.

        :returns: :py:obj:`True` if this domain is paused, \
            :py:obj:`False` otherwise.
        :rtype: bool
        '''

        return self.get_power_state() == 'Paused'

    def is_running(self):
        '''Check whether this domain is running.

        :returns: :py:obj:`True` if this domain is started, \
            :py:obj:`False` otherwise.
        :rtype: bool
        '''

        return self.get_power_state() not in ('Halted', 'NA')

    def is_networked(self):
        '''Check whether this VM can reach network (firewall notwithstanding).

        :returns: :py:obj:`True` if is machine can reach network, \
            :py:obj:`False` otherwise.
        :rtype: bool
        '''

        if self.provides_network:
            return True

        return self.netvm is not None

    @property
    def volumes(self):
        '''VM disk volumes'''
        if self._volumes is None:
            volumes_list = self.qubesd_call(
                self._method_dest, 'admin.vm.volume.List')
            self._volumes = {}
            for volname in volumes_list.decode('ascii').splitlines():
                if not volname:
                    continue
                self._volumes[volname] = qubesadmin.storage.Volume(self.app,
                    vm=self.name, vm_name=volname)
        return self._volumes

    def get_disk_utilization(self):
        '''Get total disk usage of the VM'''
        return sum(vol.usage for vol in self.volumes.values())

    def run_service(self, service, **kwargs):
        '''Run service on this VM

        :param str service: service name
        :rtype: subprocess.Popen
        '''
        return self.app.run_service(self._method_dest, service, **kwargs)

    def run_service_for_stdio(self, service, input=None, **kwargs):
        '''Run a service, pass an optional input and return (stdout, stderr).

        Raises an exception if return code != 0.

        *args* and *kwargs* are passed verbatim to :py:meth:`run_service`.

        .. warning::
            There are some combinations if stdio-related *kwargs*, which are
            not filtered for problems originating between the keyboard and the
            chair.
        '''  # pylint: disable=redefined-builtin
        p = self.run_service(service, **kwargs)

        # this one is actually a tuple, but there is no need to unpack it
        stdouterr = p.communicate(input=input)

        if p.returncode:
            exc = subprocess.CalledProcessError(p.returncode, service)
            # Python < 3.5 didn't have those
            exc.output, exc.stderr = stdouterr
            raise exc

        return stdouterr

    def prepare_input_for_vmshell(self, command, input=None):
        '''Prepare shell input for the given command and optional (real) input
        '''  # pylint: disable=redefined-builtin
        if input is None:
            input = b''
        close_shell_suffix = b'; exit\n'
        if self.features.check_with_template('os', 'Linux') == 'Windows':
            close_shell_suffix = b'& exit\n'
        return b''.join((command.rstrip('\n').encode('utf-8'),
            close_shell_suffix, input))

    def run(self, command, input=None, **kwargs):
        '''Run a shell command inside the domain using qubes.VMShell qrexec.

        '''  # pylint: disable=redefined-builtin
        try:
            return self.run_service_for_stdio('qubes.VMShell',
                input=self.prepare_input_for_vmshell(command, input), **kwargs)
        except subprocess.CalledProcessError as e:
            e.cmd = command
            raise e

    def run_with_args(self, *args, **kwargs):
        '''Run a single command inside the domain using qubes.VMShell qrexec.

        This method execute a single command, without interpreting any shell
        special characters.

        '''  # pylint: disable=redefined-builtin
        return self.run(' '.join(shlex.quote(arg) for arg in args), **kwargs)

    @property
    def appvms(self):
        ''' Returns a generator containing all domains based on the current
            TemplateVM.

            Do not check vm type of self, core (including its extentions) have
            ultimate control what can be a template of what.
        '''
        for vm in self.app.domains:
            try:
                if vm.template == self:
                    yield vm
            except AttributeError:
                pass

    @property
    def connected_vms(self):
        ''' Return a generator containing all domains connected to the current
            NetVM.
        '''
        for vm in self.app.domains:
            try:
                if vm.netvm == self:
                    yield vm
            except AttributeError:
                pass

    @property
    def klass(self):
        ''' Qube class '''
        # use cached value if available
        if self._klass is None:
            # pylint: disable=no-member
            self._klass = super(QubesVM, self).klass
        return self._klass

class DispVMWrapper(QubesVM):
    '''Wrapper class for new DispVM, supporting only service call

    Note that when running in dom0, one need to manually kill the DispVM after
    service call ends.
    '''

    def run_service(self, service, **kwargs):
        if self.app.qubesd_connection_type == 'socket':
            # create dispvm at service call
            if self._method_dest.startswith('$dispvm'):
                if self._method_dest.startswith('$dispvm:'):
                    method_dest = self._method_dest[len('$dispvm:'):]
                else:
                    method_dest = 'dom0'
                dispvm = self.app.qubesd_call(method_dest,
                    'admin.vm.CreateDisposable')
                dispvm = dispvm.decode('ascii')
                self._method_dest = dispvm
                # Service call may wait for session start, give it more time
                # than default 5s
                kwargs['connect_timeout'] = self.qrexec_timeout
        return super(DispVMWrapper, self).run_service(service, **kwargs)

    def cleanup(self):
        '''Cleanup after DispVM usage'''
        # in 'remote' case nothing is needed, as DispVM is cleaned up
        # automatically
        if self.app.qubesd_connection_type == 'socket' and \
                not self._method_dest.startswith('$dispvm'):
            try:
                self.kill()
            except qubesadmin.exc.QubesVMNotRunningError:
                pass

class DispVM(QubesVM):
    '''Disposable VM'''

    @classmethod
    def from_appvm(cls, app, appvm):
        '''Returns a wrapper for calling service in a new DispVM based on given
        AppVM. If *appvm* is none, use default DispVM template'''

        if appvm:
            method_dest = '$dispvm:' + str(appvm)
        else:
            method_dest = '$dispvm'

        wrapper = DispVMWrapper(app, method_dest)
        return wrapper
<commit_msg>Add missing docstrings<commit_after># -*- encoding: utf-8 -*-
#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2017 Marek Marczykowski-Górecki
#                               <marmarek@invisiblethingslab.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program; if not, see <http://www.gnu.org/licenses/>.

'''Qubes VM objects.'''

import logging
import shlex

import subprocess
import warnings

import qubesadmin.base
import qubesadmin.exc
import qubesadmin.storage
import qubesadmin.features
import qubesadmin.devices
import qubesadmin.firewall
import qubesadmin.tags

if not hasattr(shlex, 'quote'):
    # python2 compat
    import pipes
    shlex.quote = pipes.quote

class QubesVM(qubesadmin.base.PropertyHolder):
    '''Qubes domain.'''

    log = None

    tags = None

    features = None

    devices = None

    firewall = None

    def __init__(self, app, name, klass=None):
        super(QubesVM, self).__init__(app, 'admin.vm.property.', name)
        self._volumes = None
        self._klass = klass
        self.log = logging.getLogger(name)
        self.tags = qubesadmin.tags.Tags(self)
        self.features = qubesadmin.features.Features(self)
        self.devices = qubesadmin.devices.DeviceManager(self)
        self.firewall = qubesadmin.firewall.Firewall(self)

    @property
    def name(self):
        '''Domain name'''
        return self._method_dest

    @name.setter
    def name(self, new_value):
        self.qubesd_call(
            self._method_dest,
            self._method_prefix + 'Set',
            'name',
            str(new_value).encode('utf-8'))
        self._method_dest = new_value
        self._volumes = None
        self.app.domains.clear_cache()

    def __str__(self):
        return self._method_dest

    def __lt__(self, other):
        if isinstance(other, QubesVM):
            return self.name < other.name
        return NotImplemented

    def __eq__(self, other):
        if isinstance(other, QubesVM):
            return self.name == other.name
        if isinstance(other, str):
            return self.name == other
        return NotImplemented

    def __hash__(self):
        return hash(self.name)

    def start(self):
        '''
        Start domain.

        :return:
        '''
        self.qubesd_call(self._method_dest, 'admin.vm.Start')

    def shutdown(self, force=False):
        '''
        Shutdown domain.

        :return:
        '''
        # TODO: force parameter
        # TODO: wait parameter (using event?)
        if force:
            raise NotImplementedError
        self.qubesd_call(self._method_dest, 'admin.vm.Shutdown')

    def kill(self):
        '''
        Kill domain (forcefuly shutdown).

        :return:
        '''
        self.qubesd_call(self._method_dest, 'admin.vm.Kill')

    def force_shutdown(self):
        '''Deprecated alias for :py:meth:`kill`'''
        warnings.warn(
            'Call to deprecated function force_shutdown(), use kill() instead',
            DeprecationWarning, stacklevel=2)
        return self.kill()

    def pause(self):
        '''
        Pause domain.

        Pause its execution without any prior notification.

        :return:
        '''
        self.qubesd_call(self._method_dest, 'admin.vm.Pause')

    def unpause(self):
        '''
        Unpause domain.

        Opposite to :py:meth:`pause`.

        :return:
        '''
        self.qubesd_call(self._method_dest, 'admin.vm.Unpause')

    def get_power_state(self):
        '''Return power state description string.

        Return value may be one of those:

        =============== ========================================================
        return value    meaning
        =============== ========================================================
        ``'Halted'``    Machine is not active.
        ``'Transient'`` Machine is running, but does not have :program:`guid`
                        or :program:`qrexec` available.
        ``'Running'``   Machine is ready and running.
        ``'Paused'``    Machine is paused.
        ``'Suspended'`` Machine is S3-suspended.
        ``'Halting'``   Machine is in process of shutting down (OS shutdown).
        ``'Dying'``     Machine is in process of shutting down (cleanup).
        ``'Crashed'``   Machine crashed and is unusable.
        ``'NA'``        Machine is in unknown state.
        =============== ========================================================

        .. seealso::

            http://wiki.libvirt.org/page/VM_lifecycle
                Description of VM life cycle from the point of view of libvirt.

            https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainState
                Libvirt's enum describing precise state of a domain.

        '''

        try:
            return self._get_current_state()['power_state']
        except qubesadmin.exc.QubesDaemonNoResponseError:
            return 'NA'

    def get_mem(self):
        '''Get current memory usage from VM.'''

        return int(self._get_current_state()['mem'])

    def _get_current_state(self):
        '''Call admin.vm.CurrentState, and return the result as a dict.'''

        state = {}
        response = self.qubesd_call(self._method_dest, 'admin.vm.CurrentState')
        for part in response.decode('ascii').split():
            name, value = part.split('=', 1)
            state[name] = value
        return state

    def is_halted(self):
        ''' Check whether this domain's state is 'Halted'
            :returns: :py:obj:`True` if this domain is halted, \
                :py:obj:`False` otherwise.
            :rtype: bool
        '''
        return self.get_power_state() == 'Halted'

    def is_paused(self):
        '''Check whether this domain is paused.

        :returns: :py:obj:`True` if this domain is paused, \
            :py:obj:`False` otherwise.
        :rtype: bool
        '''

        return self.get_power_state() == 'Paused'

    def is_running(self):
        '''Check whether this domain is running.

        :returns: :py:obj:`True` if this domain is started, \
            :py:obj:`False` otherwise.
        :rtype: bool
        '''

        return self.get_power_state() not in ('Halted', 'NA')

    def is_networked(self):
        '''Check whether this VM can reach network (firewall notwithstanding).

        :returns: :py:obj:`True` if is machine can reach network, \
            :py:obj:`False` otherwise.
        :rtype: bool
        '''

        if self.provides_network:
            return True

        return self.netvm is not None

    @property
    def volumes(self):
        '''VM disk volumes'''
        if self._volumes is None:
            volumes_list = self.qubesd_call(
                self._method_dest, 'admin.vm.volume.List')
            self._volumes = {}
            for volname in volumes_list.decode('ascii').splitlines():
                if not volname:
                    continue
                self._volumes[volname] = qubesadmin.storage.Volume(self.app,
                    vm=self.name, vm_name=volname)
        return self._volumes

    def get_disk_utilization(self):
        '''Get total disk usage of the VM'''
        return sum(vol.usage for vol in self.volumes.values())

    def run_service(self, service, **kwargs):
        '''Run service on this VM

        :param str service: service name
        :rtype: subprocess.Popen
        '''
        return self.app.run_service(self._method_dest, service, **kwargs)

    def run_service_for_stdio(self, service, input=None, **kwargs):
        '''Run a service, pass an optional input and return (stdout, stderr).

        Raises an exception if return code != 0.

        *args* and *kwargs* are passed verbatim to :py:meth:`run_service`.

        .. warning::
            There are some combinations if stdio-related *kwargs*, which are
            not filtered for problems originating between the keyboard and the
            chair.
        '''  # pylint: disable=redefined-builtin
        p = self.run_service(service, **kwargs)

        # this one is actually a tuple, but there is no need to unpack it
        stdouterr = p.communicate(input=input)

        if p.returncode:
            exc = subprocess.CalledProcessError(p.returncode, service)
            # Python < 3.5 didn't have those
            exc.output, exc.stderr = stdouterr
            raise exc

        return stdouterr

    def prepare_input_for_vmshell(self, command, input=None):
        '''Prepare shell input for the given command and optional (real) input
        '''  # pylint: disable=redefined-builtin
        if input is None:
            input = b''
        close_shell_suffix = b'; exit\n'
        if self.features.check_with_template('os', 'Linux') == 'Windows':
            close_shell_suffix = b'& exit\n'
        return b''.join((command.rstrip('\n').encode('utf-8'),
            close_shell_suffix, input))

    def run(self, command, input=None, **kwargs):
        '''Run a shell command inside the domain using qubes.VMShell qrexec.

        '''  # pylint: disable=redefined-builtin
        try:
            return self.run_service_for_stdio('qubes.VMShell',
                input=self.prepare_input_for_vmshell(command, input), **kwargs)
        except subprocess.CalledProcessError as e:
            e.cmd = command
            raise e

    def run_with_args(self, *args, **kwargs):
        '''Run a single command inside the domain using qubes.VMShell qrexec.

        This method execute a single command, without interpreting any shell
        special characters.

        '''  # pylint: disable=redefined-builtin
        return self.run(' '.join(shlex.quote(arg) for arg in args), **kwargs)

    @property
    def appvms(self):
        ''' Returns a generator containing all domains based on the current
            TemplateVM.

            Do not check vm type of self, core (including its extentions) have
            ultimate control what can be a template of what.
        '''
        for vm in self.app.domains:
            try:
                if vm.template == self:
                    yield vm
            except AttributeError:
                pass

    @property
    def connected_vms(self):
        ''' Return a generator containing all domains connected to the current
            NetVM.
        '''
        for vm in self.app.domains:
            try:
                if vm.netvm == self:
                    yield vm
            except AttributeError:
                pass

    @property
    def klass(self):
        ''' Qube class '''
        # use cached value if available
        if self._klass is None:
            # pylint: disable=no-member
            self._klass = super(QubesVM, self).klass
        return self._klass

class DispVMWrapper(QubesVM):
    '''Wrapper class for new DispVM, supporting only service call

    Note that when running in dom0, one need to manually kill the DispVM after
    service call ends.
    '''

    def run_service(self, service, **kwargs):
        if self.app.qubesd_connection_type == 'socket':
            # create dispvm at service call
            if self._method_dest.startswith('$dispvm'):
                if self._method_dest.startswith('$dispvm:'):
                    method_dest = self._method_dest[len('$dispvm:'):]
                else:
                    method_dest = 'dom0'
                dispvm = self.app.qubesd_call(method_dest,
                    'admin.vm.CreateDisposable')
                dispvm = dispvm.decode('ascii')
                self._method_dest = dispvm
                # Service call may wait for session start, give it more time
                # than default 5s
                kwargs['connect_timeout'] = self.qrexec_timeout
        return super(DispVMWrapper, self).run_service(service, **kwargs)

    def cleanup(self):
        '''Cleanup after DispVM usage'''
        # in 'remote' case nothing is needed, as DispVM is cleaned up
        # automatically
        if self.app.qubesd_connection_type == 'socket' and \
                not self._method_dest.startswith('$dispvm'):
            try:
                self.kill()
            except qubesadmin.exc.QubesVMNotRunningError:
                pass

class DispVM(QubesVM):
    '''Disposable VM'''

    @classmethod
    def from_appvm(cls, app, appvm):
        '''Returns a wrapper for calling service in a new DispVM based on given
        AppVM. If *appvm* is none, use default DispVM template'''

        if appvm:
            method_dest = '$dispvm:' + str(appvm)
        else:
            method_dest = '$dispvm'

        wrapper = DispVMWrapper(app, method_dest)
        return wrapper
<|endoftext|>


<commit_before>import io
import re
from configparser import ConfigParser, NoSectionError

import click

TOX_SECTION = "tox"
ENVLIST = "envlist"
TEST_ENV_SECTION = "testenv"
TEST_ENV_DEPS = "deps"
PYTHON_SUBSTITUTE = "py38"
DJANGO_SUBSTITUTE = "django{32,40}"

DJANGO_32_DEPENDENCY = "django32: Django>=3.2,<4.0\n"
DJANGO_40_DEPENDENCY = "django40: Django>=3.3,<4.1\n"
NEW_DJANGO_DEPENDENCIES = DJANGO_32_DEPENDENCY + DJANGO_40_DEPENDENCY

SECTIONS = [TOX_SECTION, TEST_ENV_SECTION]

PYTHON_PATTERN = "(py{.*?}-?|py[0-9]+,|py[0-9]+-)"

DJANGO_PATTERN = "(django[0-9]+,|django[0-9]+\n|django{.*}\n|django{.*?}|django[0-9]+-|django{.*}-)"

DJANGO_DEPENDENCY_PATTERN = "([^\n]*django[0-9]+:.*\n?)"

class ConfigReader:
    def __init__(self, file_path=None, config_dict=None):
        self.config_dict = config_dict
        self.file_path = file_path

    def get_modernizer(self):
        config_parser = ConfigParser()
        if self.config_dict is not None:
            config_parser.read_dict(self.config_dict)
        else:
            config_parser.read(self.file_path)
        return ToxModernizer(config_parser, self.file_path)

class ToxModernizer:
    def __init__(self, config_parser, file_path):
        self.file_path = file_path
        self.config_parser = config_parser
        self._validate_tox_config_sections()

    def _validate_tox_config_sections(self):
        if not self.config_parser.sections():
            raise NoSectionError("Bad Config. No sections found.")

        if all(section not in SECTIONS for section in self.config_parser.sections()):
            raise NoSectionError("File doesn't contain required sections")

    def _update_env_list(self):
        tox_section = self.config_parser[TOX_SECTION]
        env_list = tox_section[ENVLIST]

        env_list = ToxModernizer._replace_runners(PYTHON_PATTERN, PYTHON_SUBSTITUTE, env_list)
        env_list = ToxModernizer._replace_runners(DJANGO_PATTERN, DJANGO_SUBSTITUTE, env_list)
        self.config_parser[TOX_SECTION][ENVLIST] = env_list

    @staticmethod
    def _replace_runners(pattern, substitute, env_list):
        matches = re.findall(pattern, env_list)
        if not matches:
            return env_list
        substitute = ToxModernizer._get_runner_substitute(matches, substitute)
        return ToxModernizer._replace_matches(pattern, substitute, env_list, matches)

    @staticmethod
    def _replace_matches(pattern, substitute, target, matches):
        if not matches:
            return target
        occurrences_to_replace = len(matches) - 1
        if occurrences_to_replace > 0:
            target = re.sub(pattern, '', target, occurrences_to_replace)
        target = re.sub(pattern, substitute, target)
        return target

    @staticmethod
    def _get_runner_substitute(matches, substitute):
        last_match = matches[-1]
        has_other_runners = last_match.endswith('-')
        return substitute + "-" if has_other_runners else substitute

    def _replace_django_versions(self):
        test_environment = self.config_parser[TEST_ENV_SECTION]
        dependencies = test_environment[TEST_ENV_DEPS]
        matches = re.findall(DJANGO_DEPENDENCY_PATTERN, dependencies)
        dependencies = self._replace_matches(DJANGO_DEPENDENCY_PATTERN, NEW_DJANGO_DEPENDENCIES, dependencies, matches)

        self.config_parser[TEST_ENV_SECTION][TEST_ENV_DEPS] = dependencies

    def _update_config_file(self):
        # ConfigParser insists on using tabs for output. We want spaces.
        with io.StringIO() as configw:
            self.config_parser.write(configw)
            new_ini = configw.getvalue()
        new_ini = new_ini.replace("\t", "    ")
        with open(self.file_path, 'w') as configfile:
            configfile.write(new_ini)

    def modernize(self):
        self._update_env_list()
        self._replace_django_versions()
        self._update_config_file()

@click.command()
@click.option(
    '--path', default='tox.ini',
    help="Path to target tox config file")
def main(path):
    modernizer = ConfigReader(path).get_modernizer()
    modernizer.modernize()

if __name__ == '__main__':
    main()
<commit_msg>fix: Update tox modernizer Django40 Dependency definition<commit_after>import io
import re
from configparser import ConfigParser, NoSectionError

import click

TOX_SECTION = "tox"
ENVLIST = "envlist"
TEST_ENV_SECTION = "testenv"
TEST_ENV_DEPS = "deps"
PYTHON_SUBSTITUTE = "py38"
DJANGO_SUBSTITUTE = "django{32,40}"

DJANGO_32_DEPENDENCY = "django32: Django>=3.2,<4.0\n"
DJANGO_40_DEPENDENCY = "django40: Django>=4.0,<4.1\n"
NEW_DJANGO_DEPENDENCIES = DJANGO_32_DEPENDENCY + DJANGO_40_DEPENDENCY

SECTIONS = [TOX_SECTION, TEST_ENV_SECTION]

PYTHON_PATTERN = "(py{.*?}-?|py[0-9]+,|py[0-9]+-)"

DJANGO_PATTERN = "(django[0-9]+,|django[0-9]+\n|django{.*}\n|django{.*?}|django[0-9]+-|django{.*}-)"

DJANGO_DEPENDENCY_PATTERN = "([^\n]*django[0-9]+:.*\n?)"

class ConfigReader:
    def __init__(self, file_path=None, config_dict=None):
        self.config_dict = config_dict
        self.file_path = file_path

    def get_modernizer(self):
        config_parser = ConfigParser()
        if self.config_dict is not None:
            config_parser.read_dict(self.config_dict)
        else:
            config_parser.read(self.file_path)
        return ToxModernizer(config_parser, self.file_path)

class ToxModernizer:
    def __init__(self, config_parser, file_path):
        self.file_path = file_path
        self.config_parser = config_parser
        self._validate_tox_config_sections()

    def _validate_tox_config_sections(self):
        if not self.config_parser.sections():
            raise NoSectionError("Bad Config. No sections found.")

        if all(section not in SECTIONS for section in self.config_parser.sections()):
            raise NoSectionError("File doesn't contain required sections")

    def _update_env_list(self):
        tox_section = self.config_parser[TOX_SECTION]
        env_list = tox_section[ENVLIST]

        env_list = ToxModernizer._replace_runners(PYTHON_PATTERN, PYTHON_SUBSTITUTE, env_list)
        env_list = ToxModernizer._replace_runners(DJANGO_PATTERN, DJANGO_SUBSTITUTE, env_list)
        self.config_parser[TOX_SECTION][ENVLIST] = env_list

    @staticmethod
    def _replace_runners(pattern, substitute, env_list):
        matches = re.findall(pattern, env_list)
        if not matches:
            return env_list
        substitute = ToxModernizer._get_runner_substitute(matches, substitute)
        return ToxModernizer._replace_matches(pattern, substitute, env_list, matches)

    @staticmethod
    def _replace_matches(pattern, substitute, target, matches):
        if not matches:
            return target
        occurrences_to_replace = len(matches) - 1
        if occurrences_to_replace > 0:
            target = re.sub(pattern, '', target, occurrences_to_replace)
        target = re.sub(pattern, substitute, target)
        return target

    @staticmethod
    def _get_runner_substitute(matches, substitute):
        last_match = matches[-1]
        has_other_runners = last_match.endswith('-')
        return substitute + "-" if has_other_runners else substitute

    def _replace_django_versions(self):
        test_environment = self.config_parser[TEST_ENV_SECTION]
        dependencies = test_environment[TEST_ENV_DEPS]
        matches = re.findall(DJANGO_DEPENDENCY_PATTERN, dependencies)
        dependencies = self._replace_matches(DJANGO_DEPENDENCY_PATTERN, NEW_DJANGO_DEPENDENCIES, dependencies, matches)

        self.config_parser[TEST_ENV_SECTION][TEST_ENV_DEPS] = dependencies

    def _update_config_file(self):
        # ConfigParser insists on using tabs for output. We want spaces.
        with io.StringIO() as configw:
            self.config_parser.write(configw)
            new_ini = configw.getvalue()
        new_ini = new_ini.replace("\t", "    ")
        with open(self.file_path, 'w') as configfile:
            configfile.write(new_ini)

    def modernize(self):
        self._update_env_list()
        self._replace_django_versions()
        self._update_config_file()

@click.command()
@click.option(
    '--path', default='tox.ini',
    help="Path to target tox config file")
def main(path):
    modernizer = ConfigReader(path).get_modernizer()
    modernizer.modernize()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Stdlib
from django.contrib import admin

# Django-Webstore
from .models import Cart, CartItem

class CartItemInline(admin.TabularInline):
    model = CartItem

class CartAdmin(admin.ModelAdmin):
    inlines = [
        CartItemInline
    ]

    class Meta:
        model = Cart

admin.site.register(Cart, CartAdmin)
<commit_msg>Adding comments to carts admin.<commit_after># Stdlib
from django.contrib import admin

# Django-Webstore
from .models import Cart, CartItem

# =========================================================================
# ----- Custom Admin Classes
# =========================================================================
class CartItemInline(admin.TabularInline):
    model = CartItem

class CartAdmin(admin.ModelAdmin):
    inlines = [
        CartItemInline
    ]

    class Meta:
        model = Cart

# =========================================================================
# ----- Admin Model Registration
# =========================================================================
admin.site.register(Cart, CartAdmin)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Guewen Baconnier
#    Copyright 2013 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import logging
import os
import threading
import time
import traceback
import uuid
from datetime import datetime
from StringIO import StringIO

from psycopg2 import OperationalError, ProgrammingError

import openerp
from openerp.osv.osv import PG_CONCURRENCY_ERRORS_TO_RETRY
from openerp.tools import config
from .queue import JobsQueue
from ..session import ConnectorSessionHandler
from .job import (OpenERPJobStorage,
                  PENDING,
                  DONE)
from ..exception import (NoSuchJobError,
                         NotReadableJobError,
                         RetryableJobError,
                         FailedJobError,
                         NothingToDoJob)

_logger = logging.getLogger(__name__)

WAIT_CHECK_WORKER_ALIVE = 30  # seconds
WAIT_WHEN_ONLY_AFTER_JOBS = 10  # seconds
WORKER_TIMEOUT = 5 * 60  # seconds
PG_RETRY = 5  # seconds

class Worker(threading.Thread):
    """ Post and retrieve jobs from the queue, execute them"""

    queue_class = JobsQueue
    job_storage_class = OpenERPJobStorage

    def __init__(self, db_name, watcher):
        super(Worker, self).__init__()
        self.queue = self.queue_class()
        self.db_name = db_name
        threading.current_thread().dbname = db_name
        self.uuid = unicode(uuid.uuid4())
        self.watcher = watcher

    def run_job(self, job):
        """ Execute a job """
        def retry_postpone(job, message, seconds=None):
            with session_hdl.session() as session:
                job.postpone(result=message, seconds=seconds)
                job.set_enqueued(self)
                self.job_storage_class(session).store(job)
            self.queue.enqueue(job)

        session_hdl = ConnectorSessionHandler(self.db_name,
                                              openerp.SUPERUSER_ID)
        try:
            with session_hdl.session() as session:
                job = self._load_job(session, job.uuid)
                if job is None:
                    return

            # if the job has been manually set to DONE or PENDING
            # before its execution, stop
            if job.state in (DONE, PENDING):
                return

            # the job has been enqueued in this worker but has likely be
            # modified in the database since its enqueue
            if job.worker_uuid != self.uuid:
                # put the job in pending so it can be requeued
                _logger.error('Job %s was enqueued in worker %s but '
                              'was linked to worker %s. Reset to pending.',
                              job.uuid, self.uuid, job.worker_uuid)
                with session_hdl.session() as session:
                    job.set_pending()
                    self.job_storage_class(session).store(job)
                return

            if job.eta and job.eta > datetime.now():
                # The queue is sorted by 'eta' date first
                # so if we dequeued a job expected to be run in
                # the future, we have no jobs to do right now!
                self.queue.enqueue(job)
                # Wait some time just to avoid to loop over
                # the same 'future' jobs
                _logger.debug('Wait %s seconds because the delayed '
                              'jobs have been reached',
                              WAIT_WHEN_ONLY_AFTER_JOBS)
                time.sleep(WAIT_WHEN_ONLY_AFTER_JOBS)
                return

            with session_hdl.session() as session:
                job.set_started()
                self.job_storage_class(session).store(job)

            _logger.debug('%s started', job)
            with session_hdl.session() as session:
                job.perform(session)
            _logger.debug('%s done', job)

            with session_hdl.session() as session:
                job.set_done()
                self.job_storage_class(session).store(job)

        except NothingToDoJob as err:
            if unicode(err):
                msg = unicode(err)
            else:
                msg = None
            job.cancel(msg)
            with session_hdl.session() as session:
                self.job_storage_class(session).store(job)

        except RetryableJobError as err:
            # delay the job later, requeue
            retry_postpone(job, unicode(err))
            _logger.debug('%s postponed', job)

        except OperationalError as err:
            # Automatically retry the typical transaction serialization errors
            if err.pgcode not in PG_CONCURRENCY_ERRORS_TO_RETRY:
                raise
            retry_postpone(job, unicode(err), seconds=PG_RETRY)
            _logger.debug('%s OperionalError, postponed', job)

        except (FailedJobError, Exception):
            buff = StringIO()
            traceback.print_exc(file=buff)
            _logger.error(buff.getvalue())

            job.set_failed(exc_info=buff.getvalue())
            with session_hdl.session() as session:
                self.job_storage_class(session).store(job)
            raise

    def _load_job(self, session, job_uuid):
        """ Reload a job from the backend """
        try:
            job = self.job_storage_class(session).load(job_uuid)
        except NoSuchJobError:
            # just skip it
            job = None
        except NotReadableJobError:
            _logger.exception('Could not read job: %s', job_uuid)
            raise
        return job

    def run(self):
        """ Worker's main loop

        Check if it still exists in the ``watcher``. When it does no
        longer exist, it break the loop so the thread stops properly.

        Wait for jobs and execute them sequentially.
        """
        while 1:
            # check if the worker has to exit (db destroyed, connector
            # uninstalled)
            if self.watcher.worker_lost(self):
                break
            job = self.queue.dequeue()
            try:
                self.run_job(job)
            except:
                continue

    def enqueue_job_uuid(self, job_uuid):
        """ Enqueue a job:

        It will be executed by the worker as soon as possible (according
        to the job's priority
        """
        session_hdl = ConnectorSessionHandler(self.db_name,
                                              openerp.SUPERUSER_ID)
        with session_hdl.session() as session:
            job = self._load_job(session, job_uuid)
            if job is None:
                # skip a deleted job
                return
            job.set_enqueued(self)
            self.job_storage_class(session).store(job)
        # the change of state should be commited before
        # the enqueue otherwise we may have concurrent updates
        # if the job is started directly
        self.queue.enqueue(job)
        _logger.debug('%s enqueued in %s', job, self)

class WorkerWatcher(threading.Thread):
    """ Keep a sight on the workers and signal their aliveness.

    A `WorkerWatcher` is shared between databases, so only 1 instance is
    necessary to check the aliveness of the workers for every database.
    """

    def __init__(self):
        super(WorkerWatcher, self).__init__()
        self._workers = {}

    def _new(self, db_name):
        """ Create a new worker for the database """
        if db_name in self._workers:
            raise Exception('Database %s already has a worker (%s)' %
                            (db_name, self._workers[db_name].uuid))
        worker = Worker(db_name, self)
        self._workers[db_name] = worker
        worker.daemon = True
        worker.start()

    def _delete(self, db_name):
        """ Delete a worker associated with a database """
        if db_name in self._workers:
            worker_uuid = self._workers[db_name].uuid
            # the worker will exit (it checks ``worker_lost()``)
            del self._workers[db_name]

    def worker_for_db(self, db_name):
        return self._workers.get(db_name)

    def worker_lost(self, worker):
        """ Indicate if a worker is no longer referenced by the watcher.

        Used by the worker threads to know if they have to exit.
        """
        return worker not in self._workers.itervalues()

    @staticmethod
    def available_db_names():
        """ Returns the databases for the server having
        the connector module installed.

        Available means that they can be used by a `Worker`.

        :return: database names
        :rtype: list
        """
        if config['db_name']:
            db_names = config['db_name'].split(',')
        else:
            services = openerp.netsvc.ExportService._services
            if services.get('db'):
                db_names = services['db'].exp_list(True)
            else:
                db_names = []
        available_db_names = []
        for db_name in db_names:
            session_hdl = ConnectorSessionHandler(db_name,
                                                  openerp.SUPERUSER_ID)
            with session_hdl.session() as session:
                cr = session.cr
                try:
                    cr.execute("SELECT 1 FROM ir_module_module "
                               "WHERE name = %s "
                               "AND state = %s", ('connector', 'installed'),
                               log_exceptions=False)
                except ProgrammingError as err:
                    if unicode(err).startswith('relation "ir_module_module" does not exist'):
                        _logger.debug('Database %s is not an OpenERP database,'
                                      ' connector worker not started', db_name)
                    else:
                        raise
                else:
                    if cr.fetchone():
                        available_db_names.append(db_name)
        return available_db_names

    def _update_workers(self):
        """ Refresh the list of workers according to the available
        databases and registries.

        A new database can be available, so we need to create a new
        `Worker` or a database could have been dropped, so we have to
        discard the Worker.
        """
        db_names = self.available_db_names()
        # deleted db or connector uninstalled: remove the workers
        for db_name in set(self._workers) - set(db_names):
            self._delete(db_name)

        for db_name in db_names:
            if db_name not in self._workers:
                self._new(db_name)

    def run(self):
        """ `WorkerWatcher`'s main loop """
        while 1:
            self._update_workers()
            for db_name, worker in self._workers.items():
                self.check_alive(db_name, worker)
            time.sleep(WAIT_CHECK_WORKER_ALIVE)

    def check_alive(self, db_name, worker):
        """ Check if the the worker is still alive and notify
        its aliveness.
        Check if the other workers are still alive, if they are
        dead, remove them from the worker's pool.
        """
        session_hdl = ConnectorSessionHandler(db_name,
                                              openerp.SUPERUSER_ID)
        with session_hdl.session() as session:
            if worker.is_alive():
                self._notify_alive(session, worker)
                session.commit()
            self._purge_dead_workers(session)
            session.commit()

    def _notify_alive(self, session, worker):
        _logger.debug('Worker %s is alive on process %s',
                      worker.uuid, os.getpid())
        dbworker_obj = session.pool.get('queue.worker')
        # at startup (especially when running tests),
        # registry is not yet initialized, so we just skip
        # the notify in such case
        if not dbworker_obj:
            return
        dbworker_obj._notify_alive(session.cr,
                                   session.uid,
                                   worker,
                                   context=session.context)

    def _purge_dead_workers(self, session):
        dbworker_obj = session.pool.get('queue.worker')
        # at startup (especially when running tests),
        # registry is not yet initialized, so we just skip
        # the notify in such case
        if not dbworker_obj:
            return
        dbworker_obj._purge_dead_workers(session.cr,
                                         session.uid,
                                         context=session.context)

watcher = WorkerWatcher()

def start_service():
    """ Start the watcher """
    watcher.daemon = True
    watcher.start()

# We have to launch the Jobs Workers only if:
# 1. OpenERP is used in standalone mode (monoprocess)
# 2. Or it is used in multiprocess (with option ``--workers``)
#    but the current process is a Connector Worker
#    (launched with the ``openerp-connector-worker`` script).
if (not getattr(openerp, 'multi_process', False) or
        getattr(openerp, 'worker_connector', False)):
    start_service()
<commit_msg>No longer necessary since the race condition has been fixed in the OpenERP server (see details on https://bugs.launchpad.net/bugs/1238498)<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Guewen Baconnier
#    Copyright 2013 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import logging
import os
import threading
import time
import traceback
import uuid
from datetime import datetime
from StringIO import StringIO

from psycopg2 import OperationalError, ProgrammingError

import openerp
from openerp.osv.osv import PG_CONCURRENCY_ERRORS_TO_RETRY
from openerp.tools import config
from .queue import JobsQueue
from ..session import ConnectorSessionHandler
from .job import (OpenERPJobStorage,
                  PENDING,
                  DONE)
from ..exception import (NoSuchJobError,
                         NotReadableJobError,
                         RetryableJobError,
                         FailedJobError,
                         NothingToDoJob)

_logger = logging.getLogger(__name__)

WAIT_CHECK_WORKER_ALIVE = 30  # seconds
WAIT_WHEN_ONLY_AFTER_JOBS = 10  # seconds
WORKER_TIMEOUT = 5 * 60  # seconds
PG_RETRY = 5  # seconds

class Worker(threading.Thread):
    """ Post and retrieve jobs from the queue, execute them"""

    queue_class = JobsQueue
    job_storage_class = OpenERPJobStorage

    def __init__(self, db_name, watcher):
        super(Worker, self).__init__()
        self.queue = self.queue_class()
        self.db_name = db_name
        threading.current_thread().dbname = db_name
        self.uuid = unicode(uuid.uuid4())
        self.watcher = watcher

    def run_job(self, job):
        """ Execute a job """
        def retry_postpone(job, message, seconds=None):
            with session_hdl.session() as session:
                job.postpone(result=message, seconds=seconds)
                job.set_enqueued(self)
                self.job_storage_class(session).store(job)
            self.queue.enqueue(job)

        session_hdl = ConnectorSessionHandler(self.db_name,
                                              openerp.SUPERUSER_ID)
        try:
            with session_hdl.session() as session:
                job = self._load_job(session, job.uuid)
                if job is None:
                    return

            # if the job has been manually set to DONE or PENDING
            # before its execution, stop
            if job.state in (DONE, PENDING):
                return

            # the job has been enqueued in this worker but has likely be
            # modified in the database since its enqueue
            if job.worker_uuid != self.uuid:
                # put the job in pending so it can be requeued
                _logger.error('Job %s was enqueued in worker %s but '
                              'was linked to worker %s. Reset to pending.',
                              job.uuid, self.uuid, job.worker_uuid)
                with session_hdl.session() as session:
                    job.set_pending()
                    self.job_storage_class(session).store(job)
                return

            if job.eta and job.eta > datetime.now():
                # The queue is sorted by 'eta' date first
                # so if we dequeued a job expected to be run in
                # the future, we have no jobs to do right now!
                self.queue.enqueue(job)
                # Wait some time just to avoid to loop over
                # the same 'future' jobs
                _logger.debug('Wait %s seconds because the delayed '
                              'jobs have been reached',
                              WAIT_WHEN_ONLY_AFTER_JOBS)
                time.sleep(WAIT_WHEN_ONLY_AFTER_JOBS)
                return

            with session_hdl.session() as session:
                job.set_started()
                self.job_storage_class(session).store(job)

            _logger.debug('%s started', job)
            with session_hdl.session() as session:
                job.perform(session)
            _logger.debug('%s done', job)

            with session_hdl.session() as session:
                job.set_done()
                self.job_storage_class(session).store(job)

        except NothingToDoJob as err:
            if unicode(err):
                msg = unicode(err)
            else:
                msg = None
            job.cancel(msg)
            with session_hdl.session() as session:
                self.job_storage_class(session).store(job)

        except RetryableJobError as err:
            # delay the job later, requeue
            retry_postpone(job, unicode(err))
            _logger.debug('%s postponed', job)

        except OperationalError as err:
            # Automatically retry the typical transaction serialization errors
            if err.pgcode not in PG_CONCURRENCY_ERRORS_TO_RETRY:
                raise
            retry_postpone(job, unicode(err), seconds=PG_RETRY)
            _logger.debug('%s OperionalError, postponed', job)

        except (FailedJobError, Exception):
            buff = StringIO()
            traceback.print_exc(file=buff)
            _logger.error(buff.getvalue())

            job.set_failed(exc_info=buff.getvalue())
            with session_hdl.session() as session:
                self.job_storage_class(session).store(job)
            raise

    def _load_job(self, session, job_uuid):
        """ Reload a job from the backend """
        try:
            job = self.job_storage_class(session).load(job_uuid)
        except NoSuchJobError:
            # just skip it
            job = None
        except NotReadableJobError:
            _logger.exception('Could not read job: %s', job_uuid)
            raise
        return job

    def run(self):
        """ Worker's main loop

        Check if it still exists in the ``watcher``. When it does no
        longer exist, it break the loop so the thread stops properly.

        Wait for jobs and execute them sequentially.
        """
        while 1:
            # check if the worker has to exit (db destroyed, connector
            # uninstalled)
            if self.watcher.worker_lost(self):
                break
            job = self.queue.dequeue()
            try:
                self.run_job(job)
            except:
                continue

    def enqueue_job_uuid(self, job_uuid):
        """ Enqueue a job:

        It will be executed by the worker as soon as possible (according
        to the job's priority
        """
        session_hdl = ConnectorSessionHandler(self.db_name,
                                              openerp.SUPERUSER_ID)
        with session_hdl.session() as session:
            job = self._load_job(session, job_uuid)
            if job is None:
                # skip a deleted job
                return
            job.set_enqueued(self)
            self.job_storage_class(session).store(job)
        # the change of state should be commited before
        # the enqueue otherwise we may have concurrent updates
        # if the job is started directly
        self.queue.enqueue(job)
        _logger.debug('%s enqueued in %s', job, self)

class WorkerWatcher(threading.Thread):
    """ Keep a sight on the workers and signal their aliveness.

    A `WorkerWatcher` is shared between databases, so only 1 instance is
    necessary to check the aliveness of the workers for every database.
    """

    def __init__(self):
        super(WorkerWatcher, self).__init__()
        self._workers = {}

    def _new(self, db_name):
        """ Create a new worker for the database """
        if db_name in self._workers:
            raise Exception('Database %s already has a worker (%s)' %
                            (db_name, self._workers[db_name].uuid))
        worker = Worker(db_name, self)
        self._workers[db_name] = worker
        worker.daemon = True
        worker.start()

    def _delete(self, db_name):
        """ Delete a worker associated with a database """
        if db_name in self._workers:
            worker_uuid = self._workers[db_name].uuid
            # the worker will exit (it checks ``worker_lost()``)
            del self._workers[db_name]

    def worker_for_db(self, db_name):
        return self._workers.get(db_name)

    def worker_lost(self, worker):
        """ Indicate if a worker is no longer referenced by the watcher.

        Used by the worker threads to know if they have to exit.
        """
        return worker not in self._workers.itervalues()

    @staticmethod
    def available_db_names():
        """ Returns the databases for the server having
        the connector module installed.

        Available means that they can be used by a `Worker`.

        :return: database names
        :rtype: list
        """
        if config['db_name']:
            db_names = config['db_name'].split(',')
        else:
            services = openerp.netsvc.ExportService._services
            if services.get('db'):
                db_names = services['db'].exp_list(True)
            else:
                db_names = []
        available_db_names = []
        for db_name in db_names:
            session_hdl = ConnectorSessionHandler(db_name,
                                                  openerp.SUPERUSER_ID)
            with session_hdl.session() as session:
                cr = session.cr
                try:
                    cr.execute("SELECT 1 FROM ir_module_module "
                               "WHERE name = %s "
                               "AND state = %s", ('connector', 'installed'),
                               log_exceptions=False)
                except ProgrammingError as err:
                    if unicode(err).startswith('relation "ir_module_module" does not exist'):
                        _logger.debug('Database %s is not an OpenERP database,'
                                      ' connector worker not started', db_name)
                    else:
                        raise
                else:
                    if cr.fetchone():
                        available_db_names.append(db_name)
        return available_db_names

    def _update_workers(self):
        """ Refresh the list of workers according to the available
        databases and registries.

        A new database can be available, so we need to create a new
        `Worker` or a database could have been dropped, so we have to
        discard the Worker.
        """
        db_names = self.available_db_names()
        # deleted db or connector uninstalled: remove the workers
        for db_name in set(self._workers) - set(db_names):
            self._delete(db_name)

        for db_name in db_names:
            if db_name not in self._workers:
                self._new(db_name)

    def run(self):
        """ `WorkerWatcher`'s main loop """
        while 1:
            self._update_workers()
            for db_name, worker in self._workers.items():
                self.check_alive(db_name, worker)
            time.sleep(WAIT_CHECK_WORKER_ALIVE)

    def check_alive(self, db_name, worker):
        """ Check if the the worker is still alive and notify
        its aliveness.
        Check if the other workers are still alive, if they are
        dead, remove them from the worker's pool.
        """
        session_hdl = ConnectorSessionHandler(db_name,
                                              openerp.SUPERUSER_ID)
        with session_hdl.session() as session:
            if worker.is_alive():
                self._notify_alive(session, worker)
                session.commit()
            self._purge_dead_workers(session)
            session.commit()

    def _notify_alive(self, session, worker):
        _logger.debug('Worker %s is alive on process %s',
                      worker.uuid, os.getpid())
        dbworker_obj = session.pool.get('queue.worker')
        dbworker_obj._notify_alive(session.cr,
                                   session.uid,
                                   worker,
                                   context=session.context)

    def _purge_dead_workers(self, session):
        dbworker_obj = session.pool.get('queue.worker')
        dbworker_obj._purge_dead_workers(session.cr,
                                         session.uid,
                                         context=session.context)

watcher = WorkerWatcher()

def start_service():
    """ Start the watcher """
    watcher.daemon = True
    watcher.start()

# We have to launch the Jobs Workers only if:
# 1. OpenERP is used in standalone mode (monoprocess)
# 2. Or it is used in multiprocess (with option ``--workers``)
#    but the current process is a Connector Worker
#    (launched with the ``openerp-connector-worker`` script).
if (not getattr(openerp, 'multi_process', False) or
        getattr(openerp, 'worker_connector', False)):
    start_service()
<|endoftext|>


<commit_before><commit_msg>set allowed hosts to wildcard for now<commit_after><|endoftext|>


<commit_before>import os
import logging
from datetime import datetime
import pytz

from django_select2.widgets import Select2Widget, Select2MultipleWidget
from mptt.forms import TreeNodeChoiceField

from django import forms
from django.forms.forms import BoundField
from django.core.files.uploadedfile import UploadedFile
from django.utils import simplejson

# from speeches.fields import TagField
from speeches.models import Speech, Speaker, Section, Recording, RecordingTimestamp, Tag
from speeches.widgets import AudioFileInput, BootstrapDateWidget, BootstrapTimeWidget
from speeches.utils import GroupedModelChoiceField

logger = logging.getLogger(__name__)

# For Bootstrap, which needs the label class, so monkey-patch
def add_class(f):
    def class_tag(self, contents=None, attrs=None):
        if attrs is None: attrs = {}
        attrs['class'] = 'control-label'
        return f(self, contents, attrs)
    return class_tag
BoundField.label_tag = add_class(BoundField.label_tag)

class CleanAudioMixin(object):
    def clean_audio(self):
        audio = self.cleaned_data['audio']
        if audio and isinstance(audio, UploadedFile):
            ext = os.path.splitext(audio.name)[1]
            # Check that the file is an audio file and one of the
            # filetypes we can use
            if audio.content_type[0:6] != 'audio/' or ext not in ('.ogg', '.mp3', '.wav', '.3gp'):
                raise forms.ValidationError('That file does not appear to be an audio file')
        return audio

class SpeechAudioForm(forms.ModelForm, CleanAudioMixin):
    class Meta:
        model = Speech
        fields = ( 'audio', )
        widgets = {
            'audio': AudioFileInput,
        }

class SpeechForm(forms.ModelForm, CleanAudioMixin):
    audio_filename = forms.CharField(widget=forms.HiddenInput, required=False)
    speaker = forms.ModelChoiceField(queryset=Speaker.objects.all(),
            empty_label = '',
            widget = Select2Widget(select2_options={ 'placeholder':'Choose a speaker', 'width': 'resolve' }),
            required=False)
    section = TreeNodeChoiceField(queryset=Section.objects.all(), required=False)
    start_date = forms.DateField(input_formats=['%d/%m/%Y'],
            widget=BootstrapDateWidget,
            required=False)
    start_time = forms.TimeField(input_formats=['%H:%M', '%H:%M:%S'],
            widget=BootstrapTimeWidget,
            required=False)
    end_date = forms.DateField(input_formats=['%d/%m/%Y'],
            widget=BootstrapDateWidget,
            required=False)
    end_time = forms.TimeField(input_formats=['%H:%M', '%H:%M:%S'],
            widget=BootstrapTimeWidget,
            required=False)
    #tags = TagField()
    tags = forms.ModelMultipleChoiceField(queryset=Tag.objects.all(),
            widget = Select2MultipleWidget(select2_options={ 'placeholder':'Choose tags', 'width': 'resolve' }),
            required=False)

    def clean(self):
        cleaned_data = self.cleaned_data
        if 'audio_filename' in cleaned_data and cleaned_data['audio_filename']:
            filename = cleaned_data['audio_filename']
            self.cleaned_data['audio'] = filename

        if not cleaned_data.get('text') and not cleaned_data.get('audio'):
            raise forms.ValidationError('You must provide either text or some audio')

        return cleaned_data

    def clean_start_time(self):
        if self.cleaned_data['start_time'] and not self.cleaned_data['start_date']:
            raise forms.ValidationError('If you provide a start time you must give a start date too')
        return self.cleaned_data['start_time']

    def clean_end_time(self):
        if self.cleaned_data['end_time'] and not self.cleaned_data['end_date']:
            raise forms.ValidationError('If you provide an end time you must give an end date too')
        return self.cleaned_data['end_time']

    class Meta:
        model = Speech
        widgets = {
            'audio': AudioFileInput,
            'text': forms.Textarea(attrs={'class': 'input-block-level'}),
            'event': forms.TextInput(),
            'title': forms.TextInput(),
            'location': forms.TextInput(),
            'source_url': forms.TextInput(),
        }
        exclude = ('celery_task_id', 'instance')

class RecordingAPIForm(forms.ModelForm, CleanAudioMixin):
    # Form for uploading a recording

    # Force timestamps to be a charfield so we can supply json to it
    timestamps = forms.CharField(required=False)

    def __init__(self, *args, **kwargs):
        self.request = kwargs.pop('request')
        return super(RecordingAPIForm, self).__init__(*args, **kwargs)

    def clean(self):
        cleaned_data = self.cleaned_data
        if 'audio_filename' in cleaned_data and cleaned_data['audio_filename']:
            filename = cleaned_data['audio_filename']
            self.cleaned_data['audio'] = filename
        return cleaned_data

    def clean_timestamps(self):
        # Allow uploading of timestamps which don't exist by turning the supplied
        # dictionary of timestamps into new RecordingTimestamp instances
        timestamps = []

        if not 'timestamps' in self.cleaned_data or not self.cleaned_data['timestamps']:
            return timestamps

        timestamps_json = simplejson.loads(self.cleaned_data['timestamps'])

        if not isinstance(timestamps_json, list):
            return timestamps

        for recording_timestamp in timestamps_json:
            try:
                if 'timestamp' in recording_timestamp:
                    # Note - we divide by 1000 because the time comes from javascript
                    # and is in milliseconds, but this expects the time in seconds
                    supplied_time = int(recording_timestamp['timestamp']/1000)
                    # We also make it a UTC time!
                    timestamp = datetime.utcfromtimestamp(supplied_time).replace(tzinfo=pytz.utc)
                    try:
                        speaker = Speaker.objects.get(pk=recording_timestamp['speaker'])
                    except:
                        speaker = None
                    timestamps.append(RecordingTimestamp.objects.create(speaker=speaker, timestamp=timestamp, instance=self.request.instance))
                else:
                    # Timestamp is required
                    logger.error("No timestamp supplied in request: {0}".format(recording_timestamp))
                    pass
            except ValueError:
                # Ignore this one
                logger.error("ValueError encountered parsing: {0} into speaker/timestamp".format(recording_timestamp))

        if timestamps.count == 0:
            logger.error("Timestamps parameter was given but no timestamps parsed from: {0}".format(self.cleaned_data['timestamp']))

        return timestamps

    class Meta:
        model = Recording
        exclude = 'instance'

class SectionForm(forms.ModelForm):
    class Meta:
        model = Section
        exclude = 'instance'
<commit_msg>Set input-block-level class on all Textarea widgets.<commit_after>import os
import logging
from datetime import datetime
import pytz

from django_select2.widgets import Select2Widget, Select2MultipleWidget
from mptt.forms import TreeNodeChoiceField

from django import forms
from django.forms.forms import BoundField
from django.forms.widgets import Textarea
from django.core.files.uploadedfile import UploadedFile
from django.utils import simplejson

# from speeches.fields import TagField
from speeches.models import Speech, Speaker, Section, Recording, RecordingTimestamp, Tag
from speeches.widgets import AudioFileInput, BootstrapDateWidget, BootstrapTimeWidget
from speeches.utils import GroupedModelChoiceField

logger = logging.getLogger(__name__)

def add_class(f, cl):
    def class_tag(self, *args, **kwargs):
        kwargs.setdefault('attrs', {})['class'] = cl
        return f(self, *args, **kwargs)
    return class_tag

# For Bootstrap, which needs the label class, so monkey-patch
BoundField.label_tag = add_class(BoundField.label_tag, 'control-label')
# And make all textareas be block level 100% width
Textarea.render = add_class(Textarea.render, 'input-block-level')

class CleanAudioMixin(object):
    def clean_audio(self):
        audio = self.cleaned_data['audio']
        if audio and isinstance(audio, UploadedFile):
            ext = os.path.splitext(audio.name)[1]
            # Check that the file is an audio file and one of the
            # filetypes we can use
            if audio.content_type[0:6] != 'audio/' or ext not in ('.ogg', '.mp3', '.wav', '.3gp'):
                raise forms.ValidationError('That file does not appear to be an audio file')
        return audio

class SpeechAudioForm(forms.ModelForm, CleanAudioMixin):
    class Meta:
        model = Speech
        fields = ( 'audio', )
        widgets = {
            'audio': AudioFileInput,
        }

class SpeechForm(forms.ModelForm, CleanAudioMixin):
    audio_filename = forms.CharField(widget=forms.HiddenInput, required=False)
    speaker = forms.ModelChoiceField(queryset=Speaker.objects.all(),
            empty_label = '',
            widget = Select2Widget(select2_options={ 'placeholder':'Choose a speaker', 'width': 'resolve' }),
            required=False)
    section = TreeNodeChoiceField(queryset=Section.objects.all(), required=False)
    start_date = forms.DateField(input_formats=['%d/%m/%Y'],
            widget=BootstrapDateWidget,
            required=False)
    start_time = forms.TimeField(input_formats=['%H:%M', '%H:%M:%S'],
            widget=BootstrapTimeWidget,
            required=False)
    end_date = forms.DateField(input_formats=['%d/%m/%Y'],
            widget=BootstrapDateWidget,
            required=False)
    end_time = forms.TimeField(input_formats=['%H:%M', '%H:%M:%S'],
            widget=BootstrapTimeWidget,
            required=False)
    #tags = TagField()
    tags = forms.ModelMultipleChoiceField(queryset=Tag.objects.all(),
            widget = Select2MultipleWidget(select2_options={ 'placeholder':'Choose tags', 'width': 'resolve' }),
            required=False)

    def clean(self):
        cleaned_data = self.cleaned_data
        if 'audio_filename' in cleaned_data and cleaned_data['audio_filename']:
            filename = cleaned_data['audio_filename']
            self.cleaned_data['audio'] = filename

        if not cleaned_data.get('text') and not cleaned_data.get('audio'):
            raise forms.ValidationError('You must provide either text or some audio')

        return cleaned_data

    def clean_start_time(self):
        if self.cleaned_data['start_time'] and not self.cleaned_data['start_date']:
            raise forms.ValidationError('If you provide a start time you must give a start date too')
        return self.cleaned_data['start_time']

    def clean_end_time(self):
        if self.cleaned_data['end_time'] and not self.cleaned_data['end_date']:
            raise forms.ValidationError('If you provide an end time you must give an end date too')
        return self.cleaned_data['end_time']

    class Meta:
        model = Speech
        widgets = {
            'audio': AudioFileInput,
            'event': forms.TextInput(),
            'title': forms.TextInput(),
            'location': forms.TextInput(),
            'source_url': forms.TextInput(),
        }
        exclude = ('celery_task_id', 'instance')

class RecordingAPIForm(forms.ModelForm, CleanAudioMixin):
    # Form for uploading a recording

    # Force timestamps to be a charfield so we can supply json to it
    timestamps = forms.CharField(required=False)

    def __init__(self, *args, **kwargs):
        self.request = kwargs.pop('request')
        return super(RecordingAPIForm, self).__init__(*args, **kwargs)

    def clean(self):
        cleaned_data = self.cleaned_data
        if 'audio_filename' in cleaned_data and cleaned_data['audio_filename']:
            filename = cleaned_data['audio_filename']
            self.cleaned_data['audio'] = filename
        return cleaned_data

    def clean_timestamps(self):
        # Allow uploading of timestamps which don't exist by turning the supplied
        # dictionary of timestamps into new RecordingTimestamp instances
        timestamps = []

        if not 'timestamps' in self.cleaned_data or not self.cleaned_data['timestamps']:
            return timestamps

        timestamps_json = simplejson.loads(self.cleaned_data['timestamps'])

        if not isinstance(timestamps_json, list):
            return timestamps

        for recording_timestamp in timestamps_json:
            try:
                if 'timestamp' in recording_timestamp:
                    # Note - we divide by 1000 because the time comes from javascript
                    # and is in milliseconds, but this expects the time in seconds
                    supplied_time = int(recording_timestamp['timestamp']/1000)
                    # We also make it a UTC time!
                    timestamp = datetime.utcfromtimestamp(supplied_time).replace(tzinfo=pytz.utc)
                    try:
                        speaker = Speaker.objects.get(pk=recording_timestamp['speaker'])
                    except:
                        speaker = None
                    timestamps.append(RecordingTimestamp.objects.create(speaker=speaker, timestamp=timestamp, instance=self.request.instance))
                else:
                    # Timestamp is required
                    logger.error("No timestamp supplied in request: {0}".format(recording_timestamp))
                    pass
            except ValueError:
                # Ignore this one
                logger.error("ValueError encountered parsing: {0} into speaker/timestamp".format(recording_timestamp))

        if timestamps.count == 0:
            logger.error("Timestamps parameter was given but no timestamps parsed from: {0}".format(self.cleaned_data['timestamp']))

        return timestamps

    class Meta:
        model = Recording
        exclude = 'instance'

class SectionForm(forms.ModelForm):
    class Meta:
        model = Section
        exclude = 'instance'
<|endoftext|>


<commit_before>import turtle
from time import sleep

wn = turtle.Screen()
x = 1
turtle1 = turtle.Turtle()
turtle1.color("green")
for d in range(35):
    turtle1.forward(x)
    turtle1.left(90)
    x = x * 1.1618

wn.exitonclick()
<commit_msg>fixed exit on click.<commit_after>import turtle
from time import sleep

wn = turtle.Screen()
x = 1
turtle1 = turtle.Turtle()
turtle1.color("green")
for d in range(43):
    turtle1.forward(x)
    turtle1.left(90)
    x = x * 1.1618

wn.exitonclick()
<|endoftext|>