<commit_before># Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function
import logging
import os
import signal
import sys
import time
from swift import gettext_ as _
from random import random, shuffle
from tempfile import mkstemp

from eventlet import spawn, patcher, Timeout

import swift.common.db
from swift.container.backend import ContainerBroker, DATADIR
from swift.common.bufferedhttp import http_connect
from swift.common.exceptions import ConnectionTimeout
from swift.common.ring import Ring
from swift.common.utils import get_logger, config_true_value, ismount, \
    dump_recon_cache, majority_size, Timestamp
from swift.common.daemon import Daemon
from swift.common.http import is_success, HTTP_INTERNAL_SERVER_ERROR

class ContainerUpdater(Daemon):
    """Update container information in account listings."""

    def __init__(self, conf):
        self.conf = conf
        self.logger = get_logger(conf, log_route='container-updater')
        self.devices = conf.get('devices', '/srv/node')
        self.mount_check = config_true_value(conf.get('mount_check', 'true'))
        self.swift_dir = conf.get('swift_dir', '/etc/swift')
        self.interval = int(conf.get('interval', 300))
        self.account_ring = None
        self.concurrency = int(conf.get('concurrency', 4))
        self.slowdown = float(conf.get('slowdown', 0.01))
        self.node_timeout = float(conf.get('node_timeout', 3))
        self.conn_timeout = float(conf.get('conn_timeout', 0.5))
        self.no_changes = 0
        self.successes = 0
        self.failures = 0
        self.account_suppressions = {}
        self.account_suppression_time = \
            float(conf.get('account_suppression_time', 60))
        self.new_account_suppressions = None
        swift.common.db.DB_PREALLOCATION = \
            config_true_value(conf.get('db_preallocation', 'f'))
        self.recon_cache_path = conf.get('recon_cache_path',
                                         '/var/cache/swift')
        self.rcache = os.path.join(self.recon_cache_path, "container.recon")
        self.user_agent = 'container-updater %s' % os.getpid()

    def get_account_ring(self):
        """Get the account ring.  Load it if it hasn't been yet."""
        if not self.account_ring:
            self.account_ring = Ring(self.swift_dir, ring_name='account')
        return self.account_ring

    def _listdir(self, path):
        try:
            return os.listdir(path)
        except OSError as e:
            self.logger.error(_('ERROR:  Failed to get paths to drive '
                                'partitions: %s') % e)
            return []

    def get_paths(self):
        """
        Get paths to all of the partitions on each drive to be processed.

        :returns: a list of paths
        """
        paths = []
        for device in self._listdir(self.devices):
            dev_path = os.path.join(self.devices, device)
            if self.mount_check and not ismount(dev_path):
                self.logger.warning(_('%s is not mounted'), device)
                continue
            con_path = os.path.join(dev_path, DATADIR)
            if not os.path.exists(con_path):
                continue
            for partition in self._listdir(con_path):
                paths.append(os.path.join(con_path, partition))
        shuffle(paths)
        return paths

    def _load_suppressions(self, filename):
        try:
            with open(filename, 'r') as tmpfile:
                for line in tmpfile:
                    account, until = line.split()
                    until = float(until)
                    self.account_suppressions[account] = until
        except Exception:
            self.logger.exception(
                _('ERROR with loading suppressions from %s: ') % filename)
        finally:
            os.unlink(filename)

    def run_forever(self, *args, **kwargs):
        """
        Run the updator continuously.
        """
        time.sleep(random() * self.interval)
        while True:
            self.logger.info(_('Begin container update sweep'))
            begin = time.time()
            now = time.time()
            expired_suppressions = \
                [a for a, u in self.account_suppressions.items()
                 if u < now]
            for account in expired_suppressions:
                del self.account_suppressions[account]
            pid2filename = {}
            # read from account ring to ensure it's fresh
            self.get_account_ring().get_nodes('')
            for path in self.get_paths():
                while len(pid2filename) >= self.concurrency:
                    pid = os.wait()[0]
                    try:
                        self._load_suppressions(pid2filename[pid])
                    finally:
                        del pid2filename[pid]
                fd, tmpfilename = mkstemp()
                os.close(fd)
                pid = os.fork()
                if pid:
                    pid2filename[pid] = tmpfilename
                else:
                    signal.signal(signal.SIGTERM, signal.SIG_DFL)
                    patcher.monkey_patch(all=False, socket=True, select=True,
                                         thread=True)
                    self.no_changes = 0
                    self.successes = 0
                    self.failures = 0
                    self.new_account_suppressions = open(tmpfilename, 'w')
                    forkbegin = time.time()
                    self.container_sweep(path)
                    elapsed = time.time() - forkbegin
                    self.logger.debug(
                        _('Container update sweep of %(path)s completed: '
                          '%(elapsed).02fs, %(success)s successes, %(fail)s '
                          'failures, %(no_change)s with no changes'),
                        {'path': path, 'elapsed': elapsed,
                         'success': self.successes, 'fail': self.failures,
                         'no_change': self.no_changes})
                    sys.exit()
            while pid2filename:
                pid = os.wait()[0]
                try:
                    self._load_suppressions(pid2filename[pid])
                finally:
                    del pid2filename[pid]
            elapsed = time.time() - begin
            self.logger.info(_('Container update sweep completed: %.02fs'),
                             elapsed)
            dump_recon_cache({'container_updater_sweep': elapsed},
                             self.rcache, self.logger)
            if elapsed < self.interval:
                time.sleep(self.interval - elapsed)

    def run_once(self, *args, **kwargs):
        """
        Run the updater once.
        """
        patcher.monkey_patch(all=False, socket=True, select=True, thread=True)
        self.logger.info(_('Begin container update single threaded sweep'))
        begin = time.time()
        self.no_changes = 0
        self.successes = 0
        self.failures = 0
        for path in self.get_paths():
            self.container_sweep(path)
        elapsed = time.time() - begin
        self.logger.info(_(
            'Container update single threaded sweep completed: '
            '%(elapsed).02fs, %(success)s successes, %(fail)s failures, '
            '%(no_change)s with no changes'),
            {'elapsed': elapsed, 'success': self.successes,
             'fail': self.failures, 'no_change': self.no_changes})
        dump_recon_cache({'container_updater_sweep': elapsed},
                         self.rcache, self.logger)

    def container_sweep(self, path):
        """
        Walk the path looking for container DBs and process them.

        :param path: path to walk
        """
        for root, dirs, files in os.walk(path):
            for file in files:
                if file.endswith('.db'):
                    self.process_container(os.path.join(root, file))
                    time.sleep(self.slowdown)

    def process_container(self, dbfile):
        """
        Process a container, and update the information in the account.

        :param dbfile: container DB to process
        """
        start_time = time.time()
        broker = ContainerBroker(dbfile, logger=self.logger)
        info = broker.get_info()
        # Don't send updates if the container was auto-created since it
        # definitely doesn't have up to date statistics.
        if Timestamp(info['put_timestamp']) <= 0:
            return
        if self.account_suppressions.get(info['account'], 0) > time.time():
            return
        if info['put_timestamp'] > info['reported_put_timestamp'] or \
                info['delete_timestamp'] > info['reported_delete_timestamp'] \
                or info['object_count'] != info['reported_object_count'] or \
                info['bytes_used'] != info['reported_bytes_used']:
            container = '/%s/%s' % (info['account'], info['container'])
            part, nodes = self.get_account_ring().get_nodes(info['account'])
            events = [spawn(self.container_report, node, part, container,
                            info['put_timestamp'], info['delete_timestamp'],
                            info['object_count'], info['bytes_used'],
                            info['storage_policy_index'])
                      for node in nodes]
            successes = 0
            for event in events:
                if is_success(event.wait()):
                    successes += 1
            if successes >= majority_size(len(events)):
                self.logger.increment('successes')
                self.successes += 1
                self.logger.debug(
                    _('Update report sent for %(container)s %(dbfile)s'),
                    {'container': container, 'dbfile': dbfile})
                broker.reported(info['put_timestamp'],
                                info['delete_timestamp'], info['object_count'],
                                info['bytes_used'])
            else:
                self.logger.increment('failures')
                self.failures += 1
                self.logger.debug(
                    _('Update report failed for %(container)s %(dbfile)s'),
                    {'container': container, 'dbfile': dbfile})
                self.account_suppressions[info['account']] = until = \
                    time.time() + self.account_suppression_time
                if self.new_account_suppressions:
                    print(info['account'], until,
                          file=self.new_account_suppressions)
            # Only track timing data for attempted updates:
            self.logger.timing_since('timing', start_time)
        else:
            self.logger.increment('no_changes')
            self.no_changes += 1

    def container_report(self, node, part, container, put_timestamp,
                         delete_timestamp, count, bytes,
                         storage_policy_index):
        """
        Report container info to an account server.

        :param node: node dictionary from the account ring
        :param part: partition the account is on
        :param container: container name
        :param put_timestamp: put timestamp
        :param delete_timestamp: delete timestamp
        :param count: object count in the container
        :param bytes: bytes used in the container
        :param storage_policy_index: the policy index for the container
        """
        with ConnectionTimeout(self.conn_timeout):
            try:
                headers = {
                    'X-Put-Timestamp': put_timestamp,
                    'X-Delete-Timestamp': delete_timestamp,
                    'X-Object-Count': count,
                    'X-Bytes-Used': bytes,
                    'X-Account-Override-Deleted': 'yes',
                    'X-Backend-Storage-Policy-Index': storage_policy_index,
                    'user-agent': self.user_agent}
                conn = http_connect(
                    node['ip'], node['port'], node['device'], part,
                    'PUT', container, headers=headers)
            except (Exception, Timeout):
                self.logger.exception(_(
                    'ERROR account update failed with '
                    '%(ip)s:%(port)s/%(device)s (will retry later): '), node)
                return HTTP_INTERNAL_SERVER_ERROR
        with Timeout(self.node_timeout):
            try:
                resp = conn.getresponse()
                resp.read()
                return resp.status
            except (Exception, Timeout):
                if self.logger.getEffectiveLevel() <= logging.DEBUG:
                    self.logger.exception(
                        _('Exception with %(ip)s:%(port)s/%(device)s'), node)
                return HTTP_INTERNAL_SERVER_ERROR
            finally:
                conn.close()
<commit_msg>Fixed Typo in updater.py<commit_after># Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function
import logging
import os
import signal
import sys
import time
from swift import gettext_ as _
from random import random, shuffle
from tempfile import mkstemp

from eventlet import spawn, patcher, Timeout

import swift.common.db
from swift.container.backend import ContainerBroker, DATADIR
from swift.common.bufferedhttp import http_connect
from swift.common.exceptions import ConnectionTimeout
from swift.common.ring import Ring
from swift.common.utils import get_logger, config_true_value, ismount, \
    dump_recon_cache, majority_size, Timestamp
from swift.common.daemon import Daemon
from swift.common.http import is_success, HTTP_INTERNAL_SERVER_ERROR

class ContainerUpdater(Daemon):
    """Update container information in account listings."""

    def __init__(self, conf):
        self.conf = conf
        self.logger = get_logger(conf, log_route='container-updater')
        self.devices = conf.get('devices', '/srv/node')
        self.mount_check = config_true_value(conf.get('mount_check', 'true'))
        self.swift_dir = conf.get('swift_dir', '/etc/swift')
        self.interval = int(conf.get('interval', 300))
        self.account_ring = None
        self.concurrency = int(conf.get('concurrency', 4))
        self.slowdown = float(conf.get('slowdown', 0.01))
        self.node_timeout = float(conf.get('node_timeout', 3))
        self.conn_timeout = float(conf.get('conn_timeout', 0.5))
        self.no_changes = 0
        self.successes = 0
        self.failures = 0
        self.account_suppressions = {}
        self.account_suppression_time = \
            float(conf.get('account_suppression_time', 60))
        self.new_account_suppressions = None
        swift.common.db.DB_PREALLOCATION = \
            config_true_value(conf.get('db_preallocation', 'f'))
        self.recon_cache_path = conf.get('recon_cache_path',
                                         '/var/cache/swift')
        self.rcache = os.path.join(self.recon_cache_path, "container.recon")
        self.user_agent = 'container-updater %s' % os.getpid()

    def get_account_ring(self):
        """Get the account ring.  Load it if it hasn't been yet."""
        if not self.account_ring:
            self.account_ring = Ring(self.swift_dir, ring_name='account')
        return self.account_ring

    def _listdir(self, path):
        try:
            return os.listdir(path)
        except OSError as e:
            self.logger.error(_('ERROR:  Failed to get paths to drive '
                                'partitions: %s') % e)
            return []

    def get_paths(self):
        """
        Get paths to all of the partitions on each drive to be processed.

        :returns: a list of paths
        """
        paths = []
        for device in self._listdir(self.devices):
            dev_path = os.path.join(self.devices, device)
            if self.mount_check and not ismount(dev_path):
                self.logger.warning(_('%s is not mounted'), device)
                continue
            con_path = os.path.join(dev_path, DATADIR)
            if not os.path.exists(con_path):
                continue
            for partition in self._listdir(con_path):
                paths.append(os.path.join(con_path, partition))
        shuffle(paths)
        return paths

    def _load_suppressions(self, filename):
        try:
            with open(filename, 'r') as tmpfile:
                for line in tmpfile:
                    account, until = line.split()
                    until = float(until)
                    self.account_suppressions[account] = until
        except Exception:
            self.logger.exception(
                _('ERROR with loading suppressions from %s: ') % filename)
        finally:
            os.unlink(filename)

    def run_forever(self, *args, **kwargs):
        """
        Run the updater continuously.
        """
        time.sleep(random() * self.interval)
        while True:
            self.logger.info(_('Begin container update sweep'))
            begin = time.time()
            now = time.time()
            expired_suppressions = \
                [a for a, u in self.account_suppressions.items()
                 if u < now]
            for account in expired_suppressions:
                del self.account_suppressions[account]
            pid2filename = {}
            # read from account ring to ensure it's fresh
            self.get_account_ring().get_nodes('')
            for path in self.get_paths():
                while len(pid2filename) >= self.concurrency:
                    pid = os.wait()[0]
                    try:
                        self._load_suppressions(pid2filename[pid])
                    finally:
                        del pid2filename[pid]
                fd, tmpfilename = mkstemp()
                os.close(fd)
                pid = os.fork()
                if pid:
                    pid2filename[pid] = tmpfilename
                else:
                    signal.signal(signal.SIGTERM, signal.SIG_DFL)
                    patcher.monkey_patch(all=False, socket=True, select=True,
                                         thread=True)
                    self.no_changes = 0
                    self.successes = 0
                    self.failures = 0
                    self.new_account_suppressions = open(tmpfilename, 'w')
                    forkbegin = time.time()
                    self.container_sweep(path)
                    elapsed = time.time() - forkbegin
                    self.logger.debug(
                        _('Container update sweep of %(path)s completed: '
                          '%(elapsed).02fs, %(success)s successes, %(fail)s '
                          'failures, %(no_change)s with no changes'),
                        {'path': path, 'elapsed': elapsed,
                         'success': self.successes, 'fail': self.failures,
                         'no_change': self.no_changes})
                    sys.exit()
            while pid2filename:
                pid = os.wait()[0]
                try:
                    self._load_suppressions(pid2filename[pid])
                finally:
                    del pid2filename[pid]
            elapsed = time.time() - begin
            self.logger.info(_('Container update sweep completed: %.02fs'),
                             elapsed)
            dump_recon_cache({'container_updater_sweep': elapsed},
                             self.rcache, self.logger)
            if elapsed < self.interval:
                time.sleep(self.interval - elapsed)

    def run_once(self, *args, **kwargs):
        """
        Run the updater once.
        """
        patcher.monkey_patch(all=False, socket=True, select=True, thread=True)
        self.logger.info(_('Begin container update single threaded sweep'))
        begin = time.time()
        self.no_changes = 0
        self.successes = 0
        self.failures = 0
        for path in self.get_paths():
            self.container_sweep(path)
        elapsed = time.time() - begin
        self.logger.info(_(
            'Container update single threaded sweep completed: '
            '%(elapsed).02fs, %(success)s successes, %(fail)s failures, '
            '%(no_change)s with no changes'),
            {'elapsed': elapsed, 'success': self.successes,
             'fail': self.failures, 'no_change': self.no_changes})
        dump_recon_cache({'container_updater_sweep': elapsed},
                         self.rcache, self.logger)

    def container_sweep(self, path):
        """
        Walk the path looking for container DBs and process them.

        :param path: path to walk
        """
        for root, dirs, files in os.walk(path):
            for file in files:
                if file.endswith('.db'):
                    self.process_container(os.path.join(root, file))
                    time.sleep(self.slowdown)

    def process_container(self, dbfile):
        """
        Process a container, and update the information in the account.

        :param dbfile: container DB to process
        """
        start_time = time.time()
        broker = ContainerBroker(dbfile, logger=self.logger)
        info = broker.get_info()
        # Don't send updates if the container was auto-created since it
        # definitely doesn't have up to date statistics.
        if Timestamp(info['put_timestamp']) <= 0:
            return
        if self.account_suppressions.get(info['account'], 0) > time.time():
            return
        if info['put_timestamp'] > info['reported_put_timestamp'] or \
                info['delete_timestamp'] > info['reported_delete_timestamp'] \
                or info['object_count'] != info['reported_object_count'] or \
                info['bytes_used'] != info['reported_bytes_used']:
            container = '/%s/%s' % (info['account'], info['container'])
            part, nodes = self.get_account_ring().get_nodes(info['account'])
            events = [spawn(self.container_report, node, part, container,
                            info['put_timestamp'], info['delete_timestamp'],
                            info['object_count'], info['bytes_used'],
                            info['storage_policy_index'])
                      for node in nodes]
            successes = 0
            for event in events:
                if is_success(event.wait()):
                    successes += 1
            if successes >= majority_size(len(events)):
                self.logger.increment('successes')
                self.successes += 1
                self.logger.debug(
                    _('Update report sent for %(container)s %(dbfile)s'),
                    {'container': container, 'dbfile': dbfile})
                broker.reported(info['put_timestamp'],
                                info['delete_timestamp'], info['object_count'],
                                info['bytes_used'])
            else:
                self.logger.increment('failures')
                self.failures += 1
                self.logger.debug(
                    _('Update report failed for %(container)s %(dbfile)s'),
                    {'container': container, 'dbfile': dbfile})
                self.account_suppressions[info['account']] = until = \
                    time.time() + self.account_suppression_time
                if self.new_account_suppressions:
                    print(info['account'], until,
                          file=self.new_account_suppressions)
            # Only track timing data for attempted updates:
            self.logger.timing_since('timing', start_time)
        else:
            self.logger.increment('no_changes')
            self.no_changes += 1

    def container_report(self, node, part, container, put_timestamp,
                         delete_timestamp, count, bytes,
                         storage_policy_index):
        """
        Report container info to an account server.

        :param node: node dictionary from the account ring
        :param part: partition the account is on
        :param container: container name
        :param put_timestamp: put timestamp
        :param delete_timestamp: delete timestamp
        :param count: object count in the container
        :param bytes: bytes used in the container
        :param storage_policy_index: the policy index for the container
        """
        with ConnectionTimeout(self.conn_timeout):
            try:
                headers = {
                    'X-Put-Timestamp': put_timestamp,
                    'X-Delete-Timestamp': delete_timestamp,
                    'X-Object-Count': count,
                    'X-Bytes-Used': bytes,
                    'X-Account-Override-Deleted': 'yes',
                    'X-Backend-Storage-Policy-Index': storage_policy_index,
                    'user-agent': self.user_agent}
                conn = http_connect(
                    node['ip'], node['port'], node['device'], part,
                    'PUT', container, headers=headers)
            except (Exception, Timeout):
                self.logger.exception(_(
                    'ERROR account update failed with '
                    '%(ip)s:%(port)s/%(device)s (will retry later): '), node)
                return HTTP_INTERNAL_SERVER_ERROR
        with Timeout(self.node_timeout):
            try:
                resp = conn.getresponse()
                resp.read()
                return resp.status
            except (Exception, Timeout):
                if self.logger.getEffectiveLevel() <= logging.DEBUG:
                    self.logger.exception(
                        _('Exception with %(ip)s:%(port)s/%(device)s'), node)
                return HTTP_INTERNAL_SERVER_ERROR
            finally:
                conn.close()
<|endoftext|>


<commit_before># Copyright 2015-2016 Luc Saffre
# License: BSD (see file COPYING for details)

"""
Database models for `lino.modlib.plausibility`.

.. autosummary::

"""

from __future__ import unicode_literals, print_function
import six
from builtins import object
from builtins import str
from collections import OrderedDict

from django.db import models
from django.utils import translation

from lino.core.gfks import gfk2lookup
from lino.modlib.gfks.mixins import Controllable
from lino.modlib.users.mixins import UserAuthored
from lino.core.roles import SiteStaff

from lino.api import dd, rt, _

from .choicelists import Checker, Checkers

from .roles import PlausibilityUser

class UpdateProblem(dd.Action):
    icon_name = 'bell'
    label = _("Check plausibility")
    combo_group = "plausibility"
    fix_them = False
    sort_index = 90
    # custom_handler = True
    # select_rows = False
    default_format = None

    def run_from_ui(self, ar, fix=None):
        if fix is None:
            fix = self.fix_them
        Problem = rt.modules.plausibility.Problem
        # print(20150327, ar.selected_rows)
        for obj in ar.selected_rows:
            assert isinstance(obj, Problem)
            chk = obj.checker
            owner = obj.owner
            # not tested: what happens if the following deletes
            # another obj from selected_rows?
            qs = Problem.objects.filter(
                **gfk2lookup(Problem.owner, owner, checker=chk))
            qs.delete()
            chk.update_problems(owner, False, fix)
        ar.set_response(refresh_all=True)

class FixProblem(UpdateProblem):
    label = _("Fix plausibility problems")
    fix_them = True
    sort_index = 91

class UpdateProblemsByController(dd.Action):
    """Updates the table of plausibility problems for a given database
    object, also removing those messages which no longer exist. This
    action does not change anything else in the database.

    This action is automatically being installed on each model for
    which there is at least one active :class:`Checker
    <lino.modlib.plausibility.choicelists.Checker>`.

    """
    icon_name = 'bell'
    label = _("Check plausibility")
    combo_group = "plausibility"
    fix_them = False

    def __init__(self, model):
        self.model = model
        super(UpdateProblemsByController, self).__init__()

    def run_from_ui(self, ar, fix=None):
        if fix is None:
            fix = self.fix_them
        Problem = rt.modules.plausibility.Problem
        gfk = Problem.owner
        checkers = get_checkable_models()[self.model]
        for obj in ar.selected_rows:
            assert isinstance(obj, self.model)
            qs = Problem.objects.filter(**gfk2lookup(gfk, obj))
            qs.delete()
            for chk in checkers:
                chk.update_problems(obj, False, fix)
        ar.set_response(refresh=True)

class FixProblemsByController(UpdateProblemsByController):
    """Update plausibility problems, repairing those which are
    automatically fixable.

    """
    label = _("Fix plausibility problems")
    fix_them = True

@dd.python_2_unicode_compatible
class Problem(Controllable, UserAuthored):
    """Represents a detected plausibility problem.

    Database objects of this model are considered temporary data which
    may be updated automatically without user interaction.

    .. attribute:: checker

       The :class:`Checker
       <lino.modlib.plausibility.choicelists.Checker>` which reported
       this problem.

    .. attribute:: message

       The message text. This is a concatenation of all messages that
       were yeld by the :attr:`checker`.

    .. attribute:: user

       The :class:`user <lino.modlib.users.models.User>` reponsible
       for fixing this problem.

       This field is being filled by the :meth:`get_responsible_user
       <lino.modlib.plausibility.choicelists.Checker.get_responsible_user>`
       method of the :attr:`checker`.

    """
    class Meta(object):
        app_label = 'plausibility'
        verbose_name = _("Plausibility problem")
        verbose_name_plural = _("Plausibility problems")
        ordering = ['owner_type', 'owner_id', 'checker']

    # problem_type = ProblemTypes.field()
    checker = Checkers.field(verbose_name=_("Checker"))
    # severity = Severities.field()
    # feedback = Feedbacks.field(blank=True)
    message = models.CharField(_("Message"), max_length=250)
    # fixable = models.BooleanField(_("Fixable"), default=False)

    update_problem = UpdateProblem()
    fix_problem = FixProblem()

    # no longer needed after 20170826
    # @classmethod
    # def setup_parameters(cls, **fields):
    #     fields.update(checker=Checkers.field(
    #         blank=True, help_text=_("Only problems by this checker.")))
    #     return fields

    def __str__(self):
        return self.message

    @classmethod
    def get_simple_parameters(cls):
        for p in super(Problem, cls).get_simple_parameters():
            yield p
        # yield 'user'
        yield 'checker'

dd.update_field(Problem, 'user', verbose_name=_("Responsible"))
Problem.set_widget_options('checker', width=10)
Problem.set_widget_options('user', width=10)
Problem.set_widget_options('message', width=50)
Problem.update_controller_field(verbose_name = _('Database object'))

class Problems(dd.Table):
    "The base table for :class:`Problem` objects."
    model = 'plausibility.Problem'
    column_names = "user owner message #fixable checker *"
    auto_fit_column_widths = True
    editable = False
    cell_edit = False
    # parameters = dict(
    #     # user=models.ForeignKey(
    #     #     'users.User', blank=True, null=True,
    #     #     verbose_name=_("Responsible"),
    #     #     help_text=_("""Only problems for this responsible.""")),
    #     )
    params_layout = "user checker"

    # simple_parameters = ('user', 'checker')
    detail_layout = dd.DetailLayout("""
    checker
    owner
    message
    user id
    """, window_size=(70, 'auto'))

class AllProblems(Problems):
    """Show all plausibility problems.

    This table can be opened by system managers using
    :menuselection:`Explorer --> System --> Plausibility problems`.

    """
    required_roles = dd.login_required(SiteStaff)

class ProblemsByOwner(Problems):
    """Show data problems related to this database object.

    """
    master_key = 'owner'
    column_names = "message checker user #fixable *"
    slave_grid_format = 'summary'

class ProblemsByChecker(Problems):
    """Show the plausibility problems by checker.

    This was the first use case of a slave table with a master which
    is something else than a model instance.

    """
    master_key = 'checker'

    column_names = "user owner message #fixable *"

    @classmethod
    def get_master_instance(cls, ar, model, pk):
        return Checkers.get_by_value(pk)

    @classmethod
    def get_filter_kw(self, ar, **kw):
        kw.update(checker=ar.master_instance)
        return kw

class MyProblems(Problems):
    """Shows the plausibility problems assigned to this user.

    """
    required_roles = dd.login_required(PlausibilityUser)
    label = _("Plausibility problems assigned to me")

    @classmethod
    def param_defaults(self, ar, **kw):
        kw = super(MyProblems, self).param_defaults(ar, **kw)
        kw.update(user=ar.get_user())
        return kw

    @classmethod
    def get_welcome_messages(cls, ar, **kw):
        sar = ar.spawn(cls)
        if not sar.get_permission():
            return
        count = sar.get_total_count()
        if count > 0:
            msg = _("There are {0} plausibility problems assigned to you.")
            msg = msg.format(count)
            yield ar.href_to_request(sar, msg)

@dd.receiver(dd.pre_analyze)
def set_plausibility_actions(sender, **kw):
    """Installs the :class:`UpdateProblemsByController` action on every
    model for which there is at least one Checker

    """
    for m in list(get_checkable_models().keys()):
        assert m is not Problem
        m.define_action(check_plausibility=UpdateProblemsByController(m))
        m.define_action(fix_problems=FixProblemsByController(m))
        if False:
            # don't add it automatically because appdev might prefer
            # to show it in a detail_layout:
            m.define_action(show_problems=dd.ShowSlaveTable(
                ProblemsByOwner,
                icon_name = 'bell', combo_group = "plausibility"))

def get_checkable_models(*args):
    """Return an `OrderedDict` mapping each model which has at least one
    checker to a list of these checkers.

    The dict is ordered to avoid that checkers run in a random order.

    """
    checkable_models = OrderedDict()
    for chk in Checkers.objects():
        if len(args):
            skip = True
            for arg in args:
                if arg in chk.value:
                    skip = False
            if skip:
                continue
        for m in chk.get_checkable_models():
            lst = checkable_models.setdefault(m, [])
            lst.append(chk)
    return checkable_models

def check_plausibility(args=[], fix=True):
    """Called by :manage:`checkdata`. See there."""
    Problem = rt.modules.plausibility.Problem
    mc = get_checkable_models(*args)
    if len(mc) == 0 and len(args) > 0:
        raise Exception("No checker matches {0}".format(args))
    final_sums = [0, 0, 0]
    with translation.override('en'):
        for m, checkers in mc.items():
            ct = rt.modules.contenttypes.ContentType.objects.get_for_model(m)
            Problem.objects.filter(owner_type=ct).delete()
            name = str(m._meta.verbose_name_plural)
            qs = m.objects.all()
            msg = "Running {0} data checkers on {1} {2}...".format(
                len(checkers), qs.count(), name)
            dd.logger.debug(msg)
            sums = [0, 0, name]
            for obj in qs:
                for chk in checkers:
                    todo, done = chk.update_problems(obj, False, fix)
                    sums[0] += len(todo)
                    sums[1] += len(done)
            if sums[0] or sums[1]:
                msg = "Found {0} and fixed {1} data problems in {2}."
                dd.logger.info(msg.format(*sums))
            else:
                dd.logger.debug(
                    "No data problems found in {0}.".format(name))
            final_sums[0] += 1
            final_sums[1] += sums[0]
            final_sums[2] += sums[1]
    msg = "Done %d checkers, found %d and fixed %d problems."
    dd.logger.info(msg, *final_sums)

@dd.schedule_daily()
def checkdata():
    """Run all data checkers."""
    check_plausibility(fix=False)
    # rt.login().run(settings.SITE.site_config.run_checkdata)
<commit_msg>http://luc.lino-framework.org/blog/2017/1010.html<commit_after># Copyright 2015-2017 Luc Saffre
# License: BSD (see file COPYING for details)

"""
Database models for `lino.modlib.plausibility`.

.. autosummary::

"""

from __future__ import unicode_literals, print_function
import six
from builtins import object
from builtins import str
from collections import OrderedDict

from django.db import models
from django.utils import translation

from lino.core.gfks import gfk2lookup
from lino.modlib.gfks.mixins import Controllable
from lino.modlib.users.mixins import UserAuthored
from lino.core.roles import SiteStaff

from lino.api import dd, rt, _

from .choicelists import Checker, Checkers

from .roles import PlausibilityUser

class UpdateProblem(dd.Action):
    icon_name = 'bell'
    label = _("Check plausibility")
    combo_group = "plausibility"
    fix_them = False
    sort_index = 90
    # custom_handler = True
    # select_rows = False
    default_format = None

    def run_from_ui(self, ar, fix=None):
        if fix is None:
            fix = self.fix_them
        Problem = rt.modules.plausibility.Problem
        # print(20150327, ar.selected_rows)
        for obj in ar.selected_rows:
            assert isinstance(obj, Problem)
            chk = obj.checker
            owner = obj.owner
            # not tested: what happens if the following deletes
            # another obj from selected_rows?
            qs = Problem.objects.filter(
                **gfk2lookup(Problem.owner, owner, checker=chk))
            qs.delete()
            chk.update_problems(owner, False, fix)
        ar.set_response(refresh_all=True)

class FixProblem(UpdateProblem):
    label = _("Fix plausibility problems")
    fix_them = True
    sort_index = 91

class UpdateProblemsByController(dd.Action):
    """Updates the table of plausibility problems for a given database
    object, also removing those messages which no longer exist. This
    action does not change anything else in the database.

    This action is automatically being installed on each model for
    which there is at least one active :class:`Checker
    <lino.modlib.plausibility.choicelists.Checker>`.

    """
    icon_name = 'bell'
    label = _("Check plausibility")
    combo_group = "plausibility"
    fix_them = False

    def __init__(self, model):
        self.model = model
        super(UpdateProblemsByController, self).__init__()

    def run_from_ui(self, ar, fix=None):
        if fix is None:
            fix = self.fix_them
        Problem = rt.modules.plausibility.Problem
        gfk = Problem.owner
        checkers = get_checkable_models()[self.model]
        for obj in ar.selected_rows:
            assert isinstance(obj, self.model)
            qs = Problem.objects.filter(**gfk2lookup(gfk, obj))
            qs.delete()
            for chk in checkers:
                chk.update_problems(obj, False, fix)
        ar.set_response(refresh=True)

class FixProblemsByController(UpdateProblemsByController):
    """Update plausibility problems, repairing those which are
    automatically fixable.

    """
    label = _("Fix plausibility problems")
    fix_them = True

@dd.python_2_unicode_compatible
class Problem(Controllable, UserAuthored):
    """Represents a detected plausibility problem.

    Database objects of this model are considered temporary data which
    may be updated automatically without user interaction.

    .. attribute:: checker

       The :class:`Checker
       <lino.modlib.plausibility.choicelists.Checker>` which reported
       this problem.

    .. attribute:: message

       The message text. This is a concatenation of all messages that
       were yeld by the :attr:`checker`.

    .. attribute:: user

       The :class:`user <lino.modlib.users.models.User>` reponsible
       for fixing this problem.

       This field is being filled by the :meth:`get_responsible_user
       <lino.modlib.plausibility.choicelists.Checker.get_responsible_user>`
       method of the :attr:`checker`.

    """
    class Meta(object):
        app_label = 'plausibility'
        verbose_name = _("Plausibility problem")
        verbose_name_plural = _("Plausibility problems")
        ordering = ['owner_type', 'owner_id', 'checker']

    # problem_type = ProblemTypes.field()
    checker = Checkers.field(verbose_name=_("Checker"))
    # severity = Severities.field()
    # feedback = Feedbacks.field(blank=True)
    message = models.CharField(_("Message"), max_length=250)
    # fixable = models.BooleanField(_("Fixable"), default=False)

    update_problem = UpdateProblem()
    fix_problem = FixProblem()

    # no longer needed after 20170826
    # @classmethod
    # def setup_parameters(cls, **fields):
    #     fields.update(checker=Checkers.field(
    #         blank=True, help_text=_("Only problems by this checker.")))
    #     return fields

    def __str__(self):
        return self.message

    @classmethod
    def get_simple_parameters(cls):
        for p in super(Problem, cls).get_simple_parameters():
            yield p
        # yield 'user'
        yield 'checker'

dd.update_field(Problem, 'user', verbose_name=_("Responsible"))
Problem.set_widget_options('checker', width=10)
Problem.set_widget_options('user', width=10)
Problem.set_widget_options('message', width=50)
Problem.update_controller_field(verbose_name = _('Database object'))

class Problems(dd.Table):
    "The base table for :class:`Problem` objects."
    model = 'plausibility.Problem'
    column_names = "user owner message #fixable checker *"
    auto_fit_column_widths = True
    editable = False
    cell_edit = False
    # parameters = dict(
    #     # user=models.ForeignKey(
    #     #     'users.User', blank=True, null=True,
    #     #     verbose_name=_("Responsible"),
    #     #     help_text=_("""Only problems for this responsible.""")),
    #     )
    params_layout = "user checker"

    # simple_parameters = ('user', 'checker')
    detail_layout = dd.DetailLayout("""
    checker
    owner
    message
    user id
    """, window_size=(70, 'auto'))

class AllProblems(Problems):
    """Show all plausibility problems.

    This table can be opened by system managers using
    :menuselection:`Explorer --> System --> Plausibility problems`.

    """
    required_roles = dd.login_required(SiteStaff)

class ProblemsByOwner(Problems):
    """Show data problems related to this database object.

    """
    master_key = 'owner'
    column_names = "message checker user #fixable *"
    slave_grid_format = 'summary'

class ProblemsByChecker(Problems):
    """Show the plausibility problems by checker.

    This was the first use case of a slave table with a master which
    is something else than a model instance.

    """
    master_key = 'checker'

    column_names = "user owner message #fixable *"

    @classmethod
    def get_master_instance(cls, ar, model, pk):
        return Checkers.get_by_value(pk)

    @classmethod
    def get_filter_kw(self, ar, **kw):
        kw.update(checker=ar.master_instance)
        return kw

class MyProblems(Problems):
    """Shows the plausibility problems assigned to this user.

    """
    required_roles = dd.login_required(PlausibilityUser)
    label = _("Plausibility problems assigned to me")

    @classmethod
    def param_defaults(self, ar, **kw):
        kw = super(MyProblems, self).param_defaults(ar, **kw)
        kw.update(user=ar.get_user())
        return kw

    @classmethod
    def get_welcome_messages(cls, ar, **kw):
        sar = ar.spawn(cls)
        if not sar.get_permission():
            return
        count = sar.get_total_count()
        if count > 0:
            msg = _("There are {0} plausibility problems assigned to you.")
            msg = msg.format(count)
            yield ar.href_to_request(sar, msg)

@dd.receiver(dd.pre_analyze)
def set_plausibility_actions(sender, **kw):
    """Installs the :class:`UpdateProblemsByController` action on every
    model for which there is at least one Checker

    """
    for m in list(get_checkable_models().keys()):
        assert m is not Problem
        m.define_action(check_plausibility=UpdateProblemsByController(m))
        m.define_action(fix_problems=FixProblemsByController(m))
        if False:
            # don't add it automatically because appdev might prefer
            # to show it in a detail_layout:
            m.define_action(show_problems=dd.ShowSlaveTable(
                ProblemsByOwner,
                icon_name = 'bell', combo_group = "plausibility"))

def get_checkable_models(*args):
    """Return an `OrderedDict` mapping each model which has at least one
    checker to a list of these checkers.

    The dict is ordered to avoid that checkers run in a random order.

    """
    checkable_models = OrderedDict()
    for chk in Checkers.objects():
        if len(args):
            skip = True
            for arg in args:
                if arg in chk.value:
                    skip = False
            if skip:
                continue
        for m in chk.get_checkable_models():
            lst = checkable_models.setdefault(m, [])
            lst.append(chk)
    return checkable_models

def check_plausibility(args=[], fix=True):
    """Called by :manage:`checkdata`. See there."""
    Problem = rt.modules.plausibility.Problem
    mc = get_checkable_models(*args)
    if len(mc) == 0 and len(args) > 0:
        raise Exception("No checker matches {0}".format(args))
    final_sums = [0, 0, 0]
    with translation.override('en'):
        for m, checkers in mc.items():
            ct = rt.modules.contenttypes.ContentType.objects.get_for_model(m)
            Problem.objects.filter(owner_type=ct).delete()
            name = str(m._meta.verbose_name_plural)
            qs = m.objects.all()
            msg = "Running {0} data checkers on {1} {2}...".format(
                len(checkers), qs.count(), name)
            dd.logger.debug(msg)
            sums = [0, 0, name]
            for obj in qs:
                for chk in checkers:
                    todo, done = chk.update_problems(obj, False, fix)
                    sums[0] += len(todo)
                    sums[1] += len(done)
            if sums[0] or sums[1]:
                msg = "Found {0} and fixed {1} data problems in {2}."
                dd.logger.info(msg.format(*sums))
            else:
                dd.logger.debug(
                    "No data problems found in {0}.".format(name))
            final_sums[0] += 1
            final_sums[1] += sums[0]
            final_sums[2] += sums[1]
    msg = "Done %d checkers, found %d and fixed %d problems."
    dd.logger.info(msg, *final_sums)

@dd.schedule_daily()
def checkdata():
    """Run all data checkers."""
    check_plausibility(fix=False)
    # rt.login().run(settings.SITE.site_config.run_checkdata)
<|endoftext|>


<commit_before>import datetime

from django.core.management.base import BaseCommand

from lowfat.models import Claimant, Fund, Expense

class Command(BaseCommand):
    help = "CW19 funding request"

    def handle(self, *args, **options):
        for claimant in Claimant.objects.filter(
                application_year=2019
        ):
            candidate = Claimant.objects.get(
                email=claimant.email,
                application_year=2018
            )
            for fund in Fund.objects.filter(
                    claimant=claimant
            ):
                print("Transfering {} from {} to {} ...".format(
                    fund,
                    claimant,
                    candidate
                    )
                )
                fund.claimant = candidate
                fund.save()
                print("Transfer complete.")
            print("Removing {} ...".format(claimant))
            claimant.delete()
            print("Remove successful.")
<commit_msg>Fix PyLint errors<commit_after>from django.core.management.base import BaseCommand

from lowfat.models import Claimant, Fund

class Command(BaseCommand):
    help = "CW19 funding request"

    def handle(self, *args, **options):
        for claimant in Claimant.objects.filter(
                application_year=2019
        ):
            candidate = Claimant.objects.get(
                email=claimant.email,
                application_year=2018
            )
            for fund in Fund.objects.filter(
                    claimant=claimant
            ):
                print("Transfering {} from {} to {} ...".format(
                    fund,
                    claimant,
                    candidate
                ))
                fund.claimant = candidate
                fund.save()
                print("Transfer complete.")
            print("Removing {} ...".format(claimant))
            claimant.delete()
            print("Remove successful.")
<|endoftext|>


<commit_before>#!/usr/bin/python2
# pylint: disable=C,R
# -*- encoding: utf8 -*-
#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2016 Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

'''Qubes volume and block device managment'''

from __future__ import print_function

import sys

import qubes
import qubes.exc
import qubes.tools

def prepare_table(vd_list, full=False):
    ''' Converts a list of :py:class:`VolumeData` objects to a list of tupples
        for the :py:func:`qubes.tools.print_table`.

        If :program:`qvm-block` is running in a TTY, it will ommit duplicate
        data.

        :param list vd_list: List of :py:class:`VolumeData` objects.
        :param bool full:    If set to true duplicate data is printed even when
                             running from TTY.
        :returns: list of tupples
    '''
    output = []
    if sys.stdout.isatty():
        output += [('POOL:VOLUME', 'VMNAME', 'VOLUME_NAME')]  # NOQA

    for volume in vd_list:
        if volume.domains:
            vmname, volume_name = volume.domains.pop()
            output += [(str(volume), vmname, volume_name, volume.revisions)]
            for tupple in volume.domains:
                vmname, volume_name = tupple
                if full or not sys.stdout.isatty():
                    output += [(str(volume), vmname, volume_name,
                            volume.revisions)]
                else:
                    output += [('', vmname, volume_name, '', volume.revisions)]
        else:
            output += [(str(volume), "")]

    return output

class VolumeData(object):
    ''' Wrapper object around :py:class:`qubes.storage.Volume`, mainly to track
        the domains a volume is attached to.
    '''
    # pylint: disable=too-few-public-methods
    def __init__(self, volume):
        self.name = volume.name
        self.pool = volume.pool
        self.vid = volume.vid
        if volume.revisions != {}:
            self.revisions = 'Yes'
        else:
            self.revisions = 'No'
        self.domains = []

    def __str__(self):
        return "{!s}:{!s}".format(self.pool, self.vid)

def list_volumes(args):
    ''' Called by the parser to execute the qubes-block list subcommand. '''
    app = args.app

    if args.pools:
        pools = args.pools  # only specified pools
    else:
        pools = app.pools.values()  # all pools

    volumes = [v for p in pools for v in p.volumes]

    if not args.internal:  # hide internal volumes
        volumes = [v for v in volumes if not v.internal]

    vd_dict = {}

    for volume in volumes:
        volume_data = VolumeData(volume)
        try:
            vd_dict[volume.pool][volume.vid] = volume_data
        except KeyError:
            vd_dict[volume.pool] = {volume.vid: volume_data}

    if hasattr(args, 'domains') and args.domains:
        domains = args.domains
    else:
        domains = args.app.domains
    for domain in domains:  # gather the domain names
        try:
            for volume in domain.attached_volumes:
                try:
                    volume_data = vd_dict[volume.pool][volume.vid]
                    volume_data.domains += [(domain.name, volume.name)]
                except KeyError:
                    # Skipping volume
                    continue
        except AttributeError:
            # Skipping domain without volumes
            continue

    if hasattr(args, 'domains') and args.domains:
        result = [x  # reduce to only VolumeData with assigned domains
                  for p in vd_dict.itervalues() for x in p.itervalues()
                  if x.domains]
    else:
        result = [x for p in vd_dict.itervalues() for x in p.itervalues()]
    qubes.tools.print_table(prepare_table(result, full=args.full))

def revert_volume(args):
    volume = args.volume
    app = args.app
    try:
        pool = app.pools[volume.pool]
        pool.revert(volume)
    except qubes.storage.StoragePoolException as e:
        print(e.message, file=sys.stderr)
        sys.exit(1)

def attach_volumes(args):
    ''' Called by the parser to execute the :program:`qvm-block attach`
        subcommand.
    '''
    volume = args.volume
    vm = args.domains[0]
    try:
        rw = not args.ro
        vm.storage.attach(volume, rw=rw)
    except qubes.storage.StoragePoolException as e:
        print(e.message, file=sys.stderr)
        sys.exit(1)

def detach_volumes(args):
    ''' Called by the parser to execute the :program:`qvm-block detach`
        subcommand.
    '''
    volume = args.volume
    vm = args.domains[0]
    try:
        vm.storage.detach(volume)
    except qubes.storage.StoragePoolException as e:
        print(e.message, file=sys.stderr)
        sys.exit(1)

def init_list_parser(sub_parsers):
    ''' Configures the parser for the :program:`qvm-block list` subcommand '''
    # pylint: disable=protected-access
    list_parser = sub_parsers.add_parser('list', aliases=('ls', 'l'),
                                         help='list block devices')
    list_parser.add_argument('-p', '--pool', dest='pools',
                             action=qubes.tools.PoolsAction)
    list_parser.add_argument('-i', '--internal', action='store_true',
                             help='Show internal volumes')
    list_parser.add_argument(
        '--full', action='store_true',
        help='print full line for each POOL_NAME:VOLUME_ID & vm combination')

    vm_name_group = qubes.tools.VmNameGroup(
        list_parser, required=False, vm_action=qubes.tools.VmNameAction,
        help='list volumes from specified domain(s)')
    list_parser._mutually_exclusive_groups.append(vm_name_group)
    list_parser.set_defaults(func=list_volumes)

def init_revert_parser(sub_parsers):
    revert_parser = sub_parsers.add_parser(
        'revert', aliases=('rv', 'r'),
        help='revert volume to previous revision')
    revert_parser.add_argument(metavar='POOL_NAME:VOLUME_ID', dest='volume',
                               action=qubes.tools.VolumeAction)
    revert_parser.set_defaults(func=revert_volume)

def get_parser():
    '''Create :py:class:`argparse.ArgumentParser` suitable for
    :program:`qvm-block`.
    '''
    parser = qubes.tools.QubesArgumentParser(description=__doc__, want_app=True)
    parser.register('action', 'parsers', qubes.tools.AliasedSubParsersAction)
    sub_parsers = parser.add_subparsers(
        title='commands',
        description="For more information see qvm-block command -h",
        dest='command')
    init_list_parser(sub_parsers)
    init_revert_parser(sub_parsers)
    attach_parser = sub_parsers.add_parser(
        'attach', help="Attach volume to domain", aliases=('at', 'a'))
    attach_parser.add_argument('--ro', help='attach device read-only',
                               action='store_true')
    attach_parser.add_argument('VMNAME', action=qubes.tools.RunningVmNameAction)
    attach_parser.add_argument(metavar='POOL_NAME:VOLUME_ID', dest='volume',
                               action=qubes.tools.VolumeAction)
    attach_parser.set_defaults(func=attach_volumes)
    detach_parser = sub_parsers.add_parser(
        "detach", help="Detach volume from domain", aliases=('d', 'dt'))
    detach_parser.add_argument('VMNAME', action=qubes.tools.RunningVmNameAction)
    detach_parser.add_argument(metavar='POOL_NAME:VOLUME_ID', dest='volume',
                               action=qubes.tools.VolumeAction)
    detach_parser.set_defaults(func=detach_volumes)

    return parser

def main(args=None):
    '''Main routine of :program:`qvm-block`.'''
    args = get_parser().parse_args(args)
    args.func(args)

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>qvm-block refactor attach/detach sub_parser init<commit_after>#!/usr/bin/python2
# pylint: disable=C,R
# -*- encoding: utf8 -*-
#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2016 Bahtiar `kalkin-` Gadimov <bahtiar@gadimov.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

'''Qubes volume and block device managment'''

from __future__ import print_function

import sys

import qubes
import qubes.exc
import qubes.tools

def prepare_table(vd_list, full=False):
    ''' Converts a list of :py:class:`VolumeData` objects to a list of tupples
        for the :py:func:`qubes.tools.print_table`.

        If :program:`qvm-block` is running in a TTY, it will ommit duplicate
        data.

        :param list vd_list: List of :py:class:`VolumeData` objects.
        :param bool full:    If set to true duplicate data is printed even when
                             running from TTY.
        :returns: list of tupples
    '''
    output = []
    if sys.stdout.isatty():
        output += [('POOL:VOLUME', 'VMNAME', 'VOLUME_NAME')]  # NOQA

    for volume in vd_list:
        if volume.domains:
            vmname, volume_name = volume.domains.pop()
            output += [(str(volume), vmname, volume_name, volume.revisions)]
            for tupple in volume.domains:
                vmname, volume_name = tupple
                if full or not sys.stdout.isatty():
                    output += [(str(volume), vmname, volume_name,
                            volume.revisions)]
                else:
                    output += [('', vmname, volume_name, '', volume.revisions)]
        else:
            output += [(str(volume), "")]

    return output

class VolumeData(object):
    ''' Wrapper object around :py:class:`qubes.storage.Volume`, mainly to track
        the domains a volume is attached to.
    '''
    # pylint: disable=too-few-public-methods
    def __init__(self, volume):
        self.name = volume.name
        self.pool = volume.pool
        self.vid = volume.vid
        if volume.revisions != {}:
            self.revisions = 'Yes'
        else:
            self.revisions = 'No'
        self.domains = []

    def __str__(self):
        return "{!s}:{!s}".format(self.pool, self.vid)

def list_volumes(args):
    ''' Called by the parser to execute the qubes-block list subcommand. '''
    app = args.app

    if args.pools:
        pools = args.pools  # only specified pools
    else:
        pools = app.pools.values()  # all pools

    volumes = [v for p in pools for v in p.volumes]

    if not args.internal:  # hide internal volumes
        volumes = [v for v in volumes if not v.internal]

    vd_dict = {}

    for volume in volumes:
        volume_data = VolumeData(volume)
        try:
            vd_dict[volume.pool][volume.vid] = volume_data
        except KeyError:
            vd_dict[volume.pool] = {volume.vid: volume_data}

    if hasattr(args, 'domains') and args.domains:
        domains = args.domains
    else:
        domains = args.app.domains
    for domain in domains:  # gather the domain names
        try:
            for volume in domain.attached_volumes:
                try:
                    volume_data = vd_dict[volume.pool][volume.vid]
                    volume_data.domains += [(domain.name, volume.name)]
                except KeyError:
                    # Skipping volume
                    continue
        except AttributeError:
            # Skipping domain without volumes
            continue

    if hasattr(args, 'domains') and args.domains:
        result = [x  # reduce to only VolumeData with assigned domains
                  for p in vd_dict.itervalues() for x in p.itervalues()
                  if x.domains]
    else:
        result = [x for p in vd_dict.itervalues() for x in p.itervalues()]
    qubes.tools.print_table(prepare_table(result, full=args.full))

def revert_volume(args):
    volume = args.volume
    app = args.app
    try:
        pool = app.pools[volume.pool]
        pool.revert(volume)
    except qubes.storage.StoragePoolException as e:
        print(e.message, file=sys.stderr)
        sys.exit(1)

def attach_volumes(args):
    ''' Called by the parser to execute the :program:`qvm-block attach`
        subcommand.
    '''
    volume = args.volume
    vm = args.domains[0]
    try:
        rw = not args.ro
        vm.storage.attach(volume, rw=rw)
    except qubes.storage.StoragePoolException as e:
        print(e.message, file=sys.stderr)
        sys.exit(1)

def detach_volumes(args):
    ''' Called by the parser to execute the :program:`qvm-block detach`
        subcommand.
    '''
    volume = args.volume
    vm = args.domains[0]
    try:
        vm.storage.detach(volume)
    except qubes.storage.StoragePoolException as e:
        print(e.message, file=sys.stderr)
        sys.exit(1)

def init_list_parser(sub_parsers):
    ''' Configures the parser for the :program:`qvm-block list` subcommand '''
    # pylint: disable=protected-access
    list_parser = sub_parsers.add_parser('list', aliases=('ls', 'l'),
                                         help='list block devices')
    list_parser.add_argument('-p', '--pool', dest='pools',
                             action=qubes.tools.PoolsAction)
    list_parser.add_argument('-i', '--internal', action='store_true',
                             help='Show internal volumes')
    list_parser.add_argument(
        '--full', action='store_true',
        help='print full line for each POOL_NAME:VOLUME_ID & vm combination')

    vm_name_group = qubes.tools.VmNameGroup(
        list_parser, required=False, vm_action=qubes.tools.VmNameAction,
        help='list volumes from specified domain(s)')
    list_parser._mutually_exclusive_groups.append(vm_name_group)
    list_parser.set_defaults(func=list_volumes)

def init_revert_parser(sub_parsers):
    revert_parser = sub_parsers.add_parser(
        'revert', aliases=('rv', 'r'),
        help='revert volume to previous revision')
    revert_parser.add_argument(metavar='POOL_NAME:VOLUME_ID', dest='volume',
                               action=qubes.tools.VolumeAction)
    revert_parser.set_defaults(func=revert_volume)

def init_attach_parser(sub_parsers):
    attach_parser = sub_parsers.add_parser(
        'attach', help="Attach volume to domain", aliases=('at', 'a'))
    attach_parser.add_argument('--ro', help='attach device read-only',
                               action='store_true')
    attach_parser.add_argument('VMNAME', action=qubes.tools.RunningVmNameAction)
    attach_parser.add_argument(metavar='POOL_NAME:VOLUME_ID', dest='volume',
                               action=qubes.tools.VolumeAction)
    attach_parser.set_defaults(func=attach_volumes)

def init_dettach_parser(sub_parsers):
    detach_parser = sub_parsers.add_parser(
        "detach", help="Detach volume from domain", aliases=('d', 'dt'))
    detach_parser.add_argument('VMNAME', action=qubes.tools.RunningVmNameAction)
    detach_parser.add_argument(metavar='POOL_NAME:VOLUME_ID', dest='volume',
                               action=qubes.tools.VolumeAction)
    detach_parser.set_defaults(func=detach_volumes)

def get_parser():
    '''Create :py:class:`argparse.ArgumentParser` suitable for
    :program:`qvm-block`.
    '''
    parser = qubes.tools.QubesArgumentParser(description=__doc__, want_app=True)
    parser.register('action', 'parsers', qubes.tools.AliasedSubParsersAction)
    sub_parsers = parser.add_subparsers(
        title='commands',
        description="For more information see qvm-block command -h",
        dest='command')
    init_attach_parser(sub_parsers)
    init_dettach_parser(sub_parsers)
    init_list_parser(sub_parsers)
    init_revert_parser(sub_parsers)

    return parser

def main(args=None):
    '''Main routine of :program:`qvm-block`.'''
    args = get_parser().parse_args(args)
    args.func(args)

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>"""
sylloparser.py - pulls the subject and object out of a proposition.
"""
import spacy
nlp = spacy.load('en')

# This helper function returns the words/phrases that match a given list of
# dependencies
def getTokensByDep(doc, deps):
    filtered = [t for t in doc if t.dep_ in deps]
    for token in filtered:
        return ' '.join([t.lemma_ for t in token.subtree
                         if t.dep_ in deps or t.dep_ == 'amod'])
    return []

# This helper function returns the words that match a given list of parts of
# speech
def getTokensByPOS(doc, pos):
    return [t for t in doc if t.pos_ in pos]

# The parsing function to extract relevant information from a proposition.
def sylloparse(sentence):

    # Strip the sentence to remove extra whitespace tokens
    sentence = sentence.strip()

    # Run the spaCy parser on the sentence
    parse = nlp(sentence)

    # Confirm that the sentence is actually a proposition. Currently, we only
    # support sentences that include a form of the verb 'be'
    verbs = getTokensByPOS(parse, ['VERB'])
    if not any([v.lemma_ == 'be' for v in verbs]):
        print('Warning: not using "be" verb. Ignored Sentence "%s"' % parse.text)
        return None

    # We have a valid proposition, so we need to get the entity for the subject
    # and the entity for the object of the proposition. The subject implies the
    # object.
    subj = getTokensByDep(parse, ['nsubj'])
    obj = getTokensByDep(parse, ['attr', 'acomp'])

    # Check to see if there is a form of not in the sentence. If there is, then
    # we need to perform additional checks to determine the logic of this
    # proposition. Currently, we negate the object of the proposition if we
    # detect the presence of a negation.
    negs = getTokensByDep(parse, ['neg'])
    if negs:
        obj = '!' + obj

    # Return the subject, object dependency as an array.
    return [subj, obj]
<commit_msg>print warning to stderr<commit_after>"""
sylloparser.py - pulls the subject and object out of a proposition.
"""
import spacy
from sys import stderr
nlp = spacy.load('en')

def getTokensByDep(doc, deps):
    '''Returns the words/phrases that match a given list of dependencies'''
    filtered = [t for t in doc if t.dep_ in deps]
    for token in filtered:
        return ' '.join([t.lemma_ for t in token.subtree
                         if t.dep_ in deps or t.dep_ == 'amod'])
    return []

def getTokensByPOS(doc, pos):
    '''Returns the words that match a given list of parts of speech'''
    return [t for t in doc if t.pos_ in pos]

def sylloparse(sentence):
    '''Extracts relevant information from a proposition'''

    # Strip the sentence to remove extra whitespace tokens
    sentence = sentence.strip()

    # Run the spaCy parser on the sentence
    parse = nlp(sentence)

    # Confirm that the sentence is actually a proposition. Currently, we only
    # support sentences that include a form of the verb 'be'
    verbs = getTokensByPOS(parse, ['VERB'])
    if not any([v.lemma_ == 'be' for v in verbs]):
        msg = 'Warning: "{}" does not have "be" as a verb. Ignoring sentence.'.format(parse.text)
        print(msg, file=stderr)
        return None

    # We have a valid proposition, so we need to get the entity for the
    # subject and the entity for the object of the proposition. The subject
    # implies the object.
    subj = getTokensByDep(parse, ['nsubj'])
    obj = getTokensByDep(parse, ['attr', 'acomp'])

    # Check to see if there is a form of not in the sentence. If there is,
    # then we need to perform additional checks to determine the logic of this
    # proposition. Currently, we negate the object of the proposition if we
    # detect the presence of a negation.
    negs = getTokensByDep(parse, ['neg'])
    if negs:
        obj = '!' + obj

    # Return the subject, object dependency as an array.
    return [subj, obj]
<|endoftext|>


<commit_before># -*- encoding: utf8 -*-
#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2017 Marek Marczykowski-Górecki
#                               <marmarek@invisiblethingslab.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program; if not, see <http://www.gnu.org/licenses/>.

'''Qubes VM objects.'''

import qubesmgmt.base
import qubesmgmt.storage

class QubesVM(qubesmgmt.base.PropertyHolder):
    '''Qubes domain.'''
    def __init__(self, app, name):
        super(QubesVM, self).__init__(app, 'mgmt.vm.property.', name)
        self._volumes = None

    @property
    def name(self):
        '''Domain name'''
        return self._method_dest

    @name.setter
    def name(self, new_value):
        self.qubesd_call(
            self._method_dest,
            self._method_prefix + 'Set',
            'name',
            str(new_value).encode('utf-8'))
        self._method_dest = new_value
        self._volumes = None
        self.app.domains.clear_cache()

    def start(self):
        '''
        Start domain.

        :return:
        '''
        self.qubesd_call(self._method_dest, 'mgmt.vm.Start')

    def shutdown(self, force=False):
        '''
        Shutdown domain.

        :return:
        '''
        # TODO: force parameter
        # TODO: wait parameter (using event?)
        if force:
            raise NotImplementedError
        self.qubesd_call(self._method_dest, 'mgmt.vm.Shutdown')

    def kill(self):
        '''
        Kill domain (forcefuly shutdown).

        :return:
        '''
        self.qubesd_call(self._method_dest, 'mgmt.vm.Kill')

    def pause(self):
        '''
        Pause domain.

        Pause its execution without any prior notification.

        :return:
        '''
        self.qubesd_call(self._method_dest, 'mgmt.vm.Pause')

    def unpause(self):
        '''
        Unpause domain.

        Opposite to :py:meth:`pause`.

        :return:
        '''
        self.qubesd_call(self._method_dest, 'mgmt.vm.Unpause')

    def suspend(self):
        '''
        Suspend domain.

        Give domain a chance to prepare for suspend - for example suspend
        used PCI devices.

        :return:
        '''
        raise NotImplementedError
        #self.qubesd_call(self._method_dest, 'mgmt.vm.Suspend')

    def resume(self):
        '''
        Resume domain.

        Opposite to :py:meth:`suspend`.

        :return:
        '''
        raise NotImplementedError
        #self.qubesd_call(self._method_dest, 'mgmt.vm.Resume')

    def get_power_state(self):
        '''Return power state description string.

        Return value may be one of those:

        =============== ========================================================
        return value    meaning
        =============== ========================================================
        ``'Halted'``    Machine is not active.
        ``'Transient'`` Machine is running, but does not have :program:`guid`
                        or :program:`qrexec` available.
        ``'Running'``   Machine is ready and running.
        ``'Paused'``    Machine is paused.
        ``'Suspended'`` Machine is S3-suspended.
        ``'Halting'``   Machine is in process of shutting down (OS shutdown).
        ``'Dying'``     Machine is in process of shutting down (cleanup).
        ``'Crashed'``   Machine crashed and is unusable.
        ``'NA'``        Machine is in unknown state.
        =============== ========================================================

        .. seealso::

            http://wiki.libvirt.org/page/VM_lifecycle
                Description of VM life cycle from the point of view of libvirt.

            https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainState
                Libvirt's enum describing precise state of a domain.

        '''

        vm_list_info = self.qubesd_call(
            self._method_dest, 'mgmt.vm.List', None, None).decode('ascii')
        #  name class=... state=... other=...
        vm_state = vm_list_info.partition('state=')[2].split(' ')[0]
        return vm_state

    @property
    def volumes(self):
        '''VM disk volumes'''
        if self._volumes is None:
            volumes_list = self.qubesd_call(
                self._method_dest, 'mgmt.vm.volume.List')
            self._volumes = {}
            for volname in volumes_list.decode('ascii').splitlines():
                if not volname:
                    continue
                self._volumes[volname] = qubesmgmt.storage.Volume(self.app,
                    vm=self.name, vm_name=volname)
        return self._volumes

# pylint: disable=abstract-method
class AdminVM(QubesVM):
    '''Dom0'''
    pass

class AppVM(QubesVM):
    '''Application VM'''
    pass

class StandaloneVM(QubesVM):
    '''Standalone Application VM'''
    pass

class TemplateVM(QubesVM):
    '''Template for AppVM'''
    pass

class DispVM(QubesVM):
    '''Disposable VM'''
    pass
<commit_msg>Make QubesVM objects comparable and str()-able<commit_after># -*- encoding: utf8 -*-
#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2017 Marek Marczykowski-Górecki
#                               <marmarek@invisiblethingslab.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program; if not, see <http://www.gnu.org/licenses/>.

'''Qubes VM objects.'''

import qubesmgmt.base
import qubesmgmt.storage

class QubesVM(qubesmgmt.base.PropertyHolder):
    '''Qubes domain.'''
    def __init__(self, app, name):
        super(QubesVM, self).__init__(app, 'mgmt.vm.property.', name)
        self._volumes = None

    @property
    def name(self):
        '''Domain name'''
        return self._method_dest

    @name.setter
    def name(self, new_value):
        self.qubesd_call(
            self._method_dest,
            self._method_prefix + 'Set',
            'name',
            str(new_value).encode('utf-8'))
        self._method_dest = new_value
        self._volumes = None
        self.app.domains.clear_cache()

    def __str__(self):
        return self._method_dest

    def __lt__(self, other):
        if isinstance(other, QubesVM):
            return self.name < other.name
        return NotImplemented

    def start(self):
        '''
        Start domain.

        :return:
        '''
        self.qubesd_call(self._method_dest, 'mgmt.vm.Start')

    def shutdown(self, force=False):
        '''
        Shutdown domain.

        :return:
        '''
        # TODO: force parameter
        # TODO: wait parameter (using event?)
        if force:
            raise NotImplementedError
        self.qubesd_call(self._method_dest, 'mgmt.vm.Shutdown')

    def kill(self):
        '''
        Kill domain (forcefuly shutdown).

        :return:
        '''
        self.qubesd_call(self._method_dest, 'mgmt.vm.Kill')

    def pause(self):
        '''
        Pause domain.

        Pause its execution without any prior notification.

        :return:
        '''
        self.qubesd_call(self._method_dest, 'mgmt.vm.Pause')

    def unpause(self):
        '''
        Unpause domain.

        Opposite to :py:meth:`pause`.

        :return:
        '''
        self.qubesd_call(self._method_dest, 'mgmt.vm.Unpause')

    def suspend(self):
        '''
        Suspend domain.

        Give domain a chance to prepare for suspend - for example suspend
        used PCI devices.

        :return:
        '''
        raise NotImplementedError
        #self.qubesd_call(self._method_dest, 'mgmt.vm.Suspend')

    def resume(self):
        '''
        Resume domain.

        Opposite to :py:meth:`suspend`.

        :return:
        '''
        raise NotImplementedError
        #self.qubesd_call(self._method_dest, 'mgmt.vm.Resume')

    def get_power_state(self):
        '''Return power state description string.

        Return value may be one of those:

        =============== ========================================================
        return value    meaning
        =============== ========================================================
        ``'Halted'``    Machine is not active.
        ``'Transient'`` Machine is running, but does not have :program:`guid`
                        or :program:`qrexec` available.
        ``'Running'``   Machine is ready and running.
        ``'Paused'``    Machine is paused.
        ``'Suspended'`` Machine is S3-suspended.
        ``'Halting'``   Machine is in process of shutting down (OS shutdown).
        ``'Dying'``     Machine is in process of shutting down (cleanup).
        ``'Crashed'``   Machine crashed and is unusable.
        ``'NA'``        Machine is in unknown state.
        =============== ========================================================

        .. seealso::

            http://wiki.libvirt.org/page/VM_lifecycle
                Description of VM life cycle from the point of view of libvirt.

            https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainState
                Libvirt's enum describing precise state of a domain.

        '''

        vm_list_info = self.qubesd_call(
            self._method_dest, 'mgmt.vm.List', None, None).decode('ascii')
        #  name class=... state=... other=...
        vm_state = vm_list_info.partition('state=')[2].split(' ')[0]
        return vm_state

    @property
    def volumes(self):
        '''VM disk volumes'''
        if self._volumes is None:
            volumes_list = self.qubesd_call(
                self._method_dest, 'mgmt.vm.volume.List')
            self._volumes = {}
            for volname in volumes_list.decode('ascii').splitlines():
                if not volname:
                    continue
                self._volumes[volname] = qubesmgmt.storage.Volume(self.app,
                    vm=self.name, vm_name=volname)
        return self._volumes

# pylint: disable=abstract-method
class AdminVM(QubesVM):
    '''Dom0'''
    pass

class AppVM(QubesVM):
    '''Application VM'''
    pass

class StandaloneVM(QubesVM):
    '''Standalone Application VM'''
    pass

class TemplateVM(QubesVM):
    '''Template for AppVM'''
    pass

class DispVM(QubesVM):
    '''Disposable VM'''
    pass
<|endoftext|>