<commit_before># -*- encoding: utf8 -*-
#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2017 Marek Marczykowski-Górecki
#                               <marmarek@invisiblethingslab.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, see <http://www.gnu.org/licenses/>.

''' Qubes block devices extensions '''
import re
import string
import lxml.etree

import qubes.devices
import qubes.ext

name_re = re.compile(r"^[a-z0-9-]{1,12}$")
device_re = re.compile(r"^[a-z0-9/-]{1,64}$")
# FIXME: any better idea of desc_re?
desc_re = re.compile(r"^.{1,255}$")
mode_re = re.compile(r"^[rw]$")

# all frontends, prefer xvdi
# TODO: get this from libvirt driver?
AVAILABLE_FRONTENDS = ['xvd'+c for c in
                       string.ascii_lowercase[8:]+string.ascii_lowercase[:8]]

SYSTEM_DISKS = ('xvda', 'xvdb', 'xvdc', 'xvdd')

class BlockDevice(qubes.devices.DeviceInfo):
    def __init__(self, backend_domain, ident):
        super(BlockDevice, self).__init__(backend_domain=backend_domain,
            ident=ident)
        self._description = None
        self._mode = None
        self._size = None

    @property
    def description(self):
        '''Human readable device description'''
        if self._description is None:
            if not self.backend_domain.is_running():
                return self.ident
            safe_set = {ord(c) for c in
                string.ascii_letters + string.digits + '()+,-.:=_/ '}
            untrusted_desc = self.backend_domain.untrusted_qdb.read(
                '/qubes-block-devices/{}/desc'.format(self.ident))
            desc = ''.join((chr(c) if c in safe_set else '_')
                for c in untrusted_desc)
            self._description = desc
        return self._description

    @property
    def mode(self):
        '''Device mode, either 'w' for read-write, or 'r' for read-only'''
        if self._mode is None:
            if not self.backend_domain.is_running():
                return 'w'
            untrusted_mode = self.backend_domain.untrusted_qdb.read(
                '/qubes-block-devices/{}/mode'.format(self.ident))
            if untrusted_mode is None:
                self._mode = 'w'
            elif untrusted_mode not in (b'w', b'r'):
                self.backend_domain.log.warning(
                    'Device {} has invalid mode'.format(self.ident))
                self._mode = 'w'
            else:
                self._mode = untrusted_mode.decode()
        return self._mode

    @property
    def size(self):
        '''Device size in bytes'''
        if self._size is None:
            if not self.backend_domain.is_running():
                return None
            untrusted_size = self.backend_domain.untrusted_qdb.read(
                '/qubes-block-devices/{}/size'.format(self.ident))
            if untrusted_size is None:
                self._size = 0
            elif not untrusted_size.isdigit():
                self.backend_domain.log.warning(
                    'Device {} has invalid size'.format(self.ident))
                self._size = 0
            else:
                self._size = int(untrusted_size)
        return self._size

    @property
    def device_node(self):
        '''Device node in backend domain'''
        return '/dev/' + self.ident.replace('_', '/')

class BlockDeviceExtension(qubes.ext.Extension):
    @qubes.ext.handler('domain-init', 'domain-load')
    def on_domain_init_load(self, vm, event):
        '''Initialize watching for changes'''
        # pylint: disable=unused-argument,no-self-use
        vm.watch_qdb_path('/qubes-block-devices')

    @qubes.ext.handler('domain-qdb-change:/qubes-block-devices')
    def on_qdb_change(self, vm, event, path):
        '''A change in QubesDB means a change in device list'''
        # pylint: disable=unused-argument,no-self-use
        vm.fire_event('device-list-change:block')

    def device_get(self, vm, ident):
        # pylint: disable=no-self-use
        '''Read information about device from QubesDB

        :param vm: backend VM object
        :param ident: device identifier
        :returns BlockDevice'''

        untrusted_qubes_device_attrs = vm.untrusted_qdb.list(
            '/qubes-block-devices/{}/'.format(ident))
        if not untrusted_qubes_device_attrs:
            return None
        return BlockDevice(vm, ident)

    @qubes.ext.handler('device-list:block')
    def on_device_list_block(self, vm, event):
        # pylint: disable=unused-argument,no-self-use

        safe_set = string.ascii_letters + string.digits
        if not vm.is_running():
            return
        untrusted_qubes_devices = vm.untrusted_qdb.list('/qubes-block-devices/')
        untrusted_idents = set(untrusted_path.split('/', 3)[2]
            for untrusted_path in untrusted_qubes_devices)
        for untrusted_ident in untrusted_idents:
            if not all(c in safe_set for c in untrusted_ident):
                msg = ("%s vm's device path name contains unsafe characters. "
                       "Skipping it.")
                vm.log.warning(msg % vm.name)
                continue

            ident = untrusted_ident

            device_info = self.device_get(vm, ident)
            if device_info:
                yield device_info

    @qubes.ext.handler('device-get:block')
    def on_device_get_block(self, vm, event, ident):
        # pylint: disable=unused-argument,no-self-use
        if not vm.is_running():
            return
        if not vm.app.vmm.offline_mode:
            yield self.device_get(vm, ident)

    @qubes.ext.handler('device-list-attached:block')
    def on_device_list_attached(self, vm, event, **kwargs):
        # pylint: disable=unused-argument,no-self-use
        if not vm.is_running():
            return

        xml_desc = lxml.etree.fromstring(vm.libvirt_domain.XMLDesc())

        for disk in xml_desc.findall('devices/disk'):
            if disk.get('type') != 'block':
                continue
            dev_path_node = disk.find('source')
            if dev_path_node is None:
                continue
            dev_path = dev_path_node.get('dev')

            target_node = disk.find('target')
            if target_node is not None:
                frontend_dev = target_node.get('dev')
                if not frontend_dev:
                    continue
                if frontend_dev in SYSTEM_DISKS:
                    continue
            else:
                continue

            backend_domain_node = disk.find('backenddomain')
            if backend_domain_node is not None:
                backend_domain = vm.app.domains[backend_domain_node.get('name')]
            else:
                backend_domain = vm.app.domains[0]

            options = {}
            read_only_node = disk.find('readonly')
            if read_only_node is not None:
                options['read-only'] = 'yes'
            else:
                options['read-only'] = 'no'
            options['frontend-dev'] = frontend_dev

            if dev_path.startswith('/dev/'):
                ident = dev_path[len('/dev/'):]
            else:
                ident = dev_path

            ident = ident.replace('/', '_')

            yield (BlockDevice(backend_domain, ident), options)

    def find_unused_frontend(self, vm):
        # pylint: disable=no-self-use
        '''Find unused block frontend device node for <target dev=.../>
        parameter'''
        assert vm.is_running()

        xml = vm.libvirt_domain.XMLDesc()
        parsed_xml = lxml.etree.fromstring(xml)
        used = [target.get('dev', None) for target in
            parsed_xml.xpath("//domain/devices/disk/target")]
        for dev in AVAILABLE_FRONTENDS:
            if dev not in used:
                return dev
        return None

    @qubes.ext.handler('device-pre-attach:block')
    def on_device_pre_attached_block(self, vm, event, device, options):
        # pylint: disable=unused-argument

        # validate options
        for option, value in options.items():
            if option == 'frontend-dev':
                if not value.startswith('xvd') and not value.startswith('sd'):
                    raise qubes.exc.QubesValueError(
                        'Invalid frontend-dev option value: ' + value)
            elif option == 'read-only':
                if value not in ('yes', 'no'):
                    raise qubes.exc.QubesValueError(
                        'read-only option can only have '
                        '\'yes\' or \'no\' value')
            else:
                raise qubes.exc.QubesValueError(
                    'Unsupported option {}'.format(option))

        if 'read-only' not in options:
            options['read-only'] = 'yes' if device.mode == 'r' else 'no'
        if options.get('read-only', 'no') == 'no' and device.mode == 'r':
            raise qubes.exc.QubesValueError(
                'This device can be attached only read-only')

        if not vm.is_running():
            return

        if not device.backend_domain.is_running():
            raise qubes.exc.QubesVMNotRunningError(device.backend_domain,
                'Domain {} needs to be running to attach device from '
                'it'.format(device.backend_domain.name))

        if 'frontend-dev' not in options:
            options['frontend-dev'] = self.find_unused_frontend(vm)

        vm.libvirt_domain.attachDevice(
            vm.app.env.get_template('libvirt/devices/block.xml').render(
                device=device, vm=vm, options=options))

    @qubes.ext.handler('device-pre-detach:block')
    def on_device_pre_detached_block(self, vm, event, device):
        # pylint: disable=unused-argument,no-self-use
        if not vm.is_running():
            return

        # need to enumerate attached device to find frontend_dev option (at
        # least)
        for attached_device, options in self.on_device_list_attached(vm, event):
            if attached_device == device:
                vm.libvirt_domain.detachDevice(
                    vm.app.env.get_template('libvirt/devices/block.xml').render(
                        device=device, vm=vm, options=options))
                break
<commit_msg>block: add support for devtype option<commit_after># -*- encoding: utf8 -*-
#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2017 Marek Marczykowski-Górecki
#                               <marmarek@invisiblethingslab.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, see <http://www.gnu.org/licenses/>.

''' Qubes block devices extensions '''
import re
import string
import lxml.etree

import qubes.devices
import qubes.ext

name_re = re.compile(r"^[a-z0-9-]{1,12}$")
device_re = re.compile(r"^[a-z0-9/-]{1,64}$")
# FIXME: any better idea of desc_re?
desc_re = re.compile(r"^.{1,255}$")
mode_re = re.compile(r"^[rw]$")

# all frontends, prefer xvdi
# TODO: get this from libvirt driver?
AVAILABLE_FRONTENDS = ['xvd'+c for c in
                       string.ascii_lowercase[8:]+string.ascii_lowercase[:8]]

SYSTEM_DISKS = ('xvda', 'xvdb', 'xvdc', 'xvdd')

class BlockDevice(qubes.devices.DeviceInfo):
    def __init__(self, backend_domain, ident):
        super(BlockDevice, self).__init__(backend_domain=backend_domain,
            ident=ident)
        self._description = None
        self._mode = None
        self._size = None

    @property
    def description(self):
        '''Human readable device description'''
        if self._description is None:
            if not self.backend_domain.is_running():
                return self.ident
            safe_set = {ord(c) for c in
                string.ascii_letters + string.digits + '()+,-.:=_/ '}
            untrusted_desc = self.backend_domain.untrusted_qdb.read(
                '/qubes-block-devices/{}/desc'.format(self.ident))
            desc = ''.join((chr(c) if c in safe_set else '_')
                for c in untrusted_desc)
            self._description = desc
        return self._description

    @property
    def mode(self):
        '''Device mode, either 'w' for read-write, or 'r' for read-only'''
        if self._mode is None:
            if not self.backend_domain.is_running():
                return 'w'
            untrusted_mode = self.backend_domain.untrusted_qdb.read(
                '/qubes-block-devices/{}/mode'.format(self.ident))
            if untrusted_mode is None:
                self._mode = 'w'
            elif untrusted_mode not in (b'w', b'r'):
                self.backend_domain.log.warning(
                    'Device {} has invalid mode'.format(self.ident))
                self._mode = 'w'
            else:
                self._mode = untrusted_mode.decode()
        return self._mode

    @property
    def size(self):
        '''Device size in bytes'''
        if self._size is None:
            if not self.backend_domain.is_running():
                return None
            untrusted_size = self.backend_domain.untrusted_qdb.read(
                '/qubes-block-devices/{}/size'.format(self.ident))
            if untrusted_size is None:
                self._size = 0
            elif not untrusted_size.isdigit():
                self.backend_domain.log.warning(
                    'Device {} has invalid size'.format(self.ident))
                self._size = 0
            else:
                self._size = int(untrusted_size)
        return self._size

    @property
    def device_node(self):
        '''Device node in backend domain'''
        return '/dev/' + self.ident.replace('_', '/')

class BlockDeviceExtension(qubes.ext.Extension):
    @qubes.ext.handler('domain-init', 'domain-load')
    def on_domain_init_load(self, vm, event):
        '''Initialize watching for changes'''
        # pylint: disable=unused-argument,no-self-use
        vm.watch_qdb_path('/qubes-block-devices')

    @qubes.ext.handler('domain-qdb-change:/qubes-block-devices')
    def on_qdb_change(self, vm, event, path):
        '''A change in QubesDB means a change in device list'''
        # pylint: disable=unused-argument,no-self-use
        vm.fire_event('device-list-change:block')

    def device_get(self, vm, ident):
        # pylint: disable=no-self-use
        '''Read information about device from QubesDB

        :param vm: backend VM object
        :param ident: device identifier
        :returns BlockDevice'''

        untrusted_qubes_device_attrs = vm.untrusted_qdb.list(
            '/qubes-block-devices/{}/'.format(ident))
        if not untrusted_qubes_device_attrs:
            return None
        return BlockDevice(vm, ident)

    @qubes.ext.handler('device-list:block')
    def on_device_list_block(self, vm, event):
        # pylint: disable=unused-argument,no-self-use

        safe_set = string.ascii_letters + string.digits
        if not vm.is_running():
            return
        untrusted_qubes_devices = vm.untrusted_qdb.list('/qubes-block-devices/')
        untrusted_idents = set(untrusted_path.split('/', 3)[2]
            for untrusted_path in untrusted_qubes_devices)
        for untrusted_ident in untrusted_idents:
            if not all(c in safe_set for c in untrusted_ident):
                msg = ("%s vm's device path name contains unsafe characters. "
                       "Skipping it.")
                vm.log.warning(msg % vm.name)
                continue

            ident = untrusted_ident

            device_info = self.device_get(vm, ident)
            if device_info:
                yield device_info

    @qubes.ext.handler('device-get:block')
    def on_device_get_block(self, vm, event, ident):
        # pylint: disable=unused-argument,no-self-use
        if not vm.is_running():
            return
        if not vm.app.vmm.offline_mode:
            yield self.device_get(vm, ident)

    @qubes.ext.handler('device-list-attached:block')
    def on_device_list_attached(self, vm, event, **kwargs):
        # pylint: disable=unused-argument,no-self-use
        if not vm.is_running():
            return

        xml_desc = lxml.etree.fromstring(vm.libvirt_domain.XMLDesc())

        for disk in xml_desc.findall('devices/disk'):
            if disk.get('type') != 'block':
                continue
            dev_path_node = disk.find('source')
            if dev_path_node is None:
                continue
            dev_path = dev_path_node.get('dev')

            target_node = disk.find('target')
            if target_node is not None:
                frontend_dev = target_node.get('dev')
                if not frontend_dev:
                    continue
                if frontend_dev in SYSTEM_DISKS:
                    continue
            else:
                continue

            backend_domain_node = disk.find('backenddomain')
            if backend_domain_node is not None:
                backend_domain = vm.app.domains[backend_domain_node.get('name')]
            else:
                backend_domain = vm.app.domains[0]

            options = {}
            read_only_node = disk.find('readonly')
            if read_only_node is not None:
                options['read-only'] = 'yes'
            else:
                options['read-only'] = 'no'
            options['frontend-dev'] = frontend_dev

            if dev_path.startswith('/dev/'):
                ident = dev_path[len('/dev/'):]
            else:
                ident = dev_path

            ident = ident.replace('/', '_')

            yield (BlockDevice(backend_domain, ident), options)

    def find_unused_frontend(self, vm):
        # pylint: disable=no-self-use
        '''Find unused block frontend device node for <target dev=.../>
        parameter'''
        assert vm.is_running()

        xml = vm.libvirt_domain.XMLDesc()
        parsed_xml = lxml.etree.fromstring(xml)
        used = [target.get('dev', None) for target in
            parsed_xml.xpath("//domain/devices/disk/target")]
        for dev in AVAILABLE_FRONTENDS:
            if dev not in used:
                return dev
        return None

    @qubes.ext.handler('device-pre-attach:block')
    def on_device_pre_attached_block(self, vm, event, device, options):
        # pylint: disable=unused-argument

        # validate options
        for option, value in options.items():
            if option == 'frontend-dev':
                if not value.startswith('xvd') and not value.startswith('sd'):
                    raise qubes.exc.QubesValueError(
                        'Invalid frontend-dev option value: ' + value)
            elif option == 'read-only':
                if value not in ('yes', 'no'):
                    raise qubes.exc.QubesValueError(
                        'read-only option can only have '
                        '\'yes\' or \'no\' value')
            elif option == 'devtype':
                if value not in ('disk', 'cdrom'):
                    raise qubes.exc.QubesValueError(
                        'devtype option can only have '
                        '\'disk\' or \'cdrom\' value')
            else:
                raise qubes.exc.QubesValueError(
                    'Unsupported option {}'.format(option))

        if 'read-only' not in options:
            options['read-only'] = 'yes' if device.mode == 'r' else 'no'
        if options.get('read-only', 'no') == 'no' and device.mode == 'r':
            raise qubes.exc.QubesValueError(
                'This device can be attached only read-only')

        if not vm.is_running():
            return

        if not device.backend_domain.is_running():
            raise qubes.exc.QubesVMNotRunningError(device.backend_domain,
                'Domain {} needs to be running to attach device from '
                'it'.format(device.backend_domain.name))

        if 'frontend-dev' not in options:
            options['frontend-dev'] = self.find_unused_frontend(vm)

        vm.libvirt_domain.attachDevice(
            vm.app.env.get_template('libvirt/devices/block.xml').render(
                device=device, vm=vm, options=options))

    @qubes.ext.handler('device-pre-detach:block')
    def on_device_pre_detached_block(self, vm, event, device):
        # pylint: disable=unused-argument,no-self-use
        if not vm.is_running():
            return

        # need to enumerate attached device to find frontend_dev option (at
        # least)
        for attached_device, options in self.on_device_list_attached(vm, event):
            if attached_device == device:
                vm.libvirt_domain.detachDevice(
                    vm.app.env.get_template('libvirt/devices/block.xml').render(
                        device=device, vm=vm, options=options))
                break
<|endoftext|>


<commit_before># -*- encoding: utf8 -*-
#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2017 Marek Marczykowski-Górecki
#                               <marmarek@invisiblethingslab.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, see <http://www.gnu.org/licenses/>.
import argparse
import logging
import logging.handlers

import sys

import qubespolicy

parser = argparse.ArgumentParser(description="Evaluate qrexec policy")

parser.add_argument("--assume-yes-for-ask", action="store_true",
    dest="assume_yes_for_ask", default=False,
    help="Allow run of service without confirmation if policy say 'ask'")
parser.add_argument("--just-evaluate", action="store_true",
    dest="just_evaluate", default=False,
    help="Do not run the service, only evaluate policy; "
         "retcode=0 means 'allow'")
parser.add_argument('domain_id', metavar='src-domain-id',
    help='Source domain ID (Xen ID or similar, not Qubes ID)')
parser.add_argument('domain', metavar='src-domain-name',
    help='Source domain name')
parser.add_argument('target', metavar='dst-domain-name',
    help='Target domain name')
parser.add_argument('service_name', metavar='service-name',
    help='Service name')
parser.add_argument('process_ident', metavar='process-ident',
    help='Qrexec process identifier - for connecting data channel')

def main(args=None):
    args = parser.parse_args(args)

    # Add source domain information, required by qrexec-client for establishing
    # connection
    caller_ident = args.process_ident + "," + args.domain + "," + args.domain_id
    log = logging.getLogger('qubespolicy')
    log.setLevel(logging.INFO)
    handler = logging.handlers.SysLogHandler(address='/dev/log')
    log.addHandler(handler)
    log_prefix = 'qrexec: {}: {} -> {}: '.format(
        args.service_name, args.domain, args.target)
    try:
        system_info = qubespolicy.get_system_info()
    except qubespolicy.QubesMgmtException as e:
        log.error(log_prefix + 'error getting system info: ' + str(e))
        return 1
    try:
        policy = qubespolicy.Policy(args.service_name)
        action = policy.evaluate(system_info, args.domain, args.target)
        if action.action == qubespolicy.Action.ask:
            # late import to save on time for allow/deny actions
            import pydbus
            bus = pydbus.SystemBus()
            proxy = bus.get('org.qubesos.PolicyAgent',
                '/org/qubesos/PolicyAgent')

            icons = {name: system_info['domains'][name]['icon']
                for name in system_info['domains'].keys()}
            for dispvm_base in system_info['domains']:
                if not system_info['domains'][dispvm_base]['dispvm_allowed']:
                    continue
                dispvm_api_name = '$dispvm:' + dispvm_base
                icons[dispvm_api_name] = \
                    system_info['domains'][dispvm_base]['icon']
                icons[dispvm_api_name] = \
                    icons[dispvm_api_name].replace('app', 'disp')

            response = proxy.Ask(args.domain, args.service_name,
                action.targets_for_ask, action.target or '', icons)
            if response:
                action.handle_user_response(True, response)
            else:
                action.handle_user_response(False)
        log.info(log_prefix + 'allowed to {}'.format(action.target))
        action.execute(caller_ident)
    except qubespolicy.PolicySyntaxError as e:
        log.error(log_prefix + 'error loading policy: ' + str(e))
        return 1
    except qubespolicy.AccessDenied as e:
        log.info(log_prefix + 'denied: ' + str(e))
        return 1
    return 0

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>qubespolicy: implement --just-evaluate and --assume-yes-for-ask options<commit_after># -*- encoding: utf8 -*-
#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2017 Marek Marczykowski-Górecki
#                               <marmarek@invisiblethingslab.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, see <http://www.gnu.org/licenses/>.
import argparse
import logging
import logging.handlers

import sys

import qubespolicy

parser = argparse.ArgumentParser(description="Evaluate qrexec policy")

parser.add_argument("--assume-yes-for-ask", action="store_true",
    dest="assume_yes_for_ask", default=False,
    help="Allow run of service without confirmation if policy say 'ask'")
parser.add_argument("--just-evaluate", action="store_true",
    dest="just_evaluate", default=False,
    help="Do not run the service, only evaluate policy; "
         "retcode=0 means 'allow'")
parser.add_argument('domain_id', metavar='src-domain-id',
    help='Source domain ID (Xen ID or similar, not Qubes ID)')
parser.add_argument('domain', metavar='src-domain-name',
    help='Source domain name')
parser.add_argument('target', metavar='dst-domain-name',
    help='Target domain name')
parser.add_argument('service_name', metavar='service-name',
    help='Service name')
parser.add_argument('process_ident', metavar='process-ident',
    help='Qrexec process identifier - for connecting data channel')

def main(args=None):
    args = parser.parse_args(args)

    # Add source domain information, required by qrexec-client for establishing
    # connection
    caller_ident = args.process_ident + "," + args.domain + "," + args.domain_id
    log = logging.getLogger('qubespolicy')
    log.setLevel(logging.INFO)
    handler = logging.handlers.SysLogHandler(address='/dev/log')
    log.addHandler(handler)
    log_prefix = 'qrexec: {}: {} -> {}: '.format(
        args.service_name, args.domain, args.target)
    try:
        system_info = qubespolicy.get_system_info()
    except qubespolicy.QubesMgmtException as e:
        log.error(log_prefix + 'error getting system info: ' + str(e))
        return 1
    try:
        policy = qubespolicy.Policy(args.service_name)
        action = policy.evaluate(system_info, args.domain, args.target)
        if args.assume_yes_for_ask and action.action == qubespolicy.Action.ask:
            action.action = qubespolicy.Action.allow
        if args.just_evaluate:
            return {
                qubespolicy.Action.allow: 0,
                qubespolicy.Action.deny: 1,
                qubespolicy.Action.ask: 1,
            }[action.action]
        if action.action == qubespolicy.Action.ask:
            # late import to save on time for allow/deny actions
            import pydbus
            bus = pydbus.SystemBus()
            proxy = bus.get('org.qubesos.PolicyAgent',
                '/org/qubesos/PolicyAgent')

            icons = {name: system_info['domains'][name]['icon']
                for name in system_info['domains'].keys()}
            for dispvm_base in system_info['domains']:
                if not system_info['domains'][dispvm_base]['dispvm_allowed']:
                    continue
                dispvm_api_name = '$dispvm:' + dispvm_base
                icons[dispvm_api_name] = \
                    system_info['domains'][dispvm_base]['icon']
                icons[dispvm_api_name] = \
                    icons[dispvm_api_name].replace('app', 'disp')

            response = proxy.Ask(args.domain, args.service_name,
                action.targets_for_ask, action.target or '', icons)
            if response:
                action.handle_user_response(True, response)
            else:
                action.handle_user_response(False)
        log.info(log_prefix + 'allowed to {}'.format(action.target))
        action.execute(caller_ident)
    except qubespolicy.PolicySyntaxError as e:
        log.error(log_prefix + 'error loading policy: ' + str(e))
        return 1
    except qubespolicy.AccessDenied as e:
        log.info(log_prefix + 'denied: ' + str(e))
        return 1
    return 0

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>#!/usr/bin/env python2
# -*- coding: utf-8-*-
import os
import sys
import traceback
import shutil
import logging

import yaml
import argparse

from client.diagnose import Diagnostics
from client import vocabcompiler, tts, stt, jasperpath

# Add jasperpath.LIB_PATH to sys.path
sys.path.append(jasperpath.LIB_PATH)

from client.conversation import Conversation

parser = argparse.ArgumentParser(description='Jasper Voice Control Center')
parser.add_argument('--local', action='store_true', help='Use text input instead of a real microphone')
parser.add_argument('--no-network-check', action='store_true', help='Disable the network connection check')
parser.add_argument('--debug', action='store_true', help='Show debug messages')
args = parser.parse_args()

if args.local:
    from client.local_mic import Mic
else:
    from client.mic import Mic

# Change CWD to jasperpath.LIB_PATH
os.chdir(jasperpath.LIB_PATH)

class Jasper(object):
    def __init__(self):
        self._logger = logging.getLogger(__name__)

        # Create config dir if it does not exist yet
        if not os.path.exists(jasperpath.CONFIG_PATH):
            try:
                os.makedirs(jasperpath.CONFIG_PATH)
            except OSError:
                self._logger.error("Could not create config dir: '%s'", jasperpath.CONFIG_PATH, exc_info=True)
                raise

        # Check if config dir is writable
        if not os.access(jasperpath.CONFIG_PATH, os.W_OK):
            self._logger.critical("Config dir %s is not writable. Jasper won't work correctly.")

        # FIXME: For backwards compatibility, move old config file to newly created config dir
        old_configfile = os.path.join(jasperpath.LIB_PATH, 'profile.yml')
        new_configfile = jasperpath.config('profile.yml')
        if os.path.exists(old_configfile):
            if os.path.exists(new_configfile):
                self._logger.warning("Deprecated profile file found: '%s'. Please remove it.", old_configfile)
            else:
                self._logger.warning("Deprecated profile file found: '%s'. Trying to copy it to new location '%s'.", old_configfile, new_configfile)
                try:
                    shutil.copy2(old_configfile, new_configfile)
                except shutil.Error:
                    self._logger.error("Unable to copy config file. Please copy it manually.", exc_info=True)
                    raise

        # Read config
        self._logger.debug("Trying to read config file: '%s'", new_configfile)
        try:
            with open(new_configfile, "r") as f:
                self.config = yaml.safe_load(f)
        except OSError:
            self._logger.error("Can't open config file: '%s'", new_configfile)
            raise

        try:
            api_key = self.config['keys']['GOOGLE_SPEECH']
        except KeyError:
            api_key = None

        try:
            stt_engine_type = self.config['stt_engine']
        except KeyError:
            stt_engine_type = "sphinx"
            self._logger.warning("stt_engine not specified in profile, defaulting to '%s'", stt_engine_type)

        try:
            tts_engine_slug = self.config['tts_engine']
        except KeyError:
            tts_engine_slug = tts.get_default_engine_slug()
            logger.warning("tts_engine not specified in profile, defaulting to '%s'", tts_engine_slug)
        tts_engine_class = tts.get_engine_by_slug(tts_engine_slug)

        # Compile dictionary
        sentences, dictionary, languagemodel = [jasperpath.config(filename) for filename in ("sentences.txt", "dictionary.dic", "languagemodel.lm")]
        vocabcompiler.compile(sentences, dictionary, languagemodel)

        # Initialize Mic
        self.mic = Mic(tts_engine_class(), stt.PocketSphinxSTT(), stt.newSTTEngine(stt_engine_type, api_key=api_key))

    def run(self):
        if 'first_name' in self.config:
            salutation = "How can I be of service, %s?" % self.config["first_name"]
        else:
            salutation = "How can I be of service?"
        self.mic.say(salutation)

        conversation = Conversation("JASPER", self.mic, self.config)
        conversation.handleForever()

if __name__ == "__main__":

    print "==========================================================="
    print " JASPER The Talking Computer                               "
    print " Copyright 2013 Shubhro Saha & Charlie Marsh               "
    print "==========================================================="

    logging.basicConfig()
    logger = logging.getLogger()

    if args.debug:
        logger.setLevel(logging.DEBUG)

    if not args.no_network_check and not Diagnostics.check_network_connection():
        logger.warning("Network not connected. This may prevent Jasper from running properly.")

    try:
        app = Jasper()
    except Exception:
        logger.exception("Error occured!", exc_info=True)
        sys.exit(1)

    app.run()
<commit_msg>Remove chdir from jasper.py<commit_after>#!/usr/bin/env python2
# -*- coding: utf-8-*-
import os
import sys
import traceback
import shutil
import logging

import yaml
import argparse

from client.diagnose import Diagnostics
from client import vocabcompiler, tts, stt, jasperpath

# Add jasperpath.LIB_PATH to sys.path
sys.path.append(jasperpath.LIB_PATH)

from client.conversation import Conversation

parser = argparse.ArgumentParser(description='Jasper Voice Control Center')
parser.add_argument('--local', action='store_true', help='Use text input instead of a real microphone')
parser.add_argument('--no-network-check', action='store_true', help='Disable the network connection check')
parser.add_argument('--debug', action='store_true', help='Show debug messages')
args = parser.parse_args()

if args.local:
    from client.local_mic import Mic
else:
    from client.mic import Mic

class Jasper(object):
    def __init__(self):
        self._logger = logging.getLogger(__name__)

        # Create config dir if it does not exist yet
        if not os.path.exists(jasperpath.CONFIG_PATH):
            try:
                os.makedirs(jasperpath.CONFIG_PATH)
            except OSError:
                self._logger.error("Could not create config dir: '%s'", jasperpath.CONFIG_PATH, exc_info=True)
                raise

        # Check if config dir is writable
        if not os.access(jasperpath.CONFIG_PATH, os.W_OK):
            self._logger.critical("Config dir %s is not writable. Jasper won't work correctly.")

        # FIXME: For backwards compatibility, move old config file to newly created config dir
        old_configfile = os.path.join(jasperpath.LIB_PATH, 'profile.yml')
        new_configfile = jasperpath.config('profile.yml')
        if os.path.exists(old_configfile):
            if os.path.exists(new_configfile):
                self._logger.warning("Deprecated profile file found: '%s'. Please remove it.", old_configfile)
            else:
                self._logger.warning("Deprecated profile file found: '%s'. Trying to copy it to new location '%s'.", old_configfile, new_configfile)
                try:
                    shutil.copy2(old_configfile, new_configfile)
                except shutil.Error:
                    self._logger.error("Unable to copy config file. Please copy it manually.", exc_info=True)
                    raise

        # Read config
        self._logger.debug("Trying to read config file: '%s'", new_configfile)
        try:
            with open(new_configfile, "r") as f:
                self.config = yaml.safe_load(f)
        except OSError:
            self._logger.error("Can't open config file: '%s'", new_configfile)
            raise

        try:
            api_key = self.config['keys']['GOOGLE_SPEECH']
        except KeyError:
            api_key = None

        try:
            stt_engine_type = self.config['stt_engine']
        except KeyError:
            stt_engine_type = "sphinx"
            self._logger.warning("stt_engine not specified in profile, defaulting to '%s'", stt_engine_type)

        try:
            tts_engine_slug = self.config['tts_engine']
        except KeyError:
            tts_engine_slug = tts.get_default_engine_slug()
            logger.warning("tts_engine not specified in profile, defaulting to '%s'", tts_engine_slug)
        tts_engine_class = tts.get_engine_by_slug(tts_engine_slug)

        # Compile dictionary
        sentences, dictionary, languagemodel = [jasperpath.config(filename) for filename in ("sentences.txt", "dictionary.dic", "languagemodel.lm")]
        vocabcompiler.compile(sentences, dictionary, languagemodel)

        # Initialize Mic
        self.mic = Mic(tts_engine_class(), stt.PocketSphinxSTT(), stt.newSTTEngine(stt_engine_type, api_key=api_key))

    def run(self):
        if 'first_name' in self.config:
            salutation = "How can I be of service, %s?" % self.config["first_name"]
        else:
            salutation = "How can I be of service?"
        self.mic.say(salutation)

        conversation = Conversation("JASPER", self.mic, self.config)
        conversation.handleForever()

if __name__ == "__main__":

    print "==========================================================="
    print " JASPER The Talking Computer                               "
    print " Copyright 2013 Shubhro Saha & Charlie Marsh               "
    print "==========================================================="

    logging.basicConfig()
    logger = logging.getLogger()

    if args.debug:
        logger.setLevel(logging.DEBUG)

    if not args.no_network_check and not Diagnostics.check_network_connection():
        logger.warning("Network not connected. This may prevent Jasper from running properly.")

    try:
        app = Jasper()
    except Exception:
        logger.exception("Error occured!", exc_info=True)
        sys.exit(1)

    app.run()
<|endoftext|>


<commit_before># coding=UTF8
__author__ = 'daedaluschan'

import sys, time
import telepot, pprint
from telepot.delegate import pave_event_space, per_chat_id, create_open

from datetime import date
from types import *
from chk_n_conv import  chkNConv
from enum import Enum
import re

to_buy_list = []
white_list = [161517202, 197627552]
file_name = 'list.txt'
pp = pprint.PrettyPrinter(indent=4)

class ConverType(Enum):
    nothing = 1
    asking_date = 2
    adding_to_buy = 3

class Jinny:
  myName = u'Jinny Chan'
  myDOB = date(2015,6,24)

  @staticmethod
  def getNumOfDays():
    currentDate = date.today()
    return (currentDate - Jinny.myDOB).days + 1

  @staticmethod
  def getNumOfDaysSpecific(whichDateIn):
    if isinstance(whichDateIn, str):
    # if type(whichDateIn) is StringType or type(whichDateIn) is UnicodeType:
      whichDate = u''
      for charact in whichDateIn:
        if charact.isdigit():
          whichDate = whichDate + charact
      theDate = date(int(whichDate[0:4]),int(whichDate[4:6]),int(whichDate[6:8]))
    else:
      theDate = whichDateIn

    return (theDate - Jinny.myDOB).days + 1

class AngryJinny(telepot.helper.ChatHandler):
    #def __init__(self, seed_tuple, timeout):
    #    super(AngryJinny, self).__init__(seed_tuple, timeout)
    #    self._convert_type = ConverType.nothing
    #    print('constructor is being called')

    def __init__(self, *args, **kwargs):
        super(AngryJinny, self).__init__(*args, **kwargs)
        self._convert_type = ConverType.nothing
        print('constructor is being called')

    def genKeyboard(self):
        print('generate keyboard')
        show_keyboard = {'keyboard': [[u'今日 day 幾',u'某日係 day 幾'], [u'有乜未買？', u'有野要買']]}

        for idx, item in enumerate(to_buy_list):
            show_keyboard['keyboard'].append([u'買左[' + idx.__str__() + u']： ' + item])
        return show_keyboard

    def genBuyList(self):
        buy_list = u'要買呢D：\n\n'
        for item in to_buy_list:
            buy_list = buy_list + u'- ' + item + u'\n'

        return buy_list

    def doneWithBuyList(self):
        self.sender.sendMessage(text=u'Bye')
        for white_user in white_list:
            self.bot.sendMessage(white_user, text=self.genBuyList(), reply_markup=self.genKeyboard())

    def writeListToFile(self):
        with open(file_name, mode='w') as f:
            for item in to_buy_list:
                # f.write(item.encode('utf-8'))
                f.write(item)
                # f.write(u'\n'.encode('utf-8'))
                f.write(u'\n')
        f.close()

    def boughtItem(self, del_index):
        item_name = to_buy_list[del_index]
        del to_buy_list[del_index]
        self.writeListToFile()
        # self.sender.sendMessage(text=u'Okay。依家仲' + self.genBuyList(), reply_markup=self.genKeyboard())
        for white_user in white_list:
            self.bot.sendMessage(white_user, text=u'買左：' + item_name, reply_markup=self.genKeyboard())

    def on_message(self, msg):
        print('on_message() is being called')
        flavor = telepot.flavor(msg)

        # normal message
        if flavor == 'chat':
            content_type, chat_type, _chat_id = telepot.glance(msg)
            print('Normal Message:', content_type, chat_type, _chat_id,)
            print('=============================\n\nmessage content: ')
            pp.pprint(msg)

            if _chat_id in white_list :

                if self._convert_type == ConverType.nothing:
                    if msg['text'] == u'/start' or msg['text'] == u'/today' or msg['text'] == u'今日 day 幾':
                        self.sender.sendMessage(text=u'Today is ' + str(date.today()) + u'. \n' +
                                                     u'It is my day ' + str(Jinny.getNumOfDays()) + u'. \n' +
                                                     u'Use /help for more options',
                                                reply_markup=self.genKeyboard())
                    elif msg['text'] == u'/help':
                        self.sender.sendMessage(text=u'/today - get today\'s date and my day count. \n' +
                                                     u'/help - for those who have bad memory. \n' +
                                                     u'/query - check my day count for a particular date. \n',
                                                reply_markup=self.genKeyboard())
                    elif msg['text'] == u'/query' or msg['text'] == u'某日係 day 幾':
                        self._convert_type = ConverType.asking_date
                        self.sender.sendMessage(text=u'邊日呀 ?', reply_markup={'hide_keyboard': True})
                    elif msg['text'] == u'/No' :
                        self._convert_type = ConverType.nothing
                        self.sender.sendMessage(text=u'Bye !',
                                                reply_markup=self.genKeyboard())
                    elif msg['text'] == u'有野要買':
                        self._convert_type = ConverType.adding_to_buy
                        self.sender.sendMessage(text=u'買乜？', reply_markup={'hide_keyboard': True})
                    elif msg['text'] == u'有乜未買？':
                        self.sender.sendMessage(text=self.genBuyList(), reply_markup=self.genKeyboard())
                    elif re.compile('買左\[\d+\].*').match(msg['text']) is not None:
                        print('MATCHED.')
                        match_obj = re.compile(u'買左\[(\d+)\].*').match(msg['text'])
                        del_index = match_obj.group(1)
                        self.boughtItem(del_index=int(del_index))

                    else:
                        self.sender.sendMessage(text=u'我唔明呀。\n' +
                                                     u'你試多次啦。或者用 /help 我就教你用。')
                elif self._convert_type == ConverType.asking_date:
                    try :
                        self.sender.sendMessage(text=u'For ' + msg['text'] + u', It is my day ' +
                                                     str(Jinny.getNumOfDaysSpecific(msg['text'])) +
                                                     u'. Any other date to ask ? /No ?')
                        self._convert_type = ConverType.nothing;
                    except BaseException :
                        self.sender.sendMessage(text=u'Not a date that I can understand ! \n' +
                                                     u'What date (YYYYMMDD) ? \nOr you done ? ( /No )')
                elif self._convert_type == ConverType.adding_to_buy:
                    if msg['text'] == u'/done':
                        self._convert_type = ConverType.nothing
                        self.doneWithBuyList()
                        self.writeListToFile()
                    else:
                        to_buy_list.append(msg['text'])
                        self.sender.sendMessage(text=u'仲有冇？如果冇，就用 /done 完結。', reply_markup={'hide_keyboard': True})
                else:
                    self.sender.sendMessage(text=u'我唔明呀。\n' +
                                                 u'你試多次啦。或者用 /help 我就教你用。')

            else:
                self.sender.sendMessage(text=u'你邊位呀？')
        else:
            raise telepot.exception.BadFlavor(msg)

        print(u'to buy list: ' + self.genBuyList())

TOKEN = sys.argv[1]  # get token from command-line

with open(file=file_name, mode='r') as f:
    lines = f.read().splitlines()
f.close()
to_buy_list = lines

# bot = telepot.DelegatorBot(TOKEN, [
#     (per_chat_id(), create_open(AngryJinny, timeout=120)),])
bot = telepot.DelegatorBot(TOKEN, [
    pave_event_space()(
        per_chat_id(), create_open, AngryJinny, timeout=120),
])
# print('Listening ...')
# bot.notifyOnMessage(run_forever=True)
bot.message_loop(run_forever='Listening ...')

<commit_msg>file path as env variable.<commit_after># coding=UTF8
__author__ = 'daedaluschan'

import sys, time
import telepot, pprint
from telepot.delegate import pave_event_space, per_chat_id, create_open

from datetime import date
from types import *
from chk_n_conv import  chkNConv
from enum import Enum
import re

to_buy_list = []
white_list = [161517202, 197627552]
# file_name = 'list.txt'
file_name = sys.argv[2]
pp = pprint.PrettyPrinter(indent=4)

class ConverType(Enum):
    nothing = 1
    asking_date = 2
    adding_to_buy = 3

class Jinny:
  myName = u'Jinny Chan'
  myDOB = date(2015,6,24)

  @staticmethod
  def getNumOfDays():
    currentDate = date.today()
    return (currentDate - Jinny.myDOB).days + 1

  @staticmethod
  def getNumOfDaysSpecific(whichDateIn):
    if isinstance(whichDateIn, str):
    # if type(whichDateIn) is StringType or type(whichDateIn) is UnicodeType:
      whichDate = u''
      for charact in whichDateIn:
        if charact.isdigit():
          whichDate = whichDate + charact
      theDate = date(int(whichDate[0:4]),int(whichDate[4:6]),int(whichDate[6:8]))
    else:
      theDate = whichDateIn

    return (theDate - Jinny.myDOB).days + 1

class AngryJinny(telepot.helper.ChatHandler):
    #def __init__(self, seed_tuple, timeout):
    #    super(AngryJinny, self).__init__(seed_tuple, timeout)
    #    self._convert_type = ConverType.nothing
    #    print('constructor is being called')

    def __init__(self, *args, **kwargs):
        super(AngryJinny, self).__init__(*args, **kwargs)
        self._convert_type = ConverType.nothing
        print('constructor is being called')

    def genKeyboard(self):
        print('generate keyboard')
        show_keyboard = {'keyboard': [[u'今日 day 幾',u'某日係 day 幾'], [u'有乜未買？', u'有野要買']]}

        for idx, item in enumerate(to_buy_list):
            show_keyboard['keyboard'].append([u'買左[' + idx.__str__() + u']： ' + item])
        return show_keyboard

    def genBuyList(self):
        buy_list = u'要買呢D：\n\n'
        for item in to_buy_list:
            buy_list = buy_list + u'- ' + item + u'\n'

        return buy_list

    def doneWithBuyList(self):
        self.sender.sendMessage(text=u'Bye')
        for white_user in white_list:
            self.bot.sendMessage(white_user, text=self.genBuyList(), reply_markup=self.genKeyboard())

    def writeListToFile(self):
        with open(file_name, mode='w') as f:
            for item in to_buy_list:
                # f.write(item.encode('utf-8'))
                f.write(item)
                # f.write(u'\n'.encode('utf-8'))
                f.write(u'\n')
        f.close()

    def boughtItem(self, del_index):
        item_name = to_buy_list[del_index]
        del to_buy_list[del_index]
        self.writeListToFile()
        # self.sender.sendMessage(text=u'Okay。依家仲' + self.genBuyList(), reply_markup=self.genKeyboard())
        for white_user in white_list:
            self.bot.sendMessage(white_user, text=u'買左：' + item_name, reply_markup=self.genKeyboard())

    def on_message(self, msg):
        print('on_message() is being called')
        flavor = telepot.flavor(msg)

        # normal message
        if flavor == 'chat':
            content_type, chat_type, _chat_id = telepot.glance(msg)
            print('Normal Message:', content_type, chat_type, _chat_id,)
            print('=============================\n\nmessage content: ')
            pp.pprint(msg)

            if _chat_id in white_list :

                if self._convert_type == ConverType.nothing:
                    if msg['text'] == u'/start' or msg['text'] == u'/today' or msg['text'] == u'今日 day 幾':
                        self.sender.sendMessage(text=u'Today is ' + str(date.today()) + u'. \n' +
                                                     u'It is my day ' + str(Jinny.getNumOfDays()) + u'. \n' +
                                                     u'Use /help for more options',
                                                reply_markup=self.genKeyboard())
                    elif msg['text'] == u'/help':
                        self.sender.sendMessage(text=u'/today - get today\'s date and my day count. \n' +
                                                     u'/help - for those who have bad memory. \n' +
                                                     u'/query - check my day count for a particular date. \n',
                                                reply_markup=self.genKeyboard())
                    elif msg['text'] == u'/query' or msg['text'] == u'某日係 day 幾':
                        self._convert_type = ConverType.asking_date
                        self.sender.sendMessage(text=u'邊日呀 ?', reply_markup={'hide_keyboard': True})
                    elif msg['text'] == u'/No' :
                        self._convert_type = ConverType.nothing
                        self.sender.sendMessage(text=u'Bye !',
                                                reply_markup=self.genKeyboard())
                    elif msg['text'] == u'有野要買':
                        self._convert_type = ConverType.adding_to_buy
                        self.sender.sendMessage(text=u'買乜？', reply_markup={'hide_keyboard': True})
                    elif msg['text'] == u'有乜未買？':
                        self.sender.sendMessage(text=self.genBuyList(), reply_markup=self.genKeyboard())
                    elif re.compile('買左\[\d+\].*').match(msg['text']) is not None:
                        print('MATCHED.')
                        match_obj = re.compile(u'買左\[(\d+)\].*').match(msg['text'])
                        del_index = match_obj.group(1)
                        self.boughtItem(del_index=int(del_index))

                    else:
                        self.sender.sendMessage(text=u'我唔明呀。\n' +
                                                     u'你試多次啦。或者用 /help 我就教你用。')
                elif self._convert_type == ConverType.asking_date:
                    try :
                        self.sender.sendMessage(text=u'For ' + msg['text'] + u', It is my day ' +
                                                     str(Jinny.getNumOfDaysSpecific(msg['text'])) +
                                                     u'. Any other date to ask ? /No ?')
                        self._convert_type = ConverType.nothing;
                    except BaseException :
                        self.sender.sendMessage(text=u'Not a date that I can understand ! \n' +
                                                     u'What date (YYYYMMDD) ? \nOr you done ? ( /No )')
                elif self._convert_type == ConverType.adding_to_buy:
                    if msg['text'] == u'/done':
                        self._convert_type = ConverType.nothing
                        self.doneWithBuyList()
                        self.writeListToFile()
                    else:
                        to_buy_list.append(msg['text'])
                        self.sender.sendMessage(text=u'仲有冇？如果冇，就用 /done 完結。', reply_markup={'hide_keyboard': True})
                else:
                    self.sender.sendMessage(text=u'我唔明呀。\n' +
                                                 u'你試多次啦。或者用 /help 我就教你用。')

            else:
                self.sender.sendMessage(text=u'你邊位呀？')
        else:
            raise telepot.exception.BadFlavor(msg)

        print(u'to buy list: ' + self.genBuyList())

TOKEN = sys.argv[1]  # get token from command-line

with open(file=file_name, mode='r') as f:
    lines = f.read().splitlines()
f.close()
to_buy_list = lines

# bot = telepot.DelegatorBot(TOKEN, [
#     (per_chat_id(), create_open(AngryJinny, timeout=120)),])
bot = telepot.DelegatorBot(TOKEN, [
    pave_event_space()(
        per_chat_id(), create_open, AngryJinny, timeout=120),
])
# print('Listening ...')
# bot.notifyOnMessage(run_forever=True)
bot.message_loop(run_forever='Listening ...')

<|endoftext|>


<commit_before>"""Quokka CMS!"""

__version__ = '0.3.4'

from pathlib import Path
from quokka.core.app import QuokkaApp
from quokka.core import configure_extensions, configure_extension
from quokka.core.flask_dynaconf import configure_dynaconf

def create_app_base(test=False, ext_list=None, **settings):
    """Creates basic app only with extensions provided in ext_list
    useful for testing."""

    app = QuokkaApp('quokka')
    if settings:
        app.config.update(settings)
    configure_dynaconf(app)

    if test or app.config.get('TESTING'):
        app.testing = True

    app.static_folder = Path.cwd() / Path('static')
    # Path(app.config['PROJECT_ROOT'])

    if ext_list:
        for ext in ext_list:
            configure_extension(ext, app=app)

    return app

def create_app(test=False, **settings):
    """Creates full app with all extensions loaded"""
    app = create_app_base(test=test, **settings)
    configure_extensions(app)
    return app
<commit_msg>removed unused __version__<commit_after>"""Quokka CMS!"""

from pathlib import Path
from quokka.core.app import QuokkaApp
from quokka.core import configure_extensions, configure_extension
from quokka.core.flask_dynaconf import configure_dynaconf

def create_app_base(test=False, ext_list=None, **settings):
    """Creates basic app only with extensions provided in ext_list
    useful for testing."""

    app = QuokkaApp('quokka')
    if settings:
        app.config.update(settings)
    configure_dynaconf(app)

    if test or app.config.get('TESTING'):
        app.testing = True

    app.static_folder = Path.cwd() / Path('static')
    # Path(app.config['PROJECT_ROOT'])

    if ext_list:
        for ext in ext_list:
            configure_extension(ext, app=app)

    return app

def create_app(test=False, **settings):
    """Creates full app with all extensions loaded"""
    app = create_app_base(test=test, **settings)
    configure_extensions(app)
    return app
<|endoftext|>


<commit_before>import numpy as np
from pca import PCA
import copy

class pca_test():

    def setup(self):
        self.rng = np.random.RandomState(0)
        a = np.array([1.,0.,.1])
        b = np.array([1.,-1.,0])
        self.data = np.array([x*a+y*b for x,y in zip(self.rng.rand(10)-.5,self.rng.rand(10)-.5)])
        return

    def change_test(self):
        data_init = copy.deepcopy(self.data)
        p = PCA()
        p.fit(self.data)
        assert np.allclose(self.data,data_init)
        p.fit_transform(self.data)
        assert np.allclose(self.data,data_init)
        p.inv_transform(self.data)
        assert np.allclose(self.data,data_init)

    def dimreduce_test(self):
        p = PCA()
        new = p.fit_transform(self.data,dim=2)
        new = p.inv_transform(new)
        print self.data
        print new
        assert np.allclose(new,self.data)

<commit_msg>more tests pass<commit_after>import numpy as np
from pca import PCA
import copy

class pca_test():

    def setup(self):
        self.rng = np.random.RandomState(0)
        a = np.array([1.,0.,.1])
        b = np.array([1.,-1.,0])
        self.data = np.array([x*a+y*b for x,y in zip(self.rng.rand(10)-.5,self.rng.rand(10)-.5)])
        return

    def change_test(self):
        data_init = copy.deepcopy(self.data)
        p = PCA()
        p.fit(self.data)
        assert np.allclose(self.data,data_init)
        p.fit_transform(self.data)
        assert np.allclose(self.data,data_init)
        p.inv_transform(self.data)
        assert np.allclose(self.data,data_init)
        return

    def dimreduce_test(self):
        p = PCA()
        new = p.fit_transform(self.data,dim=2)
        new = p.inv_transform(new)
        print self.data
        print new
        assert np.allclose(new,self.data)
        return

    def whiten_test(self):
        data = self.data+self.rng.rand(*self.data.shape)
        p = PCA()
        new = p.fit_transform(data, whiten=True)
        cov = new.T.dot(new)
        assert np.allclose(cov,np.eye(data.shape[1]))
        return

<|endoftext|>


<commit_before>#!/usr/bin/python
# This script emulates network jitter in UDP sockets

from socket import *
from time import time
from random import random, gauss
from Queue import PriorityQueue

LOCAL = ('', 5000)
HOST = ('127.0.0.1', 5002)
DELAY = 0.5
JITTER = 0.1
LOSS = 0.1

sock = socket(AF_INET, SOCK_DGRAM)
sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
sock.bind(LOCAL)
sock.connect(HOST)
queue = PriorityQueue()

print "Local  %s:%d" % (LOCAL[0], LOCAL[1])
print "Host   %s:%d" % (HOST[0], HOST[1])
print "Delay  %dms" % (DELAY * 1000)
print "Jitter %dms" % (JITTER * 1000)
print "Loss   %d%%" % (LOSS * 100)

while True:
    while True:
        if queue.empty():
            sock.settimeout(None)
            item = None
            break;
        else:
            item = queue.get()
            delta = item[0] - time()
            if delta > 0:
                sock.settimeout(delta)
                break;
            else:
                sock.send(item[1])
    try:
        data = sock.recv(1024)
        if not random() < LOSS:
            queue.put((time() + gauss(DELAY, JITTER), data))
        if item:
            queue.put(item)
    except timeout:
        sock.send(item[1])
<commit_msg>do not connect UDP sockets<commit_after>#!/usr/bin/python
# This script emulates network jitter in UDP sockets

from socket import *
from time import time
from random import random, gauss
from Queue import PriorityQueue

LOCAL = ('', 5000)
HOST = ('127.0.0.1', 5002)
DELAY = 0.5
JITTER = 0.1
LOSS = 0.1

sock = socket(AF_INET, SOCK_DGRAM)
sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
sock.bind(LOCAL)
queue = PriorityQueue()

print "Local  %s:%d" % (LOCAL[0], LOCAL[1])
print "Host   %s:%d" % (HOST[0], HOST[1])
print "Delay  %dms" % (DELAY * 1000)
print "Jitter %dms" % (JITTER * 1000)
print "Loss   %d%%" % (LOSS * 100)

while True:
    while True:
        if queue.empty():
            sock.settimeout(None)
            item = None
            break;
        else:
            item = queue.get()
            delta = item[0] - time()
            if delta > 0:
                sock.settimeout(delta)
                break;
            else:
                sock.sendto(item[1], HOST)
    try:
        data = sock.recv(1024)
        if not random() < LOSS:
            queue.put((time() + gauss(DELAY, JITTER), data))
        if item:
            queue.put(item)
    except timeout:
        sock.sendto(item[1], HOST)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from binascii import hexlify
from ethereum.utils import denoms, int_to_big_endian

INITIAL_PORT = 40001

CACHE_TTL = 60
ESTIMATED_BLOCK_TIME = 7
GAS_LIMIT = 3141592  # Morden's gasLimit.
GAS_LIMIT_HEX = '0x' + hexlify(int_to_big_endian(GAS_LIMIT))
GAS_PRICE = denoms.shannon * 20

DEFAULT_PROTOCOL_RETRIES_BEFORE_BACKOFF = 5
DEFAULT_PROTOCOL_THROTTLE_CAPACITY = 10.
DEFAULT_PROTOCOL_THROTTLE_FILL_RATE = 10.
DEFAULT_PROTOCOL_RETRY_INTERVAL = 1.

DEFAULT_REVEAL_TIMEOUT = 10
DEFAULT_SETTLE_TIMEOUT = DEFAULT_REVEAL_TIMEOUT * 9
DEFAULT_EVENTS_POLL_TIMEOUT = 0.5
DEFAULT_POLL_TIMEOUT = 180
DEFAULT_JOINABLE_FUNDS_TARGET = 0.4
DEFAULT_INITIAL_CHANNEL_TARGET = 3
DEFAULT_WAIT_FOR_SETTLE = True

DEFAULT_NAT_KEEPALIVE_RETRIES = 2
DEFAULT_NAT_KEEPALIVE_TIMEOUT = 10
DEFAULT_NAT_INVITATION_TIMEOUT = 180

DEFAULT_SHUTDOWN_TIMEOUT = 2

ORACLE_BLOCKNUMBER_DRIFT_TOLERANCE = 3
ETHERSCAN_API = 'https://{network}.etherscan.io/api?module=proxy&action={action}'
<commit_msg>more frequent defaults for healthchecking<commit_after># -*- coding: utf-8 -*-
from binascii import hexlify
from ethereum.utils import denoms, int_to_big_endian

INITIAL_PORT = 40001

CACHE_TTL = 60
ESTIMATED_BLOCK_TIME = 7
GAS_LIMIT = 3141592  # Morden's gasLimit.
GAS_LIMIT_HEX = '0x' + hexlify(int_to_big_endian(GAS_LIMIT))
GAS_PRICE = denoms.shannon * 20

DEFAULT_PROTOCOL_RETRIES_BEFORE_BACKOFF = 5
DEFAULT_PROTOCOL_THROTTLE_CAPACITY = 10.
DEFAULT_PROTOCOL_THROTTLE_FILL_RATE = 10.
DEFAULT_PROTOCOL_RETRY_INTERVAL = 1.

DEFAULT_REVEAL_TIMEOUT = 10
DEFAULT_SETTLE_TIMEOUT = DEFAULT_REVEAL_TIMEOUT * 9
DEFAULT_EVENTS_POLL_TIMEOUT = 0.5
DEFAULT_POLL_TIMEOUT = 180
DEFAULT_JOINABLE_FUNDS_TARGET = 0.4
DEFAULT_INITIAL_CHANNEL_TARGET = 3
DEFAULT_WAIT_FOR_SETTLE = True

DEFAULT_NAT_KEEPALIVE_RETRIES = 5
DEFAULT_NAT_KEEPALIVE_TIMEOUT = 5
DEFAULT_NAT_INVITATION_TIMEOUT = 15

DEFAULT_SHUTDOWN_TIMEOUT = 2

ORACLE_BLOCKNUMBER_DRIFT_TOLERANCE = 3
ETHERSCAN_API = 'https://{network}.etherscan.io/api?module=proxy&action={action}'
<|endoftext|>


<commit_before>import logging
import logging.config
import json
import argparse

import jobs
import utils

logger = logging.getLogger(__name__)
run_logger = logging.getLogger('josync_run')

def main():
    parser = argparse.ArgumentParser(description='Scripted backup using rsync on Windows.')
    parser.add_argument('jobfile',help='path to job file specifying josync job',type=str)
    args = parser.parse_args()
    jobfile = args.jobfile

    logger.info("Session started. Josync version {}.".format(42))

    # enumerate net drives
    utils.enumerate_net_drives()

    # parse global settings file
    utils.read_config(default_cfg='default.josync-config',user_cfg='user.josync-config')

    # parse and execute job file
    try:
        job = jobs.create_job_from_file(args.jobfile)
        job.run()
        run_logger.info("A josync job was run from {}. No errors encountered.".format(args.jobfile))
    except Exception as e:
        run_logger.exception(e)

    logger.info("Session ended.")

if __name__ == '__main__':
    with open('default.josync-logging') as f:
        log_config = json.loads(f.read())
    logging.config.dictConfig(log_config)
    logging.getLogger().setLevel(logging.INFO)

    main()
<commit_msg>automagically add .josync-job to command line argument<commit_after>import logging
import logging.config
import json
import argparse

import jobs
import utils

logger = logging.getLogger(__name__)
run_logger = logging.getLogger('josync_run')

def main():
    parser = argparse.ArgumentParser(description='Scripted backup using rsync on Windows.')
    parser.add_argument('jobfile',help='path to job file specifying josync job',type=str)
    args = parser.parse_args()
    jobfile = args.jobfile

    if not jobfile.endswith('.josync-job'):
        jobfile = jobfile + '.josync-job'

    logger.info("Session started. Josync version {}.".format(42))

    # enumerate net drives
    utils.enumerate_net_drives()

    # parse global settings file
    utils.read_config(default_cfg='default.josync-config',user_cfg='user.josync-config')

    # parse and execute job file
    try:
        job = jobs.create_job_from_file(jobfile)
        job.run()
        run_logger.info("A josync job was run from {}. No errors encountered.".format(jobfile))
    except Exception as e:
        run_logger.exception(e)

    logger.info("Session ended.")

if __name__ == '__main__':
    with open('default.josync-logging') as f:
        log_config = json.loads(f.read())
    logging.config.dictConfig(log_config)
    logging.getLogger().setLevel(logging.INFO)

    main()
<|endoftext|>


<commit_before>import argparse
import logging
import os
import tempfile
import typing
from typing import Dict, Optional, Text, Union, List

from rasa.constants import NUMBER_OF_TRAINING_STORIES_FILE
from rasa.core.domain import Domain
from rasa.utils.common import TempDirectoryPath

if typing.TYPE_CHECKING:
    from rasa.core.interpreter import NaturalLanguageInterpreter
    from rasa.core.utils import AvailableEndpoints
    from rasa.importers.importer import TrainingDataImporter

logger = logging.getLogger(__name__)

async def train(
    domain_file: Union[Domain, Text],
    training_resource: Union[Text, "TrainingDataImporter"],
    output_path: Text,
    interpreter: Optional["NaturalLanguageInterpreter"] = None,
    endpoints: "AvailableEndpoints" = None,
    dump_stories: bool = False,
    policy_config: Optional[Union[Text, Dict]] = None,
    exclusion_percentage: int = None,
    kwargs: Optional[Dict] = None,
):
    from rasa.core.agent import Agent
    from rasa.core import config, utils
    from rasa.core.utils import AvailableEndpoints

    if not endpoints:
        endpoints = AvailableEndpoints()

    if not kwargs:
        kwargs = {}

    policies = config.load(policy_config)

    agent = Agent(
        domain_file,
        generator=endpoints.nlg,
        action_endpoint=endpoints.action,
        interpreter=interpreter,
        policies=policies,
    )

    data_load_args, kwargs = utils.extract_args(
        kwargs,
        {
            "use_story_concatenation",
            "unique_last_num_states",
            "augmentation_factor",
            "remove_duplicates",
            "debug_plots",
        },
    )

    training_data = await agent.load_data(
        training_resource, exclusion_percentage=exclusion_percentage, **data_load_args
    )
    agent.train(training_data, **kwargs)
    agent.persist(output_path, dump_stories)

    return agent

async def train_comparison_models(
    story_file: Text,
    domain: Text,
    output_path: Text = "",
    exclusion_percentages: Optional[List] = None,
    policy_configs: Optional[List] = None,
    runs: int = 1,
    dump_stories: bool = False,
    kwargs: Optional[Dict] = None,
):
    """Train multiple models for comparison of policies"""
    from rasa.core import config
    from rasa import model
    from rasa.importers.importer import TrainingDataImporter

    exclusion_percentages = exclusion_percentages or []
    policy_configs = policy_configs or []

    for r in range(runs):
        logging.info("Starting run {}/{}".format(r + 1, runs))

        for current_run, percentage in enumerate(exclusion_percentages, 1):
            for policy_config in policy_configs:
                policies = config.load(policy_config)

                if len(policies) > 1:
                    raise ValueError(
                        "You can only specify one policy per model for comparison"
                    )

                file_importer = TrainingDataImporter.load_core_importer_from_config(
                    policy_config, domain, [story_file]
                )

                policy_name = type(policies[0]).__name__
                logging.info(
                    "Starting to train {} round {}/{}"
                    " with {}% exclusion"
                    "".format(
                        policy_name, current_run, len(exclusion_percentages), percentage
                    )
                )

                with TempDirectoryPath(tempfile.mkdtemp()) as train_path:
                    await train(
                        domain,
                        file_importer,
                        train_path,
                        policy_config=policy_config,
                        exclusion_percentage=current_run,
                        kwargs=kwargs,
                        dump_stories=dump_stories,
                    )

                    new_fingerprint = await model.model_fingerprint(file_importer)

                    output_dir = os.path.join(output_path, "run_" + str(r + 1))
                    model_name = policy_name + str(current_run)
                    model.package_model(
                        fingerprint=new_fingerprint,
                        output_directory=output_dir,
                        train_path=train_path,
                        fixed_model_name=model_name,
                    )

async def get_no_of_stories(story_file: Text, domain: Text) -> int:
    """Get number of stories in a file."""
    from rasa.core.domain import TemplateDomain
    from rasa.core.training.dsl import StoryFileReader

    stories = await StoryFileReader.read_from_folder(
        story_file, TemplateDomain.load(domain)
    )
    return len(stories)

async def do_compare_training(
    args: argparse.Namespace,
    story_file: Text,
    additional_arguments: Optional[Dict] = None,
):
    from rasa.core import utils

    await train_comparison_models(
        story_file,
        args.domain,
        args.out,
        args.percentages,
        args.config,
        args.runs,
        args.dump_stories,
        additional_arguments,
    )

    no_stories = await get_no_of_stories(args.stories, args.domain)

    # store the list of the number of stories present at each exclusion
    # percentage
    story_range = [
        no_stories - round((x / 100.0) * no_stories) for x in args.percentages
    ]

    training_stories_per_model_file = os.path.join(
        args.out, NUMBER_OF_TRAINING_STORIES_FILE
    )
    utils.dump_obj_as_json_to_file(training_stories_per_model_file, story_range)

def do_interactive_learning(
    args: argparse.Namespace,
    stories: Optional[Text] = None,
    additional_arguments: Dict[Text, typing.Any] = None,
):
    from rasa.core.training import interactive

    interactive.run_interactive_learning(
        stories,
        skip_visualization=args.skip_visualization,
        server_args=args.__dict__,
        additional_arguments=additional_arguments,
    )

if __name__ == "__main__":
    raise RuntimeError(
        "Calling `rasa.core.train` directly is no longer supported. Please use "
        "`rasa train` to train a combined Core and NLU model or `rasa train core` "
        "to train a Core model."
    )
<commit_msg>Add missing fix for rasa train<commit_after>import argparse
import logging
import os
import tempfile
import typing
from typing import Dict, Optional, Text, Union, List

from rasa.constants import NUMBER_OF_TRAINING_STORIES_FILE
from rasa.core.domain import Domain
from rasa.utils.common import TempDirectoryPath

if typing.TYPE_CHECKING:
    from rasa.core.interpreter import NaturalLanguageInterpreter
    from rasa.core.utils import AvailableEndpoints
    from rasa.importers.importer import TrainingDataImporter

logger = logging.getLogger(__name__)

async def train(
    domain_file: Union[Domain, Text],
    training_resource: Union[Text, "TrainingDataImporter"],
    output_path: Text,
    interpreter: Optional["NaturalLanguageInterpreter"] = None,
    endpoints: "AvailableEndpoints" = None,
    dump_stories: bool = False,
    policy_config: Optional[Union[Text, Dict]] = None,
    exclusion_percentage: int = None,
    kwargs: Optional[Dict] = None,
):
    from rasa.core.agent import Agent
    from rasa.core import config, utils
    from rasa.core.utils import AvailableEndpoints

    if not endpoints:
        endpoints = AvailableEndpoints()

    if not kwargs:
        kwargs = {}

    policies = config.load(policy_config)

    agent = Agent(
        domain_file,
        generator=endpoints.nlg,
        action_endpoint=endpoints.action,
        interpreter=interpreter,
        policies=policies,
    )

    data_load_args, kwargs = utils.extract_args(
        kwargs,
        {
            "use_story_concatenation",
            "unique_last_num_states",
            "augmentation_factor",
            "remove_duplicates",
            "debug_plots",
        },
    )

    training_data = await agent.load_data(
        training_resource, exclusion_percentage=exclusion_percentage, **data_load_args
    )
    agent.train(training_data, **kwargs)
    agent.persist(output_path, dump_stories)

    return agent

async def train_comparison_models(
    story_file: Text,
    domain: Text,
    output_path: Text = "",
    exclusion_percentages: Optional[List] = None,
    policy_configs: Optional[List] = None,
    runs: int = 1,
    dump_stories: bool = False,
    kwargs: Optional[Dict] = None,
):
    """Train multiple models for comparison of policies"""
    from rasa.core import config
    from rasa import model
    from rasa.importers.importer import TrainingDataImporter

    exclusion_percentages = exclusion_percentages or []
    policy_configs = policy_configs or []

    for r in range(runs):
        logging.info("Starting run {}/{}".format(r + 1, runs))

        for current_run, percentage in enumerate(exclusion_percentages, 1):
            for policy_config in policy_configs:
                policies = config.load(policy_config)

                if len(policies) > 1:
                    raise ValueError(
                        "You can only specify one policy per model for comparison"
                    )

                file_importer = TrainingDataImporter.load_core_importer_from_config(
                    policy_config, domain, [story_file]
                )

                policy_name = type(policies[0]).__name__
                logging.info(
                    "Starting to train {} round {}/{}"
                    " with {}% exclusion"
                    "".format(
                        policy_name, current_run, len(exclusion_percentages), percentage
                    )
                )

                with TempDirectoryPath(tempfile.mkdtemp()) as train_path:
                    await train(
                        domain,
                        file_importer,
                        train_path,
                        policy_config=policy_config,
                        exclusion_percentage=percentage,
                        kwargs=kwargs,
                        dump_stories=dump_stories,
                    )

                    new_fingerprint = await model.model_fingerprint(file_importer)

                    output_dir = os.path.join(output_path, "run_" + str(r + 1))
                    model_name = policy_name + str(current_run)
                    model.package_model(
                        fingerprint=new_fingerprint,
                        output_directory=output_dir,
                        train_path=train_path,
                        fixed_model_name=model_name,
                    )

async def get_no_of_stories(story_file: Text, domain: Text) -> int:
    """Get number of stories in a file."""
    from rasa.core.domain import TemplateDomain
    from rasa.core.training.dsl import StoryFileReader

    stories = await StoryFileReader.read_from_folder(
        story_file, TemplateDomain.load(domain)
    )
    return len(stories)

async def do_compare_training(
    args: argparse.Namespace,
    story_file: Text,
    additional_arguments: Optional[Dict] = None,
):
    from rasa.core import utils

    await train_comparison_models(
        story_file,
        args.domain,
        args.out,
        args.percentages,
        args.config,
        args.runs,
        args.dump_stories,
        additional_arguments,
    )

    no_stories = await get_no_of_stories(args.stories, args.domain)

    # store the list of the number of stories present at each exclusion
    # percentage
    story_range = [
        no_stories - round((x / 100.0) * no_stories) for x in args.percentages
    ]

    training_stories_per_model_file = os.path.join(
        args.out, NUMBER_OF_TRAINING_STORIES_FILE
    )
    utils.dump_obj_as_json_to_file(training_stories_per_model_file, story_range)

def do_interactive_learning(
    args: argparse.Namespace,
    stories: Optional[Text] = None,
    additional_arguments: Dict[Text, typing.Any] = None,
):
    from rasa.core.training import interactive

    interactive.run_interactive_learning(
        stories,
        skip_visualization=args.skip_visualization,
        server_args=args.__dict__,
        additional_arguments=additional_arguments,
    )

if __name__ == "__main__":
    raise RuntimeError(
        "Calling `rasa.core.train` directly is no longer supported. Please use "
        "`rasa train` to train a combined Core and NLU model or `rasa train core` "
        "to train a Core model."
    )
<|endoftext|>


<commit_before>from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
import argparse
import json
import logging
import os
from functools import wraps

from klein import Klein
from twisted.internet import reactor, threads
from twisted.internet.defer import inlineCallbacks, returnValue, maybeDeferred

from rasa_nlu.config import RasaNLUConfig
from rasa_nlu.data_router import DataRouter, InvalidModelError
from rasa_nlu.version import __version__

logger = logging.getLogger(__name__)

def create_argparser():
    parser = argparse.ArgumentParser(description='parse incoming text')
    parser.add_argument('-c', '--config',
                        help="config file, all the command line options can also be passed via a (json-formatted) " +
                             "config file. NB command line args take precedence")
    parser.add_argument('-d', '--server_model_dirs',
                        help='directory containing model to for parser to use')
    parser.add_argument('-e', '--emulate', choices=['wit', 'luis', 'api'],
                        help='which service to emulate (default: None i.e. use simple built in format)')
    parser.add_argument('-l', '--language', choices=['de', 'en'], help="model and data language")
    parser.add_argument('-m', '--mitie_file',
                        help='file with mitie total_word_feature_extractor')
    parser.add_argument('-p', '--path', help="path where model files will be saved")
    parser.add_argument('--pipeline', help="The pipeline to use. Either a pipeline template name or a list of " +
                                           "components separated by comma")
    parser.add_argument('-P', '--port', type=int, help='port on which to run server')
    parser.add_argument('-t', '--token',
                        help="auth token. If set, reject requests which don't provide this token as a query parameter")
    parser.add_argument('-w', '--write', help='file where logs will be saved')

    return parser

def requires_auth(f):
    """Wraps a request handler with token authentication."""

    @wraps(f)
    def decorated(*args, **kwargs):
        self = args[0]
        request = args[1]
        token = request.args.get('token', [''])[0]
        if self.data_router.token is None or token == self.data_router.token:
            return f(*args, **kwargs)
        return "unauthorized", 401

    return decorated

class RasaNLU(object):
    """Class representing Rasa NLU http server"""

    app = Klein()

    def __init__(self, config, component_builder=None, testing=False):
        logging.basicConfig(filename=config['log_file'], level=config['log_level'])
        logging.captureWarnings(True)
        logger.debug("Configuration: " + config.view())

        logger.debug("Creating a new data router")
        self.config = config
        self.data_router = DataRouter(config, component_builder)
        self.testing = testing
        reactor.suggestThreadPoolSize(config['num_threads'] * 5)

    @app.route("/", methods=['GET'])
    def hello(self, request):
        """Main Rasa route to check if the server is online"""

        return "hello from Rasa NLU: " + __version__

    @app.route("/parse", methods=['GET', 'POST'])
    @requires_auth
    @inlineCallbacks
    def parse_get(self, request):
        request.setHeader('Content-Type', 'application/json')
        if request.method.decode('utf-8', 'strict') == 'GET':
            request_params = {key.decode('utf-8', 'strict'): value[0].decode('utf-8', 'strict') for key, value in
                              request.args.items()}
        else:
            request_params = json.loads(request.content.read().decode('utf-8', 'strict'))
        if 'q' not in request_params:
            request.setResponseCode(404)
            returnValue(json.dumps({"error": "Invalid parse parameter specified"}))
        else:
            data = self.data_router.extract(request_params)
            try:
                request.setResponseCode(200)
                response = yield (maybeDeferred(self.data_router.parse, data)
                                  if self.testing
                                  else threads.deferToThread(self.data_router.parse, data))
                returnValue(json.dumps(response))
            except InvalidModelError as e:
                request.setResponseCode(404)
                returnValue(json.dumps({"error": "{}".format(e)}))

    @app.route("/version", methods=['GET'])
    @requires_auth
    def version(self, request):
        """Returns the Rasa server's version"""

        request.setHeader('Content-Type', 'application/json')
        return json.dumps({'version': __version__})

    @app.route("/config", methods=['GET'])
    @requires_auth
    def rasaconfig(self, request):
        """Returns the in-memory configuration of the Rasa server"""

        request.setHeader('Content-Type', 'application/json')
        return json.dumps(self.config.as_dict())

    @app.route("/status", methods=['GET'])
    @requires_auth
    def status(self, request):
        request.setHeader('Content-Type', 'application/json')
        return json.dumps(self.data_router.get_status())

    @app.route("/train", methods=['POST'])
    @requires_auth
    def train(self, request):
        def errback(f):
            f.trap(ValueError)
            logger.debug("error: {}".format(f.getErrorMessage()))

        data_string = request.content.read().decode('utf-8', 'strict')
        kwargs = {key.decode('utf-8', 'strict'): value[0].decode('utf-8', 'strict') for key, value in
                  request.args.items()}
        request.setHeader('Content-Type', 'application/json')
        request.setResponseCode(200)

        async_training = self.data_router.start_train_process(data_string, kwargs)
        async_training.addErrback(errback)

        return json.dumps({"info": "training started."})

if __name__ == '__main__':
    # Running as standalone python application
    arg_parser = create_argparser()
    cmdline_args = {key: val for key, val in list(vars(arg_parser.parse_args()).items()) if val is not None}
    rasa_nlu_config = RasaNLUConfig(cmdline_args.get("config"), os.environ, cmdline_args)
    rasa = RasaNLU(rasa_nlu_config)
    logger.info('Started http server on port %s' % rasa_nlu_config['port'])
    rasa.app.run('0.0.0.0', rasa_nlu_config['port'])
<commit_msg>Removed useless maybeDeferred call.<commit_after>from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
import argparse
import json
import logging
import os
from functools import wraps

from klein import Klein
from twisted.internet import reactor, threads
from twisted.internet.defer import inlineCallbacks, returnValue, maybeDeferred

from rasa_nlu.config import RasaNLUConfig
from rasa_nlu.data_router import DataRouter, InvalidModelError
from rasa_nlu.version import __version__

logger = logging.getLogger(__name__)

def create_argparser():
    parser = argparse.ArgumentParser(description='parse incoming text')
    parser.add_argument('-c', '--config',
                        help="config file, all the command line options can also be passed via a (json-formatted) " +
                             "config file. NB command line args take precedence")
    parser.add_argument('-d', '--server_model_dirs',
                        help='directory containing model to for parser to use')
    parser.add_argument('-e', '--emulate', choices=['wit', 'luis', 'api'],
                        help='which service to emulate (default: None i.e. use simple built in format)')
    parser.add_argument('-l', '--language', choices=['de', 'en'], help="model and data language")
    parser.add_argument('-m', '--mitie_file',
                        help='file with mitie total_word_feature_extractor')
    parser.add_argument('-p', '--path', help="path where model files will be saved")
    parser.add_argument('--pipeline', help="The pipeline to use. Either a pipeline template name or a list of " +
                                           "components separated by comma")
    parser.add_argument('-P', '--port', type=int, help='port on which to run server')
    parser.add_argument('-t', '--token',
                        help="auth token. If set, reject requests which don't provide this token as a query parameter")
    parser.add_argument('-w', '--write', help='file where logs will be saved')

    return parser

def requires_auth(f):
    """Wraps a request handler with token authentication."""

    @wraps(f)
    def decorated(*args, **kwargs):
        self = args[0]
        request = args[1]
        token = request.args.get('token', [''])[0]
        if self.data_router.token is None or token == self.data_router.token:
            return f(*args, **kwargs)
        return "unauthorized", 401

    return decorated

class RasaNLU(object):
    """Class representing Rasa NLU http server"""

    app = Klein()

    def __init__(self, config, component_builder=None, testing=False):
        logging.basicConfig(filename=config['log_file'], level=config['log_level'])
        logging.captureWarnings(True)
        logger.debug("Configuration: " + config.view())

        logger.debug("Creating a new data router")
        self.config = config
        self.data_router = DataRouter(config, component_builder)
        self.testing = testing
        reactor.suggestThreadPoolSize(config['num_threads'] * 5)

    @app.route("/", methods=['GET'])
    def hello(self, request):
        """Main Rasa route to check if the server is online"""

        return "hello from Rasa NLU: " + __version__

    @app.route("/parse", methods=['GET', 'POST'])
    @requires_auth
    @inlineCallbacks
    def parse_get(self, request):
        request.setHeader('Content-Type', 'application/json')
        if request.method.decode('utf-8', 'strict') == 'GET':
            request_params = {key.decode('utf-8', 'strict'): value[0].decode('utf-8', 'strict') for key, value in
                              request.args.items()}
        else:
            request_params = json.loads(request.content.read().decode('utf-8', 'strict'))
        if 'q' not in request_params:
            request.setResponseCode(404)
            returnValue(json.dumps({"error": "Invalid parse parameter specified"}))
        else:
            data = self.data_router.extract(request_params)
            try:
                request.setResponseCode(200)
                response = yield (self.data_router.parse(data) if self.testing
                                  else threads.deferToThread(self.data_router.parse, data))
                returnValue(json.dumps(response))
            except InvalidModelError as e:
                request.setResponseCode(404)
                returnValue(json.dumps({"error": "{}".format(e)}))

    @app.route("/version", methods=['GET'])
    @requires_auth
    def version(self, request):
        """Returns the Rasa server's version"""

        request.setHeader('Content-Type', 'application/json')
        return json.dumps({'version': __version__})

    @app.route("/config", methods=['GET'])
    @requires_auth
    def rasaconfig(self, request):
        """Returns the in-memory configuration of the Rasa server"""

        request.setHeader('Content-Type', 'application/json')
        return json.dumps(self.config.as_dict())

    @app.route("/status", methods=['GET'])
    @requires_auth
    def status(self, request):
        request.setHeader('Content-Type', 'application/json')
        return json.dumps(self.data_router.get_status())

    @app.route("/train", methods=['POST'])
    @requires_auth
    def train(self, request):
        def errback(f):
            f.trap(ValueError)
            logger.debug("error: {}".format(f.getErrorMessage()))

        data_string = request.content.read().decode('utf-8', 'strict')
        kwargs = {key.decode('utf-8', 'strict'): value[0].decode('utf-8', 'strict') for key, value in
                  request.args.items()}
        request.setHeader('Content-Type', 'application/json')
        request.setResponseCode(200)

        async_training = self.data_router.start_train_process(data_string, kwargs)
        async_training.addErrback(errback)

        return json.dumps({"info": "training started."})

if __name__ == '__main__':
    # Running as standalone python application
    arg_parser = create_argparser()
    cmdline_args = {key: val for key, val in list(vars(arg_parser.parse_args()).items()) if val is not None}
    rasa_nlu_config = RasaNLUConfig(cmdline_args.get("config"), os.environ, cmdline_args)
    rasa = RasaNLU(rasa_nlu_config)
    logger.info('Started http server on port %s' % rasa_nlu_config['port'])
    rasa.app.run('0.0.0.0', rasa_nlu_config['port'])
<|endoftext|>