<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    This module uses OpenERP, Open Source Management Solution Framework.
#    Copyright (C):
#        2012-Today Serpent Consulting Services (<http://www.serpentcs.com>)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>
#
##############################################################################

from openerp.exceptions import UserError
from openerp import api, fields, models, _

class MassObject(models.Model):
    _name = "mass.object"

    name = fields.Char('Name', required=True, select=1)
    model_id = fields.Many2one('ir.model', 'Model', required=True,
                               help="Model is used for Selecting Fields."
                               "This is editable until Sidebar menu is not "
                               "created.")
    field_ids = fields.Many2many('ir.model.fields', 'mass_field_rel',
                                 'mass_id', 'field_id', 'Fields')
    ref_ir_act_window = fields.Many2one('ir.actions.act_window',
                                        'Sidebar action',
                                        readonly=True,
                                        help="Sidebar action to make this "
                                        "template available on records of "
                                        "the related document model.")
    ref_ir_value = fields.Many2one('ir.values', 'Sidebar button',
                                   readonly=True,
                                   help="Sidebar button to open"
                                   "the sidebar action")
    model_list = fields.Char('Model List')

    _sql_constraints = [
        ('name_uniq', 'unique (name)', _('Name must be unique!')),
    ]

    @api.onchange('model_id')
    def onchange_model(self):
        self.field_ids = [(6, 0, [])]
        model_list = []
        if self.model_id:
            model_obj = self.env['ir.model']
            model_list = [self.model_id.id]
            active_model_obj = self.env[self.model_id.model]
            if active_model_obj._inherits:
                keys = active_model_obj._inherits.keys()
                inherits_model_list = model_obj.search([('model', 'in', keys)])
                model_list.extend((inherits_model_list and
                                   inherits_model_list.ids or []))
        self.model_list = model_list

    @api.multi
    def create_action(self):
        vals = {}
        action_obj = self.env['ir.actions.act_window']
        src_obj = self.model_id.model
        button_name = _('Mass Editing (%s)') % self.name
        vals['ref_ir_act_window'] = action_obj.create({
            'name': button_name,
            'type': 'ir.actions.act_window',
            'res_model': 'mass.editing.wizard',
            'src_model': src_obj,
            'view_type': 'form',
            'context': "{'mass_editing_object' : %d}" % (self.id),
            'view_mode': 'form, tree',
            'target': 'new',
            'auto_refresh': 1,
        }).id
        vals['ref_ir_value'] = self.env['ir.values'].create({
            'name': button_name,
            'model': src_obj,
            'key2': 'client_action_multi',
            'value': "ir.actions.act_window," + str(vals['ref_ir_act_window']),
            'object': True,
        }).id
        self.write(vals)
        return True

    @api.multi
    def unlink_action(self):
        for mass in self:
            try:
                if mass.ref_ir_act_window:
                    mass.ref_ir_act_window.unlink()
                if mass.ref_ir_value:
                    mass.ref_ir_value.unlink()
            except:
                raise UserError(_("Deletion of the action record failed."))
        return True

    @api.multi
    def unlink(self):
        self.unlink_action()
        return super(MassObject, self).unlink()

    @api.one
    def copy(self, default):
        if default is None:
            default = {}
        default.update({'name': _("%s (copy)" % self.name), 'field_ids': []})
        return super(MassObject, self).copy(default)

class IrModuleModule(models.Model):
    _inherit = 'ir.module.module'

    @api.multi
    def module_uninstall(self):
        # search window actions of mass editing  and delete it
        for record in self:
            if record.name == 'mass_editing':
                values_obj = self.env['ir.values']
                actions = self.env['ir.actions.act_window'].\
                    search([('res_model', '=', 'mass.editing.wizard')])
                for action in actions:
                    values_obj.\
                        search([('value', '=',
                                 'ir.actions.act_window,%s' % action.id)]).unlink()
                actions.unlink()
        return super(IrModuleModule, self).module_uninstall()
<commit_msg>Correction for the travis test<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    This module uses OpenERP, Open Source Management Solution Framework.
#    Copyright (C):
#        2012-Today Serpent Consulting Services (<http://www.serpentcs.com>)
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>
#
##############################################################################

from openerp.exceptions import UserError
from openerp import api, fields, models, _

class MassObject(models.Model):
    _name = "mass.object"

    name = fields.Char('Name', required=True, select=1)
    model_id = fields.Many2one('ir.model', 'Model', required=True,
                               help="Model is used for Selecting Fields."
                               "This is editable until Sidebar menu is not "
                               "created.")
    field_ids = fields.Many2many('ir.model.fields', 'mass_field_rel',
                                 'mass_id', 'field_id', 'Fields')
    ref_ir_act_window = fields.Many2one('ir.actions.act_window',
                                        'Sidebar action',
                                        readonly=True,
                                        help="Sidebar action to make this "
                                        "template available on records of "
                                        "the related document model.")
    ref_ir_value = fields.Many2one('ir.values', 'Sidebar button',
                                   readonly=True,
                                   help="Sidebar button to open"
                                   "the sidebar action")
    model_list = fields.Char('Model List')

    _sql_constraints = [
        ('name_uniq', 'unique (name)', _('Name must be unique!')),
    ]

    @api.onchange('model_id')
    def onchange_model(self):
        self.field_ids = [(6, 0, [])]
        model_list = []
        if self.model_id:
            model_obj = self.env['ir.model']
            model_list = [self.model_id.id]
            active_model_obj = self.env[self.model_id.model]
            if active_model_obj._inherits:
                keys = active_model_obj._inherits.keys()
                inherits_model_list = model_obj.search([('model', 'in', keys)])
                model_list.extend((inherits_model_list and
                                   inherits_model_list.ids or []))
        self.model_list = model_list

    @api.multi
    def create_action(self):
        vals = {}
        action_obj = self.env['ir.actions.act_window']
        src_obj = self.model_id.model
        button_name = _('Mass Editing (%s)') % self.name
        vals['ref_ir_act_window'] = action_obj.create({
            'name': button_name,
            'type': 'ir.actions.act_window',
            'res_model': 'mass.editing.wizard',
            'src_model': src_obj,
            'view_type': 'form',
            'context': "{'mass_editing_object' : %d}" % (self.id),
            'view_mode': 'form, tree',
            'target': 'new',
            'auto_refresh': 1,
        }).id
        vals['ref_ir_value'] = self.env['ir.values'].create({
            'name': button_name,
            'model': src_obj,
            'key2': 'client_action_multi',
            'value': "ir.actions.act_window," + str(vals['ref_ir_act_window']),
            'object': True,
        }).id
        self.write(vals)
        return True

    @api.multi
    def unlink_action(self):
        for mass in self:
            try:
                if mass.ref_ir_act_window:
                    mass.ref_ir_act_window.unlink()
                if mass.ref_ir_value:
                    mass.ref_ir_value.unlink()
            except:
                raise UserError(_("Deletion of the action record failed."))
        return True

    @api.multi
    def unlink(self):
        self.unlink_action()
        return super(MassObject, self).unlink()

    @api.one
    def copy(self, default):
        if default is None:
            default = {}
        default.update({'name': _("%s (copy)" % self.name), 'field_ids': []})
        return super(MassObject, self).copy(default)

class IrModuleModule(models.Model):
    _inherit = 'ir.module.module'

    @api.multi
    def module_uninstall(self):
        # search window actions of mass editing  and delete it
        for record in self:
            if record.name == 'mass_editing':
                values_obj = self.env['ir.values']
                actions = self.env['ir.actions.act_window'].\
                    search([('res_model', '=', 'mass.editing.wizard')])
                for action in actions:
                    domain = [('value', '=',
                                'ir.actions.act_window,%s' % action.id)]
                    values_obj.search(domain).unlink()
                actions.unlink()
        return super(IrModuleModule, self).module_uninstall()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
mbed SDK
Copyright (c) 2011-2014 ARM Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Author: Przemyslaw Wirkus <Przemyslaw.Wirkus@arm.com>
"""

import os
import re
import sys
import json
import optparse

from mbed_test_api import run_host_test
from mbed_test_api import run_cli_command
from mbed_test_api import TEST_RESULTS
from cmake_handlers import load_ctest_testsuite
from mbed_target_info import get_mbed_clasic_target_info
from mbed_target_info import get_mbed_supported_test

try:
    import mbed_lstools
    import mbed_host_tests
except:
    pass

MBED_LMTOOLS = 'mbed_lstools' in sys.modules
MBED_HOST_TESTS = 'mbed_host_tests' in sys.modules

# git clone repository
# cd repo
# yotta target frdm-....
# yt build
#
# yt --target=frdm-XXXX,* build
#

def main():

    if not MBED_LMTOOLS:
        print "Error: mbed-lstools mbed proprietary module not installed"
        exit(-1)

    if not MBED_HOST_TESTS:
        print "Error: mbed-host-tests mbed proprietary module not installed"
        exit(-1)

    parser = optparse.OptionParser()

    parser.add_option('', '--target',
                    dest='list_of_targets',
                    help='You can specify list of targets you want to build. Use comma to sepatate them')

    #parser.add_option('-n', '--test-by-names',
    #                dest='test_by_names',
    #                help='Runs only test enumerated it this switch')

    parser.add_option("-O", "--only-build",
                    action="store_true",
                    dest="only_build_tests",
                    default=False,
                    help="Only build repository and tests, skips actual test procedures (flashing etc.)")

    parser.add_option('', '--config',
                    dest='verbose_test_configuration_only',
                    default=False,
                    action="store_true",
                    help='Displays full test specification and MUTs configration and exits')

    parser.add_option('', '--digest',
                    dest='digest_source',
                    help='Redirect input from where test suite should take console input. You can use stdin or file name to get test case console output')

    parser.add_option('-V', '--verbose-test-result',
                    dest='verbose_test_result_only',
                    default=False,
                    action="store_true",
                    help='Prints test serial output')

    parser.add_option('-v', '--verbose',
                    dest='verbose',
                    default=False,
                    action="store_true",
                    help='Verbose mode (prints some extra information)')

    parser.description = """This automated test script is used to test mbed SDK 3.0 on mbed-enabled deviecs with support from yotta build tool"""
    parser.epilog = """Example: mbedgt --auto --target frdm-k64f-gcc"""

    (opts, args) = parser.parse_args()

    # Capture alternative test console inputs
    if opts.digest_source:
        host_test_result = run_host_test(image_path=None, disk=None, port=None,
                                    digest_source=opts.digest_source,
                                    verbose=opts.verbose_test_result_only)
        single_test_result, single_test_output, single_testduration, single_timeout = host_test_result
        status = TEST_RESULTS.index(single_test_result) if single_test_result in TEST_RESULTS else -1
        sys.exit(status)

    # mbed-enabled devices auto-detection procedures
    mbeds = mbed_lstools.create()
    mbeds_list = mbeds.list_mbeds()

    print "mbed-ls: detecting connected mbed-enabled devices... %s"% ("no devices detected" if not len(mbeds_list) else "")
    list_of_targets = opts.list_of_targets.split(',') if opts.list_of_targets is not None else None

    for mut in mbeds_list:
        print "mbed-ls: detected %s, console at: %s, mounted at: %s"% (mut['platform_name'],
            mut['serial_port'],
            mut['mount_point'])

        # Check if mbed classic target name can be translated to yotta target name
        mut_info = get_mbed_clasic_target_info(mut['platform_name'])
        if mut_info is None:
            print "mbed-ls: mbed classic target name %s is not in target database"% (mut['platform_name'])
        else:
            for yotta_target in mut_info['yotta_targets']:
                yotta_target_name = yotta_target['yotta_target']
                yotta_target_toolchain = yotta_target['mbed_toolchain']
                print "\tgot yotta target '%s'"% (yotta_target_name)

                if opts.verbose_test_configuration_only:
                    continue

                # Building sources for given target
                if list_of_targets is None or yotta_target_name in list_of_targets:
                    print "mbed-ls: calling yotta to build your sources and tests"
                    yotta_verbose = '-v' if opts.verbose is not None else ''
                    yotta_result = run_cli_command("yotta %s --target=%s,* build"% (yotta_verbose, yotta_target_name))

                    # Build phase will be followed by test execution for each target
                    if yotta_result and not opts.only_build_tests:
                        binary_type = mut_info['properties']['binary_type']
                        ctest_test_list = load_ctest_testsuite(os.path.join('.', 'build', yotta_target_name),
                            binary_type=binary_type)

                        print "mbedgt: running tests..."
                        for test_bin, image_path in ctest_test_list.iteritems():
                            test_result = 'SKIPPED'
                            if get_mbed_supported_test(test_bin):
                                disk =  mut['mount_point']
                                port =  mut['serial_port']
                                micro = mut['platform_name']
                                program_cycle_s = mut_info['properties']['program_cycle_s']
                                copy_method = mut_info['properties']['copy_method']
                                verbose = opts.verbose_test_result_only

                                host_test_result = run_host_test(image_path, disk, port,
                                    micro=micro,
                                    copy_method=copy_method,
                                    program_cycle_s=program_cycle_s,
                                    digest_source=opts.digest_source,
                                    verbose=verbose)
                                single_test_result, single_test_output, single_testduration, single_timeout = host_test_result
                                test_result = single_test_result
                            print "\ttest '%s' %s"% (test_bin, '.' * (70 - len(test_bin))),
                            print "%s"% (test_result)
<commit_msg>Added new options --debug and --release to force debug/release build<commit_after>#!/usr/bin/env python

"""
mbed SDK
Copyright (c) 2011-2014 ARM Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Author: Przemyslaw Wirkus <Przemyslaw.Wirkus@arm.com>
"""

import os
import re
import sys
import json
import optparse

from mbed_test_api import run_host_test
from mbed_test_api import run_cli_command
from mbed_test_api import TEST_RESULTS
from cmake_handlers import load_ctest_testsuite
from mbed_target_info import get_mbed_clasic_target_info
from mbed_target_info import get_mbed_supported_test

try:
    import mbed_lstools
    import mbed_host_tests
except:
    pass

MBED_LMTOOLS = 'mbed_lstools' in sys.modules
MBED_HOST_TESTS = 'mbed_host_tests' in sys.modules

# git clone repository
# cd repo
# yotta target frdm-....
# yt build
#
# yt --target=frdm-XXXX,* build
#

def main():

    if not MBED_LMTOOLS:
        print "Error: mbed-lstools mbed proprietary module not installed"
        exit(-1)

    if not MBED_HOST_TESTS:
        print "Error: mbed-host-tests mbed proprietary module not installed"
        exit(-1)

    parser = optparse.OptionParser()

    parser.add_option('', '--target',
                    dest='list_of_targets',
                    help='You can specify list of targets you want to build. Use comma to sepatate them')

    #parser.add_option('-n', '--test-by-names',
    #                dest='test_by_names',
    #                help='Runs only test enumerated it this switch')

    parser.add_option("-O", "--only-build",
                    action="store_true",
                    dest="only_build_tests",
                    default=False,
                    help="Only build repository and tests, skips actual test procedures (flashing etc.)")

    parser.add_option('', '--config',
                    dest='verbose_test_configuration_only',
                    default=False,
                    action="store_true",
                    help='Displays full test specification and MUTs configration and exits')

    parser.add_option('', '--release',
                    dest='build_to_release',
                    default=False,
                    action="store_true",
                    help='If possible force build in release mode.')

    parser.add_option('', '--debug',
                    dest='build_to_debug',
                    default=False,
                    action="store_true",
                    help='If possible force build in debug mode.')

    parser.add_option('', '--digest',
                    dest='digest_source',
                    help='Redirect input from where test suite should take console input. You can use stdin or file name to get test case console output')

    parser.add_option('-V', '--verbose-test-result',
                    dest='verbose_test_result_only',
                    default=False,
                    action="store_true",
                    help='Prints test serial output')

    parser.add_option('-v', '--verbose',
                    dest='verbose',
                    default=False,
                    action="store_true",
                    help='Verbose mode (prints some extra information)')

    parser.description = """This automated test script is used to test mbed SDK 3.0 on mbed-enabled deviecs with support from yotta build tool"""
    parser.epilog = """Example: mbedgt --auto --target frdm-k64f-gcc"""

    (opts, args) = parser.parse_args()

    # Capture alternative test console inputs
    if opts.digest_source:
        host_test_result = run_host_test(image_path=None, disk=None, port=None,
                                    digest_source=opts.digest_source,
                                    verbose=opts.verbose_test_result_only)
        single_test_result, single_test_output, single_testduration, single_timeout = host_test_result
        status = TEST_RESULTS.index(single_test_result) if single_test_result in TEST_RESULTS else -1
        sys.exit(status)

    # mbed-enabled devices auto-detection procedures
    mbeds = mbed_lstools.create()
    mbeds_list = mbeds.list_mbeds()

    print "mbed-ls: detecting connected mbed-enabled devices... %s"% ("no devices detected" if not len(mbeds_list) else "")
    list_of_targets = opts.list_of_targets.split(',') if opts.list_of_targets is not None else None

    for mut in mbeds_list:
        print "mbed-ls: detected %s, console at: %s, mounted at: %s"% (mut['platform_name'],
            mut['serial_port'],
            mut['mount_point'])

        # Check if mbed classic target name can be translated to yotta target name
        mut_info = get_mbed_clasic_target_info(mut['platform_name'])
        if mut_info is None:
            print "mbed-ls: mbed classic target name %s is not in target database"% (mut['platform_name'])
        else:
            for yotta_target in mut_info['yotta_targets']:
                yotta_target_name = yotta_target['yotta_target']
                yotta_target_toolchain = yotta_target['mbed_toolchain']
                print "\tgot yotta target '%s'"% (yotta_target_name)

                if opts.verbose_test_configuration_only:
                    continue

                # Building sources for given target
                if list_of_targets is None or yotta_target_name in list_of_targets:
                    # "yotta %s --target=%s,* build"% (yotta_verbose, yotta_target_name)
                    cmd = ['yotta']
                    if opts.verbose is not None: cmd.append('-v')
                    cmd.append('--target=%s,*' % yotta_target_name)
                    cmd.append('build')
                    if opts.build_to_release: cmd.append('-r')
                    elif opts.build_to_debug: cmd.append('-d')

                    print "mbed-ls: calling yotta to build your sources and tests: %s" % (' '.join(cmd))
                    yotta_result = run_cli_command(cmd)

                    # Build phase will be followed by test execution for each target
                    if yotta_result and not opts.only_build_tests:
                        binary_type = mut_info['properties']['binary_type']
                        ctest_test_list = load_ctest_testsuite(os.path.join('.', 'build', yotta_target_name),
                            binary_type=binary_type)

                        print "mbedgt: running tests..."
                        for test_bin, image_path in ctest_test_list.iteritems():
                            test_result = 'SKIPPED'
                            if get_mbed_supported_test(test_bin):
                                disk =  mut['mount_point']
                                port =  mut['serial_port']
                                micro = mut['platform_name']
                                program_cycle_s = mut_info['properties']['program_cycle_s']
                                copy_method = mut_info['properties']['copy_method']
                                verbose = opts.verbose_test_result_only

                                host_test_result = run_host_test(image_path, disk, port,
                                    micro=micro,
                                    copy_method=copy_method,
                                    program_cycle_s=program_cycle_s,
                                    digest_source=opts.digest_source,
                                    verbose=verbose)
                                single_test_result, single_test_output, single_testduration, single_timeout = host_test_result
                                test_result = single_test_result
                            print "\ttest '%s' %s"% (test_bin, '.' * (70 - len(test_bin))),
                            print "%s"% (test_result)
<|endoftext|>


<commit_before>import re
import random

from sqlalchemy import Column, Integer, String
from sqlalchemy.ext.declarative import declarative_base

from hamper.interfaces import ChatPlugin

SQLAlchemyBase = declarative_base()

class Factoids(ChatPlugin):
    """Learn and repeat Factoids."""
    name = 'factoids'
    priority = -1

    def setup(self, loader):
        super(Factoids, self).setup(loader)
        self.db = loader.db
        SQLAlchemyBase.metadata.create_all(self.db.engine)

        self.factoids = {}

    def message(self, bot, comm):
        subs = [self.try_add_factoid,
                self.try_forget_factoid,
                self.try_forget_factoid_mass,
                self.try_respond_to_factoid]

        ret = False
        for sub in subs:
            ret = sub(bot, comm)
            if ret:
                break
        return ret

    def try_add_factoid(self, bot, comm):
        if not comm['directed']:
            return

        msg = comm['message'].strip()
        match = re.match(r'learn(?: that)? (.+)\s+(\w+)\s+<(\w+)>\s+(.*)', msg)

        if not match:
            return

        trigger, type, action, response = match.groups()
        trigger = trigger.strip()

        if action not in ['say', 'reply', 'me']:
            bot.reply(comm, "I don't know the action {0}.".format(action))
            return
        if type not in ['is', 'triggers']:
            bot.reply(comm, "I don't the type {0}.".format(type))
            return

        self.db.session.add(Factoid(trigger, type, action, response))
        self.db.session.commit()
        bot.reply(comm, 'OK, {user}'.format(**comm))

        return True

    def try_forget_factoid(self, bot, comm):
        if not comm['directed']:
            return

        msg = comm['message'].strip()
        match = re.match(r'forget that\s+(.+)\s+is\s+(.*)', msg)

        if not match:
            return

        trigger, response = match.groups()

        factoids = (self.db.session.query(Factoid)
                    .filter(Factoid.trigger == trigger,
                            Factoid.response == response)
                    .all())
        if len(factoids) == 0:
            bot.reply(comm, "I don't have anything like that.")
            return
        for factoid in factoids:
            self.db.session.delete(factoid)
        self.db.session.commit()
        bot.reply(comm, 'Done, {user}'.format(comm))

    def try_forget_factoid_mass(self, bot, comm):
        if not comm['directed']:
            return

        msg = comm['message'].strip()
        match = re.match(r'forget all about (.+)', msg)
        if not match:
            return

        trigger = match.groups()[0]
        factoids = (self.db.session.query(Factoid)
                    .filter(Factoid.trigger == trigger)
                    .all())

        if len(factoids) == 0:
            bot.reply(comm, "I don't have anything like that.")
            return

        for factoid in factoids:
            self.db.session.delete(factoid)
        self.db.session.commit()

        bot.reply(comm, 'Done, {user}'.format(comm))

    def try_respond_to_factoid(self, bot, comm):
        msg = comm['message'].strip()

        if comm['directed']:
            msg = '!' + msg

        factoids = (self.db.session.query(Factoid)
                    .filter(Factoid.trigger == msg)
                    .all())
        if len(factoids) == 0:
            factoids = (self.db.session.query(Factoid)
                        .filter(Factoid.type == 'triggers')
                        .all())
            factoids = filter(lambda f: f.trigger in msg, factoids)

        if len(factoids) == 0:
            return

        factoid = random.choice(factoids)
        if factoid.action == 'say':
            bot.reply(comm, factoid.response)
        elif factoid.action == 'reply':
            bot.reply(comm, '{}: {}'.format(comm['user'], factoid.response))
        elif factoid.action == 'me':
            bot.me(comm, factoid.response)
        else:
            bot.reply(comm, 'Um, what is the verb {}?'.format(factoid.action))

class Factoid(SQLAlchemyBase):
    '''The object that will get persisted by the database.'''

    __tablename__ = 'factoids'

    id = Column(Integer, primary_key=True)
    type = Column(String)
    trigger = Column(String)
    action = Column(String)
    response = Column(String)

    def __init__(self, trigger, type, action, response):
        self.type = type
        self.trigger = trigger
        self.action = action
        self.response = response

factoids = Factoids()
<commit_msg>Fix reply.<commit_after>import re
import random

from sqlalchemy import Column, Integer, String
from sqlalchemy.ext.declarative import declarative_base

from hamper.interfaces import ChatPlugin

SQLAlchemyBase = declarative_base()

class Factoids(ChatPlugin):
    """Learn and repeat Factoids."""
    name = 'factoids'
    priority = -1

    def setup(self, loader):
        super(Factoids, self).setup(loader)
        self.db = loader.db
        SQLAlchemyBase.metadata.create_all(self.db.engine)

        self.factoids = {}

    def message(self, bot, comm):
        subs = [self.try_add_factoid,
                self.try_forget_factoid,
                self.try_forget_factoid_mass,
                self.try_respond_to_factoid]

        ret = False
        for sub in subs:
            ret = sub(bot, comm)
            if ret:
                break
        return ret

    def try_add_factoid(self, bot, comm):
        if not comm['directed']:
            return

        msg = comm['message'].strip()
        match = re.match(r'learn(?: that)? (.+)\s+(\w+)\s+<(\w+)>\s+(.*)', msg)

        if not match:
            return

        trigger, type, action, response = match.groups()
        trigger = trigger.strip()

        if action not in ['say', 'reply', 'me']:
            bot.reply(comm, "I don't know the action {0}.".format(action))
            return
        if type not in ['is', 'triggers']:
            bot.reply(comm, "I don't the type {0}.".format(type))
            return

        self.db.session.add(Factoid(trigger, type, action, response))
        self.db.session.commit()
        bot.reply(comm, 'OK, {user}'.format(**comm))

        return True

    def try_forget_factoid(self, bot, comm):
        if not comm['directed']:
            return

        msg = comm['message'].strip()
        match = re.match(r'forget that\s+(.+)\s+is\s+(.*)', msg)

        if not match:
            return

        trigger, response = match.groups()

        factoids = (self.db.session.query(Factoid)
                    .filter(Factoid.trigger == trigger,
                            Factoid.response == response)
                    .all())
        if len(factoids) == 0:
            bot.reply(comm, "I don't have anything like that.")
            return
        for factoid in factoids:
            self.db.session.delete(factoid)
        self.db.session.commit()
        bot.reply(comm, 'Done, {user}'.format(**comm))

    def try_forget_factoid_mass(self, bot, comm):
        if not comm['directed']:
            return

        msg = comm['message'].strip()
        match = re.match(r'forget all about (.+)', msg)
        if not match:
            return

        trigger = match.groups()[0]
        factoids = (self.db.session.query(Factoid)
                    .filter(Factoid.trigger == trigger)
                    .all())

        if len(factoids) == 0:
            bot.reply(comm, "I don't have anything like that.")
            return

        for factoid in factoids:
            self.db.session.delete(factoid)
        self.db.session.commit()

        bot.reply(comm, 'Done, {user}'.format(**comm))

    def try_respond_to_factoid(self, bot, comm):
        msg = comm['message'].strip()

        if comm['directed']:
            msg = '!' + msg

        factoids = (self.db.session.query(Factoid)
                    .filter(Factoid.trigger == msg)
                    .all())
        if len(factoids) == 0:
            factoids = (self.db.session.query(Factoid)
                        .filter(Factoid.type == 'triggers')
                        .all())
            factoids = filter(lambda f: f.trigger in msg, factoids)

        if len(factoids) == 0:
            return

        factoid = random.choice(factoids)
        if factoid.action == 'say':
            bot.reply(comm, factoid.response)
        elif factoid.action == 'reply':
            bot.reply(comm, '{}: {}'.format(comm['user'], factoid.response))
        elif factoid.action == 'me':
            bot.me(comm, factoid.response)
        else:
            bot.reply(comm, 'Um, what is the verb {}?'.format(factoid.action))

class Factoid(SQLAlchemyBase):
    '''The object that will get persisted by the database.'''

    __tablename__ = 'factoids'

    id = Column(Integer, primary_key=True)
    type = Column(String)
    trigger = Column(String)
    action = Column(String)
    response = Column(String)

    def __init__(self, trigger, type, action, response):
        self.type = type
        self.trigger = trigger
        self.action = action
        self.response = response

factoids = Factoids()
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import urllib
from collections import defaultdict
import requests
import boto
from solr import Solr, SolrException
from couchdb import Server, Database, Document

URL_SOLR = os.environ.get('URL_SOLR',
                          'http://10.0.1.13:8080/solr/dc-collection/')
URL_COUCHDB = os.environ.get('URL_COUCHDB', 'http://localhost:5984')
COUCHDB_DB = os.environ.get('COUCHDB_DB', 'ucldc')
COUCHDB_LAST_SEQ_KEY = 'couchdb_last_seq'

COUCHDOC_TO_SOLR_MAPPING = {
    'id'       : lambda d: {'id': d['_id']},
    'object'   : lambda d: {'reference_image_md5': d['object']},
    'isShownAt': lambda d: {'url_item': d['isShownAt']},
}

COUCHDOC_ORG_RECORD_TO_SOLR_MAPPING = {
    'campus'     : lambda d: {'campus': [c['@id'] for c in d['campus']],
                              'campus_name': [c['name'] for c in d['campus']]},
    'repository' : lambda d: {'repository': [r['@id'] for r in d['repository']],
                              'repository_name': [r['name'] for r in d['repository']]},
}

COUCHDOC_SRC_RESOURCE_TO_SOLR_MAPPING = {
    # assuming one collection only, may need to change
    'collection'  : lambda d: {'collection': [c['@id'] for c in d['collection']],
                               'collection_name': [c['name'] for c in d['collection']]},
    'contributor' : lambda d: {'contributor': d.get('contributor', None)},
    'coverage'    : lambda d: {'spatial': d.get('spatial', None)},
    'creator'     : lambda d: {'creator': d.get('creator', None)},
    'description' : lambda d: {'description': d.get('description', None)},
    'date'        : lambda d: {'date': d.get('date', None)},
    'identifier'  : lambda d: {'identifier': d.get('identifier', None)},
    'language'    : lambda d: {'language': d.get('language', None)},
    'publisher'   : lambda d: {'publisher': d.get('publisher', None)},
    'relation'    : lambda d: {'relation': d.get('relation', None)},
    'rights'      : lambda d: {'rights': d.get('rights', None)},
    'subject'     : lambda d: {'subject': [s['name'] for s in d['subject'] if s]},
    'title'       : lambda d: {'title': d.get('title', None)},
    'type'        : lambda d: {'type': d.get('type', None)},
    'format'      : lambda d: {'format': d.get('format', None)},
    'extent'      : lambda d: {'extent': d.get('extent', None)},
}

def map_couch_to_solr_doc(doc):
    '''Return a json document suitable for updating the solr index
    how to make schema aware mapping?'''
    solr_doc = {}
    for p in doc.keys():
        if p in COUCHDOC_TO_SOLR_MAPPING:
            solr_doc.update(COUCHDOC_TO_SOLR_MAPPING[p](doc))
    originalRecord = doc['originalRecord']
    for p in originalRecord.keys():
        if p in COUCHDOC_ORG_RECORD_TO_SOLR_MAPPING:
            solr_doc.update(COUCHDOC_ORG_RECORD_TO_SOLR_MAPPING[p](originalRecord))
    sourceResource = doc['sourceResource']
    for p in sourceResource.keys():
        if p in COUCHDOC_SRC_RESOURCE_TO_SOLR_MAPPING:
            try:
                solr_doc.update(COUCHDOC_SRC_RESOURCE_TO_SOLR_MAPPING[p](sourceResource))
            except TypeError, e:
                print('TypeError for doc {} on sourceResource {}'.format(doc['_id'], p))
                raise e
    return solr_doc

def push_doc_to_solr(solr_doc, solr_db):
    '''Push one couch doc to solr'''
    try:
        solr_db.add(solr_doc)
    except SolrException, e:
        print "ERROR:", solr_doc
        print e
        if not e.httpcode == 400:
            raise e
    return solr_doc

def set_couchdb_last_seq(seq_num):
    '''Set the value fof the last sequence from couchdb _changes api'''
    conn = boto.connect_s3()
    b = conn.get_bucket('ucldc')
    k = b.get_key(COUCHDB_LAST_SEQ_KEY)
    if not k:
        k = Key(b)
        k.key = COUCHDB_LAST_SEQ_KEY
    k.set_contents_from_string(seq_num)

def get_couchdb_last_seq():
    '''Return the value stored in the s3 bucket'''
    conn = boto.connect_s3()
    b = conn.get_bucket('ucldc')
    k = b.get_key(COUCHDB_LAST_SEQ_KEY)
    return int(k.get_contents_as_string())

def main(url_couchdb=URL_COUCHDB, dbname=COUCHDB_DB, url_solr=URL_SOLR):
    print('Attempt to connect to {0} - db:{1}'.format(url_couchdb, dbname))
    server = Server(url_couchdb)
    db = server[dbname]
    since = get_couchdb_last_seq()
    changes = db.changes(since=since)
    last_seq = int(changes['last_seq'])
    results = changes['results']
    n = 0
    solr_db = Solr(url_solr)
    for row in results:
        if '_design' in row['id']:
            print("Skip {0}".format(row['id']))
            continue
        n += 1
        print(row)
        continue
        # TODO: Handle deletions, need to look for "deleted" key in
        # change doc
        doc = db.get(row['id'])
        try:
            solr_doc = map_couch_to_solr_doc(doc)
            solr_doc = push_doc_to_solr(solr_doc, solr_db=solr_db)
        except TypeError:
            continue
    solr_db.commit() #commit updates
    #set_couchdb_last_seq(last_seq)
    print("UPDATED {0} DOCUMENTS".format(n))

if __name__ == '__main__':
    main()
<commit_msg>more work<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import urllib
import argparse
from collections import defaultdict
import requests
import boto
from solr import Solr, SolrException
from couchdb import Server, Database, Document

URL_SOLR = os.environ.get('URL_SOLR',
                          'http://10.0.1.13:8080/solr/dc-collection/')
URL_COUCHDB = os.environ.get('URL_COUCHDB', 'http://localhost:5984')
COUCHDB_DB = os.environ.get('COUCHDB_DB', 'ucldc')
COUCHDB_LAST_SEQ_KEY = 'couchdb_last_seq'

COUCHDOC_TO_SOLR_MAPPING = {
    'id'       : lambda d: {'id': d['_id']},
    'object'   : lambda d: {'reference_image_md5': d['object']},
    'isShownAt': lambda d: {'url_item': d['isShownAt']},
}

COUCHDOC_ORG_RECORD_TO_SOLR_MAPPING = {
    'campus'     : lambda d: {'campus': [c['@id'] for c in d['campus']],
                              'campus_name': [c['name'] for c in d['campus']]},
    'repository' : lambda d: {'repository': [r['@id'] for r in d['repository']],
                              'repository_name': [r['name'] for r in d['repository']]},
}

COUCHDOC_SRC_RESOURCE_TO_SOLR_MAPPING = {
    # assuming one collection only, may need to change
    'collection'  : lambda d: {'collection': [c['@id'] for c in d['collection']],
                               'collection_name': [c['name'] for c in d['collection']]},
    'contributor' : lambda d: {'contributor': d.get('contributor', None)},
    'coverage'    : lambda d: {'spatial': d.get('spatial', None)},
    'creator'     : lambda d: {'creator': d.get('creator', None)},
    'description' : lambda d: {'description': d.get('description', None)},
    'date'        : lambda d: {'date': d.get('date', None)},
    'identifier'  : lambda d: {'identifier': d.get('identifier', None)},
    'language'    : lambda d: {'language': d.get('language', None)},
    'publisher'   : lambda d: {'publisher': d.get('publisher', None)},
    'relation'    : lambda d: {'relation': d.get('relation', None)},
    'rights'      : lambda d: {'rights': d.get('rights', None)},
    'subject'     : lambda d: {'subject': [s['name'] for s in d['subject'] if s]},
    'title'       : lambda d: {'title': d.get('title', None)},
    'type'        : lambda d: {'type': d.get('type', None)},
    'format'      : lambda d: {'format': d.get('format', None)},
    'extent'      : lambda d: {'extent': d.get('extent', None)},
}

def map_couch_to_solr_doc(doc):
    '''Return a json document suitable for updating the solr index
    how to make schema aware mapping?'''
    solr_doc = {}
    for p in doc.keys():
        if p in COUCHDOC_TO_SOLR_MAPPING:
            solr_doc.update(COUCHDOC_TO_SOLR_MAPPING[p](doc))
    originalRecord = doc['originalRecord']
    for p in originalRecord.keys():
        if p in COUCHDOC_ORG_RECORD_TO_SOLR_MAPPING:
            solr_doc.update(COUCHDOC_ORG_RECORD_TO_SOLR_MAPPING[p](originalRecord))
    sourceResource = doc['sourceResource']
    for p in sourceResource.keys():
        if p in COUCHDOC_SRC_RESOURCE_TO_SOLR_MAPPING:
            try:
                solr_doc.update(COUCHDOC_SRC_RESOURCE_TO_SOLR_MAPPING[p](sourceResource))
            except TypeError, e:
                print('TypeError for doc {} on sourceResource {}'.format(doc['_id'], p))
                raise e
    return solr_doc

def push_doc_to_solr(solr_doc, solr_db):
    '''Push one couch doc to solr'''
    try:
        solr_db.add(solr_doc)
    except SolrException, e:
        print "ERROR:", solr_doc
        print e
        if not e.httpcode == 400:
            raise e
    return solr_doc

def set_couchdb_last_seq(seq_num):
    '''Set the value fof the last sequence from couchdb _changes api'''
    conn = boto.connect_s3()
    b = conn.get_bucket('ucldc')
    k = b.get_key(COUCHDB_LAST_SEQ_KEY)
    if not k:
        k = Key(b)
        k.key = COUCHDB_LAST_SEQ_KEY
    k.set_contents_from_string(seq_num)

def get_couchdb_last_seq():
    '''Return the value stored in the s3 bucket'''
    conn = boto.connect_s3()
    b = conn.get_bucket('ucldc')
    k = b.get_key(COUCHDB_LAST_SEQ_KEY)
    return int(k.get_contents_as_string())

def main(url_couchdb=URL_COUCHDB, dbname=COUCHDB_DB, url_solr=URL_SOLR):
    print('Attempt to connect to {0} - db:{1}'.format(url_couchdb, dbname))
    server = Server(url_couchdb)
    db = server[dbname]
    since = get_couchdb_last_seq()
    changes = db.changes(since=since)
    last_seq = int(changes['last_seq'])
    results = changes['results']
    n = 0
    solr_db = Solr(url_solr)
    for row in results:
        if '_design' in row['id']:
            print("Skip {0}".format(row['id']))
            continue
        n += 1
        print(row)
        continue
        # TODO: Handle deletions, need to look for "deleted" key in
        # change doc
        doc = db.get(row['id'])
        try:
            solr_doc = map_couch_to_solr_doc(doc)
            solr_doc = push_doc_to_solr(solr_doc, solr_db=solr_db)
        except TypeError:
            continue
    solr_db.commit() #commit updates
    #set_couchdb_last_seq(last_seq)
    print("UPDATED {0} DOCUMENTS".format(n))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='update a solr instance from the couchdb doc store')
    parser.add_argument('url_couchdb',
                        help='URL to couchdb (http://127.0.0.1:5984)')
    parser.add_argument('dbname', help='Couchdb database name')
    parser.add_argument('url_solr', help='URL to writeable solr instance')

    args = parser.parse_args()
    main(url_couchdb=args.url_couchdb, dbname=args.dbname,
         url_solr=args.url_solr)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""\
This module tries to detect acronyms and cables w/o subject.
"""
import os
import re
import codecs
from functools import partial
from cablemap.core import cables_from_directory
from cablemap.core.reader import parse_subject

import logging
import sys
logger = logging.getLogger('cablemap.core.reader')
logger.setLevel(logging.DEBUG)
logger.addHandler(logging.StreamHandler(sys.stdout))

_AC_PATTERN = re.compile(r'\(([A-Z\-/\.]{2,}[0-9]*)\)')

_ACRONYMS = (u'AA/S', u'ADC', u'AFM', u'AG', u'ASD/ISA', u'AU', u'AK', u'APHSCT',
             u'AF-PAK', u'AKP', u'ASD', u'AQAP', u'AQIM', u'ARENA',
             u'BBC', u'BP', u'BR-3',
             u'CMC', u'CNP', u'CODEL', u'CJCS', u'CT', u'CWS/BWC', u'CW', u'CENTCOM',
             u'CDR', u'CFE', u'CISMOA', u'CN', u'CIA', u'CTJWG', u'CG', u'CL', u'CBD',
             u'DAS', u'DCA', u'DDR', u'DEA', u'DG', u'DCM', u'DFID',
             u'DRC', u'DASD', u'DIO', u'DHS', u'DOL', u'DPRK',
             u'EFCC', u'ETA', u'EU', u'EU/US', u'EXBS', u'EUR', u'EFTA',
             u'FATF', u'FBI', u'FCO', u'FDA', u'FDP', u'FM', u'FTAA', u'FARC', u'FX2', u'FMLN',
             u'GAERC', u'GCC', u'GDRC', u'GM', u'GOAJ', u'GOB', u'GOC', u'GOE', u'GOI', u'GOK',
             u'GOL', u'GPC', u'GSL', u'GSP', u'GTMO', u'GOF', u'GOS', u'GBRV', u'GOP', u'GOU',
             u'GFA', u'G/TIP', u'GI',
             u'HMG', u'HLDG', u'HLG', u'HEU',
             u'ICTY', u'II', u'III', u'IMF', u'ITGA', u'IPR', u'IRGC', u'ICAO', u'IRA', u'ISAF',
             u'(INCSR)', u'ICRC', u'ISA', u'INR/B', u'ICC', u'ICG-G', u'ICJ', u'ILSA',
             u'JSF', u'JHA', u'JCET', u'JPMG',
             u'MDC', u'MEP', u'MFA', u'MOD', u'MI-17', u'MI-6',
             u'MRE', u'MP', u'MONUC', u'MOP-3', u'MEA',
             u'NATO', u'NDP', u'NSA', u'NGO', u'NEA', u'NEA/MAG', u'NTM-I',
             u'OIC', u'OECD', u'OAS', u'OSCE',
             u'PA', u'PD', u'PM', u'PMDB', u'PS', u'PDAS', u'PRT', u'POC', u'PRC/DPRK',
             u'PNG', u'PRC',
             u'ROK', u'RWE', u'RFG', u'RMB', u'RSO', u'RPO', u'RTG',
             u'SLA', u'SLA/U', u'SPD', u'SWIFT', u'S/WCI', u'S/CT', u'S/CRS',
             u'S/GC', u'SCSL', u'S/SRAP', u'SG',
             u'TFTP', u'TFTP/SWIFT', u'U.S.-EU', u'U.S.-UK', u'UAE', u'UK', u'UN',
             u'UNHCR', u'UNSC', u'US', u'US-CU', u'US-EU', u'USG', u'USTR', u'UNCHR',
             u'USEB', u'UNGA', u'U.S./UK', u'UNESCO', u'U/SYG', u'US-ROYG', u'UNSCR',
             u'USG-GOB', u'USD/P', u'USS', u'UNDP',
             u'VFM', u'VP', u'VI', u'VARIG', u'VOA',
             u'WEF', u'WTO',
             u'XVI',
             u'ZANU-PF')

_UNWANTED = (u'SAVE', u'CITES', u'SHARIA', u'IRAN', u'WHO', u'CAN')

def run_update(in_dir, predicate=None):
    acronyms = set(_ACRONYMS)
    subjects = set()
    for cable in cables_from_directory(in_dir, predicate):
        update_acronyms(cable, acronyms)
        update_missing_subjects(cable, subjects)
    return {'acronyms': acronyms, 'subjects': subjects}

def update_acronyms(cable, acronyms):
    if not cable.subject:
        return
    subject_words = [w for w in cable.subject.upper().split() if not w.startswith('XXXXX')]
    acronyms.update((ac for ac in _AC_PATTERN.findall(cable.content) if ac in subject_words and ac not in _UNWANTED))

def update_missing_subjects(cable, cable_refs):
    if not parse_subject(cable.content, fix_subject=False):
        cable_refs.add(cable.reference_id)

def _filename(name):
    return os.path.join(os.path.dirname(__file__), name)

def _file_in_core(name):
    return os.path.join(os.path.dirname(__file__), '..', 'cablemap.core', 'cablemap/core/', name)

def _file_as_set(filename):
    if not os.path.exists(filename):
        return set()
    f = codecs.open(filename, 'rb', 'utf-8')
    s = set((l.rstrip() for l in f))
    f.close()
    return s

def _write_set(filename, s):
    f = codecs.open(filename, 'wb', 'utf-8')
    for item in sorted(s):
        f.write(item)
        f.write('\n')
    f.close()

def update_file(filename, found):
    if not found:
        return
    current = _file_as_set(filename)
    diff = current ^ found
    if diff:
        _write_set(filename, found)
        removed = current - found
        fn = os.path.basename(filename)
        header = 'Changes in %s' % fn
        print('\n' + header)
        print('=' * len(header))
        if removed:
            print('Removed:\n%r\n' % (removed))
        added = found - current
        if added:
            print('Added:\n%r\n' % (added))

_FILE_ACRONYMS = _file_in_core('acronyms.txt')
_FILE_SUBJECTS = _filename('no_subject.txt')

if __name__ == '__main__':
    if not os.path.isdir('./cable/'):
        raise Exception('Expected a directory "cable"')
    res = run_update('./cable/')
    update_file(_FILE_ACRONYMS, res['acronyms'])
    update_file(_FILE_SUBJECTS, res['subjects'])
<commit_msg>Updated helper script to detect malformed TAGs<commit_after># -*- coding: utf-8 -*-
"""\
This module tries to detect acronyms and cables w/o subject.
"""
import os
import re
import codecs
from functools import partial
from cablemap.core import cables_from_directory
from cablemap.core.reader import parse_subject

import logging
import sys
logger = logging.getLogger('cablemap.core.reader')
logger.setLevel(logging.DEBUG)
logger.addHandler(logging.StreamHandler(sys.stdout))

_AC_PATTERN = re.compile(r'\(([A-Z\-/\.]{2,}[0-9]*)\)')

_KNOWN_TAGS = ('BIDEN, JOSEPH', "RICE, CONDOLEEZZA", "CLINTON, HILLARY", "CARSON, JOHNNIE", "BUSH, GEORGE",
                "NEW ZEALAND", "ROOD, JOHN", "ECONOMY AND FINANCE", "ECONOMIC AFFAIRS", "HUMAN RIGHTS",
                "ITALY NATIONAL ELECTIONS", "ZOELLICK, ROBERT", "ZANU-PF", "GAZA DISENGAGEMENT",
                "COUNTERTERRORISM", "IRAQI FREEDOM", "MEETINGS WITH AMBASSADOR", "EXTERNAL", "CONSULAR AFFAIRS",
               "JIMENEZ, GASPAR", "ITALIAN POLITICS", "ISRAELI PALESTINIAN AFFAIRS", "DOMESTIC POLITICS",
               "CROS, GERARD", "GLOBAL DEFENSE", "ISRAEL RELATIONS", "MILITARY RELATIONS", "POLITICS FOREIGN POLICY",
               "USEU BRUSSELS", "INDIA-BURMA", "STEINBERG, JAMES B.", "NOVO, GUILLERMO", "REMON, PEDRO",
               "COUNTRY CLEARANCE", "CARICOM", "INMARSAT", "UNESCO", "INTERPOL")

_ACRONYMS = (u'AA/S', u'ADC', u'AFM', u'AG', u'ASD/ISA', u'AU', u'AK', u'APHSCT',
             u'AF-PAK', u'AKP', u'ASD', u'AQAP', u'AQIM', u'ARENA',
             u'BBC', u'BP', u'BR-3',
             u'CMC', u'CNP', u'CODEL', u'CJCS', u'CT', u'CWS/BWC', u'CW', u'CENTCOM',
             u'CDR', u'CFE', u'CISMOA', u'CN', u'CIA', u'CTJWG', u'CG', u'CL', u'CBD',
             u'DAS', u'DCA', u'DDR', u'DEA', u'DG', u'DCM', u'DFID',
             u'DRC', u'DASD', u'DIO', u'DHS', u'DOL', u'DPRK',
             u'EFCC', u'ETA', u'EU', u'EU/US', u'EXBS', u'EUR', u'EFTA',
             u'FATF', u'FBI', u'FCO', u'FDA', u'FDP', u'FM', u'FTAA', u'FARC', u'FX2', u'FMLN',
             u'GAERC', u'GCC', u'GDRC', u'GM', u'GOAJ', u'GOB', u'GOC', u'GOE', u'GOI', u'GOK',
             u'GOL', u'GPC', u'GSL', u'GSP', u'GTMO', u'GOF', u'GOS', u'GBRV', u'GOP', u'GOU',
             u'GFA', u'G/TIP', u'GI',
             u'HMG', u'HLDG', u'HLG', u'HEU',
             u'ICTY', u'II', u'III', u'IMF', u'ITGA', u'IPR', u'IRGC', u'ICAO', u'IRA', u'ISAF',
             u'(INCSR)', u'ICRC', u'ISA', u'INR/B', u'ICC', u'ICG-G', u'ICJ', u'ILSA',
             u'JSF', u'JHA', u'JCET', u'JPMG',
             u'MDC', u'MEP', u'MFA', u'MOD', u'MI-17', u'MI-6',
             u'MRE', u'MP', u'MONUC', u'MOP-3', u'MEA',
             u'NATO', u'NDP', u'NSA', u'NGO', u'NEA', u'NEA/MAG', u'NTM-I',
             u'OIC', u'OECD', u'OAS', u'OSCE',
             u'PA', u'PD', u'PM', u'PMDB', u'PS', u'PDAS', u'PRT', u'POC', u'PRC/DPRK',
             u'PNG', u'PRC',
             u'ROK', u'RWE', u'RFG', u'RMB', u'RSO', u'RPO', u'RTG',
             u'SLA', u'SLA/U', u'SPD', u'SWIFT', u'S/WCI', u'S/CT', u'S/CRS',
             u'S/GC', u'SCSL', u'S/SRAP', u'SG',
             u'TFTP', u'TFTP/SWIFT', u'U.S.-EU', u'U.S.-UK', u'UAE', u'UK', u'UN',
             u'UNHCR', u'UNSC', u'US', u'US-CU', u'US-EU', u'USG', u'USTR', u'UNCHR',
             u'USEB', u'UNGA', u'U.S./UK', u'UNESCO', u'U/SYG', u'US-ROYG', u'UNSCR',
             u'USG-GOB', u'USD/P', u'USS', u'UNDP',
             u'VFM', u'VP', u'VI', u'VARIG', u'VOA',
             u'WEF', u'WTO',
             u'XVI',
             u'ZANU-PF')

_UNWANTED = (u'SAVE', u'CITES', u'SHARIA', u'IRAN', u'WHO', u'CAN')

def run_update(in_dir, predicate=None):
    acronyms = set(_ACRONYMS)
    subjects = set()
    tags = set()
    for cable in cables_from_directory(in_dir, predicate):
        update_acronyms(cable, acronyms)
        update_missing_subjects(cable, subjects)
        update_tags(cable, tags)
    return {'acronyms': acronyms, 'subjects': subjects}

def update_acronyms(cable, acronyms):
    if not cable.subject:
        return
    subject_words = [w for w in cable.subject.upper().split() if not w.startswith('XXXXX')]
    acronyms.update((ac for ac in _AC_PATTERN.findall(cable.content) if ac in subject_words and ac not in _UNWANTED))

def update_missing_subjects(cable, cable_refs):
    if not parse_subject(cable.content, fix_subject=False):
        cable_refs.add(cable.reference_id)

def update_tags(cable, tags):
    for tag in cable.tags:
        if len(tag) > 5 and tag not in _KNOWN_TAGS:
            print("Valid TAG? u'%s' # %s" % (tag, cable.reference_id))
        tags.add(tag)

def _filename(name):
    return os.path.join(os.path.dirname(__file__), name)

def _file_in_core(name):
    return os.path.join(os.path.dirname(__file__), '..', 'cablemap.core', 'cablemap/core/', name)

def _file_as_set(filename):
    if not os.path.exists(filename):
        return set()
    f = codecs.open(filename, 'rb', 'utf-8')
    s = set((l.rstrip() for l in f))
    f.close()
    return s

def _write_set(filename, s):
    f = codecs.open(filename, 'wb', 'utf-8')
    for item in sorted(s):
        f.write(item)
        f.write('\n')
    f.close()

def update_file(filename, found):
    if not found:
        return
    current = _file_as_set(filename)
    diff = current ^ found
    if diff:
        _write_set(filename, found)
        removed = current - found
        fn = os.path.basename(filename)
        header = 'Changes in %s' % fn
        print('\n' + header)
        print('=' * len(header))
        if removed:
            print('Removed:\n%r\n' % (removed))
        added = found - current
        if added:
            print('Added:\n%r\n' % (added))

_FILE_ACRONYMS = _file_in_core('acronyms.txt')
_FILE_SUBJECTS = _filename('no_subject.txt')

if __name__ == '__main__':
    if not os.path.isdir('./cable/'):
        raise Exception('Expected a directory "cable"')
    res = run_update('./cable/')
    update_file(_FILE_ACRONYMS, res['acronyms'])
    update_file(_FILE_SUBJECTS, res['subjects'])
<|endoftext|>


<commit_before>from __future__ import print_function
import click
import functools
import os
import sys
import unittest
import ykman.descriptor
from ykman.util import TRANSPORT
import test.util

_skip = True

_test_serials = os.environ.get('DESTRUCTIVE_TEST_YUBIKEY_SERIALS')
_no_prompt = os.environ.get('DESTRUCTIVE_TEST_DO_NOT_PROMPT') == 'TRUE'
_versions = {}

if _test_serials is not None:
    _test_serials = set(int(s) for s in _test_serials.split(','))
    _serials_present = set()

    for dev in ykman.descriptor.list_devices():
        _serials_present.add(dev.serial)
        _versions[dev.serial] = dev.version
        dev.close()

    _unwanted_serials = _serials_present.difference(_test_serials)

    if len(_unwanted_serials) != 0:
        print('Encountered YubiKeys not listed in serial numbers to be used '
              'for the test: {}'.format(_unwanted_serials),
              file=sys.stderr)
        sys.exit(1)

    if _serials_present != _test_serials:
        print('Test YubiKeys missing: {}'
              .format(_test_serials.difference(_serials_present)),
              file=sys.stderr)
        sys.exit(1)

    _skip = False

    if not _no_prompt:
        click.confirm(
            'Run integration tests? This will erase data on the YubiKeys'
            ' with serial numbers: {}. Make sure these are all keys used for'
            ' development.'.format(_serials_present),
            abort=True)

def _specialize_ykman_cli(dev, _transports):
    '''
    Creates a specialized version of ykman_cli preset with the serial number of
    the given device.
    '''
    return functools.partial(test.util.ykman_cli, '--device', dev.serial)

def _specialize_open_device(dev, transports):
    '''
    Creates a specialized version of open_device which will open the given
    device using the given transport(s).
    '''
    return functools.partial(
        ykman.descriptor.open_device,
        transports=transports,
        serial=dev.serial
    )

def _make_skipped_original_test_cases(create_test_classes):
    for test_class in create_test_classes(None):
        yield unittest.skip('No YubiKey available for test')(test_class)

def _device_satisfies_test_conditions(dev, test_method):
    if '_yubikey_conditions' in dir(test_method):
        conditions = getattr(test_method, '_yubikey_conditions')
        return all(cond(dev) for cond in conditions)
    else:
        return True

def _delete_inapplicable_test_methods(dev, test_class):
    for method_name in _get_test_method_names(test_class):
        method = getattr(test_class, method_name)
        if not _device_satisfies_test_conditions(dev, method):
            delattr(test_class, method_name)
    return test_class

def _create_test_classes_for_device(
        transport,
        dev,
        create_test_classes,
        create_test_class_context
):
    context = create_test_class_context(dev, transport)
    for test_class in create_test_classes(context):
        setattr(test_class, '_original_test_name', test_class.__qualname__)
        fw_version = '.'.join(str(v) for v in dev.version)
        test_class.__qualname__ = f'{test_class.__qualname__}_{transport.name}_{fw_version}_{dev.serial}'  # noqa: E501
        yield _delete_inapplicable_test_methods(dev, test_class)

def _get_test_method_names(test_class):
    return set(
        attr_name for attr_name in dir(test_class)
        if attr_name.startswith('test')
    )

def _multiply_test_classes_by_devices(
        transports,
        create_test_classes,
        create_test_class_context
):
    tests = []
    covered_test_names = {}

    for transport in (t for t in TRANSPORT if transports & t):
        for serial in _test_serials or []:
            with ykman.descriptor.open_device(
                    transports=transport,
                    serial=serial
            ) as dev:
                for test_class in _create_test_classes_for_device(
                        transport,
                        dev,
                        create_test_classes,
                        create_test_class_context
                ):
                    orig_name = test_class._original_test_name
                    test_names = _get_test_method_names(test_class)
                    covered_test_names[orig_name] = (
                        covered_test_names.get(orig_name, set())
                        .union(test_names))
                    for test_method_name in test_names:
                        tests.append(test_class(test_method_name))

    return tests, covered_test_names

def _make_skips_for_uncovered_tests(create_test_classes, covered_test_names):
    for original_test_class in _make_skipped_original_test_cases(
            create_test_classes):
        original_test_names = _get_test_method_names(original_test_class)
        uncovered_test_names = original_test_names.difference(
            covered_test_names.get(
                original_test_class.__qualname__, set()))

        for uncovered_test_name in uncovered_test_names:
            yield original_test_class(uncovered_test_name)

def _make_test_suite(transports, create_test_class_context):
    def decorate(create_test_classes):
        def additional_tests():
            (tests, covered_test_names) = _multiply_test_classes_by_devices(
                transports,
                create_test_classes,
                create_test_class_context
            )

            skipped_tests = _make_skips_for_uncovered_tests(
                create_test_classes, covered_test_names)

            suite = unittest.TestSuite()
            suite.addTests(tests)
            suite.addTests(skipped_tests)

            return suite
        return additional_tests
    return decorate

def device_test_suite(transports):
    if not (isinstance(transports, TRANSPORT) or isinstance(transports, int)):
        raise ValueError('Argument to @device_test_suite must be a TRANSPORT value.')  # noqa: E501
    return _make_test_suite(transports, _specialize_open_device)

def cli_test_suite(transports):
    if not (isinstance(transports, TRANSPORT) or isinstance(transports, int)):
        raise ValueError('Argument to @cli_test_suite must be a TRANSPORT value.')  # noqa: E501
    return _make_test_suite(transports, _specialize_ykman_cli)

destructive_tests_not_activated = (
    _skip, 'DESTRUCTIVE_TEST_YUBIKEY_SERIALS == None')

@unittest.skipIf(*destructive_tests_not_activated)
class DestructiveYubikeyTestCase(unittest.TestCase):
    pass
<commit_msg>Extract function _add_suffix_to_class_name<commit_after>from __future__ import print_function
import click
import functools
import os
import sys
import unittest
import ykman.descriptor
from ykman.util import TRANSPORT
import test.util

_skip = True

_test_serials = os.environ.get('DESTRUCTIVE_TEST_YUBIKEY_SERIALS')
_no_prompt = os.environ.get('DESTRUCTIVE_TEST_DO_NOT_PROMPT') == 'TRUE'
_versions = {}

if _test_serials is not None:
    _test_serials = set(int(s) for s in _test_serials.split(','))
    _serials_present = set()

    for dev in ykman.descriptor.list_devices():
        _serials_present.add(dev.serial)
        _versions[dev.serial] = dev.version
        dev.close()

    _unwanted_serials = _serials_present.difference(_test_serials)

    if len(_unwanted_serials) != 0:
        print('Encountered YubiKeys not listed in serial numbers to be used '
              'for the test: {}'.format(_unwanted_serials),
              file=sys.stderr)
        sys.exit(1)

    if _serials_present != _test_serials:
        print('Test YubiKeys missing: {}'
              .format(_test_serials.difference(_serials_present)),
              file=sys.stderr)
        sys.exit(1)

    _skip = False

    if not _no_prompt:
        click.confirm(
            'Run integration tests? This will erase data on the YubiKeys'
            ' with serial numbers: {}. Make sure these are all keys used for'
            ' development.'.format(_serials_present),
            abort=True)

def _specialize_ykman_cli(dev, _transports):
    '''
    Creates a specialized version of ykman_cli preset with the serial number of
    the given device.
    '''
    return functools.partial(test.util.ykman_cli, '--device', dev.serial)

def _specialize_open_device(dev, transports):
    '''
    Creates a specialized version of open_device which will open the given
    device using the given transport(s).
    '''
    return functools.partial(
        ykman.descriptor.open_device,
        transports=transports,
        serial=dev.serial
    )

def _make_skipped_original_test_cases(create_test_classes):
    for test_class in create_test_classes(None):
        yield unittest.skip('No YubiKey available for test')(test_class)

def _device_satisfies_test_conditions(dev, test_method):
    if '_yubikey_conditions' in dir(test_method):
        conditions = getattr(test_method, '_yubikey_conditions')
        return all(cond(dev) for cond in conditions)
    else:
        return True

def _delete_inapplicable_test_methods(dev, test_class):
    for method_name in _get_test_method_names(test_class):
        method = getattr(test_class, method_name)
        if not _device_satisfies_test_conditions(dev, method):
            delattr(test_class, method_name)
    return test_class

def _add_suffix_to_class_name(transport, dev, test_class):
    setattr(test_class, '_original_test_name', test_class.__qualname__)
    fw_version = '.'.join(str(v) for v in dev.version)
    test_class.__qualname__ = f'{test_class.__qualname__}_{transport.name}_{fw_version}_{dev.serial}'  # noqa: E501
    return test_class

def _create_test_classes_for_device(
        transport,
        dev,
        create_test_classes,
        create_test_class_context
):
    context = create_test_class_context(dev, transport)
    for test_class in create_test_classes(context):
        _delete_inapplicable_test_methods(dev, test_class)
        _add_suffix_to_class_name(transport, dev, test_class)
        yield test_class

def _get_test_method_names(test_class):
    return set(
        attr_name for attr_name in dir(test_class)
        if attr_name.startswith('test')
    )

def _multiply_test_classes_by_devices(
        transports,
        create_test_classes,
        create_test_class_context
):
    tests = []
    covered_test_names = {}

    for transport in (t for t in TRANSPORT if transports & t):
        for serial in _test_serials or []:
            with ykman.descriptor.open_device(
                    transports=transport,
                    serial=serial
            ) as dev:
                for test_class in _create_test_classes_for_device(
                        transport,
                        dev,
                        create_test_classes,
                        create_test_class_context
                ):
                    orig_name = test_class._original_test_name
                    test_names = _get_test_method_names(test_class)
                    covered_test_names[orig_name] = (
                        covered_test_names.get(orig_name, set())
                        .union(test_names))
                    for test_method_name in test_names:
                        tests.append(test_class(test_method_name))

    return tests, covered_test_names

def _make_skips_for_uncovered_tests(create_test_classes, covered_test_names):
    for original_test_class in _make_skipped_original_test_cases(
            create_test_classes):
        original_test_names = _get_test_method_names(original_test_class)
        uncovered_test_names = original_test_names.difference(
            covered_test_names.get(
                original_test_class.__qualname__, set()))

        for uncovered_test_name in uncovered_test_names:
            yield original_test_class(uncovered_test_name)

def _make_test_suite(transports, create_test_class_context):
    def decorate(create_test_classes):
        def additional_tests():
            (tests, covered_test_names) = _multiply_test_classes_by_devices(
                transports,
                create_test_classes,
                create_test_class_context
            )

            skipped_tests = _make_skips_for_uncovered_tests(
                create_test_classes, covered_test_names)

            suite = unittest.TestSuite()
            suite.addTests(tests)
            suite.addTests(skipped_tests)

            return suite
        return additional_tests
    return decorate

def device_test_suite(transports):
    if not (isinstance(transports, TRANSPORT) or isinstance(transports, int)):
        raise ValueError('Argument to @device_test_suite must be a TRANSPORT value.')  # noqa: E501
    return _make_test_suite(transports, _specialize_open_device)

def cli_test_suite(transports):
    if not (isinstance(transports, TRANSPORT) or isinstance(transports, int)):
        raise ValueError('Argument to @cli_test_suite must be a TRANSPORT value.')  # noqa: E501
    return _make_test_suite(transports, _specialize_ykman_cli)

destructive_tests_not_activated = (
    _skip, 'DESTRUCTIVE_TEST_YUBIKEY_SERIALS == None')

@unittest.skipIf(*destructive_tests_not_activated)
class DestructiveYubikeyTestCase(unittest.TestCase):
    pass
<|endoftext|>


<commit_before>from __future__ import print_function
import click
import functools
import os
import sys
import unittest
import ykman.descriptor
from ykman.util import TRANSPORT
import test.util

_skip = True

_test_serials = os.environ.get('DESTRUCTIVE_TEST_YUBIKEY_SERIALS')
_no_prompt = os.environ.get('DESTRUCTIVE_TEST_DO_NOT_PROMPT') == 'TRUE'
_versions = {}

if _test_serials is not None:
    _test_serials = set(int(s) for s in _test_serials.split(','))
    _serials_present = set()

    for dev in ykman.descriptor.list_devices():
        _serials_present.add(dev.serial)
        _versions[dev.serial] = dev.version
        dev.close()

    _unwanted_serials = _serials_present.difference(_test_serials)

    if len(_unwanted_serials) != 0:
        print('Encountered YubiKeys not listed in serial numbers to be used '
              'for the test: {}'.format(_unwanted_serials),
              file=sys.stderr)
        sys.exit(1)

    if _serials_present != _test_serials:
        print('Test YubiKeys missing: {}'
              .format(_test_serials.difference(_serials_present)),
              file=sys.stderr)
        sys.exit(1)

    _skip = False

    if not _no_prompt:
        click.confirm(
            'Run integration tests? This will erase data on the YubiKeys'
            ' with serial numbers: {}. Make sure these are all keys used for'
            ' development.'.format(_serials_present),
            abort=True)

def _specialize_ykman_cli(dev, _transports):
    '''
    Creates a specialized version of ykman_cli preset with the serial number of
    the given device.
    '''
    return functools.partial(test.util.ykman_cli, '--device', dev.serial)

def _specialize_open_device(dev, transports):
    '''
    Creates a specialized version of open_device which will open the given
    device using the given transport(s).
    '''
    return functools.partial(
        ykman.descriptor.open_device,
        transports=transports,
        serial=dev.serial
    )

def _make_skipped_original_test_cases(create_test_classes):
    for test_class in create_test_classes(None):
        yield unittest.skip('No YubiKey available for test')(test_class)

def _device_satisfies_test_conditions(dev, test_method):
    if '_yubikey_conditions' in dir(test_method):
        conditions = getattr(test_method, '_yubikey_conditions')
        return all(cond(dev) for cond in conditions)
    else:
        return True

def _make_test_classes_for_device(
        transport,
        dev,
        create_test_classes,
        create_test_class_context
):
    context = create_test_class_context(dev, transport)
    for test_class in create_test_classes(context):
        setattr(test_class, '_original_test_name', test_class.__qualname__)
        fw_version = '.'.join(str(v) for v in dev.version)
        test_class.__qualname__ = f'{test_class.__qualname__}_{transport.name}_{fw_version}_{dev.serial}'  # noqa: E501

        for attr_name in dir(test_class):
            method = getattr(test_class, attr_name)
            if attr_name.startswith('test'):
                if not _device_satisfies_test_conditions(dev, method):
                    delattr(test_class, attr_name)

        yield test_class

def _make_test_suite(transports, create_test_class_context):
    def decorate(create_test_classes):
        def additional_tests():
            suite = unittest.TestSuite()
            yubikey_test_names = {}

            for transport in (t for t in TRANSPORT if transports & t):
                for serial in _test_serials or []:
                    with ykman.descriptor.open_device(
                            transports=transport,
                            serial=serial
                    ) as dev:
                        for test_case in _make_test_classes_for_device(
                                transport,
                                dev,
                                create_test_classes,
                                create_test_class_context
                        ):
                            orig_name = test_case._original_test_name
                            for attr_name in dir(test_case):
                                if attr_name.startswith('test'):
                                    test_names = yubikey_test_names.get(
                                        orig_name, set())
                                    test_names.add(attr_name)
                                    yubikey_test_names[orig_name] = test_names
                                    suite.addTest(test_case(attr_name))

            for original_test_class in _make_skipped_original_test_cases(
                    create_test_classes):
                original_test_names = set(
                    attr_name
                    for attr_name in dir(original_test_class)
                    if attr_name.startswith('test')
                )
                uncovered_test_names = original_test_names.difference(
                    yubikey_test_names.get(
                        original_test_class.__qualname__, set()))

                for uncovered_test_name in uncovered_test_names:
                    suite.addTest(original_test_class(uncovered_test_name))

            return suite
        return additional_tests
    return decorate

def device_test_suite(transports):
    if not (isinstance(transports, TRANSPORT) or isinstance(transports, int)):
        raise ValueError('Argument to @device_test_suite must be a TRANSPORT value.')  # noqa: E501
    return _make_test_suite(transports, _specialize_open_device)

def cli_test_suite(transports):
    if not (isinstance(transports, TRANSPORT) or isinstance(transports, int)):
        raise ValueError('Argument to @cli_test_suite must be a TRANSPORT value.')  # noqa: E501
    return _make_test_suite(transports, _specialize_ykman_cli)

destructive_tests_not_activated = (
    _skip, 'DESTRUCTIVE_TEST_YUBIKEY_SERIALS == None')

@unittest.skipIf(*destructive_tests_not_activated)
class DestructiveYubikeyTestCase(unittest.TestCase):
    pass
<commit_msg>Extract function _delete_inapplicable_test_methods<commit_after>from __future__ import print_function
import click
import functools
import os
import sys
import unittest
import ykman.descriptor
from ykman.util import TRANSPORT
import test.util

_skip = True

_test_serials = os.environ.get('DESTRUCTIVE_TEST_YUBIKEY_SERIALS')
_no_prompt = os.environ.get('DESTRUCTIVE_TEST_DO_NOT_PROMPT') == 'TRUE'
_versions = {}

if _test_serials is not None:
    _test_serials = set(int(s) for s in _test_serials.split(','))
    _serials_present = set()

    for dev in ykman.descriptor.list_devices():
        _serials_present.add(dev.serial)
        _versions[dev.serial] = dev.version
        dev.close()

    _unwanted_serials = _serials_present.difference(_test_serials)

    if len(_unwanted_serials) != 0:
        print('Encountered YubiKeys not listed in serial numbers to be used '
              'for the test: {}'.format(_unwanted_serials),
              file=sys.stderr)
        sys.exit(1)

    if _serials_present != _test_serials:
        print('Test YubiKeys missing: {}'
              .format(_test_serials.difference(_serials_present)),
              file=sys.stderr)
        sys.exit(1)

    _skip = False

    if not _no_prompt:
        click.confirm(
            'Run integration tests? This will erase data on the YubiKeys'
            ' with serial numbers: {}. Make sure these are all keys used for'
            ' development.'.format(_serials_present),
            abort=True)

def _specialize_ykman_cli(dev, _transports):
    '''
    Creates a specialized version of ykman_cli preset with the serial number of
    the given device.
    '''
    return functools.partial(test.util.ykman_cli, '--device', dev.serial)

def _specialize_open_device(dev, transports):
    '''
    Creates a specialized version of open_device which will open the given
    device using the given transport(s).
    '''
    return functools.partial(
        ykman.descriptor.open_device,
        transports=transports,
        serial=dev.serial
    )

def _make_skipped_original_test_cases(create_test_classes):
    for test_class in create_test_classes(None):
        yield unittest.skip('No YubiKey available for test')(test_class)

def _device_satisfies_test_conditions(dev, test_method):
    if '_yubikey_conditions' in dir(test_method):
        conditions = getattr(test_method, '_yubikey_conditions')
        return all(cond(dev) for cond in conditions)
    else:
        return True

def _delete_inapplicable_test_methods(dev, test_class):
    for attr_name in dir(test_class):
        method = getattr(test_class, attr_name)
        if attr_name.startswith('test'):
            if not _device_satisfies_test_conditions(dev, method):
                print('Deleting test', attr_name, 'from test', test_class)
                delattr(test_class, attr_name)
    return test_class

def _make_test_classes_for_device(
        transport,
        dev,
        create_test_classes,
        create_test_class_context
):
    context = create_test_class_context(dev, transport)
    for test_class in create_test_classes(context):
        setattr(test_class, '_original_test_name', test_class.__qualname__)
        fw_version = '.'.join(str(v) for v in dev.version)
        test_class.__qualname__ = f'{test_class.__qualname__}_{transport.name}_{fw_version}_{dev.serial}'  # noqa: E501
        yield _delete_inapplicable_test_methods(dev, test_class)

def _make_test_suite(transports, create_test_class_context):
    def decorate(create_test_classes):
        def additional_tests():
            suite = unittest.TestSuite()
            yubikey_test_names = {}

            for transport in (t for t in TRANSPORT if transports & t):
                for serial in _test_serials or []:
                    with ykman.descriptor.open_device(
                            transports=transport,
                            serial=serial
                    ) as dev:
                        for test_case in _make_test_classes_for_device(
                                transport,
                                dev,
                                create_test_classes,
                                create_test_class_context
                        ):
                            orig_name = test_case._original_test_name
                            for attr_name in dir(test_case):
                                if attr_name.startswith('test'):
                                    test_names = yubikey_test_names.get(
                                        orig_name, set())
                                    test_names.add(attr_name)
                                    yubikey_test_names[orig_name] = test_names
                                    suite.addTest(test_case(attr_name))

            for original_test_class in _make_skipped_original_test_cases(
                    create_test_classes):
                original_test_names = set(
                    attr_name
                    for attr_name in dir(original_test_class)
                    if attr_name.startswith('test')
                )
                uncovered_test_names = original_test_names.difference(
                    yubikey_test_names.get(
                        original_test_class.__qualname__, set()))

                for uncovered_test_name in uncovered_test_names:
                    suite.addTest(original_test_class(uncovered_test_name))

            return suite
        return additional_tests
    return decorate

def device_test_suite(transports):
    if not (isinstance(transports, TRANSPORT) or isinstance(transports, int)):
        raise ValueError('Argument to @device_test_suite must be a TRANSPORT value.')  # noqa: E501
    return _make_test_suite(transports, _specialize_open_device)

def cli_test_suite(transports):
    if not (isinstance(transports, TRANSPORT) or isinstance(transports, int)):
        raise ValueError('Argument to @cli_test_suite must be a TRANSPORT value.')  # noqa: E501
    return _make_test_suite(transports, _specialize_ykman_cli)

destructive_tests_not_activated = (
    _skip, 'DESTRUCTIVE_TEST_YUBIKEY_SERIALS == None')

@unittest.skipIf(*destructive_tests_not_activated)
class DestructiveYubikeyTestCase(unittest.TestCase):
    pass
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

from datetime import datetime
from uuid import UUID, uuid4

from tornado import gen
from ujson import loads, dumps

from holmes.models import Review, Page
from holmes.handlers import BaseHandler

class BaseReviewHandler(BaseHandler):
    def _parse_uuid(self, uuid):
        try:
            return UUID(uuid)
        except ValueError:
            return None

class ReviewHandler(BaseReviewHandler):
    def get(self, page_uuid, review_uuid):
        review = None
        if self._parse_uuid(review_uuid):
            review = Review.by_uuid(review_uuid, self.db)

        if not review:
            self.set_status(404, 'Review with uuid of %s not found!' % review_uuid)
            return

        result = review.to_dict(self.application.fact_definitions, self.application.violation_definitions)
        result.update({
            'violationPoints': review.get_violation_points(),
            'violationCount': review.violation_count,
        })

        self.write_json(result)

    @gen.coroutine
    def post(self, page_uuid, review_uuid=None):
        page = Page.by_uuid(page_uuid, self.db)

        review_data = loads(self.get_argument('review'))

        page.expires = review_data['expires']
        if page.expires is not None:
            page.expires = datetime.utcfromtimestamp(page.expires)

        page.last_modified = review_data['lastModified']
        if page.last_modified is not None:
            page.last_modified = datetime.utcfromtimestamp(page.last_modified)

        self.db.flush()

        review = Review(
            domain=page.domain,
            page=page,
            is_active=True,
            is_complete=False,
            completed_date=datetime.utcnow(),
            uuid=uuid4(),
        )

        page.last_review_uuid = review.uuid

        self.db.add(review)
        self.db.flush()

        for fact in review_data['facts']:
            name = fact['key']
            key = self.application.fact_definitions[name]['key']
            review.add_fact(key, fact['value'])

        self.db.flush()

        for violation in review_data['violations']:
            name = violation['key']
            key = self.application.violation_definitions[name]['key']
            review.add_violation(key, violation['value'], violation['points'])

        self.db.flush()

        page.violations_count = len(review_data['violations'])

        review.is_complete = True
        self.db.flush()

        active_review = page.last_review

        if not active_review:
            yield self.cache.increment_active_review_count(page.domain)

            yield self.cache.increment_violations_count(
                page.domain,
                increment=page.violations_count
            )
        else:
            old_violations_count = len(active_review.violations)
            new_violations_count = len(review.violations)

            yield self.cache.increment_violations_count(
                page.domain,
                increment=new_violations_count - old_violations_count
            )

            active_review.is_active = False

        page.last_review = review
        page.last_review_date = review.completed_date

        self.db.flush()

        self._remove_older_reviews_with_same_day(review)

        self.application.event_bus.publish(dumps({
            'type': 'new-review',
            'reviewId': str(review.uuid)
        }))

    def _remove_older_reviews_with_same_day(self, review):
        dt = datetime.now()
        dt = dt.replace(hour=0, minute=0, second=0, microsecond=0)

        for review in self.db.query(Review) \
                .filter(Review.page == review.page) \
                .filter(Review.uuid != review.uuid) \
                .filter(Review.created_date >= dt) \
                .all():

            for fact in review.facts:
                self.db.delete(fact)

            for violation in review.violations:
                self.db.delete(violation)

            self.db.delete(review)

            self.db.flush()

class LastReviewsHandler(BaseReviewHandler):
    def get(self):
        reviews = Review.get_last_reviews(self.db)

        reviews_json = []
        for review in reviews:
            review_dict = review.to_dict(self.application.fact_definitions, self.application.violation_definitions)
            data = {
                'violationCount': review.violation_count,
            }
            review_dict.update(data)
            reviews_json.append(review_dict)

        self.write_json(reviews_json)
<commit_msg>Fixing CircularDependencyError<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

from datetime import datetime
from uuid import UUID, uuid4

from tornado import gen
from ujson import loads, dumps

from holmes.models import Review, Page
from holmes.handlers import BaseHandler

class BaseReviewHandler(BaseHandler):
    def _parse_uuid(self, uuid):
        try:
            return UUID(uuid)
        except ValueError:
            return None

class ReviewHandler(BaseReviewHandler):
    def get(self, page_uuid, review_uuid):
        review = None
        if self._parse_uuid(review_uuid):
            review = Review.by_uuid(review_uuid, self.db)

        if not review:
            self.set_status(404, 'Review with uuid of %s not found!' % review_uuid)
            return

        result = review.to_dict(self.application.fact_definitions, self.application.violation_definitions)
        result.update({
            'violationPoints': review.get_violation_points(),
            'violationCount': review.violation_count,
        })

        self.write_json(result)

    @gen.coroutine
    def post(self, page_uuid, review_uuid=None):
        page = Page.by_uuid(page_uuid, self.db)

        review_data = loads(self.get_argument('review'))

        page.expires = review_data['expires']
        if page.expires is not None:
            page.expires = datetime.utcfromtimestamp(page.expires)

        page.last_modified = review_data['lastModified']
        if page.last_modified is not None:
            page.last_modified = datetime.utcfromtimestamp(page.last_modified)

        self.db.flush()

        review = Review(
            domain=page.domain,
            page=page,
            is_active=True,
            is_complete=False,
            completed_date=datetime.utcnow(),
            uuid=uuid4(),
        )

        page.last_review_uuid = review.uuid

        self.db.add(review)
        self.db.flush()

        for fact in review_data['facts']:
            name = fact['key']
            key = self.application.fact_definitions[name]['key']
            review.add_fact(key, fact['value'])

        self.db.flush()

        for violation in review_data['violations']:
            name = violation['key']
            key = self.application.violation_definitions[name]['key']
            review.add_violation(key, violation['value'], violation['points'])

        self.db.flush()

        page.violations_count = len(review_data['violations'])

        review.is_complete = True
        self.db.flush()

        if not page.last_review:
            yield self.cache.increment_active_review_count(page.domain)

            yield self.cache.increment_violations_count(
                page.domain,
                increment=page.violations_count
            )
        else:
            old_violations_count = len(page.last_review.violations)
            new_violations_count = len(review.violations)

            yield self.cache.increment_violations_count(
                page.domain,
                increment=new_violations_count - old_violations_count
            )

            page.last_review.is_active = False
            self.db.flush()

        page.last_review = review
        page.last_review_date = review.completed_date

        self.db.flush()

        self._remove_older_reviews_with_same_day(review)

        self.application.event_bus.publish(dumps({
            'type': 'new-review',
            'reviewId': str(review.uuid)
        }))

    def _remove_older_reviews_with_same_day(self, review):
        dt = datetime.now()
        dt = dt.replace(hour=0, minute=0, second=0, microsecond=0)

        for review in self.db.query(Review) \
                .filter(Review.page == review.page) \
                .filter(Review.uuid != review.uuid) \
                .filter(Review.created_date >= dt) \
                .all():

            for fact in review.facts:
                self.db.delete(fact)

            for violation in review.violations:
                self.db.delete(violation)

            self.db.delete(review)

            self.db.flush()

class LastReviewsHandler(BaseReviewHandler):
    def get(self):
        reviews = Review.get_last_reviews(self.db)

        reviews_json = []
        for review in reviews:
            review_dict = review.to_dict(self.application.fact_definitions, self.application.violation_definitions)
            data = {
                'violationCount': review.violation_count,
            }
            review_dict.update(data)
            reviews_json.append(review_dict)

        self.write_json(reviews_json)
<|endoftext|>


<commit_before># -*- coding: iso-8859-1 -*-
# Maintainer: joaander

from hoomd import *
import hoomd;

# test the command line option
context.initialize("--nthreads=4")

import unittest
import os

# unit tests for options
class option_tests (unittest.TestCase):
    def setUp(self):
        print

    # tests that mode settings work properly
    def test_nthreads(self):
        self.assertEqual(hoomd.context.ExecutionContext().num_threads, 4);
        hoomd.context.ExecutionContext().num_threads = 2
        self.assertEqual(hoomd.context.ExecutionContext().num_threads, 2);

    def tearDown(self):
        pass;

if __name__ == '__main__':
    unittest.main(argv = ['test.py', '-v'])
<commit_msg>fix non-TBB build's unit test<commit_after># -*- coding: iso-8859-1 -*-
# Maintainer: joaander

from hoomd import *
import hoomd;

from hoomd import _hoomd
if _hoomd.is_TBB_available():
    # test the command line option
    context.initialize("--nthreads=4")

    import unittest
    import os

    # unit tests for options
    class option_tests (unittest.TestCase):
        def setUp(self):
            print

        # tests that mode settings work properly
        def test_nthreads(self):
            self.assertEqual(hoomd.context.ExecutionContext().num_threads, 4);
            hoomd.context.ExecutionContext().num_threads = 2
            self.assertEqual(hoomd.context.ExecutionContext().num_threads, 2);

        def tearDown(self):
            pass;

    if __name__ == '__main__':
        unittest.main(argv = ['test.py', '-v'])
<|endoftext|>