<commit_before># -*- coding: utf-8 -*-
'''
    :codauthor: :email:`Mike Place <mp@saltstack.com>`
'''

# Import Salt Testing libs
from salttesting import TestCase, skipIf
from salttesting.helpers import ensure_in_syspath
from salttesting.mock import MagicMock, patch, call

ensure_in_syspath('../../')

from salt.modules import ps

HAS_PSUTIL = ps.__virtual__()

if HAS_PSUTIL:
    import psutil

    STUB_CPU_TIMES = psutil._compat.namedtuple('cputimes', 'user nice system idle')(1, 2, 3, 4)
    STUB_PHY_MEM_USAGE = psutil._compat.namedtuple('usage', 'total used free percent')(1000, 500, 500, 50)
    STUB_DISK_PARTITION = psutil._compat.namedtuple('partition', 'device mountpoint fstype, opts')('/dev/disk0s2', '/',
                                                                                'hfs',
                                                                                'rw,local,rootfs,dovolfs,journaled,multilabel')
    STUB_DISK_USAGE = psutil._compat.namedtuple('usage', 'total used free percent')(1000, 500, 500, 50)
    STUB_NETWORK_IO = psutil._compat.namedtuple('iostat',
                                                'bytes_sent, bytes_recv, packets_sent, packets_recv, errin errout dropin dropout')(
        1000, 2000, 500, 600, 1, 2, 3, 4)
    STUB_DISK_IO = psutil._compat.namedtuple('iostat',
                                             'read_count, write_count, read_bytes, write_bytes, read_time, write_time')(
        1000, 2000, 500, 600, 2000, 3000)
    STUB_USER = psutil._compat.namedtuple('user', 'name, terminal, host, started')('bdobbs', 'ttys000', 'localhost',
                                                                                0.0)
else:
    (STUB_CPU_TIMES,
     STUB_PHY_MEM_USAGE,
     STUB_DISK_PARTITION,
     STUB_DISK_USAGE,
     STUB_NETWORK_IO,
     STUB_DISK_IO,
     STUB_USER) = [None for val in range(7)]

STUB_PID_LIST = [0, 1, 2, 3]
MOCK_PROC = mocked_proc = MagicMock('psutil.Process')

try:
    import utmp

    HAS_UTMP = True
except ImportError:
    HAS_UTMP = False

@skipIf(not HAS_PSUTIL, "psutils are required for this test case")
class PsTestCase(TestCase):
    def setUp(self):
        MOCK_PROC.name = 'test_mock_proc'
        MOCK_PROC.pid = 9999999999

    @patch('psutil.get_pid_list', new=MagicMock(return_value=STUB_PID_LIST))
    def test_get_pid_list(self):
        self.assertListEqual(STUB_PID_LIST, ps.get_pid_list())

    @patch('psutil.Process')
    def test_kill_pid(self, send_signal_mock):
        ps.kill_pid(0, signal=999)
        self.assertEqual(send_signal_mock.call_args, call(0))

    @patch('psutil.Process.send_signal')
    @patch('psutil.process_iter', new=MagicMock(return_value=[MOCK_PROC]))
    def test_pkill(self, send_signal_mock):
        mocked_proc.send_signal = MagicMock()
        test_signal = 1234
        ps.pkill(mocked_proc.name, signal=test_signal)
        self.assertEqual(mocked_proc.send_signal.call_args, call(test_signal))

    @patch('psutil.process_iter', new=MagicMock(return_value=[MOCK_PROC]))
    def test_pgrep(self):
        self.assertIn(MOCK_PROC.pid, ps.pgrep(MOCK_PROC.name))

    @patch('psutil.cpu_percent', new=MagicMock(return_value=1))
    def test_cpu_percent(self):
        self.assertEqual(ps.cpu_percent(), 1)

    @patch('psutil.cpu_times', new=MagicMock(return_value=STUB_CPU_TIMES))
    def test_cpu_times(self):
        self.assertDictEqual({'idle': 4, 'nice': 2, 'system': 3, 'user': 1}, ps.cpu_times())

    @patch('psutil.phymem_usage', new=MagicMock(return_value=STUB_PHY_MEM_USAGE))
    def test_physical_memory_usage(self):
        self.assertDictEqual({'used': 500, 'total': 1000, 'percent': 50, 'free': 500}, ps.physical_memory_usage())

    @patch('psutil.virtmem_usage', new=MagicMock(return_value=STUB_PHY_MEM_USAGE))
    def test_virtual_memory_usage(self):
        self.assertDictEqual({'used': 500, 'total': 1000, 'percent': 50, 'free': 500}, ps.virtual_memory_usage())

    # ps.cached_physical_memory is deprecated! See #9301
    # def test_cached_physical_memory(self):
    #    pass

    #ps.physical_memory_buffers is deprecated! See #9301
    # def test_physical_memory_buffers(self):
    #     pass

    @patch('psutil.disk_partitions', new=MagicMock(return_value=[STUB_DISK_PARTITION]))
    def test_disk_partitions(self):
        self.assertDictEqual(
            {'device': '/dev/disk0s2', 'mountpoint': '/', 'opts': 'rw,local,rootfs,dovolfs,journaled,multilabel',
             'fstype': 'hfs'},
            ps.disk_partitions()[0])

    @patch('psutil.disk_usage', new=MagicMock(return_value=STUB_DISK_USAGE))
    def test_disk_usage(self):
        self.assertDictEqual({'used': 500, 'total': 1000, 'percent': 50, 'free': 500}, ps.disk_usage('DUMMY_PATH'))

    @patch('psutil.disk_partitions', new=MagicMock(return_value=[STUB_DISK_PARTITION]))
    def test_disk_partition_usage(self):
        self.assertDictEqual(
            {'device': '/dev/disk0s2', 'mountpoint': '/', 'opts': 'rw,local,rootfs,dovolfs,journaled,multilabel',
             'fstype': 'hfs'},
            ps.disk_partitions()[0])

    ## Should only be tested in integration
    # def test_total_physical_memory(self):
    #     pass

    ## Should only be tested in integration
    # def test_num_cpus(self):
    #     pass

    ## Should only be tested in integration
    # def test_boot_time(self):
    #     pass
    @patch('psutil.network_io_counters', new=MagicMock(return_value=STUB_NETWORK_IO))
    def test_network_io_counters(self):
        self.assertDictEqual(
            {'packets_sent': 500, 'packets_recv': 600, 'bytes_recv': 2000, 'dropout': 4, 'bytes_sent': 1000,
             'errout': 2, 'errin': 1, 'dropin': 3}, ps.network_io_counters())

    @patch('psutil.disk_io_counters', new=MagicMock(return_value=STUB_DISK_IO))
    def test_disk_io_counters(self):
        self.assertDictEqual(
            {'read_time': 2000, 'write_bytes': 600, 'read_bytes': 500, 'write_time': 3000, 'read_count': 1000,
             'write_count': 2000}, ps.disk_io_counters())

    @patch('psutil.get_users', new=MagicMock(return_value=[STUB_USER]))
    def test_get_users(self):
        self.assertDictEqual({'terminal': 'ttys000', 'started': 0.0, 'host': 'localhost', 'name': 'bdobbs'},
                             ps.get_users()[0])

        ## This is commented out pending discussion on https://github.com/saltstack/salt/commit/2e5c3162ef87cca8a2c7b12ade7c7e1b32028f0a
        # @skipIf(not HAS_UTMP, "The utmp module must be installed to run test_get_users_utmp()")
        # @patch('psutil.get_users', new=MagicMock(return_value=None))  # This will force the function to use utmp
        # def test_get_users_utmp(self):
        #     pass
<commit_msg>Added unit tests for new psutil features.<commit_after># -*- coding: utf-8 -*-
'''
    :codauthor: :email:`Mike Place <mp@saltstack.com>`
'''

# Import Salt Testing libs
from salttesting import TestCase, skipIf
from salttesting.helpers import ensure_in_syspath
from salttesting.mock import MagicMock, patch, call

ensure_in_syspath('../../')

from salt.modules import ps

HAS_PSUTIL = ps.__virtual__()

if HAS_PSUTIL:
    import psutil

    STUB_CPU_TIMES = psutil._compat.namedtuple('cputimes', 'user nice system idle')(1, 2, 3, 4)
    STUB_VIRT_MEM = psutil._compat.namedtuple('vmem', 'total available percent used free')(1000, 500, 50, 500, 500)
    STUB_SWAP_MEM = psutil._compat.namedtuple('swap', 'total used free percent sin sout')(1000, 500, 500, 50, 0, 0)
    STUB_PHY_MEM_USAGE = psutil._compat.namedtuple('usage', 'total used free percent')(1000, 500, 500, 50)
    STUB_DISK_PARTITION = psutil._compat.namedtuple('partition', 'device mountpoint fstype, opts')('/dev/disk0s2', '/',
                                                                                'hfs',
                                                                                'rw,local,rootfs,dovolfs,journaled,multilabel')
    STUB_DISK_USAGE = psutil._compat.namedtuple('usage', 'total used free percent')(1000, 500, 500, 50)
    STUB_NETWORK_IO = psutil._compat.namedtuple('iostat',
                                                'bytes_sent, bytes_recv, packets_sent, packets_recv, errin errout dropin dropout')(
        1000, 2000, 500, 600, 1, 2, 3, 4)
    STUB_DISK_IO = psutil._compat.namedtuple('iostat',
                                             'read_count, write_count, read_bytes, write_bytes, read_time, write_time')(
        1000, 2000, 500, 600, 2000, 3000)
    STUB_USER = psutil._compat.namedtuple('user', 'name, terminal, host, started')('bdobbs', 'ttys000', 'localhost',
                                                                                0.0)
else:
    (STUB_CPU_TIMES,
     STUB_VIRT_MEM,
     STUB_SWAP_MEM,
     STUB_PHY_MEM_USAGE,
     STUB_DISK_PARTITION,
     STUB_DISK_USAGE,
     STUB_NETWORK_IO,
     STUB_DISK_IO,
     STUB_USER) = [None for val in range(7)]

STUB_PID_LIST = [0, 1, 2, 3]
MOCK_PROC = mocked_proc = MagicMock('psutil.Process')

try:
    import utmp

    HAS_UTMP = True
except ImportError:
    HAS_UTMP = False

@skipIf(not HAS_PSUTIL, "psutils are required for this test case")
class PsTestCase(TestCase):
    def setUp(self):
        MOCK_PROC.name = 'test_mock_proc'
        MOCK_PROC.pid = 9999999999

    @patch('psutil.get_pid_list', new=MagicMock(return_value=STUB_PID_LIST))
    def test_get_pid_list(self):
        self.assertListEqual(STUB_PID_LIST, ps.get_pid_list())

    @patch('psutil.Process')
    def test_kill_pid(self, send_signal_mock):
        ps.kill_pid(0, signal=999)
        self.assertEqual(send_signal_mock.call_args, call(0))

    @patch('psutil.Process.send_signal')
    @patch('psutil.process_iter', new=MagicMock(return_value=[MOCK_PROC]))
    def test_pkill(self, send_signal_mock):
        mocked_proc.send_signal = MagicMock()
        test_signal = 1234
        ps.pkill(mocked_proc.name, signal=test_signal)
        self.assertEqual(mocked_proc.send_signal.call_args, call(test_signal))

    @patch('psutil.process_iter', new=MagicMock(return_value=[MOCK_PROC]))
    def test_pgrep(self):
        self.assertIn(MOCK_PROC.pid, ps.pgrep(MOCK_PROC.name))

    @patch('psutil.cpu_percent', new=MagicMock(return_value=1))
    def test_cpu_percent(self):
        self.assertEqual(ps.cpu_percent(), 1)

    @patch('psutil.cpu_times', new=MagicMock(return_value=STUB_CPU_TIMES))
    def test_cpu_times(self):
        self.assertDictEqual({'idle': 4, 'nice': 2, 'system': 3, 'user': 1}, ps.cpu_times())

    @patch('psutil.virtual_memory', new=MagicMock(return_value=STUB_VIRT_MEM))
    def test_virtual_memory(self):
        self.assertDictEqual({'used': 500, 'total': 1000, 'available': 500, 'percent': 50, 'free': 500}, ps.virtual_memory())

    @patch('psutil.swap_memory', new=MagicMock(return_value=STUB_SWAP_MEM))
    def test_swap_memory(self):
        self.assertDictEqual({'used': 500, 'total': 1000, 'percent': 50, 'free': 500, 'sin': 0, 'sout': 0}, ps.swap_memory())

    @patch('psutil.phymem_usage', new=MagicMock(return_value=STUB_PHY_MEM_USAGE))
    def test_physical_memory_usage(self):
        self.assertDictEqual({'used': 500, 'total': 1000, 'percent': 50, 'free': 500}, ps.physical_memory_usage())

    @patch('psutil.virtmem_usage', new=MagicMock(return_value=STUB_PHY_MEM_USAGE))
    def test_virtual_memory_usage(self):
        self.assertDictEqual({'used': 500, 'total': 1000, 'percent': 50, 'free': 500}, ps.virtual_memory_usage())

    # ps.cached_physical_memory is deprecated! See #9301
    # def test_cached_physical_memory(self):
    #    pass

    #ps.physical_memory_buffers is deprecated! See #9301
    # def test_physical_memory_buffers(self):
    #     pass

    @patch('psutil.disk_partitions', new=MagicMock(return_value=[STUB_DISK_PARTITION]))
    def test_disk_partitions(self):
        self.assertDictEqual(
            {'device': '/dev/disk0s2', 'mountpoint': '/', 'opts': 'rw,local,rootfs,dovolfs,journaled,multilabel',
             'fstype': 'hfs'},
            ps.disk_partitions()[0])

    @patch('psutil.disk_usage', new=MagicMock(return_value=STUB_DISK_USAGE))
    def test_disk_usage(self):
        self.assertDictEqual({'used': 500, 'total': 1000, 'percent': 50, 'free': 500}, ps.disk_usage('DUMMY_PATH'))

    @patch('psutil.disk_partitions', new=MagicMock(return_value=[STUB_DISK_PARTITION]))
    def test_disk_partition_usage(self):
        self.assertDictEqual(
            {'device': '/dev/disk0s2', 'mountpoint': '/', 'opts': 'rw,local,rootfs,dovolfs,journaled,multilabel',
             'fstype': 'hfs'},
            ps.disk_partitions()[0])

    ## Should only be tested in integration
    # def test_total_physical_memory(self):
    #     pass

    ## Should only be tested in integration
    # def test_num_cpus(self):
    #     pass

    ## Should only be tested in integration
    # def test_boot_time(self):
    #     pass
    @patch('psutil.network_io_counters', new=MagicMock(return_value=STUB_NETWORK_IO))
    def test_network_io_counters(self):
        self.assertDictEqual(
            {'packets_sent': 500, 'packets_recv': 600, 'bytes_recv': 2000, 'dropout': 4, 'bytes_sent': 1000,
             'errout': 2, 'errin': 1, 'dropin': 3}, ps.network_io_counters())

    @patch('psutil.disk_io_counters', new=MagicMock(return_value=STUB_DISK_IO))
    def test_disk_io_counters(self):
        self.assertDictEqual(
            {'read_time': 2000, 'write_bytes': 600, 'read_bytes': 500, 'write_time': 3000, 'read_count': 1000,
             'write_count': 2000}, ps.disk_io_counters())

    @patch('psutil.get_users', new=MagicMock(return_value=[STUB_USER]))
    def test_get_users(self):
        self.assertDictEqual({'terminal': 'ttys000', 'started': 0.0, 'host': 'localhost', 'name': 'bdobbs'},
                             ps.get_users()[0])

        ## This is commented out pending discussion on https://github.com/saltstack/salt/commit/2e5c3162ef87cca8a2c7b12ade7c7e1b32028f0a
        # @skipIf(not HAS_UTMP, "The utmp module must be installed to run test_get_users_utmp()")
        # @patch('psutil.get_users', new=MagicMock(return_value=None))  # This will force the function to use utmp
        # def test_get_users_utmp(self):
        #     pass
<|endoftext|>


<commit_before>from fabric.api import *
from datetime import datetime
import re
import fnmatch
import utility
import utility_redcap

@task(default=True)
def upgrade(name):
    """
    Upgrade an existing redcap instance using the <name> package.

    This input file should be in the TGZ format
    as packaged by this fabfile
    """

    utility_redcap.make_upload_target()
    copy_running_code_to_backup_dir()
    utility_redcap.upload_package_and_extract(name)
    utility.write_remote_my_cnf()
    offline()
    utility_redcap.move_software_to_live()
    new = utility.extract_version_from_string(name)
    old = utility_redcap.get_current_redcap_version()
    apply_incremental_db_changes(old,new)
    online()
    # run the tests but take REDCap offline again and abort if they fail
    if not utility_redcap.test(warn_only=True):
        offline()
        utility.delete_remote_my_cnf()
        abort("One or more tests failed.  REDCap has been taken offline.")
    utility.delete_remote_my_cnf()

def copy_running_code_to_backup_dir():
    """
    Copy the running code e.g. /var/www/redcap/* to the directory from which the
    the new software will be deployed, e.g., /var/www.backup/redcap-20160117T1543/.
    This will allow the new software to be overlain on the old software without
    risk of corrupting the old software.
    """
    with settings(user=env.deploy_user):
        with settings(warn_only=True):
            if run("test -e %(live_project_full_path)s/cron.php" % env).succeeded:
                run("cp -r -P %(live_project_full_path)s/* %(upload_target_backup_dir)s" % env)

def apply_incremental_db_changes(old, new):
    """
    Upgrade the database from the <old> REDCap version to the <new> version.

    Done by applying the needed upgrade_M.NN.OO.sql and upgrade_M.NN.OO.php
    files in sequence. The arguments old and new must be version numbers (i.e., 6.11.5)
    """
    old = utility.convert_version_to_int(old)
    redcap_sql_dir = '/'.join([env.live_pre_path, env.project_path, 'redcap_v' + new, 'Resources/sql'])
    with settings(user=env.deploy_user):
        with hide('output'):
            files = run('ls -1 %s/upgrade_*.sql %s/upgrade_*.php  | sort --version-sort ' % (redcap_sql_dir, redcap_sql_dir))
    path_to_sql_generation = '/'.join([env.live_pre_path, env.project_path, 'redcap_v' + new, 'generate_upgrade_sql_from_php.php'])
    for file in files.splitlines():
        match = re.search(r"(upgrade_)(\d+.\d+.\d+)(.)(php|sql)", file)
        version = match.group(2)
        version = utility.convert_version_to_int(version)
        if version > old:
            if fnmatch.fnmatch(file, "*.php"):
                print (file + " is a php file!\n")
                with settings(user=env.deploy_user):
                    run('php %s %s | mysql' % (path_to_sql_generation,file))
            else:
                print("Executing sql file %s" % file)
                with settings(user=env.deploy_user):
                    run('mysql < %s' % file)
    # Finalize upgrade
    utility_redcap.set_redcap_config('redcap_last_install_date', datetime.now().strftime("%Y-%m-%d"))
    utility_redcap.set_redcap_config('redcap_version', new)

def offline():
    change_online_status('Offline')

def online():
    change_online_status('Online')

def change_online_status(state):
    """
    Set the online/offline status with <state>.
    """

    with settings(user=env.deploy_user):
        if state == "Online":
            offline_binary = 0
            offline_message = 'The system is online.'
        elif state == "Offline":
            offline_binary = 1
            offline_message = 'The system is offline.'
        else:
            abort("Invald state provided. Specify 'Online' or 'Offline'.")

        utility.write_remote_my_cnf()
        utility_redcap.set_redcap_config('system_offline', '%s' % offline_binary)
        utility_redcap.set_redcap_config('system_offline_message', '%s' % offline_message)
        utility.delete_remote_my_cnf()<commit_msg>Do load any database upgrade scripts with a version number higher than the version you are upgrading to<commit_after>from fabric.api import *
from datetime import datetime
import re
import fnmatch
import utility
import utility_redcap

@task(default=True)
def upgrade(name):
    """
    Upgrade an existing redcap instance using the <name> package.

    This input file should be in the TGZ format
    as packaged by this fabfile
    """

    utility_redcap.make_upload_target()
    copy_running_code_to_backup_dir()
    utility_redcap.upload_package_and_extract(name)
    utility.write_remote_my_cnf()
    offline()
    utility_redcap.move_software_to_live()
    new = utility.extract_version_from_string(name)
    old = utility_redcap.get_current_redcap_version()
    apply_incremental_db_changes(old,new)
    online()
    # run the tests but take REDCap offline again and abort if they fail
    if not utility_redcap.test(warn_only=True):
        offline()
        utility.delete_remote_my_cnf()
        abort("One or more tests failed.  REDCap has been taken offline.")
    utility.delete_remote_my_cnf()

def copy_running_code_to_backup_dir():
    """
    Copy the running code e.g. /var/www/redcap/* to the directory from which the
    the new software will be deployed, e.g., /var/www.backup/redcap-20160117T1543/.
    This will allow the new software to be overlain on the old software without
    risk of corrupting the old software.
    """
    with settings(user=env.deploy_user):
        with settings(warn_only=True):
            if run("test -e %(live_project_full_path)s/cron.php" % env).succeeded:
                run("cp -r -P %(live_project_full_path)s/* %(upload_target_backup_dir)s" % env)

def apply_incremental_db_changes(old, new):
    """
    Upgrade the database from the <old> REDCap version to the <new> version.

    Done by applying the needed upgrade_M.NN.OO.sql and upgrade_M.NN.OO.php
    files in sequence. The arguments old and new must be version numbers (i.e., 6.11.5)
    """
    old = utility.convert_version_to_int(old)
    new_as_an_int = utility.convert_version_to_int(new)
    redcap_sql_dir = '/'.join([env.live_pre_path, env.project_path, 'redcap_v' + new, 'Resources/sql'])
    with settings(user=env.deploy_user):
        with hide('output'):
            files = run('ls -1 %s/upgrade_*.sql %s/upgrade_*.php  | sort --version-sort ' % (redcap_sql_dir, redcap_sql_dir))
    path_to_sql_generation = '/'.join([env.live_pre_path, env.project_path, 'redcap_v' + new, 'generate_upgrade_sql_from_php.php'])
    for file in files.splitlines():
        match = re.search(r"(upgrade_)(\d+.\d+.\d+)(.)(php|sql)", file)
        version = match.group(2)
        version = utility.convert_version_to_int(version)
        if version > old and version <= new_as_an_int:
            if fnmatch.fnmatch(file, "*.php"):
                print (file + " is a php file!\n")
                with settings(user=env.deploy_user):
                    run('php %s %s | mysql' % (path_to_sql_generation,file))
            else:
                print("Executing sql file %s" % file)
                with settings(user=env.deploy_user):
                    run('mysql < %s' % file)
    # Finalize upgrade
    utility_redcap.set_redcap_config('redcap_last_install_date', datetime.now().strftime("%Y-%m-%d"))
    utility_redcap.set_redcap_config('redcap_version', new)

def offline():
    change_online_status('Offline')

def online():
    change_online_status('Online')

def change_online_status(state):
    """
    Set the online/offline status with <state>.
    """

    with settings(user=env.deploy_user):
        if state == "Online":
            offline_binary = 0
            offline_message = 'The system is online.'
        elif state == "Offline":
            offline_binary = 1
            offline_message = 'The system is offline.'
        else:
            abort("Invald state provided. Specify 'Online' or 'Offline'.")

        utility.write_remote_my_cnf()
        utility_redcap.set_redcap_config('system_offline', '%s' % offline_binary)
        utility_redcap.set_redcap_config('system_offline_message', '%s' % offline_message)
        utility.delete_remote_my_cnf()<|endoftext|>


<commit_before>import os
import logging
import re
from typing import Any, Dict, Optional, Text

from rasa.nlu import utils
from rasa.nlu.components import Component
from rasa.nlu.training_data import Message

logger = logging.getLogger(__name__)

if typing.TYPE_CHECKING:
    from tensorflow import Graph, Session, Tensor
    from rasa.nlu.config import RasaNLUModelConfig
    from rasa.nlu.training_data import TrainingData
    from rasa.nlu.model import Metadata
    from rasa.nlu.training_data import Message

class KeywordIntentClassifier(Component):
    """Intent classifier using simple keyword matching.

    The classifier takes a list of keywords and associated intents as an input.
    A input sentence is checked for the keywords and the intent is returned.

    """

    provides = ["intent"]

    intent_keyword_map = {}

    def train(
        self,
        training_data: "TrainingData",
        cfg: Optional["RasaNLUModelConfig"] = None,
        **kwargs: Any
    ) -> None:
        self.intent_keyword_map = {}
        for intent in training_data.intents:
            self.intent_keyword_map[intent] = [
                ex.text
                for ex in training_data.training_examples
                if ex.get("intent") == intent
            ]

    def process(self, message: Message, **kwargs: Any) -> None:
        intent_name = self.parse(message.text)
        if intent_name is not None:
            intent = {"name": intent_name, "confidence": 1.0}
            message.set("intent", intent, add_to_output=True)

    def parse(self, text: Text) -> Optional[Text]:
        _text = text.lower()

        for intent, keywords in self.intent_keyword_map.items():
            for word in keywords:
                if re.search(r"\b" + word.lower() + r"\b", _text):
                    return intent

        return None

    def persist(self, file_name: Text, model_dir: Text) -> Dict[Text, Any]:
        """Persist this model into the passed directory.

        Return the metadata necessary to load the model again.
        """
        file_name = file_name + ".json"
        keyword_file = os.path.join(model_dir, file_name)
        utils.write_json_to_file(keyword_file, self.intent_keyword_map)

        return {"file": file_name}

    @classmethod
    def load(
        self,
        meta: Optional["Metadata"] = None,
        model_dir: Optional[Text] = None,
        model_metadata: "Metadata" = None,
        cached_component: Optional["KeywordIntentClassifier"] = None,
        **kwargs: Any
    ) -> "KeywordIntentClassifier":

        file_name = meta.get("file")
        if file_name is not None:
            keyword_file = os.path.join(model_dir, file_name)
            if os.path.exists(keyword_file):
                self.intent_keyword_map = utils.read_json_file(keyword_file)
            else:
                logger.warning(
                    "Failed to load IntentKeywordClassifier, maybe "
                    "{} does not exist.".format(keyword_file)
                )
        return self()
<commit_msg>fixed typing<commit_after>import os
import logging
import re
import typing
from typing import Any, Dict, Optional, Text

from rasa.nlu import utils
from rasa.nlu.components import Component
from rasa.nlu.training_data import Message

logger = logging.getLogger(__name__)

if typing.TYPE_CHECKING:
    from rasa.nlu.config import RasaNLUModelConfig
    from rasa.nlu.training_data import TrainingData
    from rasa.nlu.model import Metadata
    from rasa.nlu.training_data import Message

class KeywordIntentClassifier(Component):
    """Intent classifier using simple keyword matching.

    The classifier takes a list of keywords and associated intents as an input.
    A input sentence is checked for the keywords and the intent is returned.

    """

    provides = ["intent"]

    intent_keyword_map = {}

    def train(
        self,
        training_data: "TrainingData",
        cfg: Optional["RasaNLUModelConfig"] = None,
        **kwargs: Any
    ) -> None:
        self.intent_keyword_map = {}
        for intent in training_data.intents:
            self.intent_keyword_map[intent] = [
                ex.text
                for ex in training_data.training_examples
                if ex.get("intent") == intent
            ]

    def process(self, message: Message, **kwargs: Any) -> None:
        intent_name = self.parse(message.text)
        if intent_name is not None:
            intent = {"name": intent_name, "confidence": 1.0}
            message.set("intent", intent, add_to_output=True)

    def parse(self, text: Text) -> Optional[Text]:
        _text = text.lower()

        for intent, keywords in self.intent_keyword_map.items():
            for word in keywords:
                if re.search(r"\b" + word.lower() + r"\b", _text):
                    return intent

        return None

    def persist(self, file_name: Text, model_dir: Text) -> Dict[Text, Any]:
        """Persist this model into the passed directory.

        Return the metadata necessary to load the model again.
        """
        file_name = file_name + ".json"
        keyword_file = os.path.join(model_dir, file_name)
        utils.write_json_to_file(keyword_file, self.intent_keyword_map)

        return {"file": file_name}

    @classmethod
    def load(
        self,
        meta: Dict[Text, Any],
        model_dir: Optional[Text] = None,
        model_metadata: "Metadata" = None,
        cached_component: Optional["KeywordIntentClassifier"] = None,
        **kwargs: Any
    ) -> "KeywordIntentClassifier":

        if model_dir and meta.get("file"):
            file_name = meta.get("file")
            keyword_file = os.path.join(model_dir, file_name)
            if os.path.exists(keyword_file):
                self.intent_keyword_map = utils.read_json_file(keyword_file)
            else:
                logger.warning(
                    "Failed to load IntentKeywordClassifier, maybe "
                    "{} does not exist.".format(keyword_file)
                )
        return self()
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
import json
import unittest
import routes
import os.path
from tests.unit import BaseTest

from webtest import TestApp
from quantum.common import extensions
from quantum.common import wsgi
from quantum.common import config

response_body = "Try to say this Mr. Knox, sir..."
test_conf_file = os.path.join(os.path.dirname(__file__), os.pardir,
                              os.pardir, 'etc', 'quantum.conf.test')

class ExtensionControllerTest(unittest.TestCase):

    def setUp(self):
        super(ExtensionControllerTest, self).setUp()
        self.test_app = setup_extensions_test_app()

    def test_index(self):
        response = self.test_app.get("/extensions")
        foxnsox = response.json["extensions"][0]
        self.assertEqual(foxnsox["alias"], "FOXNSOX")
        self.assertEqual(foxnsox["namespace"],
                         "http://www.fox.in.socks/api/ext/pie/v1.0")
        self.assertEqual(200, response.status_int)

    def test_get_by_alias(self):
        response = self.test_app.get("/extensions/FOXNSOX")
        self.assertEqual(200, response.status_int)

class ResourceExtensionTest(unittest.TestCase):

    def test_no_extension_present(self):
        test_app = setup_extensions_test_app(StubExtensionManager(None))
        response = test_app.get("/blah", status='*')
        self.assertEqual(404, response.status_int)

    def test_get_resources(self):
        res_ext = extensions.ResourceExtension('tweedles',
                                               StubController(response_body))
        test_app = setup_extensions_test_app(StubExtensionManager(res_ext))

        response = test_app.get("/tweedles")
        self.assertEqual(200, response.status_int)
        self.assertEqual(response_body, response.body)

class ExtensionManagerTest(unittest.TestCase):

    def test_get_resources(self):
        test_app = setup_extensions_test_app()
        response = test_app.get('/foxnsocks')

        self.assertEqual(200, response.status_int)
        self.assertEqual(response_body, response.body)

class ActionExtensionTest(unittest.TestCase):

    def setUp(self):
        super(ActionExtensionTest, self).setUp()
        self.test_app = setup_extensions_test_app()

    def _send_server_action_request(self, url, body):
        return self.test_app.post(url, json.dumps(body),
                                  content_type='application/json', status='*')

    def test_extended_action(self):
        body = json.dumps(dict(add_tweedle=dict(name="test")))
        response = self.test_app.post('/dummy_resources/1/action', body,
                                      content_type='application/json')
        self.assertEqual("Tweedle Beetle Added.", response.body)

        body = json.dumps(dict(delete_tweedle=dict(name="test")))
        response = self.test_app.post("/dummy_resources/1/action", body,
                                      content_type='application/json')

        self.assertEqual(200, response.status_int)
        self.assertEqual("Tweedle Beetle Deleted.", response.body)

    def test_invalid_action_body(self):
        body = json.dumps(dict(blah=dict(name="test")))  # Doesn't exist
        response = self.test_app.post("/dummy_resources/1/action", body,
                                      content_type='application/json',
                                      status='*')
        self.assertEqual(404, response.status_int)

    def test_invalid_action(self):
        body = json.dumps(dict(blah=dict(name="test")))
        response = self.test_app.post("/asdf/1/action",
                                      body, content_type='application/json',
                                      status='*')
        self.assertEqual(404, response.status_int)

class RequestExtensionTest(BaseTest):

    def test_get_resources_with_stub_mgr(self):

        def _req_handler(req, res):
            # only handle JSON responses
            data = json.loads(res.body)
            data['googoose'] = req.GET.get('chewing')
            res.body = json.dumps(data)
            return res

        req_ext = extensions.RequestExtension('GET',
                                                '/dummy_resources/:(id)',
                                                _req_handler)

        manager = StubExtensionManager(None, None, req_ext)
        app = setup_extensions_test_app(manager)

        response = app.get("/dummy_resources/1?chewing=bluegoos",
                           extra_environ={'api.version': '1.1'})

        self.assertEqual(200, response.status_int)
        response_data = json.loads(response.body)
        self.assertEqual('bluegoos', response_data['googoose'])
        self.assertEqual('knox', response_data['fort'])

    def test_get_resources_with_mgr(self):
        app = setup_extensions_test_app()

        response = app.get("/dummy_resources/1?"
                                            "chewing=newblue", status='*')

        self.assertEqual(200, response.status_int)
        response_data = json.loads(response.body)
        self.assertEqual('newblue', response_data['googoose'])
        self.assertEqual("Pig Bands!", response_data['big_bands'])

    def test_edit_previously_uneditable_field(self):

        def _update_handler(req, res):
            data = json.loads(res.body)
            data['uneditable'] = req.params['uneditable']
            res.body = json.dumps(data)
            return res

        conf, app = config.load_paste_app('extensions_test_app',
                                               {'config_file': test_conf_file}, None)
        base_app = TestApp(app)
        response = base_app.put("/dummy_resources/1", {'uneditable': "new_value"})
        self.assertEqual(response.json['uneditable'], "original_value")

        req_ext = extensions.RequestExtension('PUT',
                                                '/dummy_resources/:(id)',
                                                _update_handler)

        manager = StubExtensionManager(None, None, req_ext)
        extended_app = setup_extensions_test_app(manager)
        response = extended_app.put("/dummy_resources/1", {'uneditable': "new_value"})
        self.assertEqual(response.json['uneditable'], "new_value")

class TestExtensionMiddlewareFactory(unittest.TestCase):

    def test_app_configured_with_extensions_as_filter(self):
        conf, quantum_app = config.load_paste_app('extensions_app_with_filter',
                                                  {"config_file": test_conf_file},
                                                  None)

        response = TestApp(quantum_app).get("/extensions")
        self.assertEqual(response.status_int, 200)

class ExtensionsTestApp(wsgi.Router):

    def __init__(self, options={}):
        mapper = routes.Mapper()
        controller = StubController(response_body)
        mapper.resource("dummy_resource", "/dummy_resources",
                        controller=controller)
        super(ExtensionsTestApp, self).__init__(mapper)

class StubController(wsgi.Controller):

    def __init__(self, body):
        self.body = body

    def index(self, request):
        return self.body

    def show(self, request, id):
        return {'fort': 'knox'}

    def update(self, request, id):
        return {'uneditable': 'original_value'}

def app_factory(global_conf, **local_conf):
    conf = global_conf.copy()
    conf.update(local_conf)
    return ExtensionsTestApp(conf)

def setup_extensions_test_app(extension_manager=None):
    options = {'config_file': test_conf_file}
    conf, app = config.load_paste_app('extensions_test_app', options, None)
    extended_app = extensions.ExtensionMiddleware(app, conf, extension_manager)
    return TestApp(extended_app)

class StubExtensionManager(object):

    def __init__(self, resource_ext=None, action_ext=None, request_ext=None):
        self.resource_ext = resource_ext
        self.action_ext = action_ext
        self.request_ext = request_ext

    def get_name(self):
        return "Tweedle Beetle Extension"

    def get_alias(self):
        return "TWDLBETL"

    def get_description(self):
        return "Provides access to Tweedle Beetles"

    def get_resources(self):
        resource_exts = []
        if self.resource_ext:
            resource_exts.append(self.resource_ext)
        return resource_exts

    def get_actions(self):
        action_exts = []
        if self.action_ext:
            action_exts.append(self.action_ext)
        return action_exts

    def get_request_extensions(self):
        request_extensions = []
        if self.request_ext:
            request_extensions.append(self.request_ext)
        return request_extensions
<commit_msg>Santosh/Rajaram| added extenstion test to show header extensibility<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
import json
import unittest
import routes
import os.path
from tests.unit import BaseTest

from webtest import TestApp
from quantum.common import extensions
from quantum.common import wsgi
from quantum.common import config

response_body = "Try to say this Mr. Knox, sir..."
test_conf_file = os.path.join(os.path.dirname(__file__), os.pardir,
                              os.pardir, 'etc', 'quantum.conf.test')

class ExtensionControllerTest(unittest.TestCase):

    def setUp(self):
        super(ExtensionControllerTest, self).setUp()
        self.test_app = setup_extensions_test_app()

    def test_index(self):
        response = self.test_app.get("/extensions")
        foxnsox = response.json["extensions"][0]
        self.assertEqual(foxnsox["alias"], "FOXNSOX")
        self.assertEqual(foxnsox["namespace"],
                         "http://www.fox.in.socks/api/ext/pie/v1.0")
        self.assertEqual(200, response.status_int)

    def test_get_by_alias(self):
        response = self.test_app.get("/extensions/FOXNSOX")
        self.assertEqual(200, response.status_int)

class ResourceExtensionTest(unittest.TestCase):

    def test_no_extension_present(self):
        test_app = setup_extensions_test_app(StubExtensionManager(None))
        response = test_app.get("/blah", status='*')
        self.assertEqual(404, response.status_int)

    def test_get_resources(self):
        res_ext = extensions.ResourceExtension('tweedles',
                                               StubController(response_body))
        test_app = setup_extensions_test_app(StubExtensionManager(res_ext))

        response = test_app.get("/tweedles")
        self.assertEqual(200, response.status_int)
        self.assertEqual(response_body, response.body)

class ExtensionManagerTest(unittest.TestCase):

    def test_get_resources(self):
        test_app = setup_extensions_test_app()
        response = test_app.get('/foxnsocks')

        self.assertEqual(200, response.status_int)
        self.assertEqual(response_body, response.body)

class ActionExtensionTest(unittest.TestCase):

    def setUp(self):
        super(ActionExtensionTest, self).setUp()
        self.test_app = setup_extensions_test_app()

    def _send_server_action_request(self, url, body):
        return self.test_app.post(url, json.dumps(body),
                                  content_type='application/json', status='*')

    def test_extended_action(self):
        body = json.dumps(dict(add_tweedle=dict(name="test")))
        response = self.test_app.post('/dummy_resources/1/action', body,
                                      content_type='application/json')
        self.assertEqual("Tweedle Beetle Added.", response.body)

        body = json.dumps(dict(delete_tweedle=dict(name="test")))
        response = self.test_app.post("/dummy_resources/1/action", body,
                                      content_type='application/json')

        self.assertEqual(200, response.status_int)
        self.assertEqual("Tweedle Beetle Deleted.", response.body)

    def test_invalid_action_body(self):
        body = json.dumps(dict(blah=dict(name="test")))  # Doesn't exist
        response = self.test_app.post("/dummy_resources/1/action", body,
                                      content_type='application/json',
                                      status='*')
        self.assertEqual(404, response.status_int)

    def test_invalid_action(self):
        body = json.dumps(dict(blah=dict(name="test")))
        response = self.test_app.post("/asdf/1/action",
                                      body, content_type='application/json',
                                      status='*')
        self.assertEqual(404, response.status_int)

class RequestExtensionTest(BaseTest):

    def test_headers_extension(self):
        def _req_header_handler(req, res):
            ext_header = req.headers['X-rax-fox']
            res.headers['X-got-header'] = ext_header
            return res

        app = self._setup_app_with_request_handler(_req_header_handler,
                                                   'GET')
        response = app.get("/dummy_resources/1", headers={'X-rax-fox': "sox"})
        self.assertEqual(response.headers['X-got-header'], "sox")

    def test_get_resources_with_stub_mgr(self):
        def _req_handler(req, res):
            # only handle JSON responses
            data = json.loads(res.body)
            data['googoose'] = req.GET.get('chewing')
            res.body = json.dumps(data)
            return res

        req_ext = extensions.RequestExtension('GET',
                                                '/dummy_resources/:(id)',
                                                _req_handler)

        manager = StubExtensionManager(None, None, req_ext)
        app = setup_extensions_test_app(manager)

        response = app.get("/dummy_resources/1?chewing=bluegoos",
                           extra_environ={'api.version': '1.1'})

        self.assertEqual(200, response.status_int)
        response_data = json.loads(response.body)
        self.assertEqual('bluegoos', response_data['googoose'])
        self.assertEqual('knox', response_data['fort'])

    def test_get_resources_with_mgr(self):
        app = setup_extensions_test_app()

        response = app.get("/dummy_resources/1?"
                                            "chewing=newblue", status='*')

        self.assertEqual(200, response.status_int)
        response_data = json.loads(response.body)
        self.assertEqual('newblue', response_data['googoose'])
        self.assertEqual("Pig Bands!", response_data['big_bands'])

    def test_edit_previously_uneditable_field(self):

        def _update_handler(req, res):
            data = json.loads(res.body)
            data['uneditable'] = req.params['uneditable']
            res.body = json.dumps(data)
            return res

        conf, app = config.load_paste_app('extensions_test_app',
                                         {'config_file': test_conf_file}, None)
        base_app = TestApp(app)

        response = base_app.put("/dummy_resources/1",
                                {'uneditable': "new_value"})
        self.assertEqual(response.json['uneditable'], "original_value")

        ext_app = self._setup_app_with_request_handler(_update_handler,
                                                            'PUT')
        ext_response = ext_app.put("/dummy_resources/1",
                                    {'uneditable': "new_value"})
        self.assertEqual(ext_response.json['uneditable'], "new_value")

    def _setup_app_with_request_handler(self, handler, verb):
        req_ext = extensions.RequestExtension(verb,
                                   '/dummy_resources/:(id)', handler)

        manager = StubExtensionManager(None, None, req_ext)
        return setup_extensions_test_app(manager)

class TestExtensionMiddlewareFactory(unittest.TestCase):

    def test_app_configured_with_extensions_as_filter(self):
        conf, quantum_app = config.load_paste_app('extensions_app_with_filter',
                                        {"config_file": test_conf_file}, None)

        response = TestApp(quantum_app).get("/extensions")
        self.assertEqual(response.status_int, 200)

class ExtensionsTestApp(wsgi.Router):

    def __init__(self, options={}):
        mapper = routes.Mapper()
        controller = StubController(response_body)
        mapper.resource("dummy_resource", "/dummy_resources",
                        controller=controller)
        super(ExtensionsTestApp, self).__init__(mapper)

class StubController(wsgi.Controller):

    def __init__(self, body):
        self.body = body

    def index(self, request):
        return self.body

    def show(self, request, id):
        return {'fort': 'knox'}

    def update(self, request, id):
        return {'uneditable': 'original_value'}

def app_factory(global_conf, **local_conf):
    conf = global_conf.copy()
    conf.update(local_conf)
    return ExtensionsTestApp(conf)

def setup_extensions_test_app(extension_manager=None):
    options = {'config_file': test_conf_file}
    conf, app = config.load_paste_app('extensions_test_app', options, None)
    extended_app = extensions.ExtensionMiddleware(app, conf, extension_manager)
    return TestApp(extended_app)

class StubExtensionManager(object):

    def __init__(self, resource_ext=None, action_ext=None, request_ext=None):
        self.resource_ext = resource_ext
        self.action_ext = action_ext
        self.request_ext = request_ext

    def get_name(self):
        return "Tweedle Beetle Extension"

    def get_alias(self):
        return "TWDLBETL"

    def get_description(self):
        return "Provides access to Tweedle Beetles"

    def get_resources(self):
        resource_exts = []
        if self.resource_ext:
            resource_exts.append(self.resource_ext)
        return resource_exts

    def get_actions(self):
        action_exts = []
        if self.action_ext:
            action_exts.append(self.action_ext)
        return action_exts

    def get_request_extensions(self):
        request_extensions = []
        if self.request_ext:
            request_extensions.append(self.request_ext)
        return request_extensions
<|endoftext|>


<commit_before>import logging
import numpy as np
import os
import typing
from typing import Any, Dict, List, Optional, Text, Tuple

from rasa.nlu import utils
from rasa.nlu.classifiers import INTENT_RANKING_LENGTH
from rasa.nlu.components import Component
from rasa.nlu.config import RasaNLUModelConfig
from rasa.nlu.model import Metadata
from rasa.nlu.training_data import Message, TrainingData

logger = logging.getLogger(__name__)

if typing.TYPE_CHECKING:
    import sklearn

class SklearnIntentClassifier(Component):
    """Intent classifier using the sklearn framework"""

    provides = ["intent", "intent_ranking"]

    requires = ["text_features"]

    defaults = {
        # C parameter of the svm - cross validation will select the best value
        "C": [1, 2, 5, 10, 20, 100],
        # gamma parameter of the svm
        "gamma": [0.1],
        # the kernels to use for the svm training - cross validation will
        # decide which one of them performs best
        "kernels": ["linear"],
        # We try to find a good number of cross folds to use during
        # intent training, this specifies the max number of folds
        "max_cross_validation_folds": 5,
        # Scoring function used for evaluating the hyper parameters
        # This can be a name or a function (cfr GridSearchCV doc for more info)
        "scoring_function": "f1_weighted",
    }

    def __init__(
        self,
        component_config: Dict[Text, Any] = None,
        clf: "sklearn.model_selection.GridSearchCV" = None,
        le: Optional["sklearn.preprocessing.LabelEncoder"] = None,
    ) -> None:
        """Construct a new intent classifier using the sklearn framework."""
        from sklearn.preprocessing import LabelEncoder

        super(SklearnIntentClassifier, self).__init__(component_config)

        if le is not None:
            self.le = le
        else:
            self.le = LabelEncoder()
        self.clf = clf

    @classmethod
    def required_packages(cls) -> List[Text]:
        return ["sklearn"]

    def transform_labels_str2num(self, labels: List[Text]) -> np.ndarray:
        """Transforms a list of strings into numeric label representation.

        :param labels: List of labels to convert to numeric representation"""

        return self.le.fit_transform(labels)

    def transform_labels_num2str(self, y: np.ndarray) -> np.ndarray:
        """Transforms a list of strings into numeric label representation.

        :param y: List of labels to convert to numeric representation"""

        return self.le.inverse_transform(y)

    def train(
        self, training_data: TrainingData, cfg: RasaNLUModelConfig, **kwargs: Any
    ) -> None:
        """Train the intent classifier on a data set."""

        num_threads = kwargs.get("num_threads", 1)

        labels = [e.get("intent") for e in training_data.intent_examples]

        if len(set(labels)) < 2:
            logger.warning(
                "Can not train an intent classifier. "
                "Need at least 2 different classes. "
                "Skipping training of intent classifier."
            )
        else:
            y = self.transform_labels_str2num(labels)
            X = np.stack(
                [
                    example.get("text_features")
                    for example in training_data.intent_examples
                ]
            )

            self.clf = self._create_classifier(num_threads, y)

            self.clf.fit(X, y)

    def _num_cv_splits(self, y):
        folds = self.component_config["max_cross_validation_folds"]
        return max(2, min(folds, np.min(np.bincount(y)) // 5))

    def _create_classifier(self, num_threads, y):
        from sklearn.model_selection import GridSearchCV
        from sklearn.svm import SVC

        C = self.component_config["C"]
        kernels = self.component_config["kernels"]
        gamma = self.component_config["gamma"]
        # dirty str fix because sklearn is expecting
        # str not instance of basestr...
        tuned_parameters = [
            {"C": C, "gamma": gamma, "kernel": [str(k) for k in kernels]}
        ]

        # aim for 5 examples in each fold

        cv_splits = self._num_cv_splits(y)

        return GridSearchCV(
            SVC(C=1, probability=True, class_weight="balanced"),
            param_grid=tuned_parameters,
            n_jobs=num_threads,
            cv=cv_splits,
            scoring=self.component_config["scoring_function"],
            verbose=1,
        )

    def process(self, message: Message, **kwargs: Any) -> None:
        """Return the most likely intent and its probability for a message."""

        if not self.clf:
            # component is either not trained or didn't
            # receive enough training data
            intent = None
            intent_ranking = []
        else:
            X = message.get("text_features").reshape(1, -1)
            intent_ids, probabilities = self.predict(X)
            intents = self.transform_labels_num2str(np.ravel(intent_ids))
            # `predict` returns a matrix as it is supposed
            # to work for multiple examples as well, hence we need to flatten
            probabilities = probabilities.flatten()

            if intents.size > 0 and probabilities.size > 0:
                ranking = list(zip(list(intents), list(probabilities)))[
                    :INTENT_RANKING_LENGTH
                ]

                intent = {"name": intents[0], "confidence": probabilities[0]}

                intent_ranking = [
                    {"name": intent_name, "confidence": score}
                    for intent_name, score in ranking
                ]
            else:
                intent = {"name": None, "confidence": 0.0}
                intent_ranking = []

        message.set("intent", intent, add_to_output=True)
        message.set("intent_ranking", intent_ranking, add_to_output=True)

    def predict_prob(self, X: np.ndarray) -> np.ndarray:
        """Given a bow vector of an input text, predict the intent label.

        Return probabilities for all labels.

        :param X: bow of input text
        :return: vector of probabilities containing one entry for each label"""

        return self.clf.predict_proba(X)

    def predict(self, X: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
        """Given a bow vector of an input text, predict most probable label.

        Return only the most likely label.

        :param X: bow of input text
        :return: tuple of first, the most probable label and second,
                 its probability."""

        pred_result = self.predict_prob(X)
        # sort the probabilities retrieving the indices of
        # the elements in sorted order
        sorted_indices = np.fliplr(np.argsort(pred_result, axis=1))
        return sorted_indices, pred_result[:, sorted_indices]

    def persist(self, file_name: Text, model_dir: Text) -> Optional[Dict[Text, Any]]:
        """Persist this model into the passed directory."""

        classifier_file_name = file_name + "_classifier.pkl"
        encoder_file_name = file_name + "_encoder.pkl"
        if self.clf:
            utils.json_pickle(
                os.path.join(model_dir, encoder_file_name), self.le.classes_
            )
            utils.json_pickle(
                os.path.join(model_dir, classifier_file_name), self.clf.best_estimator_
            )
        return {"classifier": classifier_file_name, "encoder": encoder_file_name}

    @classmethod
    def load(
        cls,
        meta: Dict[Text, Any],
        model_dir: Optional[Text] = None,
        model_metadata: Optional[Metadata] = None,
        cached_component: Optional["SklearnIntentClassifier"] = None,
        **kwargs: Any
    ) -> "SklearnIntentClassifier":
        from sklearn.preprocessing import LabelEncoder

        classifier_file = os.path.join(model_dir, meta.get("classifier"))
        encoder_file = os.path.join(model_dir, meta.get("encoder"))

        if os.path.exists(classifier_file):
            classifier = utils.json_unpickle(classifier_file)
            classes = utils.json_unpickle(encoder_file)
            encoder = LabelEncoder()
            encoder.classes_ = classes
            return cls(meta, classifier, encoder)
        else:
            return cls(meta)
<commit_msg>review improvement to sklearn persist<commit_after>import logging
import numpy as np
import os
import typing
from typing import Any, Dict, List, Optional, Text, Tuple

from rasa.nlu import utils
from rasa.nlu.classifiers import INTENT_RANKING_LENGTH
from rasa.nlu.components import Component
from rasa.nlu.config import RasaNLUModelConfig
from rasa.nlu.model import Metadata
from rasa.nlu.training_data import Message, TrainingData

logger = logging.getLogger(__name__)

if typing.TYPE_CHECKING:
    import sklearn

class SklearnIntentClassifier(Component):
    """Intent classifier using the sklearn framework"""

    provides = ["intent", "intent_ranking"]

    requires = ["text_features"]

    defaults = {
        # C parameter of the svm - cross validation will select the best value
        "C": [1, 2, 5, 10, 20, 100],
        # gamma parameter of the svm
        "gamma": [0.1],
        # the kernels to use for the svm training - cross validation will
        # decide which one of them performs best
        "kernels": ["linear"],
        # We try to find a good number of cross folds to use during
        # intent training, this specifies the max number of folds
        "max_cross_validation_folds": 5,
        # Scoring function used for evaluating the hyper parameters
        # This can be a name or a function (cfr GridSearchCV doc for more info)
        "scoring_function": "f1_weighted",
    }

    def __init__(
        self,
        component_config: Dict[Text, Any] = None,
        clf: "sklearn.model_selection.GridSearchCV" = None,
        le: Optional["sklearn.preprocessing.LabelEncoder"] = None,
    ) -> None:
        """Construct a new intent classifier using the sklearn framework."""
        from sklearn.preprocessing import LabelEncoder

        super(SklearnIntentClassifier, self).__init__(component_config)

        if le is not None:
            self.le = le
        else:
            self.le = LabelEncoder()
        self.clf = clf

    @classmethod
    def required_packages(cls) -> List[Text]:
        return ["sklearn"]

    def transform_labels_str2num(self, labels: List[Text]) -> np.ndarray:
        """Transforms a list of strings into numeric label representation.

        :param labels: List of labels to convert to numeric representation"""

        return self.le.fit_transform(labels)

    def transform_labels_num2str(self, y: np.ndarray) -> np.ndarray:
        """Transforms a list of strings into numeric label representation.

        :param y: List of labels to convert to numeric representation"""

        return self.le.inverse_transform(y)

    def train(
        self, training_data: TrainingData, cfg: RasaNLUModelConfig, **kwargs: Any
    ) -> None:
        """Train the intent classifier on a data set."""

        num_threads = kwargs.get("num_threads", 1)

        labels = [e.get("intent") for e in training_data.intent_examples]

        if len(set(labels)) < 2:
            logger.warning(
                "Can not train an intent classifier. "
                "Need at least 2 different classes. "
                "Skipping training of intent classifier."
            )
        else:
            y = self.transform_labels_str2num(labels)
            X = np.stack(
                [
                    example.get("text_features")
                    for example in training_data.intent_examples
                ]
            )

            self.clf = self._create_classifier(num_threads, y)

            self.clf.fit(X, y)

    def _num_cv_splits(self, y):
        folds = self.component_config["max_cross_validation_folds"]
        return max(2, min(folds, np.min(np.bincount(y)) // 5))

    def _create_classifier(self, num_threads, y):
        from sklearn.model_selection import GridSearchCV
        from sklearn.svm import SVC

        C = self.component_config["C"]
        kernels = self.component_config["kernels"]
        gamma = self.component_config["gamma"]
        # dirty str fix because sklearn is expecting
        # str not instance of basestr...
        tuned_parameters = [
            {"C": C, "gamma": gamma, "kernel": [str(k) for k in kernels]}
        ]

        # aim for 5 examples in each fold

        cv_splits = self._num_cv_splits(y)

        return GridSearchCV(
            SVC(C=1, probability=True, class_weight="balanced"),
            param_grid=tuned_parameters,
            n_jobs=num_threads,
            cv=cv_splits,
            scoring=self.component_config["scoring_function"],
            verbose=1,
        )

    def process(self, message: Message, **kwargs: Any) -> None:
        """Return the most likely intent and its probability for a message."""

        if not self.clf:
            # component is either not trained or didn't
            # receive enough training data
            intent = None
            intent_ranking = []
        else:
            X = message.get("text_features").reshape(1, -1)
            intent_ids, probabilities = self.predict(X)
            intents = self.transform_labels_num2str(np.ravel(intent_ids))
            # `predict` returns a matrix as it is supposed
            # to work for multiple examples as well, hence we need to flatten
            probabilities = probabilities.flatten()

            if intents.size > 0 and probabilities.size > 0:
                ranking = list(zip(list(intents), list(probabilities)))[
                    :INTENT_RANKING_LENGTH
                ]

                intent = {"name": intents[0], "confidence": probabilities[0]}

                intent_ranking = [
                    {"name": intent_name, "confidence": score}
                    for intent_name, score in ranking
                ]
            else:
                intent = {"name": None, "confidence": 0.0}
                intent_ranking = []

        message.set("intent", intent, add_to_output=True)
        message.set("intent_ranking", intent_ranking, add_to_output=True)

    def predict_prob(self, X: np.ndarray) -> np.ndarray:
        """Given a bow vector of an input text, predict the intent label.

        Return probabilities for all labels.

        :param X: bow of input text
        :return: vector of probabilities containing one entry for each label"""

        return self.clf.predict_proba(X)

    def predict(self, X: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
        """Given a bow vector of an input text, predict most probable label.

        Return only the most likely label.

        :param X: bow of input text
        :return: tuple of first, the most probable label and second,
                 its probability."""

        pred_result = self.predict_prob(X)
        # sort the probabilities retrieving the indices of
        # the elements in sorted order
        sorted_indices = np.fliplr(np.argsort(pred_result, axis=1))
        return sorted_indices, pred_result[:, sorted_indices]

    def persist(self, file_name: Text, model_dir: Text) -> Optional[Dict[Text, Any]]:
        """Persist this model into the passed directory."""

        classifier_file_name = file_name + "_classifier.pkl"
        encoder_file_name = file_name + "_encoder.pkl"
        if self.clf and self.le:
            utils.json_pickle(
                os.path.join(model_dir, encoder_file_name), self.le.classes_
            )
            utils.json_pickle(
                os.path.join(model_dir, classifier_file_name), self.clf.best_estimator_
            )
        return {"classifier": classifier_file_name, "encoder": encoder_file_name}

    @classmethod
    def load(
        cls,
        meta: Dict[Text, Any],
        model_dir: Optional[Text] = None,
        model_metadata: Optional[Metadata] = None,
        cached_component: Optional["SklearnIntentClassifier"] = None,
        **kwargs: Any
    ) -> "SklearnIntentClassifier":
        from sklearn.preprocessing import LabelEncoder

        classifier_file = os.path.join(model_dir, meta.get("classifier"))
        encoder_file = os.path.join(model_dir, meta.get("encoder"))

        if os.path.exists(classifier_file):
            classifier = utils.json_unpickle(classifier_file)
            classes = utils.json_unpickle(encoder_file)
            encoder = LabelEncoder()
            encoder.classes_ = classes
            return cls(meta, classifier, encoder)
        else:
            return cls(meta)
<|endoftext|>


<commit_before>'''
A Q-Learning algorithm learns the state-action values for Tic-Tac-Toe board
positions against a deterministic Alpha-Beta player. Aftere the values are
learned, we use a Greedy Player to play 100 games against AlphaBeta.

Since Q-learning is a model-free algorithm, i.e. it does not require a model,
it takes environment and it does not have access to the transition and
reward dynamics.

A greedy player that uses the learned state-action values should always
draw against Alpha-Beta assuming the Q-learning algorithm ran for a high
number of episodes.

Note that Q-learning runs to only learn the values for board-moves for
the first player. If we want to learn this to play as the second player,
we need to run the algorithm against using an MDP where opp_idx = 0.
'''
from capstone.algorithms import Sarsa
from capstone.environment import Environment
from capstone.game import TicTacToe
from capstone.mdp import GameMDP
from capstone.player import AlphaBeta, GreedyQF
from capstone.util import play_series

board = 'XOXO-O--X'
game = TicTacToe(board)
ab = AlphaBeta()
mdp = GameMDP(game, ab, 1)
env = Environment(mdp)
qf = {}
Sarsa(env, qf=qf, n_episodes=1000).learn()
players = [GreedyQF(qf), ab]
play_series(TicTacToe(board), players, n_matches=100)
<commit_msg>Update SARSA<commit_after>'''
A SARSA Q-Learning algorithm learns the state-action values for Tic-Tac-Toe board
positions against a deterministic Alpha-Beta player. After the values are
learned, we use a Greedy Player to play 100 games against AlphaBeta.
'''

from capstone.algorithms import Sarsa
from capstone.environment import Environment
from capstone.game import TicTacToe
from capstone.mdp import GameMDP
from capstone.player import AlphaBeta, GreedyQF
from capstone.util import play_series

board = 'XOXO-O--X'
game = TicTacToe(board)
ab = AlphaBeta()
mdp = GameMDP(game, ab, 1)
env = Environment(mdp)
qf = {}
Sarsa(env, qf=qf, n_episodes=1000).learn()
players = [GreedyQF(qf), ab]
play_series(TicTacToe(board), players, n_matches=100)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Bo Maryniuk <bo@suse.de>`
'''
from __future__ import absolute_import, print_function, unicode_literals
import datetime

from tests.support.unit import TestCase, skipIf
from tests.support.mock import (
    NO_MOCK,
    NO_MOCK_REASON,
    MagicMock,
    patch)

from salt.ext.six.moves import zip
from salt.ext import six

import salt.exceptions
from salt.utils import thin
import salt.utils.stringutils

try:
    import pytest
except ImportError:
    pytest = None

@skipIf(NO_MOCK, NO_MOCK_REASON)
@skipIf(pytest is None, 'PyTest is missing')
class SSHThinTestCase(TestCase):
    '''
    TestCase for SaltSSH-related parts.
    '''

    @patch('salt.exceptions.SaltSystemExit', Exception)
    @patch('salt.utils.thin.log', MagicMock())
    def test_get_ext_tops_cfg_missing_dependencies(self):
        '''
        Test thin.get_ext_tops contains all required dependencies.

        :return:
        '''
        cfg = [
            {'namespace': {'py-version': [0, 0], 'path': '/foo', 'dependencies': []}},
        ]
        with pytest.raises(Exception) as err:
            thin.get_ext_tops(cfg)
        assert 'Missing dependencies' in str(err)
        assert thin.log.error.called
        assert 'Missing dependencies' in thin.log.error.call_args[0][0]
        assert 'jinja2, yaml, tornado, msgpack' in thin.log.error.call_args[0][0]

    @patch('salt.exceptions.SaltSystemExit', Exception)
    @patch('salt.utils.thin.log', MagicMock())
    def test_get_ext_tops_cfg_missing_interpreter(self):
        '''
        Test thin.get_ext_tops contains interpreter configuration.

        :return:
        '''
        cfg = [
            {'namespace': {'path': '/foo',
                           'dependencies': []}},
        ]
        with pytest.raises(salt.exceptions.SaltSystemExit) as err:
            thin.get_ext_tops(cfg)
        assert 'missing specific locked Python version' in str(err)

    @patch('salt.exceptions.SaltSystemExit', Exception)
    @patch('salt.utils.thin.log', MagicMock())
    def test_get_ext_tops_cfg_wrong_interpreter(self):
        '''
        Test thin.get_ext_tops contains correct interpreter configuration.

        :return:
        '''
        cfg = [
            {'namespace': {'path': '/foo',
                           'py-version': 2,
                           'dependencies': []}},
        ]
        with pytest.raises(salt.exceptions.SaltSystemExit) as err:
            thin.get_ext_tops(cfg)
        assert 'specific locked Python version should be a list of major/minor version' in str(err)

    @patch('salt.exceptions.SaltSystemExit', Exception)
    @patch('salt.utils.thin.log', MagicMock())
    def test_get_ext_tops_cfg_interpreter(self):
        '''
        Test thin.get_ext_tops interpreter configuration.

        :return:
        '''
        cfg = [
            {'namespace': {'path': '/foo',
                           'py-version': [2, 6],
                           'dependencies': {'jinja2': '',
                                            'yaml': '',
                                            'tornado': '',
                                            'msgpack': ''}}},
        ]
        with pytest.raises(salt.exceptions.SaltSystemExit) as err:
            thin.get_ext_tops(cfg)
        assert len(thin.log.warning.mock_calls) == 4
        assert sorted([x[1][1] for x in thin.log.warning.mock_calls]) == ['jinja2', 'msgpack', 'tornado', 'yaml']
        assert 'Module test has missing configuration' == thin.log.warning.mock_calls[0][1][0] % 'test'
<commit_msg>Mock os.path.isfile function<commit_after># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Bo Maryniuk <bo@suse.de>`
'''
from __future__ import absolute_import, print_function, unicode_literals
import datetime

from tests.support.unit import TestCase, skipIf
from tests.support.mock import (
    NO_MOCK,
    NO_MOCK_REASON,
    MagicMock,
    patch)

from salt.ext.six.moves import zip
from salt.ext import six

import salt.exceptions
from salt.utils import thin
import salt.utils.stringutils

try:
    import pytest
except ImportError:
    pytest = None

@skipIf(NO_MOCK, NO_MOCK_REASON)
@skipIf(pytest is None, 'PyTest is missing')
class SSHThinTestCase(TestCase):
    '''
    TestCase for SaltSSH-related parts.
    '''

    @patch('salt.exceptions.SaltSystemExit', Exception)
    @patch('salt.utils.thin.log', MagicMock())
    @patch('salt.utils.thin.os.path.isfile', MagicMock(return_value=False))
    def test_get_ext_tops_cfg_missing_dependencies(self):
        '''
        Test thin.get_ext_tops contains all required dependencies.

        :return:
        '''
        cfg = [
            {'namespace': {'py-version': [0, 0], 'path': '/foo', 'dependencies': []}},
        ]
        with pytest.raises(Exception) as err:
            thin.get_ext_tops(cfg)
        assert 'Missing dependencies' in str(err)
        assert thin.log.error.called
        assert 'Missing dependencies' in thin.log.error.call_args[0][0]
        assert 'jinja2, yaml, tornado, msgpack' in thin.log.error.call_args[0][0]

    @patch('salt.exceptions.SaltSystemExit', Exception)
    @patch('salt.utils.thin.log', MagicMock())
    @patch('salt.utils.thin.os.path.isfile', MagicMock(return_value=False))
    def test_get_ext_tops_cfg_missing_interpreter(self):
        '''
        Test thin.get_ext_tops contains interpreter configuration.

        :return:
        '''
        cfg = [
            {'namespace': {'path': '/foo',
                           'dependencies': []}},
        ]
        with pytest.raises(salt.exceptions.SaltSystemExit) as err:
            thin.get_ext_tops(cfg)
        assert 'missing specific locked Python version' in str(err)

    @patch('salt.exceptions.SaltSystemExit', Exception)
    @patch('salt.utils.thin.log', MagicMock())
    @patch('salt.utils.thin.os.path.isfile', MagicMock(return_value=False))
    def test_get_ext_tops_cfg_wrong_interpreter(self):
        '''
        Test thin.get_ext_tops contains correct interpreter configuration.

        :return:
        '''
        cfg = [
            {'namespace': {'path': '/foo',
                           'py-version': 2,
                           'dependencies': []}},
        ]
        with pytest.raises(salt.exceptions.SaltSystemExit) as err:
            thin.get_ext_tops(cfg)
        assert 'specific locked Python version should be a list of major/minor version' in str(err)

    @patch('salt.exceptions.SaltSystemExit', Exception)
    @patch('salt.utils.thin.log', MagicMock())
    @patch('salt.utils.thin.os.path.isfile', MagicMock(return_value=False))
    def test_get_ext_tops_cfg_interpreter(self):
        '''
        Test thin.get_ext_tops interpreter configuration.

        :return:
        '''
        cfg = [
            {'namespace': {'path': '/foo',
                           'py-version': [2, 6],
                           'dependencies': {'jinja2': '',
                                            'yaml': '',
                                            'tornado': '',
                                            'msgpack': ''}}},
        ]
        with pytest.raises(salt.exceptions.SaltSystemExit) as err:
            thin.get_ext_tops(cfg)
        assert len(thin.log.warning.mock_calls) == 4
        assert sorted([x[1][1] for x in thin.log.warning.mock_calls]) == ['jinja2', 'msgpack', 'tornado', 'yaml']
        assert 'Module test has missing configuration' == thin.log.warning.mock_calls[0][1][0] % 'test'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import (
    absolute_import, division, print_function, unicode_literals)

import mock
from django.contrib.auth.models import User
from django.test import TestCase
from django.test.utils import override_settings
from django_dynamic_fixture import get
from textclassifier.validators import ClassifierValidator

from readthedocs.projects.exceptions import ProjectSpamError
from readthedocs.projects.forms import (
    ProjectBasicsForm, ProjectExtraForm, TranslationForm, UpdateProjectForm)
from readthedocs.projects.models import Project

class TestProjectForms(TestCase):
    @mock.patch.object(ClassifierValidator, '__call__')
    def test_form_spam(self, mocked_validator):
        """Form description field fails spam validation."""
        mocked_validator.side_effect = ProjectSpamError

        data = {
            'description': 'foo',
            'documentation_type': 'sphinx',
            'language': 'en',
        }
        form = ProjectExtraForm(data)
        with self.assertRaises(ProjectSpamError):
            form.is_valid()

    def test_import_repo_url(self):
        """Validate different type of repository URLs on importing a Project."""

        common_urls = [
            # Invalid
            ('./path/to/relative/folder', False),
            ('../../path/to/relative/folder', False),
            ('../../path/to/@/folder', False),
            ('/path/to/local/folder', False),
            ('/path/to/@/folder', False),
            ('file:///path/to/local/folder', False),
            ('file:///path/to/@/folder', False),
            ('github.com/humitos/foo', False),
            ('https://github.com/|/foo', False),
            ('git://github.com/&&/foo', False),
            # Valid
            ('git://github.com/humitos/foo', True),
            ('http://github.com/humitos/foo', True),
            ('https://github.com/humitos/foo', True),
            ('http://gitlab.com/humitos/foo', True),
            ('http://bitbucket.com/humitos/foo', True),
            ('ftp://ftpserver.com/humitos/foo', True),
            ('ftps://ftpserver.com/humitos/foo', True),
            ('lp:zaraza', True),
        ]

        public_urls = [
            ('git@github.com:humitos/foo', False),
            ('ssh://git@github.com/humitos/foo', False),
            ('ssh+git://github.com/humitos/foo', False),
            ('strangeuser@bitbucket.org:strangeuser/readthedocs.git', False),
            ('user@one-ssh.domain.com:22/_ssh/docs', False),
        ] + common_urls

        private_urls = [
            ('git@github.com:humitos/foo', True),
            ('ssh://git@github.com/humitos/foo', True),
            ('ssh+git://github.com/humitos/foo', True),
            ('strangeuser@bitbucket.org:strangeuser/readthedocs.git', True),
            ('user@one-ssh.domain.com:22/_ssh/docs', True),
         ] + common_urls

        with override_settings(ALLOW_PRIVATE_REPOS=False):
            for url, valid in public_urls:
                initial = {
                    'name': 'foo',
                    'repo_type': 'git',
                    'repo': url,
                }
                form = ProjectBasicsForm(initial)
                self.assertEqual(form.is_valid(), valid, msg=url)

        with override_settings(ALLOW_PRIVATE_REPOS=True):
            for url, valid in private_urls:
                initial = {
                    'name': 'foo',
                    'repo_type': 'git',
                    'repo': url,
                }
                form = ProjectBasicsForm(initial)
                self.assertEqual(form.is_valid(), valid, msg=url)

    def test_empty_slug(self):
        initial = {
            'name': "''",
        }
        form = ProjectBasicsForm(initial)
        self.assertFalse(form.is_valid())

class TestTranslationForms(TestCase):

    def setUp(self):
        self.user_a = get(User)
        self.project_a_es = self.get_project(lang='es', users=[self.user_a])
        self.project_b_en = self.get_project(lang='en', users=[self.user_a])
        self.project_c_br = self.get_project(lang='br', users=[self.user_a])
        self.project_d_ar = self.get_project(lang='ar', users=[self.user_a])
        self.project_e_en = self.get_project(lang='en', users=[self.user_a])

        self.user_b = get(User)
        self.project_f_ar = self.get_project(lang='ar', users=[self.user_b])
        self.project_g_ga = self.get_project(lang='ga', users=[self.user_b])

        self.project_s_fr = self.get_project(
            lang='fr',
            users=[self.user_b, self.user_a]
        )

    def get_project(self, lang, users, **kwargs):
        return get(
            Project, language=lang, users=users,
            main_language_project=None, **kwargs
        )

    def test_list_only_owner_projects(self):
        form = TranslationForm(
            {'project': self.project_b_en.slug},
            parent=self.project_a_es,
            user=self.user_a,
        )
        self.assertTrue(form.is_valid())
        expected_projects = [
            self.project_b_en,
            self.project_c_br,
            self.project_d_ar,
            self.project_e_en,
            self.project_s_fr,
        ]
        self.assertEqual(
            {proj_slug for proj_slug, _ in form.fields['project'].choices},
            {project.slug for project in expected_projects}
        )

        form = TranslationForm(
            {'project': self.project_g_ga.slug},
            parent=self.project_f_ar,
            user=self.user_b,
        )
        self.assertTrue(form.is_valid())
        expected_projects = [
            self.project_g_ga,
            self.project_s_fr,
        ]
        self.assertEqual(
            {proj_slug for proj_slug, _ in form.fields['project'].choices},
            {project.slug for project in expected_projects}
        )

    def test_excludes_existing_translations(self):
        self.project_a_es.translations.add(self.project_b_en)
        self.project_a_es.translations.add(self.project_c_br)
        self.project_a_es.save()

        form = TranslationForm(
            {'project': self.project_d_ar.slug},
            parent=self.project_a_es,
            user=self.user_a,
        )
        self.assertTrue(form.is_valid())
        expected_projects = [
            self.project_d_ar,
            self.project_e_en,
            self.project_s_fr,
        ]
        self.assertEqual(
            {proj_slug for proj_slug, _ in form.fields['project'].choices},
            {project.slug for project in expected_projects}
        )

    def test_user_cant_add_other_user_project(self):
        form = TranslationForm(
            {'project': self.project_f_ar.slug},
            parent=self.project_b_en,
            user=self.user_a,
        )
        self.assertFalse(form.is_valid())
        self.assertIn(
            'Select a valid choice',
            ''.join(form.errors['project'])
        )
        self.assertNotIn(
            self.project_f_ar,
            [proj_slug for proj_slug, _ in form.fields['project'].choices]
        )

    def test_user_cant_add_project_with_same_lang(self):
        form = TranslationForm(
            {'project': self.project_b_en.slug},
            parent=self.project_e_en,
            user=self.user_a,
        )
        self.assertFalse(form.is_valid())
        self.assertIn(
            'Both projects can not have the same language (English).',
            ''.join(form.errors['project'])
        )

    def test_user_cant_add_project_with_same_lang_of_other_translation(self):
        self.project_a_es.translations.add(self.project_b_en)
        self.project_a_es.save()

        form = TranslationForm(
            {'project': self.project_e_en.slug},
            parent=self.project_a_es,
            user=self.user_a,
        )
        self.assertFalse(form.is_valid())
        self.assertIn(
            'This project already has a translation for English.',
            ''.join(form.errors['project'])
        )

    def test_no_nesting_translation(self):
        self.project_a_es.translations.add(self.project_b_en)
        self.project_a_es.save()

        form = TranslationForm(
            {'project': self.project_b_en.slug},
            parent=self.project_c_br,
            user=self.user_a,
        )
        self.assertFalse(form.is_valid())
        self.assertIn(
            'Select a valid choice',
            ''.join(form.errors['project'])
        )

    def test_no_nesting_translation_case_2(self):
        self.project_a_es.translations.add(self.project_b_en)
        self.project_a_es.save()

        form = TranslationForm(
            {'project': self.project_a_es.slug},
            parent=self.project_c_br,
            user=self.user_a,
        )
        self.assertFalse(form.is_valid())
        self.assertIn(
            'A project with existing translations can not',
            ''.join(form.errors['project'])
        )

    def test_not_already_translation(self):
        self.project_a_es.translations.add(self.project_b_en)
        self.project_a_es.save()

        form = TranslationForm(
            {'project': self.project_c_br.slug},
            parent=self.project_b_en,
            user=self.user_a,
        )
        self.assertFalse(form.is_valid())
        self.assertIn(
            'is already a translation',
            ''.join(form.errors['project'])
        )

    def test_cant_change_language_to_translation_lang(self):
        self.project_a_es.translations.add(self.project_b_en)
        self.project_a_es.translations.add(self.project_c_br)
        self.project_a_es.save()

        # Parent project tries to change lang
        form = UpdateProjectForm(
            {
                'documentation_type': 'sphinx',
                'language': 'en',
            },
            instance=self.project_a_es
        )
        self.assertFalse(form.is_valid())
        self.assertIn(
            'There is already a "en" translation',
            ''.join(form.errors['language'])
        )

        # Translation tries to change lang
        form = UpdateProjectForm(
            {
                'documentation_type': 'sphinx',
                'language': 'es',
            },
            instance=self.project_b_en
        )
        self.assertFalse(form.is_valid())
        self.assertIn(
            'There is already a "es" translation',
            ''.join(form.errors['language'])
        )

        # Translation tries to change lang
        # to the same as its sibling
        form = UpdateProjectForm(
            {
                'documentation_type': 'sphinx',
                'language': 'br',
            },
            instance=self.project_b_en
        )
        self.assertFalse(form.is_valid())
        self.assertIn(
            'There is already a "br" translation',
            ''.join(form.errors['language'])
        )

    def test_can_change_language_to_self_lang(self):
        self.project_a_es.translations.add(self.project_b_en)
        self.project_a_es.translations.add(self.project_c_br)
        self.project_a_es.save()

        # Parent project tries to change lang
        form = UpdateProjectForm(
            {
                'repo': 'https://github.com/test/test',
                'repo_type': self.project_a_es.repo_type,
                'name': self.project_a_es.name,
                'documentation_type': 'sphinx',
                'language': 'es',
            },
            instance=self.project_a_es
        )
        self.assertTrue(form.is_valid())

        # Translation tries to change lang
        form = UpdateProjectForm(
            {
                'repo': 'https://github.com/test/test',
                'repo_type': self.project_b_en.repo_type,
                'name': self.project_b_en.name,
                'documentation_type': 'sphinx',
                'language': 'en',
            },
            instance=self.project_b_en
        )
        self.assertTrue(form.is_valid())
<commit_msg>Make test more robust<commit_after># -*- coding: utf-8 -*-

from __future__ import (
    absolute_import, division, print_function, unicode_literals)

import mock
from django.contrib.auth.models import User
from django.test import TestCase
from django.test.utils import override_settings
from django_dynamic_fixture import get
from textclassifier.validators import ClassifierValidator

from readthedocs.projects.exceptions import ProjectSpamError
from readthedocs.projects.forms import (
    ProjectBasicsForm, ProjectExtraForm, TranslationForm, UpdateProjectForm)
from readthedocs.projects.models import Project

class TestProjectForms(TestCase):
    @mock.patch.object(ClassifierValidator, '__call__')
    def test_form_spam(self, mocked_validator):
        """Form description field fails spam validation."""
        mocked_validator.side_effect = ProjectSpamError

        data = {
            'description': 'foo',
            'documentation_type': 'sphinx',
            'language': 'en',
        }
        form = ProjectExtraForm(data)
        with self.assertRaises(ProjectSpamError):
            form.is_valid()

    def test_import_repo_url(self):
        """Validate different type of repository URLs on importing a Project."""

        common_urls = [
            # Invalid
            ('./path/to/relative/folder', False),
            ('../../path/to/relative/folder', False),
            ('../../path/to/@/folder', False),
            ('/path/to/local/folder', False),
            ('/path/to/@/folder', False),
            ('file:///path/to/local/folder', False),
            ('file:///path/to/@/folder', False),
            ('github.com/humitos/foo', False),
            ('https://github.com/|/foo', False),
            ('git://github.com/&&/foo', False),
            # Valid
            ('git://github.com/humitos/foo', True),
            ('http://github.com/humitos/foo', True),
            ('https://github.com/humitos/foo', True),
            ('http://gitlab.com/humitos/foo', True),
            ('http://bitbucket.com/humitos/foo', True),
            ('ftp://ftpserver.com/humitos/foo', True),
            ('ftps://ftpserver.com/humitos/foo', True),
            ('lp:zaraza', True),
        ]

        public_urls = [
            ('git@github.com:humitos/foo', False),
            ('ssh://git@github.com/humitos/foo', False),
            ('ssh+git://github.com/humitos/foo', False),
            ('strangeuser@bitbucket.org:strangeuser/readthedocs.git', False),
            ('user@one-ssh.domain.com:22/_ssh/docs', False),
        ] + common_urls

        private_urls = [
            ('git@github.com:humitos/foo', True),
            ('ssh://git@github.com/humitos/foo', True),
            ('ssh+git://github.com/humitos/foo', True),
            ('strangeuser@bitbucket.org:strangeuser/readthedocs.git', True),
            ('user@one-ssh.domain.com:22/_ssh/docs', True),
         ] + common_urls

        with override_settings(ALLOW_PRIVATE_REPOS=False):
            for url, valid in public_urls:
                initial = {
                    'name': 'foo',
                    'repo_type': 'git',
                    'repo': url,
                }
                form = ProjectBasicsForm(initial)
                self.assertEqual(form.is_valid(), valid, msg=url)

        with override_settings(ALLOW_PRIVATE_REPOS=True):
            for url, valid in private_urls:
                initial = {
                    'name': 'foo',
                    'repo_type': 'git',
                    'repo': url,
                }
                form = ProjectBasicsForm(initial)
                self.assertEqual(form.is_valid(), valid, msg=url)

    def test_empty_slug(self):
        initial = {
            'name': "''",
            'repo_type': 'git',
            'repo': 'https://github.com/user/repository',
        }
        form = ProjectBasicsForm(initial)
        self.assertFalse(form.is_valid())
        self.assertIn('name', form.errors)

class TestTranslationForms(TestCase):

    def setUp(self):
        self.user_a = get(User)
        self.project_a_es = self.get_project(lang='es', users=[self.user_a])
        self.project_b_en = self.get_project(lang='en', users=[self.user_a])
        self.project_c_br = self.get_project(lang='br', users=[self.user_a])
        self.project_d_ar = self.get_project(lang='ar', users=[self.user_a])
        self.project_e_en = self.get_project(lang='en', users=[self.user_a])

        self.user_b = get(User)
        self.project_f_ar = self.get_project(lang='ar', users=[self.user_b])
        self.project_g_ga = self.get_project(lang='ga', users=[self.user_b])

        self.project_s_fr = self.get_project(
            lang='fr',
            users=[self.user_b, self.user_a]
        )

    def get_project(self, lang, users, **kwargs):
        return get(
            Project, language=lang, users=users,
            main_language_project=None, **kwargs
        )

    def test_list_only_owner_projects(self):
        form = TranslationForm(
            {'project': self.project_b_en.slug},
            parent=self.project_a_es,
            user=self.user_a,
        )
        self.assertTrue(form.is_valid())
        expected_projects = [
            self.project_b_en,
            self.project_c_br,
            self.project_d_ar,
            self.project_e_en,
            self.project_s_fr,
        ]
        self.assertEqual(
            {proj_slug for proj_slug, _ in form.fields['project'].choices},
            {project.slug for project in expected_projects}
        )

        form = TranslationForm(
            {'project': self.project_g_ga.slug},
            parent=self.project_f_ar,
            user=self.user_b,
        )
        self.assertTrue(form.is_valid())
        expected_projects = [
            self.project_g_ga,
            self.project_s_fr,
        ]
        self.assertEqual(
            {proj_slug for proj_slug, _ in form.fields['project'].choices},
            {project.slug for project in expected_projects}
        )

    def test_excludes_existing_translations(self):
        self.project_a_es.translations.add(self.project_b_en)
        self.project_a_es.translations.add(self.project_c_br)
        self.project_a_es.save()

        form = TranslationForm(
            {'project': self.project_d_ar.slug},
            parent=self.project_a_es,
            user=self.user_a,
        )
        self.assertTrue(form.is_valid())
        expected_projects = [
            self.project_d_ar,
            self.project_e_en,
            self.project_s_fr,
        ]
        self.assertEqual(
            {proj_slug for proj_slug, _ in form.fields['project'].choices},
            {project.slug for project in expected_projects}
        )

    def test_user_cant_add_other_user_project(self):
        form = TranslationForm(
            {'project': self.project_f_ar.slug},
            parent=self.project_b_en,
            user=self.user_a,
        )
        self.assertFalse(form.is_valid())
        self.assertIn(
            'Select a valid choice',
            ''.join(form.errors['project'])
        )
        self.assertNotIn(
            self.project_f_ar,
            [proj_slug for proj_slug, _ in form.fields['project'].choices]
        )

    def test_user_cant_add_project_with_same_lang(self):
        form = TranslationForm(
            {'project': self.project_b_en.slug},
            parent=self.project_e_en,
            user=self.user_a,
        )
        self.assertFalse(form.is_valid())
        self.assertIn(
            'Both projects can not have the same language (English).',
            ''.join(form.errors['project'])
        )

    def test_user_cant_add_project_with_same_lang_of_other_translation(self):
        self.project_a_es.translations.add(self.project_b_en)
        self.project_a_es.save()

        form = TranslationForm(
            {'project': self.project_e_en.slug},
            parent=self.project_a_es,
            user=self.user_a,
        )
        self.assertFalse(form.is_valid())
        self.assertIn(
            'This project already has a translation for English.',
            ''.join(form.errors['project'])
        )

    def test_no_nesting_translation(self):
        self.project_a_es.translations.add(self.project_b_en)
        self.project_a_es.save()

        form = TranslationForm(
            {'project': self.project_b_en.slug},
            parent=self.project_c_br,
            user=self.user_a,
        )
        self.assertFalse(form.is_valid())
        self.assertIn(
            'Select a valid choice',
            ''.join(form.errors['project'])
        )

    def test_no_nesting_translation_case_2(self):
        self.project_a_es.translations.add(self.project_b_en)
        self.project_a_es.save()

        form = TranslationForm(
            {'project': self.project_a_es.slug},
            parent=self.project_c_br,
            user=self.user_a,
        )
        self.assertFalse(form.is_valid())
        self.assertIn(
            'A project with existing translations can not',
            ''.join(form.errors['project'])
        )

    def test_not_already_translation(self):
        self.project_a_es.translations.add(self.project_b_en)
        self.project_a_es.save()

        form = TranslationForm(
            {'project': self.project_c_br.slug},
            parent=self.project_b_en,
            user=self.user_a,
        )
        self.assertFalse(form.is_valid())
        self.assertIn(
            'is already a translation',
            ''.join(form.errors['project'])
        )

    def test_cant_change_language_to_translation_lang(self):
        self.project_a_es.translations.add(self.project_b_en)
        self.project_a_es.translations.add(self.project_c_br)
        self.project_a_es.save()

        # Parent project tries to change lang
        form = UpdateProjectForm(
            {
                'documentation_type': 'sphinx',
                'language': 'en',
            },
            instance=self.project_a_es
        )
        self.assertFalse(form.is_valid())
        self.assertIn(
            'There is already a "en" translation',
            ''.join(form.errors['language'])
        )

        # Translation tries to change lang
        form = UpdateProjectForm(
            {
                'documentation_type': 'sphinx',
                'language': 'es',
            },
            instance=self.project_b_en
        )
        self.assertFalse(form.is_valid())
        self.assertIn(
            'There is already a "es" translation',
            ''.join(form.errors['language'])
        )

        # Translation tries to change lang
        # to the same as its sibling
        form = UpdateProjectForm(
            {
                'documentation_type': 'sphinx',
                'language': 'br',
            },
            instance=self.project_b_en
        )
        self.assertFalse(form.is_valid())
        self.assertIn(
            'There is already a "br" translation',
            ''.join(form.errors['language'])
        )

    def test_can_change_language_to_self_lang(self):
        self.project_a_es.translations.add(self.project_b_en)
        self.project_a_es.translations.add(self.project_c_br)
        self.project_a_es.save()

        # Parent project tries to change lang
        form = UpdateProjectForm(
            {
                'repo': 'https://github.com/test/test',
                'repo_type': self.project_a_es.repo_type,
                'name': self.project_a_es.name,
                'documentation_type': 'sphinx',
                'language': 'es',
            },
            instance=self.project_a_es
        )
        self.assertTrue(form.is_valid())

        # Translation tries to change lang
        form = UpdateProjectForm(
            {
                'repo': 'https://github.com/test/test',
                'repo_type': self.project_b_en.repo_type,
                'name': self.project_b_en.name,
                'documentation_type': 'sphinx',
                'language': 'en',
            },
            instance=self.project_b_en
        )
        self.assertTrue(form.is_valid())
<|endoftext|>