<commit_before># -*- coding: utf-8 -*-

# FOGLAMP_BEGIN
# See: http://foglamp.readthedocs.io/
# FOGLAMP_END

import datetime
import uuid

from aiohttp import web

from foglamp.common import utils
from foglamp.common import logger
from foglamp.common.configuration_manager import ConfigurationManager
from foglamp.common.storage_client.payload_builder import PayloadBuilder
from foglamp.common.storage_client.exceptions import StorageServerError

from foglamp.services.core import server
from foglamp.services.core import connect
from foglamp.services.core.scheduler.entities import Schedule, TimedSchedule, IntervalSchedule, ManualSchedule
from foglamp.services.core.api import utils as apiutils

__author__ = "Amarendra K Sinha"
__copyright__ = "Copyright (c) 2018 OSIsoft, LLC"
__license__ = "Apache 2.0"
__version__ = "${VERSION}"

_help = """
    -------------------------------------------------------------------------------
    | POST                 | /foglamp/scheduled/task                              |
    | DELETE               | /foglamp/scheduled/task/{task_name}                  |
    -------------------------------------------------------------------------------
"""

_logger = logger.setup()

async def add_task(request):
    """ Create a new task to run a specific plugin

    :Example:
     curl -X POST http://localhost:8081/foglamp/scheduled/task -d
     '{
        "name": "North Readings to PI",
        "plugin": "pi_server",
        "type": "north",
        "schedule_type": 3,
        "schedule_day": 0,
        "schedule_time": 0,
        "schedule_repeat": 30,
        "schedule_enabled": true
     }'

     curl -sX POST http://localhost:8081/foglamp/scheduled/task -d
     '{"name": "PI-2",
     "plugin": "pi_server",
     "type": "north",
     "schedule_type": 3,
     "schedule_day": 0,
     "schedule_time": 0,
     "schedule_repeat": 30,
     "schedule_enabled": true,
     "config": {
        "producerToken": {"value": "uid=180905062754237&sig=kx5l+"},
        "URL": {"value": "https://10.2.5.22:5460/ingress/messages"}}}'
    """

    try:
        data = await request.json()
        if not isinstance(data, dict):
            raise ValueError('Data payload must be a valid JSON')

        name = data.get('name', None)
        plugin = data.get('plugin', None)
        task_type = data.get('type', None)

        schedule_type = data.get('schedule_type', None)
        schedule_day = data.get('schedule_day', None)
        schedule_time = data.get('schedule_time', None)
        schedule_repeat = data.get('schedule_repeat', None)
        enabled = data.get('schedule_enabled', None)
        config = data.get('config', None)

        if name is None:
            raise web.HTTPBadRequest(reason='Missing name property in payload.')
        if plugin is None:
            raise web.HTTPBadRequest(reason='Missing plugin property in payload.')
        if task_type is None:
            raise web.HTTPBadRequest(reason='Missing type property in payload.')
        if utils.check_reserved(name) is False:
            raise web.HTTPBadRequest(reason='Invalid name property in payload.')
        if utils.check_reserved(plugin) is False:
            raise web.HTTPBadRequest(reason='Invalid plugin property in payload.')
        if task_type not in ['north']:
            raise web.HTTPBadRequest(reason='Only north type is supported.')

        if schedule_type is None:
            raise web.HTTPBadRequest(reason='schedule_type is mandatory')
        if not isinstance(schedule_type, int) and not schedule_type.isdigit():
            raise web.HTTPBadRequest(reason='Error in schedule_type: {}'.format(schedule_type))
        if int(schedule_type) not in list(Schedule.Type):
            raise web.HTTPBadRequest(reason='schedule_type error: {}'.format(schedule_type))
        if int(schedule_type) == Schedule.Type.STARTUP:
            raise web.HTTPBadRequest(reason='schedule_type cannot be STARTUP: {}'.format(schedule_type))

        schedule_type = int(schedule_type)

        if schedule_day is not None:
            if isinstance(schedule_day, float) or (isinstance(schedule_day, str) and (schedule_day.strip() != "" and not schedule_day.isdigit())):
                raise web.HTTPBadRequest(reason='Error in schedule_day: {}'.format(schedule_day))
        else:
            schedule_day = int(schedule_day) if schedule_day is not None else None

        if schedule_time is not None and (not isinstance(schedule_time, int) and not schedule_time.isdigit()):
            raise web.HTTPBadRequest(reason='Error in schedule_time: {}'.format(schedule_time))
        else:
            schedule_time = int(schedule_time) if schedule_time is not None else None

        if schedule_repeat is not None and (not isinstance(schedule_repeat, int) and not schedule_repeat.isdigit()):
            raise web.HTTPBadRequest(reason='Error in schedule_repeat: {}'.format(schedule_repeat))
        else:
            schedule_repeat = int(schedule_repeat) if schedule_repeat is not None else None

        if schedule_type == Schedule.Type.TIMED:
            if not schedule_time:
                raise web.HTTPBadRequest(reason='schedule_time cannot be empty/None for TIMED schedule.')
            if schedule_day is not None and (schedule_day < 1 or schedule_day > 7):
                raise web.HTTPBadRequest(reason='schedule_day {} must either be None or must be an integer, 1(Monday) '
                                                'to 7(Sunday).'.format(schedule_day))
            if schedule_time < 0 or schedule_time > 86399:
                raise web.HTTPBadRequest(reason='schedule_time {} must be an integer and in range 0-86399.'.format(schedule_time))

        if schedule_type == Schedule.Type.INTERVAL:
            if schedule_repeat is None:
                raise web.HTTPBadRequest(reason='schedule_repeat {} is required for INTERVAL schedule_type.'.format(schedule_repeat))
            elif not isinstance(schedule_repeat, int):
                raise web.HTTPBadRequest(reason='schedule_repeat {} must be an integer.'.format(schedule_repeat))

        if enabled is not None:
            if enabled not in ['true', 'false', True, False]:
                raise web.HTTPBadRequest(reason='Only "true", "false", true, false are allowed for value of enabled.')
        is_enabled = True if ((type(enabled) is str and enabled.lower() in ['true']) or (
            (type(enabled) is bool and enabled is True))) else False

        # Check if a valid plugin has been provided
        try:
            # "plugin_module_path" is fixed by design. It is MANDATORY to keep the plugin in the exactly similar named
            # folder, within the plugin_module_path.
            # if multiple plugin with same name are found, then python plugin import will be tried first
            plugin_module_path = "foglamp.plugins.{}".format(task_type)
            import_file_name = "{path}.{dir}.{file}".format(path=plugin_module_path, dir=plugin, file=plugin)
            _plugin = __import__(import_file_name, fromlist=[''])

            script = '["tasks/north"]'
            # Fetch configuration from the configuration defined in the plugin
            plugin_info = _plugin.plugin_info()
            if plugin_info['type'] != task_type:
                msg = "Plugin of {} type is not supported".format(plugin_info['type'])
                _logger.exception(msg)
                return web.HTTPBadRequest(reason=msg)
            plugin_config = plugin_info['config']
            process_name = 'north'
        except ImportError as ex:
            # Checking for C-type plugins
            script = '["tasks/north_c"]'
            plugin_info = apiutils.get_plugin_info(plugin)
            if plugin_info['type'] != task_type:
                msg = "Plugin of {} type is not supported".format(plugin_info['type'])
                _logger.exception(msg)
                return web.HTTPBadRequest(reason=msg)
            plugin_config = plugin_info['config']
            process_name = 'north_c'
            if not plugin_config:
                _logger.exception("Plugin %s import problem from path %s. %s", plugin, plugin_module_path, str(ex))
                raise web.HTTPNotFound(reason='Plugin "{}" import problem from path "{}"'.format(plugin, plugin_module_path))
        except Exception as ex:
            _logger.exception("Failed to fetch plugin configuration. %s", str(ex))
            raise web.HTTPInternalServerError(reason='Failed to fetch plugin configuration.')

        storage = connect.get_storage_async()

        # Check that the schedule name is not already registered
        count = await check_schedules(storage, name)
        if count != 0:
            raise web.HTTPBadRequest(reason='A north instance with this name already exists')

        # Check that the process name is not already registered
        count = await check_scheduled_processes(storage, process_name)
        if count == 0:  # Create the scheduled process entry for the new task
            payload = PayloadBuilder().INSERT(name=process_name, script=script).payload()
            try:
                res = await storage.insert_into_tbl("scheduled_processes", payload)
            except StorageServerError as ex:
                _logger.exception("Failed to create scheduled process. %s", ex.error)
                raise web.HTTPInternalServerError(reason='Failed to create north instance.')
            except Exception as ex:
                _logger.exception("Failed to create scheduled process. %s", ex)
                raise web.HTTPInternalServerError(reason='Failed to create north instance.')

        # If successful then create a configuration entry from plugin configuration
        try:
            # Create a configuration category from the configuration defined in the plugin
            category_desc = plugin_config['plugin']['description']
            config_mgr = ConfigurationManager(storage)
            await config_mgr.create_category(category_name=name,
                                             category_description=category_desc,
                                             category_value=plugin_config,
                                             keep_original_items=True)
            # Create the parent category for all North tasks
            await config_mgr.create_category("North", {}, 'North tasks', True)
            await config_mgr.create_child_category("North", [name])

            # If config is in POST data, then update the value for each config item
            if config is not None:
                if not isinstance(config, dict):
                    raise ValueError('Config must be a JSON object')
                for k, v in config.items():
                    await config_mgr.set_category_item_value_entry(name, k, v['value'])
        except Exception as ex:
            await delete_configuration(storage, name)  # Revert configuration entry
            await delete_parent_child_configuration(storage, name)
            _logger.exception("Failed to create plugin configuration. %s", str(ex))
            raise web.HTTPInternalServerError(reason='Failed to create plugin configuration.')

        # If all successful then lastly add a schedule to run the new task at startup
        try:
            schedule = TimedSchedule() if schedule_type == Schedule.Type.TIMED else \
                       IntervalSchedule() if schedule_type == Schedule.Type.INTERVAL else \
                       ManualSchedule()
            schedule.name = name
            schedule.process_name = process_name
            schedule.day = schedule_day
            m, s = divmod(schedule_time if schedule_time is not None else 0, 60)
            h, m = divmod(m, 60)
            schedule.time = datetime.time().replace(hour=h, minute=m, second=s)
            schedule.repeat = datetime.timedelta(seconds=schedule_repeat if schedule_repeat is not None else 0)
            schedule.exclusive = True
            schedule.enabled = False  # if "enabled" is supplied, it gets activated in save_schedule() via is_enabled flag

            # Save schedule
            await server.Server.scheduler.save_schedule(schedule, is_enabled)
            schedule = await server.Server.scheduler.get_schedule_by_name(name)
        except StorageServerError as ex:
            await delete_configuration(storage, name)  # Revert configuration entry
            await delete_parent_child_configuration(storage, name)
            _logger.exception("Failed to create schedule. %s", ex.error)
            raise web.HTTPInternalServerError(reason='Failed to create north instance.')
        except Exception as ex:
            await delete_configuration(storage, name)  # Revert configuration entry
            await delete_parent_child_configuration(storage, name)
            _logger.exception("Failed to create schedule. %s", str(ex))
            raise web.HTTPInternalServerError(reason='Failed to create north instance.')

    except ValueError as e:
        raise web.HTTPBadRequest(reason=str(e))
    else:
        return web.json_response({'name': name, 'id': str(schedule.schedule_id)})

async def delete_task(request):
    """ Delete a north plugin instance task

        :Example:
            curl -X DELETE http://localhost:8081/foglamp/scheduled/task/<task name>
    """
    try:
        north_instance = request.match_info.get('task_name', None)

        if north_instance is None or north_instance.strip() == '':
            raise web.HTTPBadRequest(reason='Missing task_name in requested URL')

        storage = connect.get_storage_async()

        result = await get_schedule(storage, north_instance)
        if result['count'] == 0:
            raise web.HTTPBadRequest(reason='A north instance task with this name does not exist.')

        north_instance_schedule = result['rows'][0]
        sch_id = uuid.UUID(north_instance_schedule['id'])
        if north_instance_schedule['enabled'].lower() == 't':
            # disable it
            await server.Server.scheduler.disable_schedule(sch_id)
        # delete it
        await server.Server.scheduler.delete_schedule(sch_id)

        # delete all configuration for the service name
        await delete_configuration(storage, north_instance)

        # delete statistics key
        await delete_statistics_key(storage, north_instance)

    except Exception as ex:
        raise web.HTTPInternalServerError(reason=ex)
    else:
        return web.json_response({'result': 'North instance {} deleted successfully.'.format(north_instance)})

async def get_schedule(storage, schedule_name):
    payload = PayloadBuilder().SELECT(["id", "enabled"]).WHERE(['schedule_name', '=', schedule_name]).payload()
    result = await storage.query_tbl_with_payload('schedules', payload)
    return result

async def check_scheduled_processes(storage, process_name):
    payload = PayloadBuilder().SELECT("name").WHERE(['name', '=', process_name]).payload()
    result = await storage.query_tbl_with_payload('scheduled_processes', payload)
    return result['count']

async def check_schedules(storage, schedule_name):
    payload = PayloadBuilder().SELECT("schedule_name").WHERE(['schedule_name', '=', schedule_name]).payload()
    result = await storage.query_tbl_with_payload('schedules', payload)
    return result['count']

async def delete_configuration(storage, key):
    payload = PayloadBuilder().WHERE(['key', '=', key]).payload()
    await storage.delete_from_tbl('configuration', payload)
    # Removed key from configuration cache
    config_mgr = ConfigurationManager(storage)
    config_mgr._cacheManager.remove(key)

async def delete_parent_child_configuration(storage, key):
    payload = PayloadBuilder().WHERE(['parent', '=', "North"]).AND_WHERE(['child', '=', key]).payload()
    await storage.delete_from_tbl('category_children', payload)

async def delete_statistics_key(storage, key):
    payload = PayloadBuilder().WHERE(['key', '=', key]).payload()
    await storage.delete_from_tbl('statistics', payload)
<commit_msg>delete North relationship entry from category_children table<commit_after># -*- coding: utf-8 -*-

# FOGLAMP_BEGIN
# See: http://foglamp.readthedocs.io/
# FOGLAMP_END

import datetime
import uuid

from aiohttp import web

from foglamp.common import utils
from foglamp.common import logger
from foglamp.common.configuration_manager import ConfigurationManager
from foglamp.common.storage_client.payload_builder import PayloadBuilder
from foglamp.common.storage_client.exceptions import StorageServerError

from foglamp.services.core import server
from foglamp.services.core import connect
from foglamp.services.core.scheduler.entities import Schedule, TimedSchedule, IntervalSchedule, ManualSchedule
from foglamp.services.core.api import utils as apiutils

__author__ = "Amarendra K Sinha"
__copyright__ = "Copyright (c) 2018 OSIsoft, LLC"
__license__ = "Apache 2.0"
__version__ = "${VERSION}"

_help = """
    -------------------------------------------------------------------------------
    | POST                 | /foglamp/scheduled/task                              |
    | DELETE               | /foglamp/scheduled/task/{task_name}                  |
    -------------------------------------------------------------------------------
"""

_logger = logger.setup()

async def add_task(request):
    """ Create a new task to run a specific plugin

    :Example:
     curl -X POST http://localhost:8081/foglamp/scheduled/task -d
     '{
        "name": "North Readings to PI",
        "plugin": "pi_server",
        "type": "north",
        "schedule_type": 3,
        "schedule_day": 0,
        "schedule_time": 0,
        "schedule_repeat": 30,
        "schedule_enabled": true
     }'

     curl -sX POST http://localhost:8081/foglamp/scheduled/task -d
     '{"name": "PI-2",
     "plugin": "pi_server",
     "type": "north",
     "schedule_type": 3,
     "schedule_day": 0,
     "schedule_time": 0,
     "schedule_repeat": 30,
     "schedule_enabled": true,
     "config": {
        "producerToken": {"value": "uid=180905062754237&sig=kx5l+"},
        "URL": {"value": "https://10.2.5.22:5460/ingress/messages"}}}'
    """

    try:
        data = await request.json()
        if not isinstance(data, dict):
            raise ValueError('Data payload must be a valid JSON')

        name = data.get('name', None)
        plugin = data.get('plugin', None)
        task_type = data.get('type', None)

        schedule_type = data.get('schedule_type', None)
        schedule_day = data.get('schedule_day', None)
        schedule_time = data.get('schedule_time', None)
        schedule_repeat = data.get('schedule_repeat', None)
        enabled = data.get('schedule_enabled', None)
        config = data.get('config', None)

        if name is None:
            raise web.HTTPBadRequest(reason='Missing name property in payload.')
        if plugin is None:
            raise web.HTTPBadRequest(reason='Missing plugin property in payload.')
        if task_type is None:
            raise web.HTTPBadRequest(reason='Missing type property in payload.')
        if utils.check_reserved(name) is False:
            raise web.HTTPBadRequest(reason='Invalid name property in payload.')
        if utils.check_reserved(plugin) is False:
            raise web.HTTPBadRequest(reason='Invalid plugin property in payload.')
        if task_type not in ['north']:
            raise web.HTTPBadRequest(reason='Only north type is supported.')

        if schedule_type is None:
            raise web.HTTPBadRequest(reason='schedule_type is mandatory')
        if not isinstance(schedule_type, int) and not schedule_type.isdigit():
            raise web.HTTPBadRequest(reason='Error in schedule_type: {}'.format(schedule_type))
        if int(schedule_type) not in list(Schedule.Type):
            raise web.HTTPBadRequest(reason='schedule_type error: {}'.format(schedule_type))
        if int(schedule_type) == Schedule.Type.STARTUP:
            raise web.HTTPBadRequest(reason='schedule_type cannot be STARTUP: {}'.format(schedule_type))

        schedule_type = int(schedule_type)

        if schedule_day is not None:
            if isinstance(schedule_day, float) or (isinstance(schedule_day, str) and (schedule_day.strip() != "" and not schedule_day.isdigit())):
                raise web.HTTPBadRequest(reason='Error in schedule_day: {}'.format(schedule_day))
        else:
            schedule_day = int(schedule_day) if schedule_day is not None else None

        if schedule_time is not None and (not isinstance(schedule_time, int) and not schedule_time.isdigit()):
            raise web.HTTPBadRequest(reason='Error in schedule_time: {}'.format(schedule_time))
        else:
            schedule_time = int(schedule_time) if schedule_time is not None else None

        if schedule_repeat is not None and (not isinstance(schedule_repeat, int) and not schedule_repeat.isdigit()):
            raise web.HTTPBadRequest(reason='Error in schedule_repeat: {}'.format(schedule_repeat))
        else:
            schedule_repeat = int(schedule_repeat) if schedule_repeat is not None else None

        if schedule_type == Schedule.Type.TIMED:
            if not schedule_time:
                raise web.HTTPBadRequest(reason='schedule_time cannot be empty/None for TIMED schedule.')
            if schedule_day is not None and (schedule_day < 1 or schedule_day > 7):
                raise web.HTTPBadRequest(reason='schedule_day {} must either be None or must be an integer, 1(Monday) '
                                                'to 7(Sunday).'.format(schedule_day))
            if schedule_time < 0 or schedule_time > 86399:
                raise web.HTTPBadRequest(reason='schedule_time {} must be an integer and in range 0-86399.'.format(schedule_time))

        if schedule_type == Schedule.Type.INTERVAL:
            if schedule_repeat is None:
                raise web.HTTPBadRequest(reason='schedule_repeat {} is required for INTERVAL schedule_type.'.format(schedule_repeat))
            elif not isinstance(schedule_repeat, int):
                raise web.HTTPBadRequest(reason='schedule_repeat {} must be an integer.'.format(schedule_repeat))

        if enabled is not None:
            if enabled not in ['true', 'false', True, False]:
                raise web.HTTPBadRequest(reason='Only "true", "false", true, false are allowed for value of enabled.')
        is_enabled = True if ((type(enabled) is str and enabled.lower() in ['true']) or (
            (type(enabled) is bool and enabled is True))) else False

        # Check if a valid plugin has been provided
        try:
            # "plugin_module_path" is fixed by design. It is MANDATORY to keep the plugin in the exactly similar named
            # folder, within the plugin_module_path.
            # if multiple plugin with same name are found, then python plugin import will be tried first
            plugin_module_path = "foglamp.plugins.{}".format(task_type)
            import_file_name = "{path}.{dir}.{file}".format(path=plugin_module_path, dir=plugin, file=plugin)
            _plugin = __import__(import_file_name, fromlist=[''])

            script = '["tasks/north"]'
            # Fetch configuration from the configuration defined in the plugin
            plugin_info = _plugin.plugin_info()
            if plugin_info['type'] != task_type:
                msg = "Plugin of {} type is not supported".format(plugin_info['type'])
                _logger.exception(msg)
                return web.HTTPBadRequest(reason=msg)
            plugin_config = plugin_info['config']
            process_name = 'north'
        except ImportError as ex:
            # Checking for C-type plugins
            script = '["tasks/north_c"]'
            plugin_info = apiutils.get_plugin_info(plugin)
            if plugin_info['type'] != task_type:
                msg = "Plugin of {} type is not supported".format(plugin_info['type'])
                _logger.exception(msg)
                return web.HTTPBadRequest(reason=msg)
            plugin_config = plugin_info['config']
            process_name = 'north_c'
            if not plugin_config:
                _logger.exception("Plugin %s import problem from path %s. %s", plugin, plugin_module_path, str(ex))
                raise web.HTTPNotFound(reason='Plugin "{}" import problem from path "{}"'.format(plugin, plugin_module_path))
        except Exception as ex:
            _logger.exception("Failed to fetch plugin configuration. %s", str(ex))
            raise web.HTTPInternalServerError(reason='Failed to fetch plugin configuration.')

        storage = connect.get_storage_async()

        # Check that the schedule name is not already registered
        count = await check_schedules(storage, name)
        if count != 0:
            raise web.HTTPBadRequest(reason='A north instance with this name already exists')

        # Check that the process name is not already registered
        count = await check_scheduled_processes(storage, process_name)
        if count == 0:  # Create the scheduled process entry for the new task
            payload = PayloadBuilder().INSERT(name=process_name, script=script).payload()
            try:
                res = await storage.insert_into_tbl("scheduled_processes", payload)
            except StorageServerError as ex:
                _logger.exception("Failed to create scheduled process. %s", ex.error)
                raise web.HTTPInternalServerError(reason='Failed to create north instance.')
            except Exception as ex:
                _logger.exception("Failed to create scheduled process. %s", ex)
                raise web.HTTPInternalServerError(reason='Failed to create north instance.')

        # If successful then create a configuration entry from plugin configuration
        try:
            # Create a configuration category from the configuration defined in the plugin
            category_desc = plugin_config['plugin']['description']
            config_mgr = ConfigurationManager(storage)
            await config_mgr.create_category(category_name=name,
                                             category_description=category_desc,
                                             category_value=plugin_config,
                                             keep_original_items=True)
            # Create the parent category for all North tasks
            await config_mgr.create_category("North", {}, 'North tasks', True)
            await config_mgr.create_child_category("North", [name])

            # If config is in POST data, then update the value for each config item
            if config is not None:
                if not isinstance(config, dict):
                    raise ValueError('Config must be a JSON object')
                for k, v in config.items():
                    await config_mgr.set_category_item_value_entry(name, k, v['value'])
        except Exception as ex:
            await delete_configuration(storage, name)  # Revert configuration entry
            await delete_parent_child_configuration(storage, name)
            _logger.exception("Failed to create plugin configuration. %s", str(ex))
            raise web.HTTPInternalServerError(reason='Failed to create plugin configuration.')

        # If all successful then lastly add a schedule to run the new task at startup
        try:
            schedule = TimedSchedule() if schedule_type == Schedule.Type.TIMED else \
                       IntervalSchedule() if schedule_type == Schedule.Type.INTERVAL else \
                       ManualSchedule()
            schedule.name = name
            schedule.process_name = process_name
            schedule.day = schedule_day
            m, s = divmod(schedule_time if schedule_time is not None else 0, 60)
            h, m = divmod(m, 60)
            schedule.time = datetime.time().replace(hour=h, minute=m, second=s)
            schedule.repeat = datetime.timedelta(seconds=schedule_repeat if schedule_repeat is not None else 0)
            schedule.exclusive = True
            schedule.enabled = False  # if "enabled" is supplied, it gets activated in save_schedule() via is_enabled flag

            # Save schedule
            await server.Server.scheduler.save_schedule(schedule, is_enabled)
            schedule = await server.Server.scheduler.get_schedule_by_name(name)
        except StorageServerError as ex:
            await delete_configuration(storage, name)  # Revert configuration entry
            await delete_parent_child_configuration(storage, name)
            _logger.exception("Failed to create schedule. %s", ex.error)
            raise web.HTTPInternalServerError(reason='Failed to create north instance.')
        except Exception as ex:
            await delete_configuration(storage, name)  # Revert configuration entry
            await delete_parent_child_configuration(storage, name)
            _logger.exception("Failed to create schedule. %s", str(ex))
            raise web.HTTPInternalServerError(reason='Failed to create north instance.')

    except ValueError as e:
        raise web.HTTPBadRequest(reason=str(e))
    else:
        return web.json_response({'name': name, 'id': str(schedule.schedule_id)})

async def delete_task(request):
    """ Delete a north plugin instance task

        :Example:
            curl -X DELETE http://localhost:8081/foglamp/scheduled/task/<task name>
    """
    try:
        north_instance = request.match_info.get('task_name', None)

        if north_instance is None or north_instance.strip() == '':
            raise web.HTTPBadRequest(reason='Missing task_name in requested URL')

        storage = connect.get_storage_async()

        result = await get_schedule(storage, north_instance)
        if result['count'] == 0:
            raise web.HTTPBadRequest(reason='A north instance task with this name does not exist.')

        north_instance_schedule = result['rows'][0]
        sch_id = uuid.UUID(north_instance_schedule['id'])
        if north_instance_schedule['enabled'].lower() == 't':
            # disable it
            await server.Server.scheduler.disable_schedule(sch_id)
        # delete it
        await server.Server.scheduler.delete_schedule(sch_id)

        # delete all configuration for the north task instance name
        await delete_configuration(storage, north_instance)
        await delete_parent_child_configuration(storage, north_instance)

        # delete statistics key
        await delete_statistics_key(storage, north_instance)

    except Exception as ex:
        raise web.HTTPInternalServerError(reason=ex)
    else:
        return web.json_response({'result': 'North instance {} deleted successfully.'.format(north_instance)})

async def get_schedule(storage, schedule_name):
    payload = PayloadBuilder().SELECT(["id", "enabled"]).WHERE(['schedule_name', '=', schedule_name]).payload()
    result = await storage.query_tbl_with_payload('schedules', payload)
    return result

async def check_scheduled_processes(storage, process_name):
    payload = PayloadBuilder().SELECT("name").WHERE(['name', '=', process_name]).payload()
    result = await storage.query_tbl_with_payload('scheduled_processes', payload)
    return result['count']

async def check_schedules(storage, schedule_name):
    payload = PayloadBuilder().SELECT("schedule_name").WHERE(['schedule_name', '=', schedule_name]).payload()
    result = await storage.query_tbl_with_payload('schedules', payload)
    return result['count']

async def delete_configuration(storage, key):
    payload = PayloadBuilder().WHERE(['key', '=', key]).payload()
    await storage.delete_from_tbl('configuration', payload)
    # Removed key from configuration cache
    config_mgr = ConfigurationManager(storage)
    config_mgr._cacheManager.remove(key)

async def delete_parent_child_configuration(storage, key):
    payload = PayloadBuilder().WHERE(['parent', '=', "North"]).AND_WHERE(['child', '=', key]).payload()
    await storage.delete_from_tbl('category_children', payload)

async def delete_statistics_key(storage, key):
    payload = PayloadBuilder().WHERE(['key', '=', key]).payload()
    await storage.delete_from_tbl('statistics', payload)
<|endoftext|>


<commit_before>from time import sleep
from lettuce import step, world
from questionnaire.features.pages.questions import QuestionListingPage, CreateQuestionPage
from questionnaire.models import Question, Theme, Questionnaire, Section, SubSection, QuestionGroup, QuestionGroupOrder, QuestionOption, TextAnswer, Answer, AnswerGroup

@step(u'And I have 100 questions')
def and_i_have_100_questions(step):
    for i in range(0, 100):
        Question.objects.create(text="When will you be %s years old" % i, export_label="Export text for %s" % i,
                                instructions="question %s answer sensibly" % i, UID="%s" % i)

@step(u'I visit the question listing page')
def and_i_visit_the_question_listing_page(step):
    world.page = QuestionListingPage(world.browser)
    world.page.visit()

@step(u'And I have two themes')
def and_i_have_two_themes(step):
    world.theme1 = Theme.objects.create(name="Theme 1")
    world.theme2 = Theme.objects.create(name="Theme 2")

@step(u'Then I should see all questions paginated')
def then_i_should_see_all_questions_paginated(step):
    for i in range(0, 4):
        world.page.is_text_present("Export text for %s" % i, "%s" % i)
    world.page.click_link_by_text("2")

@step(u'And I click add new question page')
def and_i_click_add_new_question_page(step):
    world.page.click_by_id('id-add-new-question-link')

    world.data = {'text': 'How many measles cases did you find this year',
                  'instructions': 'Just give an answer',
                  'export_label': 'blah',
                  'theme': world.theme1.id}

@step(u'And I fill in the question details')
def and_i_fill_in_the_question_details(step):
    world.page.fill_form(world.data)
    world.page.select('answer_type', 'Number')

@step(u'Then I should see the question created')
def then_i_should_see_the_question_created(step):
    world.page.is_text_present("Question successfully created")

@step(u'And I click save question button')
def and_i_click_save_question_button(step):
    world.page.click_by_css('.submit')

@step(u'And I select Multi-choice answer type')
def and_i_select_multi_choice_answer_type(step):
    world.page.select('answer_type', 'MultiChoice')

@step(u'Then I should see the option field')
def then_i_should_see_the_option_field(step):
    world.page.is_text_present('Option 1')

@step(u'When Fill in the option')
def when_fill_in_the_option(step):
    world.page = CreateQuestionPage(world.browser)
    world.page.fill_first_visible_option('options', 'Yes')

@step(u'When I click add more button')
def when_i_click_add_more_button(step):
    world.page.click_by_css('.add-option')

@step(u'Then I should see another option field')
def then_i_should_see_another_option_field(step):
    world.page.is_text_present('Option 2')

@step(u'When I click remove the added option field')
def when_i_click_remove_the_added_option_field(step):
    world.page.remove_option_field('.remove-option', 1)

@step(u'Then I should not see that option field')
def then_i_should_not_see_that_option_field(step):
    world.page.is_text_present('Option 2', status=False)

@step(u'And I fill in the multichoice question form data')
def and_i_fill_in_the_multichoice_question_form_data(step):
    data = {'text': 'How many measles cases did you find this year',
            'instructions': 'Just give an answer',
            'export_label': 'blah',
            'theme': world.theme2.id}
    world.page.fill_form(data)

@step(u'And I check custom option')
def and_i_check_custom_option(step):
    world.page.check('custom')

@step(u'And I have a question without answers')
def and_i_have_a_question_without_answers(step):
    data = {'text': 'B. Number of cases tested',
            'instructions': "Enter the total number of cases",
            'UID': '00001', 'answer_type': 'Number',
            'theme': world.theme1}
    world.question = Question.objects.create(**data)

@step(u'And I click delete on that question')
def and_i_click_delete_on_that_question(step):
    world.page.click_by_id('delete-question-%s' % world.question.id)

@step(u'Then I should see that question was deleted successfully')
def then_i_should_see_that_question_was_deleted_successfully(step):
    world.page.is_text_present("Question was deleted successfully")

@step(u'Then I should see a delete confirmation modal')
def then_i_should_see_a_delete_confirmation_modal(step):
    world.page.confirm_delete('question')

@step(u'When I confirm delete')
def when_i_confirm_delete(step):
    world.page.click_by_css('.confirm-delete')

@step(u'And I have a core question')
def and_i_have_a_core_question(step):
    world.core_question = Question.objects.create(text='Core Question Display Label', UID='C0001',
                                                  answer_type='MultiChoice', export_label='Core Question Export Label',
                                                  instructions='Sample Instructions')
    world.option1 = QuestionOption.objects.create(text="Option 1", question=world.core_question)
    world.option2 = QuestionOption.objects.create(text="Option 2", question=world.core_question)

@step(u'And that core question is used in a questionnaire with submitted responses')
def and_that_core_question_is_used_in_a_questionnaire_with_submitted_responses(step):
    world.questionnaire_status = Questionnaire.objects.create(name="JRF Core Status",
                                                              description="Questionnaire Status",
                                                              status=Questionnaire.PUBLISHED)
    world.section_status = Section.objects.create(order=1, title="Section AFRO", description="Description",
                                                  questionnaire=world.questionnaire_status, name="Cover page")
    world.subsection_status = SubSection.objects.create(order=1, section=world.section_status, title='AFRO Subsection')
    world.parent = QuestionGroup.objects.create(subsection=world.subsection_status, order=1)
    world.parent.question.add(world.core_question)
    QuestionGroupOrder.objects.create(question=world.core_question, question_group=world.parent, order=1)

    text_answer = TextAnswer.objects.create(question=world.core_question,
                              questionnaire=world.questionnaire_status,
                              status=Answer.SUBMITTED_STATUS, response='First Version Response')
    answerGroup = AnswerGroup.objects.create(grouped_question=world.parent)
    answerGroup.answer.add(text_answer)

@step(u'And I select the option to edit that question')
def and_i_select_the_option_to_edit_that_question(step):
    world.page.click_link_by_partial_href('/questions/%s/edit/' % world.core_question.id)

@step(u'The question should be displayed with all its attributes')
def the_question_should_be_displayed_with_all_its_attributes(step):
    world.page.validate_question_attributes(world.core_question)

@step(u'When I update the question with invalid details')
def when_i_update_the_question_with_invalid_details(step):
    question_data = {'text': '',
                     'instructions': '',
                     'export_label': '',
                     'theme': ''}
    world.page.fill_form(question_data)
    step.given('And I click save question button')

@step(u'I should see an error message')
def i_should_see_an_error_message(step):
    world.page.is_text_present('Question NOT updated. See errors below')
    world.page.is_text_present('This field is required')
    world.page.is_text_present('All questions must have export label')

@step(u'When I update the question with valid details')
def when_i_update_the_question_with_valid_details(step):
    world.theme = Theme.objects.create(name='Sample Theme')
    world.page.browser.reload()
    question_data = {'text': 'Updated Question Text',
                     'instructions': 'Updated Question Instructions',
                     'export_label': 'Updated Question Export Label',
                     'theme': world.theme.id}
    world.page.fill_form(question_data)
    world.page.submit()

@step(u'Then I should see a message that the question was successfully updated')
def then_i_should_see_a_message_that_the_question_was_successfully_updated(step):
    world.page.is_text_present('Question successfully updated')

@step(u'And I should see the updated details')
def and_i_should_see_the_updated_details(step):
    world.page.is_text_present('Updated Question Export Label')

@step(u'When I preview the submitted questionnaire where the question was used')
def when_i_preview_the_submitted_questionnaire_where_the_question_was_used(step):
    world.page.visit_url('/questionnaire/entry/%s/section/%s/?preview=1' % (world.questionnaire_status.id, world.section_status.id))

@step(u'I should see the earlier question display label')
def i_should_see_the_earlier_question_display_label(step):
    world.page.is_text_present(world.core_question.text)<commit_msg>Fixing Failing IE test<commit_after>from time import sleep
from lettuce import step, world
from questionnaire.features.pages.questions import QuestionListingPage, CreateQuestionPage
from questionnaire.models import Question, Theme, Questionnaire, Section, SubSection, QuestionGroup, QuestionGroupOrder, QuestionOption, TextAnswer, Answer, AnswerGroup

@step(u'And I have 100 questions')
def and_i_have_100_questions(step):
    for i in range(0, 100):
        Question.objects.create(text="When will you be %s years old" % i, export_label="Export text for %s" % i,
                                instructions="question %s answer sensibly" % i, UID="%s" % i)

@step(u'I visit the question listing page')
def and_i_visit_the_question_listing_page(step):
    world.page = QuestionListingPage(world.browser)
    world.page.visit()

@step(u'And I have two themes')
def and_i_have_two_themes(step):
    world.theme1 = Theme.objects.create(name="Theme 1")
    world.theme2 = Theme.objects.create(name="Theme 2")

@step(u'Then I should see all questions paginated')
def then_i_should_see_all_questions_paginated(step):
    for i in range(0, 4):
        world.page.is_text_present("Export text for %s" % i, "%s" % i)
    world.page.click_link_by_text("2")

@step(u'And I click add new question page')
def and_i_click_add_new_question_page(step):
    world.page.click_by_id('id-add-new-question-link')

    world.data = {'text': 'How many measles cases did you find this year',
                  'instructions': 'Just give an answer',
                  'export_label': 'blah',
                  'theme': world.theme1.id}

@step(u'And I fill in the question details')
def and_i_fill_in_the_question_details(step):
    world.page.fill_form(world.data)
    world.page.select('answer_type', 'Number')

@step(u'Then I should see the question created')
def then_i_should_see_the_question_created(step):
    world.page.is_text_present("Question successfully created")

@step(u'And I click save question button')
def and_i_click_save_question_button(step):
    world.page.click_by_css('.submit')

@step(u'And I select Multi-choice answer type')
def and_i_select_multi_choice_answer_type(step):
    world.page.select('answer_type', 'MultiChoice')

@step(u'Then I should see the option field')
def then_i_should_see_the_option_field(step):
    world.page.is_text_present('Option 1')

@step(u'When Fill in the option')
def when_fill_in_the_option(step):
    world.page = CreateQuestionPage(world.browser)
    world.page.fill_first_visible_option('options', 'Yes')

@step(u'When I click add more button')
def when_i_click_add_more_button(step):
    world.page.click_by_css('.add-option')

@step(u'Then I should see another option field')
def then_i_should_see_another_option_field(step):
    world.page.is_text_present('Option 2')

@step(u'When I click remove the added option field')
def when_i_click_remove_the_added_option_field(step):
    world.page.remove_option_field('.remove-option', 1)

@step(u'Then I should not see that option field')
def then_i_should_not_see_that_option_field(step):
    world.page.is_text_present('Option 2', status=False)

@step(u'And I fill in the multichoice question form data')
def and_i_fill_in_the_multichoice_question_form_data(step):
    data = {'text': 'How many measles cases did you find this year',
            'instructions': 'Just give an answer',
            'export_label': 'blah',
            'theme': world.theme2.id}
    world.page.fill_form(data)

@step(u'And I check custom option')
def and_i_check_custom_option(step):
    world.page.check('custom')

@step(u'And I have a question without answers')
def and_i_have_a_question_without_answers(step):
    data = {'text': 'B. Number of cases tested',
            'instructions': "Enter the total number of cases",
            'UID': '00001', 'answer_type': 'Number',
            'theme': world.theme1}
    world.question = Question.objects.create(**data)

@step(u'And I click delete on that question')
def and_i_click_delete_on_that_question(step):
    world.page.click_by_id('delete-question-%s' % world.question.id)

@step(u'Then I should see that question was deleted successfully')
def then_i_should_see_that_question_was_deleted_successfully(step):
    world.page.is_text_present("Question was deleted successfully")

@step(u'Then I should see a delete confirmation modal')
def then_i_should_see_a_delete_confirmation_modal(step):
    world.page.confirm_delete('question')

@step(u'When I confirm delete')
def when_i_confirm_delete(step):
    world.page.click_by_css('.confirm-delete')

@step(u'And I have a core question')
def and_i_have_a_core_question(step):
    world.core_question = Question.objects.create(text='Core Question Display Label', UID='C0001',
                                                  answer_type='MultiChoice', export_label='Core Question Export Label',
                                                  instructions='Sample Instructions')
    world.option1 = QuestionOption.objects.create(text="Option 1", question=world.core_question)
    world.option2 = QuestionOption.objects.create(text="Option 2", question=world.core_question)

@step(u'And that core question is used in a questionnaire with submitted responses')
def and_that_core_question_is_used_in_a_questionnaire_with_submitted_responses(step):
    world.questionnaire_status = Questionnaire.objects.create(name="JRF Core Status",
                                                              description="Questionnaire Status",
                                                              status=Questionnaire.PUBLISHED)
    world.section_status = Section.objects.create(order=1, title="Section AFRO", description="Description",
                                                  questionnaire=world.questionnaire_status, name="Cover page")
    world.subsection_status = SubSection.objects.create(order=1, section=world.section_status, title='AFRO Subsection')
    world.parent = QuestionGroup.objects.create(subsection=world.subsection_status, order=1)
    world.parent.question.add(world.core_question)
    QuestionGroupOrder.objects.create(question=world.core_question, question_group=world.parent, order=1)

    text_answer = TextAnswer.objects.create(question=world.core_question,
                              questionnaire=world.questionnaire_status,
                              status=Answer.SUBMITTED_STATUS, response='First Version Response')
    answerGroup = AnswerGroup.objects.create(grouped_question=world.parent)
    answerGroup.answer.add(text_answer)

@step(u'And I select the option to edit that question')
def and_i_select_the_option_to_edit_that_question(step):
    world.page.click_link_by_partial_href('/questions/%s/edit/' % world.core_question.id)

@step(u'The question should be displayed with all its attributes')
def the_question_should_be_displayed_with_all_its_attributes(step):
    world.page.validate_question_attributes(world.core_question)

@step(u'When I update the question with invalid details')
def when_i_update_the_question_with_invalid_details(step):
    world.theme = Theme.objects.create(name='Sample Theme')
    question_data = {'text': '',
                     'instructions': '',
                     'export_label': '',
                     'theme': ''}
    world.page.fill_form(question_data)
    step.given('And I click save question button')

@step(u'I should see an error message')
def i_should_see_an_error_message(step):
    world.page.is_text_present('Question NOT updated. See errors below')
    world.page.is_text_present('This field is required')
    world.page.is_text_present('All questions must have export label')

@step(u'When I update the question with valid details')
def when_i_update_the_question_with_valid_details(step):
    question_data = {'text': 'Updated Question Text',
                     'instructions': 'Updated Question Instructions',
                     'export_label': 'Updated Question Export Label',
                     'theme': world.theme.id}
    world.page.fill_form(question_data)
    world.page.submit()

@step(u'Then I should see a message that the question was successfully updated')
def then_i_should_see_a_message_that_the_question_was_successfully_updated(step):
    world.page.is_text_present('Question successfully updated')

@step(u'And I should see the updated details')
def and_i_should_see_the_updated_details(step):
    world.page.is_text_present('Updated Question Export Label')

@step(u'When I preview the submitted questionnaire where the question was used')
def when_i_preview_the_submitted_questionnaire_where_the_question_was_used(step):
    world.page.visit_url('/questionnaire/entry/%s/section/%s/?preview=1' % (world.questionnaire_status.id, world.section_status.id))

@step(u'I should see the earlier question display label')
def i_should_see_the_earlier_question_display_label(step):
    world.page.is_text_present(world.core_question.text)<|endoftext|>


<commit_before>import json
import logging
import os
from typing import List, Text

from rasa_core import utils
from rasa_core.actions.action import (ACTION_REVERT_FALLBACK_EVENTS_NAME,
                                      ACTION_DEFAULT_FALLBACK_NAME,
                                      ACTION_DEFAULT_ASK_REPHRASE_NAME,
                                      ACTION_DEFAULT_ASK_AFFIRMATION_NAME,
                                      ACTION_LISTEN_NAME)
from rasa_core.constants import (FALLBACK_SCORE, USER_INTENT_AFFIRM,
                                 USER_INTENT_DENY)
from rasa_core.domain import Domain, InvalidDomain
from rasa_core.policies.fallback import FallbackPolicy
from rasa_core.policies.policy import confidence_scores_for
from rasa_core.trackers import DialogueStateTracker

logger = logging.getLogger(__name__)

class TwoStageFallbackPolicy(FallbackPolicy):
    """ This policy handles low NLU confidence in multiple stages.

        If a NLU prediction has a low confidence score,
        the user is asked to affirm whether they really had this intent.
        If they affirm, the story continues as if the intent was classified
        with high confidence from the beginning.
        If they deny, the user is asked to rephrase his intent.
        If the classification for the rephrased intent was confident, the story
        continues as if the user had this intent from the beginning.
        If the rephrased intent was not classified with high confidence,
        the user is asked to affirm the classified intent.
        If the user affirm the intent, the story continues as if the user had
        this intent from the beginning.
        If the user denies, an ultimate fallback action is triggered
        (e.g. a handoff to a human).
    """

    def __init__(self,
                 nlu_threshold: float = 0.3,
                 core_threshold: float = 0.3,
                 fallback_action_name: Text = ACTION_DEFAULT_FALLBACK_NAME,
                 ) -> None:
        """Create a new Two-stage Fallback policy.

        Args:
            nlu_threshold: minimum threshold for NLU confidence.
                If intent prediction confidence is lower than this,
                predict fallback action with confidence 1.0.
            core_threshold: if NLU confidence threshold is met,
                predict fallback action with confidence
                `core_threshold`. If this is the highest confidence in
                the ensemble, the fallback action will be executed.
            fallback_action_name: This action is executed if the user
                denies the recognised intent for the second time.
        """
        super(TwoStageFallbackPolicy, self).__init__()

        self.nlu_threshold = nlu_threshold
        self.core_threshold = core_threshold
        self.fallback_action_name = fallback_action_name

    def predict_action_probabilities(self,
                                     tracker: DialogueStateTracker,
                                     domain: Domain) -> List[float]:
        """Predicts the next action if NLU confidence is low.
        """

        if (USER_INTENT_AFFIRM not in domain.intents or
                USER_INTENT_DENY not in domain.intents):
            raise InvalidDomain('The intents {} and {} must be present in the '
                                'domain file to use this policy.'.format(
                                    USER_INTENT_AFFIRM, USER_INTENT_AFFIRM))

        nlu_data = tracker.latest_message.parse_data
        nlu_confidence = nlu_data["intent"].get("confidence", 1.0)
        last_intent_name = nlu_data['intent'].get('name', None)
        should_fallback = self.should_fallback(nlu_confidence,
                                               tracker.latest_action_name)
        user_rephrased = has_user_rephrased(tracker)

        if self._is_user_input_expected(tracker):
            result = confidence_scores_for(ACTION_LISTEN_NAME, FALLBACK_SCORE,
                                           domain)
        elif _has_user_denied(last_intent_name, tracker):
            logger.debug("User '{}' denied suggested intents.".format(
                tracker.sender_id))
            result = self._results_for_user_denied(tracker, domain)
        elif user_rephrased and should_fallback:
            logger.debug("Ambiguous rephrasing of user '{}' "
                         "for intent '{}'".format(tracker.sender_id,
                                                  last_intent_name))
            result = confidence_scores_for(ACTION_DEFAULT_ASK_AFFIRMATION_NAME,
                                           FALLBACK_SCORE,
                                           domain)
        elif has_user_affirmed(last_intent_name, tracker) or user_rephrased:
            logger.debug("User '{}' affirmed intent by affirmation or "
                         "rephrasing.".format(tracker.sender_id))
            result = confidence_scores_for(ACTION_REVERT_FALLBACK_EVENTS_NAME,
                                           FALLBACK_SCORE, domain)
        elif tracker.last_executed_action_has(
                ACTION_DEFAULT_ASK_AFFIRMATION_NAME):
            if not should_fallback:
                logger.debug("User '{}' rephrased intent '{}' instead "
                             "of affirming.".format(tracker.sender_id,
                                                    last_intent_name))
                result = confidence_scores_for(
                    ACTION_REVERT_FALLBACK_EVENTS_NAME,
                    FALLBACK_SCORE, domain)
            else:
                result = confidence_scores_for(self.fallback_action_name,
                                               FALLBACK_SCORE, domain)
        elif should_fallback:
            logger.debug("User '{}' has to affirm intent '{}'.".format(
                tracker.sender_id, last_intent_name))
            result = confidence_scores_for(ACTION_DEFAULT_ASK_AFFIRMATION_NAME,
                                           FALLBACK_SCORE,
                                           domain)
        else:
            result = self.fallback_scores(domain, self.core_threshold)

        return result

    def _is_user_input_expected(self, tracker: DialogueStateTracker) -> bool:
        return tracker.latest_action_name in [
            ACTION_DEFAULT_ASK_AFFIRMATION_NAME,
            ACTION_DEFAULT_ASK_REPHRASE_NAME,
            self.fallback_action_name]

    def _results_for_user_denied(self, tracker: DialogueStateTracker,
                                 domain: Domain) -> List[float]:
        has_denied_before = tracker.last_executed_action_has(
            ACTION_DEFAULT_ASK_REPHRASE_NAME,
            skip=1)

        if has_denied_before:
            return confidence_scores_for(self.fallback_action_name,
                                         FALLBACK_SCORE, domain)
        else:
            return confidence_scores_for(ACTION_DEFAULT_ASK_REPHRASE_NAME,
                                         FALLBACK_SCORE, domain)

    def persist(self, path: Text) -> None:
        """Persists the policy to storage."""
        config_file = os.path.join(path, 'two_stage_fallback_policy.json')
        meta = {
            "nlu_threshold": self.nlu_threshold,
            "core_threshold": self.core_threshold,
            "fallback_action_name": self.fallback_action_name,
        }
        utils.create_dir_for_file(config_file)
        utils.dump_obj_as_json_to_file(config_file, meta)

    @classmethod
    def load(cls, path: Text) -> 'FallbackPolicy':
        meta = {}
        if os.path.exists(path):
            meta_path = os.path.join(path, "two_stage_fallback_policy.json")
            if os.path.isfile(meta_path):
                meta = json.loads(utils.read_file(meta_path))

        return cls(**meta)

def has_user_affirmed(last_intent: Text,
                      tracker: DialogueStateTracker) -> bool:
    return (tracker.last_executed_action_has(
        ACTION_DEFAULT_ASK_AFFIRMATION_NAME) and
        last_intent == USER_INTENT_AFFIRM)

def _has_user_denied(last_intent: Text,
                     tracker: DialogueStateTracker) -> bool:
    return (tracker.last_executed_action_has(
        ACTION_DEFAULT_ASK_AFFIRMATION_NAME) and
        last_intent == USER_INTENT_DENY)

def has_user_rephrased(tracker: DialogueStateTracker) -> bool:
    return tracker.last_executed_action_has(
        ACTION_DEFAULT_ASK_REPHRASE_NAME)
<commit_msg>reuse constructor of parent class<commit_after>import json
import logging
import os
from typing import List, Text

from rasa_core import utils
from rasa_core.actions.action import (ACTION_REVERT_FALLBACK_EVENTS_NAME,
                                      ACTION_DEFAULT_FALLBACK_NAME,
                                      ACTION_DEFAULT_ASK_REPHRASE_NAME,
                                      ACTION_DEFAULT_ASK_AFFIRMATION_NAME,
                                      ACTION_LISTEN_NAME)
from rasa_core.constants import (FALLBACK_SCORE, USER_INTENT_AFFIRM,
                                 USER_INTENT_DENY)
from rasa_core.domain import Domain, InvalidDomain
from rasa_core.policies.fallback import FallbackPolicy
from rasa_core.policies.policy import confidence_scores_for
from rasa_core.trackers import DialogueStateTracker

logger = logging.getLogger(__name__)

class TwoStageFallbackPolicy(FallbackPolicy):
    """ This policy handles low NLU confidence in multiple stages.

        If a NLU prediction has a low confidence score,
        the user is asked to affirm whether they really had this intent.
        If they affirm, the story continues as if the intent was classified
        with high confidence from the beginning.
        If they deny, the user is asked to rephrase his intent.
        If the classification for the rephrased intent was confident, the story
        continues as if the user had this intent from the beginning.
        If the rephrased intent was not classified with high confidence,
        the user is asked to affirm the classified intent.
        If the user affirm the intent, the story continues as if the user had
        this intent from the beginning.
        If the user denies, an ultimate fallback action is triggered
        (e.g. a handoff to a human).
    """

    def __init__(self,
                 nlu_threshold: float = 0.3,
                 core_threshold: float = 0.3,
                 fallback_action_name: Text = ACTION_DEFAULT_FALLBACK_NAME,
                 ) -> None:
        """Create a new Two-stage Fallback policy.

        Args:
            nlu_threshold: minimum threshold for NLU confidence.
                If intent prediction confidence is lower than this,
                predict fallback action with confidence 1.0.
            core_threshold: if NLU confidence threshold is met,
                predict fallback action with confidence
                `core_threshold`. If this is the highest confidence in
                the ensemble, the fallback action will be executed.
            fallback_action_name: This action is executed if the user
                denies the recognised intent for the second time.
        """
        super(TwoStageFallbackPolicy, self).__init__(
            nlu_threshold,
            core_threshold,
            fallback_action_name)

    def predict_action_probabilities(self,
                                     tracker: DialogueStateTracker,
                                     domain: Domain) -> List[float]:
        """Predicts the next action if NLU confidence is low.
        """

        if (USER_INTENT_AFFIRM not in domain.intents or
                USER_INTENT_DENY not in domain.intents):
            raise InvalidDomain('The intents {} and {} must be present in the '
                                'domain file to use this policy.'.format(
                                    USER_INTENT_AFFIRM, USER_INTENT_AFFIRM))

        nlu_data = tracker.latest_message.parse_data
        nlu_confidence = nlu_data["intent"].get("confidence", 1.0)
        last_intent_name = nlu_data['intent'].get('name', None)
        should_fallback = self.should_fallback(nlu_confidence,
                                               tracker.latest_action_name)
        user_rephrased = has_user_rephrased(tracker)

        if self._is_user_input_expected(tracker):
            result = confidence_scores_for(ACTION_LISTEN_NAME, FALLBACK_SCORE,
                                           domain)
        elif _has_user_denied(last_intent_name, tracker):
            logger.debug("User '{}' denied suggested intents.".format(
                tracker.sender_id))
            result = self._results_for_user_denied(tracker, domain)
        elif user_rephrased and should_fallback:
            logger.debug("Ambiguous rephrasing of user '{}' "
                         "for intent '{}'".format(tracker.sender_id,
                                                  last_intent_name))
            result = confidence_scores_for(ACTION_DEFAULT_ASK_AFFIRMATION_NAME,
                                           FALLBACK_SCORE,
                                           domain)
        elif has_user_affirmed(last_intent_name, tracker) or user_rephrased:
            logger.debug("User '{}' affirmed intent by affirmation or "
                         "rephrasing.".format(tracker.sender_id))
            result = confidence_scores_for(ACTION_REVERT_FALLBACK_EVENTS_NAME,
                                           FALLBACK_SCORE, domain)
        elif tracker.last_executed_action_has(
                ACTION_DEFAULT_ASK_AFFIRMATION_NAME):
            if not should_fallback:
                logger.debug("User '{}' rephrased intent '{}' instead "
                             "of affirming.".format(tracker.sender_id,
                                                    last_intent_name))
                result = confidence_scores_for(
                    ACTION_REVERT_FALLBACK_EVENTS_NAME,
                    FALLBACK_SCORE, domain)
            else:
                result = confidence_scores_for(self.fallback_action_name,
                                               FALLBACK_SCORE, domain)
        elif should_fallback:
            logger.debug("User '{}' has to affirm intent '{}'.".format(
                tracker.sender_id, last_intent_name))
            result = confidence_scores_for(ACTION_DEFAULT_ASK_AFFIRMATION_NAME,
                                           FALLBACK_SCORE,
                                           domain)
        else:
            result = self.fallback_scores(domain, self.core_threshold)

        return result

    def _is_user_input_expected(self, tracker: DialogueStateTracker) -> bool:
        return tracker.latest_action_name in [
            ACTION_DEFAULT_ASK_AFFIRMATION_NAME,
            ACTION_DEFAULT_ASK_REPHRASE_NAME,
            self.fallback_action_name]

    def _results_for_user_denied(self, tracker: DialogueStateTracker,
                                 domain: Domain) -> List[float]:
        has_denied_before = tracker.last_executed_action_has(
            ACTION_DEFAULT_ASK_REPHRASE_NAME,
            skip=1)

        if has_denied_before:
            return confidence_scores_for(self.fallback_action_name,
                                         FALLBACK_SCORE, domain)
        else:
            return confidence_scores_for(ACTION_DEFAULT_ASK_REPHRASE_NAME,
                                         FALLBACK_SCORE, domain)

    def persist(self, path: Text) -> None:
        """Persists the policy to storage."""
        config_file = os.path.join(path, 'two_stage_fallback_policy.json')
        meta = {
            "nlu_threshold": self.nlu_threshold,
            "core_threshold": self.core_threshold,
            "fallback_action_name": self.fallback_action_name,
        }
        utils.create_dir_for_file(config_file)
        utils.dump_obj_as_json_to_file(config_file, meta)

    @classmethod
    def load(cls, path: Text) -> 'FallbackPolicy':
        meta = {}
        if os.path.exists(path):
            meta_path = os.path.join(path, "two_stage_fallback_policy.json")
            if os.path.isfile(meta_path):
                meta = json.loads(utils.read_file(meta_path))

        return cls(**meta)

def has_user_affirmed(last_intent: Text,
                      tracker: DialogueStateTracker) -> bool:
    return (tracker.last_executed_action_has(
        ACTION_DEFAULT_ASK_AFFIRMATION_NAME) and
        last_intent == USER_INTENT_AFFIRM)

def _has_user_denied(last_intent: Text,
                     tracker: DialogueStateTracker) -> bool:
    return (tracker.last_executed_action_has(
        ACTION_DEFAULT_ASK_AFFIRMATION_NAME) and
        last_intent == USER_INTENT_DENY)

def has_user_rephrased(tracker: DialogueStateTracker) -> bool:
    return tracker.last_executed_action_has(
        ACTION_DEFAULT_ASK_REPHRASE_NAME)
<|endoftext|>


<commit_before>from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import

import typing
from builtins import map
from builtins import range
import logging
import os
import re
import time
import warnings
import io
from collections import Counter
from string import punctuation
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from future.utils import PY3
from typing import Text

from rasa_nlu.components import Component
from rasa_nlu.config import RasaNLUConfig
from rasa_nlu.featurizers import Featurizer
from rasa_nlu.training_data import Message
from rasa_nlu.training_data import TrainingData

logger = logging.getLogger(__name__)

if typing.TYPE_CHECKING:
    from spacy.language import Language
    import numpy as np
    from rasa_nlu.model import Metadata

class NGramFeaturizer(Featurizer):
    name = "intent_featurizer_ngrams"

    provides = ["text_features"]

    requires = ["spacy_doc"]

    n_gram_min_length = 3

    n_gram_max_length = 17

    n_gram_min_occurrences = 5

    min_intent_examples_for_ngram_classification = 4

    def __init__(self):
        self.best_num_ngrams = None
        self.all_ngrams = None

    @classmethod
    def required_packages(cls):
        # type: () -> List[Text]
        return ["spacy", "numpy", "sklearn", "cloudpickle"]

    def train(self, training_data, config, **kwargs):
        # type: (TrainingData, RasaNLUConfig, **Any) -> None

        start = time.time()
        self.train_on_sentences(training_data.intent_examples, config["max_number_of_ngrams"])
        logger.debug("Ngram collection took {} seconds".format(time.time() - start))

        for example in training_data.training_examples:
            updated = self._text_features_with_ngrams(example, self.best_num_ngrams)
            example.set("text_features", updated)

    def process(self, message, **kwargs):
        # type: (Message, **Any) -> None

        updated = self._text_features_with_ngrams(message, self.best_num_ngrams)
        message.set("text_features", updated)

    def _text_features_with_ngrams(self, message, max_ngrams):
        import numpy as np

        ngrams_to_use = self._ngrams_to_use(max_ngrams)

        if ngrams_to_use is not None:
            extras = np.array(self._ngrams_in_sentence(message, ngrams_to_use))
            return self._combine_with_existing_text_features(message, extras)
        else:
            return message.get("text_features")

    @classmethod
    def load(cls, model_dir=None, model_metadata=None, cached_component=None, **kwargs):
        # type: (Text, Metadata, Optional[Component], **Any) -> NGramFeaturizer
        import cloudpickle

        if model_dir and model_metadata.get("ngram_featurizer"):
            classifier_file = os.path.join(model_dir, model_metadata.get("ngram_featurizer"))
            with io.open(classifier_file, 'rb') as f:   # pramga: no cover
                if PY3:
                    return cloudpickle.load(f, encoding="latin-1")
                else:
                    return cloudpickle.load(f)
        else:
            return NGramFeaturizer()

    def persist(self, model_dir):
        # type: (Text) -> Dict[Text, Any]
        """Persist this model into the passed directory. Returns the metadata necessary to load the model again."""
        import cloudpickle

        classifier_file = os.path.join(model_dir, "ngram_featurizer.pkl")
        with io.open(classifier_file, 'wb') as f:
            cloudpickle.dump(self, f)

        return {
            "ngram_featurizer": "ngram_featurizer.pkl"
        }

    def train_on_sentences(self, examples, max_number_of_ngrams):
        labels = [e.get("intent") for e in examples]
        self.all_ngrams = self._get_best_ngrams(examples, labels)
        self.best_num_ngrams = self._cross_validation(examples, labels, max_number_of_ngrams)

    def _ngrams_to_use(self, num_ngrams):
        if num_ngrams == 0 or self.all_ngrams is None:
            return []
        elif num_ngrams is not None:
            return self.all_ngrams[:num_ngrams]
        else:
            return self.all_ngrams

    def _get_best_ngrams(self, examples, labels):
        """Returns an ordered list of the best character ngrams for an intent classification problem"""

        oov_strings = self._remove_in_vocab_words(examples)
        ngrams = self._generate_all_ngrams(oov_strings)
        return self._sort_applicable_ngrams(ngrams, examples, labels)

    def _remove_in_vocab_words(self, examples):
        """Automatically removes words with digits in them, that may be a
        hyperlink or that _are_ in vocabulary for the nlp"""

        new_sents = []
        for example in examples:
            new_sents.append(self._remove_in_vocab_words_from_sentence(example))
        return new_sents

    def _remove_in_vocab_words_from_sentence(self, example):
        """Automatically removes words with digits in them, hyperlink and in-vocab-words."""

        cleaned_tokens = []
        for token in example.get("spacy_doc"):
            if not token.has_vector and not token.like_url and \
                    not token.like_num and not token.like_email and not token.is_punct:
                cleaned_tokens.append(token)

        # keep only out-of-vocab 'non_word' words
        non_words = ' '.join([t.text for t in cleaned_tokens])

        # remove digits and extra spaces
        non_words = ''.join([letter for letter in non_words if not letter.isdigit()])
        non_words = ' '.join([word for word in non_words.split(' ') if word != ''])

        # add cleaned sentence to list of these sentences
        return non_words

    def _sort_applicable_ngrams(self, list_of_ngrams, examples, labels):
        """Given an intent classification problem and a list of ngrams, creates ordered list of most useful ngrams."""

        if list_of_ngrams:
            from sklearn import linear_model, preprocessing
            import numpy as np

            # filter examples where we do not have enough labeled instances for cv
            usable_labels = []
            for label in np.unique(labels):
                lab_sents = np.array(examples)[np.array(labels) == label]
                if len(lab_sents) < self.min_intent_examples_for_ngram_classification:
                    continue
                usable_labels.append(label)

            mask = [label in usable_labels for label in labels]
            if any(mask) and len(usable_labels) >= 2:
                try:
                    examples = np.array(examples)[mask]
                    labels = np.array(labels)[mask]

                    X = np.array(self._ngrams_in_sentences(examples, list_of_ngrams))
                    intent_encoder = preprocessing.LabelEncoder()
                    intent_encoder.fit(labels)
                    y = intent_encoder.transform(labels)

                    clf = linear_model.RandomizedLogisticRegression(C=1)
                    clf.fit(X, y)
                    scores = clf.scores_
                    sort_idx = [i[0] for i in sorted(enumerate(scores), key=lambda x: -1 * x[1])]

                    return np.array(list_of_ngrams)[sort_idx]
                except ValueError as e:
                    if "needs samples of at least 2 classes" in str(e):
                        # we got unlucky during the random sampling :( and selected a slice that only contains one class
                        return []
                    else:
                        raise e
            else:
                # there is no example we can use for the cross validation
                return []
        else:
            return []

    def _ngrams_in_sentences(self, examples, ngrams):
        """Given a set of sentences, returns a feature vector for each sentence.

        The first $k$ elements are from the `intent_features`,
        the rest are {1,0} elements denoting whether an ngram is in sentence."""

        all_vectors = []
        for example in examples:
            presence_vector = self._ngrams_in_sentence(example, ngrams)
            all_vectors.append(presence_vector)
        return all_vectors

    def _ngrams_in_sentence(self, example, ngrams):
        """Given a set of sentences, returns a vector of {1,0} values indicating ngram presence"""

        import numpy as np

        cleaned_sentence = self._remove_in_vocab_words_from_sentence(example)
        presence_vector = np.zeros(len(ngrams))
        idx_array = [idx for idx in range(len(ngrams)) if ngrams[idx] in cleaned_sentence]
        presence_vector[idx_array] = 1
        return presence_vector

    def _generate_all_ngrams(self, list_of_strings):
        """Takes a list of strings and generates all character ngrams.

        Generated ngrams are at least 3 characters (and at most 17), occur at least 5 times
        and occur independently of longer superset ngrams at least once."""

        features = {}
        counters = {self.n_gram_min_length - 1: Counter()}

        for n in range(self.n_gram_min_length, self.n_gram_max_length):
            candidates = []
            features[n] = []
            counters[n] = Counter()

            # generate all possible n length ngrams
            for text in list_of_strings:
                text = text.replace(punctuation, ' ')
                for word in text.lower().split(' '):
                    cands = [word[i:i + n] for i in range(len(word) - n)]
                    for cand in cands:
                        counters[n][cand] += 1
                        if cand not in candidates:
                            candidates.append(cand)

            # iterate over these candidates picking only the applicable ones
            for can in candidates:
                if counters[n][can] >= self.n_gram_min_occurrences:
                    features[n].append(can)
                    begin = can[:-1]
                    end = can[1:]
                    if n >= self.n_gram_min_length:
                        if counters[n - 1][begin] == counters[n][can] and begin in features[n - 1]:
                            features[n - 1].remove(begin)
                        if counters[n - 1][end] == counters[n][can] and end in features[n - 1]:
                            features[n - 1].remove(end)

        return [item for sublist in list(features.values()) for item in sublist]

    def _cross_validation(self, examples, labels, max_ngrams):
        """Choose the best number of ngrams to include in bow.

        Given an intent classification problem and a set of ordered ngrams
        (ordered in terms of importance by pick_applicable_ngrams) we
        choose the best number of ngrams to include in our bow vecs
        by cross validation."""

        from sklearn import preprocessing
        from sklearn.linear_model import LogisticRegression
        from sklearn.model_selection import cross_val_score
        import numpy as np

        if examples:
            collected_features = [e.get("text_features")
                                  for e in examples
                                  if e.get("text_features") is not None]
        else:
            collected_features = []

        if collected_features:
            existing_text_features = np.stack(collected_features)
        else:
            existing_text_features = None

        def features_with_ngrams(max_ngrams):
            ngrams_to_use = self._ngrams_to_use(max_ngrams)
            extras = np.array(self._ngrams_in_sentences(examples,
                                                        ngrams_to_use))
            if existing_text_features is not None:
                return np.hstack((existing_text_features, extras))
            else:
                return extras

        clf2 = LogisticRegression(class_weight='balanced')
        intent_encoder = preprocessing.LabelEncoder()
        intent_encoder.fit(labels)
        y = intent_encoder.transform(labels)
        cv_splits = min(10, np.min(np.bincount(y))) if y.size > 0 else 0
        if cv_splits >= 3:
            logger.debug("Started ngram cross-validation to find b"
                         "est number of ngrams to use...")
            possible_ngrams = np.linspace(1, max_ngrams, 8)
            num_ngrams = np.unique(list(map(int, np.floor(possible_ngrams))))
            if existing_text_features is not None:
                no_ngrams_X = features_with_ngrams(max_ngrams=0)
                no_ngrams_score = np.mean(cross_val_score(clf2, no_ngrams_X, y,
                                                          cv=cv_splits))
            else:
                no_ngrams_score = 0.0
            scores = []
            for n in num_ngrams:
                X = features_with_ngrams(max_ngrams=n)
                score = np.mean(cross_val_score(clf2, X, y, cv=cv_splits))
                scores.append(score)
                logger.debug("Evaluating usage of {} ngrams. "
                             "Score: {}".format(n, score))
            n_top = num_ngrams[np.argmax(scores)]
            logger.debug("Score without ngrams: "
                         "{}".format(no_ngrams_score))
            logger.info("Best score with {} ngrams: "
                        "{}".format(n_top, np.max(scores)))
            return n_top
        else:
            warnings.warn("Can't cross-validate ngram featurizer. "
                          "There aren't enough examples per intent "
                          "(at least 3)")
            return max_ngrams
<commit_msg>cleanups<commit_after>from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import

import typing
from builtins import map
from builtins import range
import logging
import os
import re
import time
import warnings
import io
from collections import Counter
from string import punctuation
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from future.utils import PY3
from typing import Text

from rasa_nlu.components import Component
from rasa_nlu.config import RasaNLUConfig
from rasa_nlu.featurizers import Featurizer
from rasa_nlu.training_data import Message
from rasa_nlu.training_data import TrainingData

logger = logging.getLogger(__name__)

if typing.TYPE_CHECKING:
    from spacy.language import Language
    import numpy as np
    from rasa_nlu.model import Metadata

class NGramFeaturizer(Featurizer):
    name = "intent_featurizer_ngrams"

    provides = ["text_features"]

    requires = ["spacy_doc"]

    n_gram_min_length = 3

    n_gram_max_length = 17

    n_gram_min_occurrences = 5

    min_intent_examples_for_ngram_classification = 4

    def __init__(self):
        self.best_num_ngrams = None
        self.all_ngrams = None

    @classmethod
    def required_packages(cls):
        # type: () -> List[Text]
        return ["spacy", "numpy", "sklearn", "cloudpickle"]

    def train(self, training_data, config, **kwargs):
        # type: (TrainingData, RasaNLUConfig, **Any) -> None

        start = time.time()
        self.train_on_sentences(training_data.intent_examples, config["max_number_of_ngrams"])
        logger.debug("Ngram collection took {} seconds".format(time.time() - start))

        for example in training_data.training_examples:
            updated = self._text_features_with_ngrams(example, self.best_num_ngrams)
            example.set("text_features", updated)

    def process(self, message, **kwargs):
        # type: (Message, **Any) -> None

        updated = self._text_features_with_ngrams(message, self.best_num_ngrams)
        message.set("text_features", updated)

    def _text_features_with_ngrams(self, message, max_ngrams):
        import numpy as np

        ngrams_to_use = self._ngrams_to_use(max_ngrams)

        if ngrams_to_use is not None:
            extras = np.array(self._ngrams_in_sentence(message, ngrams_to_use))
            return self._combine_with_existing_text_features(message, extras)
        else:
            return message.get("text_features")

    @classmethod
    def load(cls, model_dir=None, model_metadata=None, cached_component=None, **kwargs):
        # type: (Text, Metadata, Optional[Component], **Any) -> NGramFeaturizer
        import cloudpickle

        if model_dir and model_metadata.get("ngram_featurizer"):
            classifier_file = os.path.join(model_dir, model_metadata.get("ngram_featurizer"))
            with io.open(classifier_file, 'rb') as f:   # pramga: no cover
                if PY3:
                    return cloudpickle.load(f, encoding="latin-1")
                else:
                    return cloudpickle.load(f)
        else:
            return NGramFeaturizer()

    def persist(self, model_dir):
        # type: (Text) -> Dict[Text, Any]
        """Persist this model into the passed directory. Returns the metadata necessary to load the model again."""
        import cloudpickle

        classifier_file = os.path.join(model_dir, "ngram_featurizer.pkl")
        with io.open(classifier_file, 'wb') as f:
            cloudpickle.dump(self, f)

        return {
            "ngram_featurizer": "ngram_featurizer.pkl"
        }

    def train_on_sentences(self, examples, max_number_of_ngrams):
        labels = [e.get("intent") for e in examples]
        self.all_ngrams = self._get_best_ngrams(examples, labels)
        self.best_num_ngrams = self._cross_validation(examples, labels, max_number_of_ngrams)

    def _ngrams_to_use(self, num_ngrams):
        if num_ngrams == 0 or self.all_ngrams is None:
            return []
        elif num_ngrams is not None:
            return self.all_ngrams[:num_ngrams]
        else:
            return self.all_ngrams

    def _get_best_ngrams(self, examples, labels):
        """Returns an ordered list of the best character ngrams for an intent classification problem"""

        oov_strings = self._remove_in_vocab_words(examples)
        ngrams = self._generate_all_ngrams(oov_strings)
        return self._sort_applicable_ngrams(ngrams, examples, labels)

    def _remove_in_vocab_words(self, examples):
        """Automatically removes words with digits in them, that may be a
        hyperlink or that _are_ in vocabulary for the nlp"""

        new_sents = []
        for example in examples:
            new_sents.append(self._remove_in_vocab_words_from_sentence(example))
        return new_sents

    def _remove_in_vocab_words_from_sentence(self, example):
        """Automatically removes words with digits in them, hyperlink and in-vocab-words."""

        cleaned_tokens = []
        for token in example.get("spacy_doc"):
            if (not token.has_vector and not token.like_url
                    and token.like_num and not token.like_email
                    and not token.is_punct):
                cleaned_tokens.append(token)

        # keep only out-of-vocab 'non_word' words
        non_words = ' '.join([t.text for t in cleaned_tokens])

        # remove digits and extra spaces
        non_words = ''.join([letter
                             for letter in non_words
                             if not letter.isdigit()])
        non_words = ' '.join([word
                              for word in non_words.split(' ')
                              if word != ''])

        # add cleaned sentence to list of these sentences
        return non_words

    def _sort_applicable_ngrams(self, list_of_ngrams, examples, labels):
        """Given an intent classification problem and a list of ngrams,

        creates ordered list of most useful ngrams."""

        if list_of_ngrams:
            from sklearn import linear_model, preprocessing
            import numpy as np

            # filter examples where we do not have enough labeled instances for cv
            usable_labels = []
            for label in np.unique(labels):
                lab_sents = np.array(examples)[np.array(labels) == label]
                if len(lab_sents) < self.min_intent_examples_for_ngram_classification:
                    continue
                usable_labels.append(label)

            mask = [label in usable_labels for label in labels]
            if any(mask) and len(usable_labels) >= 2:
                try:
                    examples = np.array(examples)[mask]
                    labels = np.array(labels)[mask]

                    X = np.array(self._ngrams_in_sentences(examples, list_of_ngrams))
                    intent_encoder = preprocessing.LabelEncoder()
                    intent_encoder.fit(labels)
                    y = intent_encoder.transform(labels)

                    clf = linear_model.RandomizedLogisticRegression(C=1)
                    clf.fit(X, y)
                    scores = clf.scores_
                    sort_idx = [i[0] for i in sorted(enumerate(scores), key=lambda x: -1 * x[1])]

                    return np.array(list_of_ngrams)[sort_idx]
                except ValueError as e:
                    if "needs samples of at least 2 classes" in str(e):
                        # we got unlucky during the random sampling :( and selected a slice that only contains one class
                        return []
                    else:
                        raise e
            else:
                # there is no example we can use for the cross validation
                return []
        else:
            return []

    def _ngrams_in_sentences(self, examples, ngrams):
        """Given a set of sentences, returns a feature vector for each sentence.

        The first $k$ elements are from the `intent_features`,
        the rest are {1,0} elements denoting whether an ngram is in sentence."""

        all_vectors = []
        for example in examples:
            presence_vector = self._ngrams_in_sentence(example, ngrams)
            all_vectors.append(presence_vector)
        return all_vectors

    def _ngrams_in_sentence(self, example, ngrams):
        """Given a set of sentences, returns a vector of {1,0} values indicating ngram presence"""

        import numpy as np

        cleaned_sentence = self._remove_in_vocab_words_from_sentence(example)
        presence_vector = np.zeros(len(ngrams))
        idx_array = [idx for idx in range(len(ngrams)) if ngrams[idx] in cleaned_sentence]
        presence_vector[idx_array] = 1
        return presence_vector

    def _generate_all_ngrams(self, list_of_strings):
        """Takes a list of strings and generates all character ngrams.

        Generated ngrams are at least 3 characters (and at most 17), occur at least 5 times
        and occur independently of longer superset ngrams at least once."""

        features = {}
        counters = {self.n_gram_min_length - 1: Counter()}

        for n in range(self.n_gram_min_length, self.n_gram_max_length):
            candidates = []
            features[n] = []
            counters[n] = Counter()

            # generate all possible n length ngrams
            for text in list_of_strings:
                text = text.replace(punctuation, ' ')
                for word in text.lower().split(' '):
                    cands = [word[i:i + n] for i in range(len(word) - n)]
                    for cand in cands:
                        counters[n][cand] += 1
                        if cand not in candidates:
                            candidates.append(cand)

            # iterate over these candidates picking only the applicable ones
            for can in candidates:
                if counters[n][can] >= self.n_gram_min_occurrences:
                    features[n].append(can)
                    begin = can[:-1]
                    end = can[1:]
                    if n >= self.n_gram_min_length:
                        if counters[n - 1][begin] == counters[n][can] and begin in features[n - 1]:
                            features[n - 1].remove(begin)
                        if counters[n - 1][end] == counters[n][can] and end in features[n - 1]:
                            features[n - 1].remove(end)

        return [item for sublist in list(features.values()) for item in sublist]

    def _cross_validation(self, examples, labels, max_ngrams):
        """Choose the best number of ngrams to include in bow.

        Given an intent classification problem and a set of ordered ngrams
        (ordered in terms of importance by pick_applicable_ngrams) we
        choose the best number of ngrams to include in our bow vecs
        by cross validation."""

        from sklearn import preprocessing
        from sklearn.linear_model import LogisticRegression
        from sklearn.model_selection import cross_val_score
        import numpy as np

        if examples:
            collected_features = [e.get("text_features")
                                  for e in examples
                                  if e.get("text_features") is not None]
        else:
            collected_features = []

        if collected_features:
            existing_text_features = np.stack(collected_features)
        else:
            existing_text_features = None

        def features_with_ngrams(max_ngrams):
            ngrams_to_use = self._ngrams_to_use(max_ngrams)
            extras = np.array(self._ngrams_in_sentences(examples,
                                                        ngrams_to_use))
            if existing_text_features is not None:
                return np.hstack((existing_text_features, extras))
            else:
                return extras

        clf2 = LogisticRegression(class_weight='balanced')
        intent_encoder = preprocessing.LabelEncoder()
        intent_encoder.fit(labels)
        y = intent_encoder.transform(labels)
        cv_splits = min(10, np.min(np.bincount(y))) if y.size > 0 else 0
        if cv_splits >= 3:
            logger.debug("Started ngram cross-validation to find b"
                         "est number of ngrams to use...")
            possible_ngrams = np.linspace(1, max_ngrams, 8)
            num_ngrams = np.unique(list(map(int, np.floor(possible_ngrams))))
            if existing_text_features is not None:
                no_ngrams_X = features_with_ngrams(max_ngrams=0)
                no_ngrams_score = np.mean(cross_val_score(clf2, no_ngrams_X, y,
                                                          cv=cv_splits))
            else:
                no_ngrams_score = 0.0
            scores = []
            for n in num_ngrams:
                X = features_with_ngrams(max_ngrams=n)
                score = np.mean(cross_val_score(clf2, X, y, cv=cv_splits))
                scores.append(score)
                logger.debug("Evaluating usage of {} ngrams. "
                             "Score: {}".format(n, score))
            n_top = num_ngrams[np.argmax(scores)]
            logger.debug("Score without ngrams: "
                         "{}".format(no_ngrams_score))
            logger.info("Best score with {} ngrams: "
                        "{}".format(n_top, np.max(scores)))
            return n_top
        else:
            warnings.warn("Can't cross-validate ngram featurizer. "
                          "There aren't enough examples per intent "
                          "(at least 3)")
            return max_ngrams
<|endoftext|>