<commit_before># -*- coding: utf-8 -*-
# !/usr/bin/python

import re
import time

import mavri

wiki = 'tr.wikipedia'
xx = mavri.login(wiki, 'KET Bot')
title = 'Vikipedi:Kullanıcı engelleme talepleri'
version = 'V2.1'
summary_ek = " (WMF-Labs, " + version + ")"
section = 1

while 1:

    content = mavri.content_of_section(wiki, title, section, xx)

    if content != '':
        vandal = re.findall('\{\{\s*[Vv]andal\s*\|\s*([^\}]*)\s*\}\}', content)

        if vandal:
            timestamp = re.findall('\{\{\s*KET Bot\s*\|\s*([^\|\}]*)\s*\|\s*[^\|\}]*\s*\}\}', content)[0]
            informer = re.findall('\{\{\s*KET Bot\s*\|\s*[^\|\}]*\s*\|\s*([^\|\}]*)\s*\}\}', content)[0]
            vandal = vandal[0]
            blocked = mavri.blocked(wiki, vandal)
            if blocked.json()['query']['blocks']:
                by = blocked.json()['query']['blocks'][0]['by']
                reason = blocked.json()['query']['blocks'][0]['reason']
                summary = '[[Özel:Katkılar/' + vandal + '|' + vandal + ']] çıkartıldı. [[Kullanıcı:' + by + '|' + by + ']] - ' + reason + summary_ek
                mavri.section_clear(wiki, title, section, summary, xx)
                message = '\n*Merhaba. [[Özel:Katkılar/' + vandal + '|' + vandal + ']], [[Kullanıcı mesaj:' + by + '|' + by + ']] tarafından engellendi. Engel açıklaması:' + reason + ' Bildirimde bulunduğunuz için teşekkürler --~~~~'
                summary = '[[Özel:Katkılar/' + vandal + '|' + vandal + ']], [[Kullanıcı mesaj:' + by + '|' + by + ']] tarafından engellendi.' + summary_ek
                mavri.sent_message(wiki, 'Kullanıcı mesaj:' + informer, message, summary, xx)
        else:
            print mavri.section_clear(wiki, title, section, '{{Vandal|XXXX}} içermeyen başlık kaldırıldı.' + summary_ek, xx).text
        section += 1
    else:
        section = 1
        time.sleep(60)
<commit_msg>V2.2 Zaman aşımına uğramış bildirimleri kaldırma özelliği eklendi<commit_after># -*- coding: utf-8 -*-
# !/usr/bin/python

import re
import socket
import time
from datetime import datetime

import mavri

wiki = 'tr.wikipedia'
xx = mavri.login(wiki, 'KET Bot')
title = 'Vikipedi:Kullanıcı engelleme talepleri'
version = 'V2.2'
summary_ek = " (WMF-Labs, " + version + ")"
section = 1
mpa = dict.fromkeys(range(32))

while 1:
    content = mavri.content_of_section(wiki, title, section, xx)

    if content != '':
        vandal = re.findall('\{\{\s*[Vv]andal\s*\|\s*([^\}]*)\s*\}\}', content)

        if vandal:
            timestamp = re.findall('\{\{\s*KET Bot\s*\|\s*([^\|\}]*)\s*\|\s*[^\|\}]*\s*\}\}', content)[0]
            informer = re.findall('\{\{\s*KET Bot\s*\|\s*[^\|\}]*\s*\|\s*([^\|\}]*)\s*\}\}', content)[0]
            vandal = vandal[0]
            blocked = mavri.blocked(wiki, vandal)
            not_time = datetime(int(timestamp[:4]), int(timestamp[4:6]), int(timestamp[6:8]), int(timestamp[8:10]),
                                int(timestamp[10:12]), int(timestamp[12:14]))
            vandal = vandal.translate(mpa)

            try:
                socket.inet_aton(vandal)
                IP = 1
            except socket.error:
                IP = 0

            if blocked.json()['query']['blocks']:
                timestamp = blocked.json()['query']['blocks'][0]['timestamp']
                blocked_time = datetime(int(timestamp[:4]), int(timestamp[5:7]), int(timestamp[8:10]),
                                        int(timestamp[11:13]), int(timestamp[14:16]), int(timestamp[17:19]))
                elapsed_time = str(blocked_time - not_time).replace('days,', 'gün')
                by = blocked.json()['query']['blocks'][0]['by']
                reason = blocked.json()['query']['blocks'][0]['reason']

                summary = '[[Özel:Katkılar/' + vandal + '|' + vandal + ']] çıkartıldı. [[Kullanıcı:' + by + '|' + by + ']] - ' + reason + summary_ek
                mavri.section_clear(wiki, title, section, summary, xx)

                message = '\n* Merhaba. [[Özel:Katkılar/' + vandal + '|' + vandal + ']], siz bildirim yaptıktan ' + elapsed_time + ' saat sonra [[Kullanıcı mesaj:' + by + '|' + by + ']] tarafından engellendi. Engel açıklaması:' + reason + ' Bildirimde bulunduğunuz için teşekkürler. --~~~~'
                summary = '[[Özel:Katkılar/' + vandal + '|' + vandal + ']], [[Kullanıcı mesaj:' + by + '|' + by + ']] tarafından engellendi.' + summary_ek
                mavri.sent_message(wiki, 'Kullanıcı mesaj:' + informer, message, summary, xx)
            else:
                now = datetime.now()
                diff = now - not_time

                if IP and diff.total_seconds() > 60 * 60 * 24:
                    summary = '[[Özel:Katkılar/' + vandal + '|' + vandal + ']] çıkartıldı. Bildirim zaman aşımına uğradı.' + summary_ek
                    mavri.section_clear(wiki, title, section, summary, xx)

                    message = '\n* Merhaba. [[Özel:Katkılar/' + vandal + '|' + vandal + ']], siz bildirim yaptıktan sonra 24 saat geçmesine rağmen engellenmediği için sayfadan çıkartıldı. Bildirimde bulunduğunuz için teşekkürler. --~~~~'
                    summary = '[[Özel:Katkılar/' + vandal + '|' + vandal + ']] bildirimi zaman aşımına uğradı.' + summary_ek
                    mavri.sent_message(wiki, 'Kullanıcı mesaj:' + informer, message, summary, xx)
                if IP == 0 and diff.total_seconds() > 60 * 60 * 24 * 5:
                    summary = '[[Özel:Katkılar/' + vandal + '|' + vandal + ']] çıkartıldı. Bildirim zaman aşımına uğradı.' + summary_ek
                    mavri.section_clear(wiki, title, section, summary, xx)

                    message = '\n* Merhaba. [[Özel:Katkılar/' + vandal + '|' + vandal + ']], siz bildirim yaptıktan sonra 5 gün geçmesine rağmen engellenmediği için sayfadan çıkartıldı. Bildirimde bulunduğunuz için teşekkürler. --~~~~'
                    summary = '[[Özel:Katkılar/' + vandal + '|' + vandal + ']] bildirimi zaman aşımına uğradı.' + summary_ek
                    mavri.sent_message(wiki, 'Kullanıcı mesaj:' + informer, message, summary, xx)
        else:
            mavri.section_clear(wiki, title, section, '{{Vandal|XXXX}} içermeyen başlık kaldırıldı.' + summary_ek, xx)
        section += 1
    else:
        section = 1
        time.sleep(60)
<|endoftext|>


<commit_before>import argparse
import asyncio
import importlib.util
import logging
import os
import signal
import traceback
import typing
from multiprocessing import get_context
from typing import List, Text, Optional, Tuple, Union, Iterable

import aiohttp
import ruamel.yaml as yaml

import rasa.cli.utils as cli_utils
import rasa.utils.io as io_utils
from rasa.cli.arguments import x as arguments
from rasa.constants import (
    DEFAULT_ENDPOINTS_PATH,
    DEFAULT_CREDENTIALS_PATH,
    DEFAULT_DOMAIN_PATH,
    DEFAULT_CONFIG_PATH,
    DEFAULT_LOG_LEVEL_RASA_X,
    DEFAULT_RASA_X_PORT,
    DEFAULT_RASA_PORT,
)
from rasa.utils.endpoints import EndpointConfig

logger = logging.getLogger(__name__)

DEFAULT_TRACKER_DB = "tracker.db"

if typing.TYPE_CHECKING:
    from rasa.core.utils import AvailableEndpoints

# noinspection PyProtectedMember
def add_subparser(
    subparsers: argparse._SubParsersAction, parents: List[argparse.ArgumentParser]
):
    x_parser_args = {
        "parents": parents,
        "conflict_handler": "resolve",
        "formatter_class": argparse.ArgumentDefaultsHelpFormatter,
    }

    if is_rasa_x_installed():
        # we'll only show the help msg for the command if Rasa X is actually installed
        x_parser_args["help"] = "Starts the Rasa X interface."

    shell_parser = subparsers.add_parser("x", **x_parser_args)
    shell_parser.set_defaults(func=rasa_x)

    arguments.set_x_arguments(shell_parser)

def _rasa_service(
    args: argparse.Namespace,
    endpoints: "AvailableEndpoints",
    rasa_x_url: Optional[Text] = None,
    credentials_path: Optional[Text] = None,
):
    """Starts the Rasa application."""
    from rasa.core.run import serve_application

    # needs separate logging configuration as it is started in its own process
    logging.basicConfig(level=args.loglevel)
    io_utils.configure_colored_logging(args.loglevel)
    logging.getLogger("apscheduler").setLevel(logging.WARNING)

    if not credentials_path:
        credentials_path = _prepare_credentials_for_rasa_x(
            args.credentials, rasa_x_url=rasa_x_url
        )

    serve_application(
        endpoints=endpoints,
        port=args.port,
        credentials=credentials_path,
        cors=args.cors,
        auth_token=args.auth_token,
        enable_api=True,
        jwt_secret=args.jwt_secret,
        jwt_method=args.jwt_method,
    )

def _prepare_credentials_for_rasa_x(
    credentials_path: Optional[Text], rasa_x_url: Optional[Text] = None
) -> Text:
    credentials_path = cli_utils.get_validated_path(
        credentials_path, "credentials", DEFAULT_CREDENTIALS_PATH, True
    )
    if credentials_path:
        credentials = io_utils.read_config_file(credentials_path)
    else:
        credentials = {}

    # this makes sure the Rasa X is properly configured no matter what
    if rasa_x_url:
        credentials["rasa"] = {"url": rasa_x_url}
    dumped_credentials = yaml.dump(credentials, default_flow_style=False)
    tmp_credentials = io_utils.create_temporary_file(dumped_credentials, "yml")

    return tmp_credentials

def _overwrite_endpoints_for_local_x(
    endpoints: "AvailableEndpoints", rasa_x_token: Text, rasa_x_url: Text
):
    from rasa.utils.endpoints import EndpointConfig
    import questionary

    endpoints.model = EndpointConfig(
        "{}/projects/default/models/tags/production".format(rasa_x_url),
        token=rasa_x_token,
        wait_time_between_pulls=2,
    )

    overwrite_existing_tracker_store = False
    if endpoints.tracker_store and not _is_correct_tracker_store(
        endpoints.tracker_store
    ):
        cli_utils.print_error(
            "Rasa X currently only supports a SQLite tracker store with path '{}' "
            "when running locally. You can deploy Rasa X with Docker "
            "(https://rasa.com/docs/rasa-x/deploy/) if you want to use "
            "other tracker store configurations.".format(DEFAULT_TRACKER_DB)
        )
        overwrite_existing_tracker_store = questionary.confirm(
            "Do you want to continue with the default SQLite tracker store?"
        ).ask()

        if not overwrite_existing_tracker_store:
            exit(0)

    if not endpoints.tracker_store or overwrite_existing_tracker_store:
        endpoints.tracker_store = EndpointConfig(type="sql", db=DEFAULT_TRACKER_DB)

def _is_correct_tracker_store(tracker_endpoint: EndpointConfig) -> bool:
    return (
        tracker_endpoint.type == "sql"
        and tracker_endpoint.kwargs.get("dialect", "").lower() == "sqlite"
        and tracker_endpoint.kwargs.get("db") == DEFAULT_TRACKER_DB
    )

def start_rasa_for_local_rasa_x(args: argparse.Namespace, rasa_x_token: Text):
    """Starts the Rasa X API with Rasa as a background process."""

    credentials_path, endpoints_path = _get_credentials_and_endpoints_paths(args)
    endpoints = AvailableEndpoints.read_endpoints(endpoints_path)

    rasa_x_url = "http://localhost:{}/api".format(args.rasa_x_port)
    _overwrite_endpoints_for_local_x(endpoints, rasa_x_token, rasa_x_url)

    vars(args).update(
        dict(
            nlu_model=None,
            cors="*",
            auth_token=args.auth_token,
            enable_api=True,
            endpoints=endpoints,
        )
    )

    ctx = get_context("spawn")
    p = ctx.Process(
        target=_rasa_service, args=(args, endpoints, rasa_x_url, credentials_path)
    )
    p.daemon = True
    p.start()
    return p

def is_rasa_x_installed():
    """Check if Rasa X is installed."""

    # we could also do something like checking if `import rasax` works,
    # the issue with that is that it actually does import the package and this
    # takes some time that we don't want to spend when booting the CLI
    return importlib.util.find_spec("rasax") is not None

def generate_rasa_x_token(length: int = 16):
    """Generate a hexadecimal secret token used to access the Rasa X API.

    A new token is generated on every `rasa x` command.
    """

    from secrets import token_hex

    return token_hex(length)

def _configure_logging(args: argparse.Namespace):
    from rasa.core.utils import configure_file_logging
    from rasa.utils.common import set_log_level

    log_level = args.loglevel or DEFAULT_LOG_LEVEL_RASA_X

    if isinstance(log_level, str):
        log_level = logging.getLevelName(log_level)

    logging.basicConfig(level=log_level)
    io_utils.configure_colored_logging(args.loglevel)

    set_log_level(log_level)
    configure_file_logging(args.log_file)

    logging.getLogger("werkzeug").setLevel(logging.WARNING)
    logging.getLogger("engineio").setLevel(logging.WARNING)
    logging.getLogger("pika").setLevel(logging.WARNING)
    logging.getLogger("socketio").setLevel(logging.ERROR)

    if not log_level == logging.DEBUG:
        logging.getLogger().setLevel(logging.WARNING)
        logging.getLogger("py.warnings").setLevel(logging.ERROR)

def is_rasa_project_setup(project_path: Text):
    mandatory_files = [DEFAULT_CONFIG_PATH, DEFAULT_DOMAIN_PATH]

    for f in mandatory_files:
        if not os.path.exists(os.path.join(project_path, f)):
            return False

    return True

def _validate_rasa_x_start(args: argparse.Namespace, project_path: Text):
    if not is_rasa_x_installed():
        cli_utils.print_error_and_exit(
            "Rasa X is not installed. The `rasa x` "
            "command requires an installation of Rasa X. "
            "Instructions on how to install Rasa X can be found here: "
            "https://rasa.com/docs/rasa-x/installation-and-setup/."
        )

    if args.port == args.rasa_x_port:
        cli_utils.print_error_and_exit(
            "The port for Rasa X '{}' and the port of the Rasa server '{}' are the "
            "same. We need two different ports, one to run Rasa X (e.g. delivering the "
            "UI) and another one to run a normal Rasa server.\nPlease specify two "
            "different ports using the arguments '--port' and '--rasa-x-port'.".format(
                args.rasa_x_port, args.port
            )
        )

    if not is_rasa_project_setup(project_path):
        cli_utils.print_error_and_exit(
            "This directory is not a valid Rasa project. Use 'rasa init' "
            "to create a new Rasa project or switch to a valid Rasa project "
            "directory (see http://rasa.com/docs/rasa/user-guide/"
            "rasa-tutorial/#create-a-new-project)."
        )

    _validate_domain(os.path.join(project_path, DEFAULT_DOMAIN_PATH))

    if args.data and not os.path.exists(args.data):
        cli_utils.print_warning(
            "The provided data path ('{}') does not exists. Rasa X will start "
            "without any training data.".format(args.data)
        )

def _validate_domain(domain_path: Text):
    from rasa.core.domain import Domain, InvalidDomain

    try:
        Domain.load(domain_path)
    except InvalidDomain as e:
        cli_utils.print_error_and_exit(
            "The provided domain file could not be loaded. " "Error: {}".format(e)
        )

def rasa_x(args: argparse.Namespace):
    from rasa.cli.utils import signal_handler

    signal.signal(signal.SIGINT, signal_handler)

    _configure_logging(args)

    if args.production:
        run_in_production(args)
    else:
        run_locally(args)

async def _pull_runtime_config_from_server(
    config_endpoint: Optional[Text],
    attempts: int = 30,
    wait_time_between_pulls: Union[int, float] = 0.5,
    keys: Iterable[Text] = ("endpoints", "credentials"),
) -> List[Text]:
    """Pull runtime config from `config_endpoint`.

    Returns a path to yaml dumps, each containing the contents of one of `keys`.
    """

    while attempts > 0:
        try:
            async with aiohttp.ClientSession() as session:
                async with session.get(config_endpoint) as resp:
                    if resp.status == 200:
                        rjs = await resp.json()
                        try:
                            return [
                                io_utils.dump_dict_to_temporary_yaml_file(rjs[k])
                                for k in keys
                            ]
                        except KeyError as e:
                            cli_utils.print_error_and_exit(
                                "Failed to find key '{}' in runtime config. "
                                "Exiting.".format(e)
                            )
                    else:
                        logger.debug(
                            "Failed to get a proper response from remote "
                            "server. Status Code: {}. Response: '{}'"
                            "".format(resp.status, await resp.text())
                        )
        except aiohttp.ClientError as e:
            logger.debug("Failed to connect to server. Retrying. {}".format(e))

        await asyncio.sleep(wait_time_between_pulls)
        attempts -= 1

    cli_utils.print_error_and_exit(
        "Could not fetch runtime config from server at '{}'. "
        "Exiting.".format(config_endpoint)
    )

def run_in_production(args: argparse.Namespace):
    from rasa.cli.utils import print_success

    print_success("Starting Rasa X in production mode... 🚀")

    credentials_path, endpoints_path = _get_credentials_and_endpoints_paths(args)
    endpoints = AvailableEndpoints.read_endpoints(endpoints_path)

    _rasa_service(args, endpoints, None, credentials_path)

def _get_credentials_and_endpoints_paths(args) -> Tuple[Optional[Text], Optional[Text]]:
    config_endpoint = args.config_endpoint
    if config_endpoint:
        loop = asyncio.get_event_loop()
        endpoints_config_path, credentials_path = loop.run_until_complete(
            _pull_runtime_config_from_server(config_endpoint)
        )

    else:
        endpoints_config_path = cli_utils.get_validated_path(
            args.endpoints, "endpoints", DEFAULT_ENDPOINTS_PATH, True
        )
        credentials_path = None

    return credentials_path, endpoints_config_path

def run_locally(args: argparse.Namespace):
    # noinspection PyUnresolvedReferences
    from rasax.community import local  # pytype: disable=import-error

    args.rasa_x_port = args.rasa_x_port or DEFAULT_RASA_X_PORT
    args.port = args.port or DEFAULT_RASA_PORT

    project_path = "."

    _validate_rasa_x_start(args, project_path)

    local.check_license_and_metrics(args)
    rasa_x_token = generate_rasa_x_token()
    process = start_rasa_for_local_rasa_x(args, rasa_x_token=rasa_x_token)

    try:
        local.main(args, project_path, args.data, token=rasa_x_token)
    except Exception:
        print (traceback.format_exc())
        cli_utils.print_error(
            "Sorry, something went wrong (see error above). Make sure to start "
            "Rasa X with valid data and valid domain and config files. Please, "
            "also check any warnings that popped up.\nIf you need help fixing "
            "the issue visit our forum: https://forum.rasa.com/."
        )
    finally:
        process.terminate()
<commit_msg>one more type<commit_after>import argparse
import asyncio
import importlib.util
import logging
import os
import signal
import traceback
import typing
from multiprocessing import get_context
from typing import List, Text, Optional, Tuple, Union, Iterable

import aiohttp
import ruamel.yaml as yaml

import rasa.cli.utils as cli_utils
import rasa.utils.io as io_utils
from rasa.cli.arguments import x as arguments
from rasa.constants import (
    DEFAULT_ENDPOINTS_PATH,
    DEFAULT_CREDENTIALS_PATH,
    DEFAULT_DOMAIN_PATH,
    DEFAULT_CONFIG_PATH,
    DEFAULT_LOG_LEVEL_RASA_X,
    DEFAULT_RASA_X_PORT,
    DEFAULT_RASA_PORT,
)
from rasa.utils.endpoints import EndpointConfig

logger = logging.getLogger(__name__)

DEFAULT_TRACKER_DB = "tracker.db"

if typing.TYPE_CHECKING:
    from rasa.core.utils import AvailableEndpoints

# noinspection PyProtectedMember
def add_subparser(
    subparsers: argparse._SubParsersAction, parents: List[argparse.ArgumentParser]
):
    x_parser_args = {
        "parents": parents,
        "conflict_handler": "resolve",
        "formatter_class": argparse.ArgumentDefaultsHelpFormatter,
    }

    if is_rasa_x_installed():
        # we'll only show the help msg for the command if Rasa X is actually installed
        x_parser_args["help"] = "Starts the Rasa X interface."

    shell_parser = subparsers.add_parser("x", **x_parser_args)
    shell_parser.set_defaults(func=rasa_x)

    arguments.set_x_arguments(shell_parser)

def _rasa_service(
    args: argparse.Namespace,
    endpoints: "AvailableEndpoints",
    rasa_x_url: Optional[Text] = None,
    credentials_path: Optional[Text] = None,
):
    """Starts the Rasa application."""
    from rasa.core.run import serve_application

    # needs separate logging configuration as it is started in its own process
    logging.basicConfig(level=args.loglevel)
    io_utils.configure_colored_logging(args.loglevel)
    logging.getLogger("apscheduler").setLevel(logging.WARNING)

    if not credentials_path:
        credentials_path = _prepare_credentials_for_rasa_x(
            args.credentials, rasa_x_url=rasa_x_url
        )

    serve_application(
        endpoints=endpoints,
        port=args.port,
        credentials=credentials_path,
        cors=args.cors,
        auth_token=args.auth_token,
        enable_api=True,
        jwt_secret=args.jwt_secret,
        jwt_method=args.jwt_method,
    )

def _prepare_credentials_for_rasa_x(
    credentials_path: Optional[Text], rasa_x_url: Optional[Text] = None
) -> Text:
    credentials_path = cli_utils.get_validated_path(
        credentials_path, "credentials", DEFAULT_CREDENTIALS_PATH, True
    )
    if credentials_path:
        credentials = io_utils.read_config_file(credentials_path)
    else:
        credentials = {}

    # this makes sure the Rasa X is properly configured no matter what
    if rasa_x_url:
        credentials["rasa"] = {"url": rasa_x_url}
    dumped_credentials = yaml.dump(credentials, default_flow_style=False)
    tmp_credentials = io_utils.create_temporary_file(dumped_credentials, "yml")

    return tmp_credentials

def _overwrite_endpoints_for_local_x(
    endpoints: "AvailableEndpoints", rasa_x_token: Text, rasa_x_url: Text
):
    from rasa.utils.endpoints import EndpointConfig
    import questionary

    endpoints.model = EndpointConfig(
        "{}/projects/default/models/tags/production".format(rasa_x_url),
        token=rasa_x_token,
        wait_time_between_pulls=2,
    )

    overwrite_existing_tracker_store = False
    if endpoints.tracker_store and not _is_correct_tracker_store(
        endpoints.tracker_store
    ):
        cli_utils.print_error(
            "Rasa X currently only supports a SQLite tracker store with path '{}' "
            "when running locally. You can deploy Rasa X with Docker "
            "(https://rasa.com/docs/rasa-x/deploy/) if you want to use "
            "other tracker store configurations.".format(DEFAULT_TRACKER_DB)
        )
        overwrite_existing_tracker_store = questionary.confirm(
            "Do you want to continue with the default SQLite tracker store?"
        ).ask()

        if not overwrite_existing_tracker_store:
            exit(0)

    if not endpoints.tracker_store or overwrite_existing_tracker_store:
        endpoints.tracker_store = EndpointConfig(type="sql", db=DEFAULT_TRACKER_DB)

def _is_correct_tracker_store(tracker_endpoint: EndpointConfig) -> bool:
    return (
        tracker_endpoint.type == "sql"
        and tracker_endpoint.kwargs.get("dialect", "").lower() == "sqlite"
        and tracker_endpoint.kwargs.get("db") == DEFAULT_TRACKER_DB
    )

def start_rasa_for_local_rasa_x(args: argparse.Namespace, rasa_x_token: Text):
    """Starts the Rasa X API with Rasa as a background process."""

    credentials_path, endpoints_path = _get_credentials_and_endpoints_paths(args)
    endpoints = AvailableEndpoints.read_endpoints(endpoints_path)

    rasa_x_url = "http://localhost:{}/api".format(args.rasa_x_port)
    _overwrite_endpoints_for_local_x(endpoints, rasa_x_token, rasa_x_url)

    vars(args).update(
        dict(
            nlu_model=None,
            cors="*",
            auth_token=args.auth_token,
            enable_api=True,
            endpoints=endpoints,
        )
    )

    ctx = get_context("spawn")
    p = ctx.Process(
        target=_rasa_service, args=(args, endpoints, rasa_x_url, credentials_path)
    )
    p.daemon = True
    p.start()
    return p

def is_rasa_x_installed():
    """Check if Rasa X is installed."""

    # we could also do something like checking if `import rasax` works,
    # the issue with that is that it actually does import the package and this
    # takes some time that we don't want to spend when booting the CLI
    return importlib.util.find_spec("rasax") is not None

def generate_rasa_x_token(length: int = 16):
    """Generate a hexadecimal secret token used to access the Rasa X API.

    A new token is generated on every `rasa x` command.
    """

    from secrets import token_hex

    return token_hex(length)

def _configure_logging(args: argparse.Namespace):
    from rasa.core.utils import configure_file_logging
    from rasa.utils.common import set_log_level

    log_level = args.loglevel or DEFAULT_LOG_LEVEL_RASA_X

    if isinstance(log_level, str):
        log_level = logging.getLevelName(log_level)

    logging.basicConfig(level=log_level)
    io_utils.configure_colored_logging(args.loglevel)

    set_log_level(log_level)
    configure_file_logging(args.log_file)

    logging.getLogger("werkzeug").setLevel(logging.WARNING)
    logging.getLogger("engineio").setLevel(logging.WARNING)
    logging.getLogger("pika").setLevel(logging.WARNING)
    logging.getLogger("socketio").setLevel(logging.ERROR)

    if not log_level == logging.DEBUG:
        logging.getLogger().setLevel(logging.WARNING)
        logging.getLogger("py.warnings").setLevel(logging.ERROR)

def is_rasa_project_setup(project_path: Text):
    mandatory_files = [DEFAULT_CONFIG_PATH, DEFAULT_DOMAIN_PATH]

    for f in mandatory_files:
        if not os.path.exists(os.path.join(project_path, f)):
            return False

    return True

def _validate_rasa_x_start(args: argparse.Namespace, project_path: Text):
    if not is_rasa_x_installed():
        cli_utils.print_error_and_exit(
            "Rasa X is not installed. The `rasa x` "
            "command requires an installation of Rasa X. "
            "Instructions on how to install Rasa X can be found here: "
            "https://rasa.com/docs/rasa-x/installation-and-setup/."
        )

    if args.port == args.rasa_x_port:
        cli_utils.print_error_and_exit(
            "The port for Rasa X '{}' and the port of the Rasa server '{}' are the "
            "same. We need two different ports, one to run Rasa X (e.g. delivering the "
            "UI) and another one to run a normal Rasa server.\nPlease specify two "
            "different ports using the arguments '--port' and '--rasa-x-port'.".format(
                args.rasa_x_port, args.port
            )
        )

    if not is_rasa_project_setup(project_path):
        cli_utils.print_error_and_exit(
            "This directory is not a valid Rasa project. Use 'rasa init' "
            "to create a new Rasa project or switch to a valid Rasa project "
            "directory (see http://rasa.com/docs/rasa/user-guide/"
            "rasa-tutorial/#create-a-new-project)."
        )

    _validate_domain(os.path.join(project_path, DEFAULT_DOMAIN_PATH))

    if args.data and not os.path.exists(args.data):
        cli_utils.print_warning(
            "The provided data path ('{}') does not exists. Rasa X will start "
            "without any training data.".format(args.data)
        )

def _validate_domain(domain_path: Text):
    from rasa.core.domain import Domain, InvalidDomain

    try:
        Domain.load(domain_path)
    except InvalidDomain as e:
        cli_utils.print_error_and_exit(
            "The provided domain file could not be loaded. " "Error: {}".format(e)
        )

def rasa_x(args: argparse.Namespace):
    from rasa.cli.utils import signal_handler

    signal.signal(signal.SIGINT, signal_handler)

    _configure_logging(args)

    if args.production:
        run_in_production(args)
    else:
        run_locally(args)

async def _pull_runtime_config_from_server(
    config_endpoint: Optional[Text],
    attempts: int = 30,
    wait_time_between_pulls: Union[int, float] = 0.5,
    keys: Iterable[Text] = ("endpoints", "credentials"),
) -> Optional[List[Text]]:
    """Pull runtime config from `config_endpoint`.

    Returns a path to yaml dumps, each containing the contents of one of `keys`.
    """

    while attempts > 0:
        try:
            async with aiohttp.ClientSession() as session:
                async with session.get(config_endpoint) as resp:
                    if resp.status == 200:
                        rjs = await resp.json()
                        try:
                            return [
                                io_utils.dump_dict_to_temporary_yaml_file(rjs[k])
                                for k in keys
                            ]
                        except KeyError as e:
                            cli_utils.print_error_and_exit(
                                "Failed to find key '{}' in runtime config. "
                                "Exiting.".format(e)
                            )
                    else:
                        logger.debug(
                            "Failed to get a proper response from remote "
                            "server. Status Code: {}. Response: '{}'"
                            "".format(resp.status, await resp.text())
                        )
        except aiohttp.ClientError as e:
            logger.debug("Failed to connect to server. Retrying. {}".format(e))

        await asyncio.sleep(wait_time_between_pulls)
        attempts -= 1

    cli_utils.print_error_and_exit(
        "Could not fetch runtime config from server at '{}'. "
        "Exiting.".format(config_endpoint)
    )

def run_in_production(args: argparse.Namespace):
    from rasa.cli.utils import print_success

    print_success("Starting Rasa X in production mode... 🚀")

    credentials_path, endpoints_path = _get_credentials_and_endpoints_paths(args)
    endpoints = AvailableEndpoints.read_endpoints(endpoints_path)

    _rasa_service(args, endpoints, None, credentials_path)

def _get_credentials_and_endpoints_paths(args) -> Tuple[Optional[Text], Optional[Text]]:
    config_endpoint = args.config_endpoint
    if config_endpoint:
        loop = asyncio.get_event_loop()
        endpoints_config_path, credentials_path = loop.run_until_complete(
            _pull_runtime_config_from_server(config_endpoint)
        )

    else:
        endpoints_config_path = cli_utils.get_validated_path(
            args.endpoints, "endpoints", DEFAULT_ENDPOINTS_PATH, True
        )
        credentials_path = None

    return credentials_path, endpoints_config_path

def run_locally(args: argparse.Namespace):
    # noinspection PyUnresolvedReferences
    from rasax.community import local  # pytype: disable=import-error

    args.rasa_x_port = args.rasa_x_port or DEFAULT_RASA_X_PORT
    args.port = args.port or DEFAULT_RASA_PORT

    project_path = "."

    _validate_rasa_x_start(args, project_path)

    local.check_license_and_metrics(args)
    rasa_x_token = generate_rasa_x_token()
    process = start_rasa_for_local_rasa_x(args, rasa_x_token=rasa_x_token)

    try:
        local.main(args, project_path, args.data, token=rasa_x_token)
    except Exception:
        print (traceback.format_exc())
        cli_utils.print_error(
            "Sorry, something went wrong (see error above). Make sure to start "
            "Rasa X with valid data and valid domain and config files. Please, "
            "also check any warnings that popped up.\nIf you need help fixing "
            "the issue visit our forum: https://forum.rasa.com/."
        )
    finally:
        process.terminate()
<|endoftext|>


<commit_before>import math
import logging

import pyglet
from pyglet.window import key

import os
script_dir = os.path.dirname(__file__)

tiles = {
    "tw": 10,
    "th": 10,
    # Current template arrangement will take up to 50x50 tiles
    "width": 20,
    "height": 20
}

#fe = 1.0e-4
settings = {
    "log_level": logging.INFO,
    "window": {
        "width": 500,
        "height": 500,
        "vsync": True,
        "resizable": False
    },
    "player": {
        "radius": tiles['tw'] / 4,
        # TODO: Put all velocity settings in dict
        "top_speed": 10.0,
        "angular_velocity": 240.0 / 4,  # degrees / s
        "accel": 5.0,
        "deaccel": 5.0,
        # TODO: Refactor to action `costs`, will apply to whichever `stat`
        "battery_use": {
            "angular": 0.01,
            "linear": 0.01
        },
        "sensors": {
            "num": 9,
            "fov": 15*math.pi/180,
            "max_range": 200 / 4
        },
        "actions": [
            #['noop'],
            ['left'],
            ['up', 'left'],
            ['up'],
            ['up', 'right'],
            ['right']
        ]
    },
    "world": {
        "width": tiles['tw'] * tiles['width'],
        "height": tiles['th'] * tiles['height'],
        "bindings": {
            #0: 'noop',
            key.LEFT: 'left',
            key.RIGHT: 'right',
            key.UP: 'up',
        }
    },
    "view": {
        # as the font file is not provided it will decay to the default font;
        # the setting is retained anyway to not downgrade the code
        "font_name": 'Axaxax',
        "palette": {
            'bg': (0, 65, 133),
            'wall': (50, 50, 100), # Wall sensor colour
            'player': (237, 27, 36),
            #'wall': (247, 148, 29),
            'gate': (140, 198, 62),
            'food': (140, 198, 62),
            'poison': (198, 62, 62)
        }
    }
}

# Callbacks to test reward conditions
def __cond_action_up(world):
    # Detect other keys
    held_buttons = {k:v for k,v in world.buttons.iteritems() if v == 1}
    return len(held_buttons) == 1 and world.buttons['up'] == 1

def __cond_battery_out(world):
    return world.player.stats['battery'] <= 0

def __cond_explore_battery(world):
    return world.player.stats['battery'] > 50

def __cond_goal_battery(world):
    return world.player.stats['battery'] <= 50

modes = [
    # Mode 0
    {
        #"proximity": {
        #    "cond": __cond_action_up,
        #    "reward": 1.1 # 10% Bonus on proximity reward
        #},
        "wall": {
            "reward": -10.0,
            "terminal": False
        },
        "items": {
            "food": {
                "num": 20,
                "scale": 2.0,
                "reward": 5.0,
                "terminal": False
            },
            "poison": {
                "num": 20,
                "scale": 2.0,
                "reward": -6.0,
                "terminal": False
            },
        }
    },
    # Mode 1
    {
        "battery": {
            "cond": __cond_battery_out,
            "reward": -100.0,
            "terminal": True
        },
        "explore": {
            "cond": __cond_explore_battery,
            "reward": 1.0,
            "terminal": False
        },
        "goal": {
            "cond": __cond_goal_battery,
            "reward": 200.0,
            "terminal": True
        },
        "wall": {
            "reward": -100.0,
            "terminal": True
        },
        "items": {}
    }
]

# Ensure all modes have items for convenience
for mode in modes:
    if mode['items'] is None:
        mode['items'] = {}

# world to view scales
scale_x = settings["window"]["width"] / settings["world"]["width"]
scale_y = settings["window"]["height"] / settings["world"]["height"]

# load resources:
pics = {
    "player": pyglet.image.load(os.path.join(script_dir, 'assets', 'player7.png')),
    "food": pyglet.image.load(os.path.join(script_dir, 'assets', 'circle6.png')),
    "poison": pyglet.image.load(os.path.join(script_dir, 'assets', 'circle6.png'))
}
<commit_msg>More drag<commit_after>import math
import logging

import pyglet
from pyglet.window import key

import os
script_dir = os.path.dirname(__file__)

tiles = {
    "tw": 10,
    "th": 10,
    # Current template arrangement will take up to 50x50 tiles
    "width": 20,
    "height": 20
}

#fe = 1.0e-4
settings = {
    "log_level": logging.INFO,
    "window": {
        "width": 500,
        "height": 500,
        "vsync": True,
        "resizable": False
    },
    "player": {
        "radius": tiles['tw'] / 4,
        # TODO: Put all velocity settings in dict
        "top_speed": 10.0,
        "angular_velocity": 240.0 / 4,  # degrees / s
        "accel": 5.0,
        "deaccel": 15.0,
        # TODO: Refactor to action `costs`, will apply to whichever `stat`
        "battery_use": {
            "angular": 0.01,
            "linear": 0.01
        },
        "sensors": {
            "num": 9,
            "fov": 15*math.pi/180,
            "max_range": 200 / 4
        },
        "actions": [
            #['noop'],
            ['left'],
            ['up', 'left'],
            ['up'],
            ['up', 'right'],
            ['right']
        ]
    },
    "world": {
        "width": tiles['tw'] * tiles['width'],
        "height": tiles['th'] * tiles['height'],
        "bindings": {
            #0: 'noop',
            key.LEFT: 'left',
            key.RIGHT: 'right',
            key.UP: 'up',
        }
    },
    "view": {
        # as the font file is not provided it will decay to the default font;
        # the setting is retained anyway to not downgrade the code
        "font_name": 'Axaxax',
        "palette": {
            'bg': (0, 65, 133),
            'wall': (50, 50, 100), # Wall sensor colour
            'player': (237, 27, 36),
            #'wall': (247, 148, 29),
            'gate': (140, 198, 62),
            'food': (140, 198, 62),
            'poison': (198, 62, 62)
        }
    }
}

# Callbacks to test reward conditions
def __cond_action_up(world):
    # Detect other keys
    held_buttons = {k:v for k,v in world.buttons.iteritems() if v == 1}
    return len(held_buttons) == 1 and world.buttons['up'] == 1

def __cond_battery_out(world):
    return world.player.stats['battery'] <= 0

def __cond_explore_battery(world):
    return world.player.stats['battery'] > 50

def __cond_goal_battery(world):
    return world.player.stats['battery'] <= 50

modes = [
    # Mode 0
    {
        #"proximity": {
        #    "cond": __cond_action_up,
        #    "reward": 1.1 # 10% Bonus on proximity reward
        #},
        "wall": {
            "reward": -10.0,
            "terminal": False
        },
        "items": {
            "food": {
                "num": 20,
                "scale": 2.0,
                "reward": 5.0,
                "terminal": False
            },
            "poison": {
                "num": 20,
                "scale": 2.0,
                "reward": -6.0,
                "terminal": False
            },
        }
    },
    # Mode 1
    {
        "battery": {
            "cond": __cond_battery_out,
            "reward": -100.0,
            "terminal": True
        },
        "explore": {
            "cond": __cond_explore_battery,
            "reward": 1.0,
            "terminal": False
        },
        "goal": {
            "cond": __cond_goal_battery,
            "reward": 200.0,
            "terminal": True
        },
        "wall": {
            "reward": -100.0,
            "terminal": True
        },
        "items": {}
    }
]

# Ensure all modes have items for convenience
for mode in modes:
    if mode['items'] is None:
        mode['items'] = {}

# world to view scales
scale_x = settings["window"]["width"] / settings["world"]["width"]
scale_y = settings["window"]["height"] / settings["world"]["height"]

# load resources:
pics = {
    "player": pyglet.image.load(os.path.join(script_dir, 'assets', 'player7.png')),
    "food": pyglet.image.load(os.path.join(script_dir, 'assets', 'circle6.png')),
    "poison": pyglet.image.load(os.path.join(script_dir, 'assets', 'circle6.png'))
}
<|endoftext|>


<commit_before>import glob
import logging
import os
import shutil
import tempfile
import typing
from collections import namedtuple
from pathlib import Path
from typing import Text, Tuple, Union, Optional, List, Dict, Any

import rasa.utils.io
from rasa.cli.utils import print_success, create_output_path
from rasa.constants import (
    DEFAULT_MODELS_PATH,
    CONFIG_MANDATORY_KEYS_CORE,
    CONFIG_MANDATORY_KEYS_NLU,
    CONFIG_MANDATORY_KEYS,
    DEFAULT_DOMAIN_PATH,
    DEFAULT_CORE_SUBDIRECTORY_NAME,
)

from rasa.core.utils import get_dict_hash
from rasa.exceptions import ModelNotFound
from rasa.utils.common import TempDirectoryPath

if typing.TYPE_CHECKING:
    from rasa.importers.importer import TrainingDataImporter

logger = logging.getLogger(__name__)

# Type alias for the fingerprint
Fingerprint = Dict[Text, Union[Text, List[Text], int, float]]

FINGERPRINT_FILE_PATH = "fingerprint.json"

FINGERPRINT_CONFIG_KEY = "config"
FINGERPRINT_CONFIG_CORE_KEY = "core-config"
FINGERPRINT_CONFIG_NLU_KEY = "nlu-config"
FINGERPRINT_DOMAIN_WITHOUT_NLG_KEY = "domain"
FINGERPRINT_NLG_KEY = "nlg"
FINGERPRINT_RASA_VERSION_KEY = "version"
FINGERPRINT_STORIES_KEY = "stories"
FINGERPRINT_NLU_DATA_KEY = "messages"
FINGERPRINT_TRAINED_AT_KEY = "trained_at"

Section = namedtuple("Section", ["name", "relevant_keys"])

SECTION_CORE = Section(
    name="Core model",
    relevant_keys=[
        FINGERPRINT_CONFIG_KEY,
        FINGERPRINT_CONFIG_CORE_KEY,
        FINGERPRINT_DOMAIN_WITHOUT_NLG_KEY,
        FINGERPRINT_STORIES_KEY,
        FINGERPRINT_RASA_VERSION_KEY,
    ],
)
SECTION_NLU = Section(
    name="NLU model",
    relevant_keys=[
        FINGERPRINT_CONFIG_KEY,
        FINGERPRINT_CONFIG_NLU_KEY,
        FINGERPRINT_NLU_DATA_KEY,
        FINGERPRINT_RASA_VERSION_KEY,
    ],
)
SECTION_NLG = Section(name="NLG Templates", relevant_keys=[FINGERPRINT_NLG_KEY])

class FingerprintComparisonResult:
    def __init__(
        self,
        nlu: bool = True,
        core: bool = True,
        nlg: bool = True,
        force_train: bool = False,
    ):
        self.nlu = nlu
        self.core = core
        self.nlg = nlg
        self.force_train = force_train

    def __eq__(self, obj: Any) -> bool:
        if not isinstance(obj, FingerprintComparisonResult):
            return False
        return all(
            [
                self.nlu == obj.nlu,
                self.core == obj.core,
                self.nlg == obj.nlg,
                self.force_train == obj.force_train,
            ]
        )

    def is_training_required(self) -> bool:
        return any([self.nlg, self.nlu, self.core, self.force_train])

    def should_retrain_core(self) -> bool:
        return self.force_train or self.core

    def should_retrain_nlg(self) -> bool:
        return self.force_train or self.nlg

    def should_retrain_nlu(self) -> bool:
        return self.force_train or self.nlu

def get_model(model_path: Text = DEFAULT_MODELS_PATH) -> TempDirectoryPath:
    """Get a model and unpack it. Raises a `ModelNotFound` exception if
    no model could be found at the provided path.

    Args:
        model_path: Path to the zipped model. If it's a directory, the latest
                    trained model is returned.

    Returns:
        Path to the unpacked model.

    """
    if not model_path:
        raise ModelNotFound("No path specified.")
    elif not os.path.exists(model_path):
        raise ModelNotFound("No file or directory at '{}'.".format(model_path))

    if os.path.isdir(model_path):
        model_path = get_latest_model(model_path)
        if not model_path:
            raise ModelNotFound(
                "Could not find any Rasa model files in '{}'.".format(model_path)
            )
    elif not model_path.endswith(".tar.gz"):
        raise ModelNotFound(
            "Path '{}' does not point to a Rasa model file.".format(model_path)
        )

    return unpack_model(model_path)

def get_latest_model(model_path: Text = DEFAULT_MODELS_PATH) -> Optional[Text]:
    """Get the latest model from a path.

    Args:
        model_path: Path to a directory containing zipped models.

    Returns:
        Path to latest model in the given directory.

    """
    if not os.path.exists(model_path) or os.path.isfile(model_path):
        model_path = os.path.dirname(model_path)

    list_of_files = glob.glob(os.path.join(model_path, "*.tar.gz"))

    if len(list_of_files) == 0:
        return None

    return max(list_of_files, key=os.path.getctime)

def unpack_model(
    model_file: Text, working_directory: Optional[Union[Path, Text]] = None
) -> TempDirectoryPath:
    """Unpack a zipped Rasa model.

    Args:
        model_file: Path to zipped model.
        working_directory: Location where the model should be unpacked to.
                           If `None` a temporary directory will be created.

    Returns:
        Path to unpacked Rasa model.

    """
    import tarfile

    if working_directory is None:
        working_directory = tempfile.mkdtemp()

    # All files are in a subdirectory.
    with tarfile.open(model_file, mode="r:gz") as tar:
        tar.extractall(working_directory)
    logger.debug(f"Extracted model to '{working_directory}'.")

    return TempDirectoryPath(working_directory)

def get_model_subdirectories(
    unpacked_model_path: Text
) -> Tuple[Optional[Text], Optional[Text]]:
    """Return paths for Core and NLU model directories, if they exist.
    If neither directories exist, a `ModelNotFound` exception is raised.

    Args:
        unpacked_model_path: Path to unpacked Rasa model.

    Returns:
        Tuple (path to Core subdirectory if it exists or `None` otherwise,
               path to NLU subdirectory if it exists or `None` otherwise).

    """
    core_path = os.path.join(unpacked_model_path, DEFAULT_CORE_SUBDIRECTORY_NAME)
    nlu_path = os.path.join(unpacked_model_path, "nlu")

    if not os.path.isdir(core_path):
        core_path = None

    if not os.path.isdir(nlu_path):
        nlu_path = None

    if not core_path and not nlu_path:
        raise ModelNotFound(
            "No NLU or Core data for unpacked model at: '{}'.".format(
                unpacked_model_path
            )
        )

    return core_path, nlu_path

def create_package_rasa(
    training_directory: Text,
    output_filename: Text,
    fingerprint: Optional[Fingerprint] = None,
) -> Text:
    """Create a zipped Rasa model from trained model files.

    Args:
        training_directory: Path to the directory which contains the trained
                            model files.
        output_filename: Name of the zipped model file to be created.
        fingerprint: A unique fingerprint to identify the model version.

    Returns:
        Path to zipped model.

    """
    import tarfile

    if fingerprint:
        persist_fingerprint(training_directory, fingerprint)

    output_directory = os.path.dirname(output_filename)
    if not os.path.exists(output_directory):
        os.makedirs(output_directory)

    with tarfile.open(output_filename, "w:gz") as tar:
        for elem in os.scandir(training_directory):
            tar.add(elem.path, arcname=elem.name)

    shutil.rmtree(training_directory)
    return output_filename

async def model_fingerprint(file_importer: "TrainingDataImporter") -> Fingerprint:
    """Create a model fingerprint from its used configuration and training data.

    Args:
        file_importer: File importer which provides the training data and model config.

    Returns:
        The fingerprint.

    """
    from rasa.core.domain import Domain

    import rasa
    import time

    config = await file_importer.get_config()
    domain = await file_importer.get_domain()
    stories = await file_importer.get_stories()
    nlu_data = await file_importer.get_nlu_data()

    domain_dict = domain.as_dict()
    templates = domain_dict.pop("templates")
    domain_without_nlg = Domain.from_dict(domain_dict)

    return {
        FINGERPRINT_CONFIG_KEY: _get_hash_of_config(
            config, exclude_keys=CONFIG_MANDATORY_KEYS
        ),
        FINGERPRINT_CONFIG_CORE_KEY: _get_hash_of_config(
            config, include_keys=CONFIG_MANDATORY_KEYS_CORE
        ),
        FINGERPRINT_CONFIG_NLU_KEY: _get_hash_of_config(
            config, include_keys=CONFIG_MANDATORY_KEYS_NLU
        ),
        FINGERPRINT_DOMAIN_WITHOUT_NLG_KEY: hash(domain_without_nlg),
        FINGERPRINT_NLG_KEY: get_dict_hash(templates),
        FINGERPRINT_NLU_DATA_KEY: hash(nlu_data),
        FINGERPRINT_STORIES_KEY: hash(stories),
        FINGERPRINT_TRAINED_AT_KEY: time.time(),
        FINGERPRINT_RASA_VERSION_KEY: rasa.__version__,
    }

def _get_hash_of_config(
    config: Optional[Dict],
    include_keys: Optional[List[Text]] = None,
    exclude_keys: Optional[List[Text]] = None,
) -> Text:
    if not config:
        return ""

    keys = include_keys or list(filter(lambda k: k not in exclude_keys, config.keys()))

    sub_config = dict((k, config[k]) for k in keys if k in config)

    return get_dict_hash(sub_config)

def fingerprint_from_path(model_path: Text) -> Fingerprint:
    """Load a persisted fingerprint.

    Args:
        model_path: Path to directory containing the fingerprint.

    Returns:
        The fingerprint or an empty dict if no fingerprint was found.
    """
    if not model_path or not os.path.exists(model_path):
        return {}

    fingerprint_path = os.path.join(model_path, FINGERPRINT_FILE_PATH)

    if os.path.isfile(fingerprint_path):
        return rasa.utils.io.read_json_file(fingerprint_path)
    else:
        return {}

def persist_fingerprint(output_path: Text, fingerprint: Fingerprint):
    """Persist a model fingerprint.

    Args:
        output_path: Directory in which the fingerprint should be saved.
        fingerprint: The fingerprint to be persisted.

    """

    path = os.path.join(output_path, FINGERPRINT_FILE_PATH)
    rasa.utils.io.dump_obj_as_json_to_file(path, fingerprint)

def did_section_fingerprint_change(
    fingerprint1: Fingerprint, fingerprint2: Fingerprint, section: Section
) -> bool:
    """Check whether the fingerprint of a section has changed."""
    for k in section.relevant_keys:
        if fingerprint1.get(k) != fingerprint2.get(k):
            logger.info(f"Data ({k}) for {section.name} section changed.")
            return True
    return False

def merge_model(source: Text, target: Text) -> bool:
    """Merge two model directories.

    Args:
        source: The original folder which should be merged in another.
        target: The destination folder where it should be moved to.

    Returns:
        `True` if the merge was successful, else `False`.

    """
    try:
        shutil.move(source, target)
        return True
    except Exception as e:
        logging.debug(f"Could not merge model: {e}")
        return False

def should_retrain(
    new_fingerprint: Fingerprint, old_model: Text, train_path: Text
) -> FingerprintComparisonResult:
    """Check which components of a model should be retrained.

    Args:
        new_fingerprint: The fingerprint of the new model to be trained.
        old_model: Path to the old zipped model file.
        train_path: Path to the directory in which the new model will be trained.

    Returns:
        A ShouldRetrain object indicating whether Rasa Core and/or Rasa NLU needs
        to be retrained or not.

    """
    retrain = FingerprintComparisonResult()

    if old_model is None or not os.path.exists(old_model):
        return retrain

    with unpack_model(old_model) as unpacked:
        last_fingerprint = fingerprint_from_path(unpacked)
        old_core, old_nlu = get_model_subdirectories(unpacked)

        retrain = FingerprintComparisonResult(
            core=did_section_fingerprint_change(
                last_fingerprint, new_fingerprint, SECTION_CORE
            ),
            nlu=did_section_fingerprint_change(
                last_fingerprint, new_fingerprint, SECTION_NLU
            ),
            nlg=did_section_fingerprint_change(
                last_fingerprint, new_fingerprint, SECTION_NLG
            ),
        )

        core_merge_failed = False
        if not retrain.core:
            target_path = os.path.join(train_path, DEFAULT_CORE_SUBDIRECTORY_NAME)
            core_merge_failed = not merge_model(old_core, target_path)
            retrain.core = core_merge_failed

        if not retrain.nlg:
            retrain.nlg = core_merge_failed

        if not retrain.nlu:
            target_path = os.path.join(train_path, "nlu")
            retrain.nlu = not merge_model(old_nlu, target_path)

        return retrain

def package_model(
    fingerprint: Fingerprint,
    output_directory: Text,
    train_path: Text,
    fixed_model_name: Optional[Text] = None,
    model_prefix: Text = "",
):
    """
    Compress a trained model.

    Args:
        fingerprint: fingerprint of the model
        output_directory: path to the directory in which the model should be stored
        train_path: path to uncompressed model
        fixed_model_name: name of the compressed model file
        model_prefix: prefix of the compressed model file

    Returns: path to 'tar.gz' model file
    """
    output_directory = create_output_path(
        output_directory, prefix=model_prefix, fixed_name=fixed_model_name
    )
    create_package_rasa(train_path, output_directory, fingerprint)

    print_success(
        "Your Rasa model is trained and saved at '{}'.".format(
            os.path.abspath(output_directory)
        )
    )

    return output_directory

async def update_with_new_domain(
    importer: "TrainingDataImporter", unpacked_model_path: Union[Path, Text]
) -> None:
    """Overwrites the domain of an unpacked model with a new domain.

    Args:
        importer: Importer which provides the new domain.
        unpacked_model_path: Path to the unpacked model.
    """

    model_path = Path(unpacked_model_path) / DEFAULT_CORE_SUBDIRECTORY_NAME
    domain = await importer.get_domain()

    domain.persist(model_path / DEFAULT_DOMAIN_PATH)
    domain.persist_specification(str(model_path))
<commit_msg>add docstring<commit_after>import glob
import logging
import os
import shutil
import tempfile
import typing
from collections import namedtuple
from pathlib import Path
from typing import Text, Tuple, Union, Optional, List, Dict, Any

import rasa.utils.io
from rasa.cli.utils import print_success, create_output_path
from rasa.constants import (
    DEFAULT_MODELS_PATH,
    CONFIG_MANDATORY_KEYS_CORE,
    CONFIG_MANDATORY_KEYS_NLU,
    CONFIG_MANDATORY_KEYS,
    DEFAULT_DOMAIN_PATH,
    DEFAULT_CORE_SUBDIRECTORY_NAME,
)

from rasa.core.utils import get_dict_hash
from rasa.exceptions import ModelNotFound
from rasa.utils.common import TempDirectoryPath

if typing.TYPE_CHECKING:
    from rasa.importers.importer import TrainingDataImporter

logger = logging.getLogger(__name__)

# Type alias for the fingerprint
Fingerprint = Dict[Text, Union[Text, List[Text], int, float]]

FINGERPRINT_FILE_PATH = "fingerprint.json"

FINGERPRINT_CONFIG_KEY = "config"
FINGERPRINT_CONFIG_CORE_KEY = "core-config"
FINGERPRINT_CONFIG_NLU_KEY = "nlu-config"
FINGERPRINT_DOMAIN_WITHOUT_NLG_KEY = "domain"
FINGERPRINT_NLG_KEY = "nlg"
FINGERPRINT_RASA_VERSION_KEY = "version"
FINGERPRINT_STORIES_KEY = "stories"
FINGERPRINT_NLU_DATA_KEY = "messages"
FINGERPRINT_TRAINED_AT_KEY = "trained_at"

Section = namedtuple("Section", ["name", "relevant_keys"])

SECTION_CORE = Section(
    name="Core model",
    relevant_keys=[
        FINGERPRINT_CONFIG_KEY,
        FINGERPRINT_CONFIG_CORE_KEY,
        FINGERPRINT_DOMAIN_WITHOUT_NLG_KEY,
        FINGERPRINT_STORIES_KEY,
        FINGERPRINT_RASA_VERSION_KEY,
    ],
)
SECTION_NLU = Section(
    name="NLU model",
    relevant_keys=[
        FINGERPRINT_CONFIG_KEY,
        FINGERPRINT_CONFIG_NLU_KEY,
        FINGERPRINT_NLU_DATA_KEY,
        FINGERPRINT_RASA_VERSION_KEY,
    ],
)
SECTION_NLG = Section(name="NLG Templates", relevant_keys=[FINGERPRINT_NLG_KEY])

class FingerprintComparisonResult:
    def __init__(
        self,
        nlu: bool = True,
        core: bool = True,
        nlg: bool = True,
        force_train: bool = False,
    ):
        """Creates a `FingerprintComparisonResult` instance.

        Args:
            nlu: `True` if the NLU model should be retrained.
            core: `True` if the Core model should be retrained.
            nlg: `True` if the templates in the domain should be updated.
            force_train: `True` if a training of all parts is forced.
        """
        self.nlu = nlu
        self.core = core
        self.nlg = nlg
        self.force_train = force_train

    def is_training_required(self) -> bool:
        """Check if anything has to be retrained."""

        return any([self.nlg, self.nlu, self.core, self.force_train])

    def should_retrain_core(self) -> bool:
        """Check if the Core model has to be updated."""

        return self.force_train or self.core

    def should_retrain_nlg(self) -> bool:
        """Check if the templates have to be updated."""

        return self.force_train or self.nlg

    def should_retrain_nlu(self) -> bool:
        """Check if the NLU model has to be updated."""

        return self.force_train or self.nlu

def get_model(model_path: Text = DEFAULT_MODELS_PATH) -> TempDirectoryPath:
    """Get a model and unpack it. Raises a `ModelNotFound` exception if
    no model could be found at the provided path.

    Args:
        model_path: Path to the zipped model. If it's a directory, the latest
                    trained model is returned.

    Returns:
        Path to the unpacked model.

    """
    if not model_path:
        raise ModelNotFound("No path specified.")
    elif not os.path.exists(model_path):
        raise ModelNotFound("No file or directory at '{}'.".format(model_path))

    if os.path.isdir(model_path):
        model_path = get_latest_model(model_path)
        if not model_path:
            raise ModelNotFound(
                "Could not find any Rasa model files in '{}'.".format(model_path)
            )
    elif not model_path.endswith(".tar.gz"):
        raise ModelNotFound(
            "Path '{}' does not point to a Rasa model file.".format(model_path)
        )

    return unpack_model(model_path)

def get_latest_model(model_path: Text = DEFAULT_MODELS_PATH) -> Optional[Text]:
    """Get the latest model from a path.

    Args:
        model_path: Path to a directory containing zipped models.

    Returns:
        Path to latest model in the given directory.

    """
    if not os.path.exists(model_path) or os.path.isfile(model_path):
        model_path = os.path.dirname(model_path)

    list_of_files = glob.glob(os.path.join(model_path, "*.tar.gz"))

    if len(list_of_files) == 0:
        return None

    return max(list_of_files, key=os.path.getctime)

def unpack_model(
    model_file: Text, working_directory: Optional[Union[Path, Text]] = None
) -> TempDirectoryPath:
    """Unpack a zipped Rasa model.

    Args:
        model_file: Path to zipped model.
        working_directory: Location where the model should be unpacked to.
                           If `None` a temporary directory will be created.

    Returns:
        Path to unpacked Rasa model.

    """
    import tarfile

    if working_directory is None:
        working_directory = tempfile.mkdtemp()

    # All files are in a subdirectory.
    with tarfile.open(model_file, mode="r:gz") as tar:
        tar.extractall(working_directory)
    logger.debug(f"Extracted model to '{working_directory}'.")

    return TempDirectoryPath(working_directory)

def get_model_subdirectories(
    unpacked_model_path: Text
) -> Tuple[Optional[Text], Optional[Text]]:
    """Return paths for Core and NLU model directories, if they exist.
    If neither directories exist, a `ModelNotFound` exception is raised.

    Args:
        unpacked_model_path: Path to unpacked Rasa model.

    Returns:
        Tuple (path to Core subdirectory if it exists or `None` otherwise,
               path to NLU subdirectory if it exists or `None` otherwise).

    """
    core_path = os.path.join(unpacked_model_path, DEFAULT_CORE_SUBDIRECTORY_NAME)
    nlu_path = os.path.join(unpacked_model_path, "nlu")

    if not os.path.isdir(core_path):
        core_path = None

    if not os.path.isdir(nlu_path):
        nlu_path = None

    if not core_path and not nlu_path:
        raise ModelNotFound(
            "No NLU or Core data for unpacked model at: '{}'.".format(
                unpacked_model_path
            )
        )

    return core_path, nlu_path

def create_package_rasa(
    training_directory: Text,
    output_filename: Text,
    fingerprint: Optional[Fingerprint] = None,
) -> Text:
    """Create a zipped Rasa model from trained model files.

    Args:
        training_directory: Path to the directory which contains the trained
                            model files.
        output_filename: Name of the zipped model file to be created.
        fingerprint: A unique fingerprint to identify the model version.

    Returns:
        Path to zipped model.

    """
    import tarfile

    if fingerprint:
        persist_fingerprint(training_directory, fingerprint)

    output_directory = os.path.dirname(output_filename)
    if not os.path.exists(output_directory):
        os.makedirs(output_directory)

    with tarfile.open(output_filename, "w:gz") as tar:
        for elem in os.scandir(training_directory):
            tar.add(elem.path, arcname=elem.name)

    shutil.rmtree(training_directory)
    return output_filename

async def model_fingerprint(file_importer: "TrainingDataImporter") -> Fingerprint:
    """Create a model fingerprint from its used configuration and training data.

    Args:
        file_importer: File importer which provides the training data and model config.

    Returns:
        The fingerprint.

    """
    from rasa.core.domain import Domain

    import rasa
    import time

    config = await file_importer.get_config()
    domain = await file_importer.get_domain()
    stories = await file_importer.get_stories()
    nlu_data = await file_importer.get_nlu_data()

    domain_dict = domain.as_dict()
    templates = domain_dict.pop("templates")
    domain_without_nlg = Domain.from_dict(domain_dict)

    return {
        FINGERPRINT_CONFIG_KEY: _get_hash_of_config(
            config, exclude_keys=CONFIG_MANDATORY_KEYS
        ),
        FINGERPRINT_CONFIG_CORE_KEY: _get_hash_of_config(
            config, include_keys=CONFIG_MANDATORY_KEYS_CORE
        ),
        FINGERPRINT_CONFIG_NLU_KEY: _get_hash_of_config(
            config, include_keys=CONFIG_MANDATORY_KEYS_NLU
        ),
        FINGERPRINT_DOMAIN_WITHOUT_NLG_KEY: hash(domain_without_nlg),
        FINGERPRINT_NLG_KEY: get_dict_hash(templates),
        FINGERPRINT_NLU_DATA_KEY: hash(nlu_data),
        FINGERPRINT_STORIES_KEY: hash(stories),
        FINGERPRINT_TRAINED_AT_KEY: time.time(),
        FINGERPRINT_RASA_VERSION_KEY: rasa.__version__,
    }

def _get_hash_of_config(
    config: Optional[Dict],
    include_keys: Optional[List[Text]] = None,
    exclude_keys: Optional[List[Text]] = None,
) -> Text:
    if not config:
        return ""

    keys = include_keys or list(filter(lambda k: k not in exclude_keys, config.keys()))

    sub_config = dict((k, config[k]) for k in keys if k in config)

    return get_dict_hash(sub_config)

def fingerprint_from_path(model_path: Text) -> Fingerprint:
    """Load a persisted fingerprint.

    Args:
        model_path: Path to directory containing the fingerprint.

    Returns:
        The fingerprint or an empty dict if no fingerprint was found.
    """
    if not model_path or not os.path.exists(model_path):
        return {}

    fingerprint_path = os.path.join(model_path, FINGERPRINT_FILE_PATH)

    if os.path.isfile(fingerprint_path):
        return rasa.utils.io.read_json_file(fingerprint_path)
    else:
        return {}

def persist_fingerprint(output_path: Text, fingerprint: Fingerprint):
    """Persist a model fingerprint.

    Args:
        output_path: Directory in which the fingerprint should be saved.
        fingerprint: The fingerprint to be persisted.

    """

    path = os.path.join(output_path, FINGERPRINT_FILE_PATH)
    rasa.utils.io.dump_obj_as_json_to_file(path, fingerprint)

def did_section_fingerprint_change(
    fingerprint1: Fingerprint, fingerprint2: Fingerprint, section: Section
) -> bool:
    """Check whether the fingerprint of a section has changed."""
    for k in section.relevant_keys:
        if fingerprint1.get(k) != fingerprint2.get(k):
            logger.info(f"Data ({k}) for {section.name} section changed.")
            return True
    return False

def merge_model(source: Text, target: Text) -> bool:
    """Merge two model directories.

    Args:
        source: The original folder which should be merged in another.
        target: The destination folder where it should be moved to.

    Returns:
        `True` if the merge was successful, else `False`.

    """
    try:
        shutil.move(source, target)
        return True
    except Exception as e:
        logging.debug(f"Could not merge model: {e}")
        return False

def should_retrain(
    new_fingerprint: Fingerprint, old_model: Text, train_path: Text
) -> FingerprintComparisonResult:
    """Check which components of a model should be retrained.

    Args:
        new_fingerprint: The fingerprint of the new model to be trained.
        old_model: Path to the old zipped model file.
        train_path: Path to the directory in which the new model will be trained.

    Returns:
        A ShouldRetrain object indicating whether Rasa Core and/or Rasa NLU needs
        to be retrained or not.

    """
    retrain = FingerprintComparisonResult()

    if old_model is None or not os.path.exists(old_model):
        return retrain

    with unpack_model(old_model) as unpacked:
        last_fingerprint = fingerprint_from_path(unpacked)
        old_core, old_nlu = get_model_subdirectories(unpacked)

        retrain = FingerprintComparisonResult(
            core=did_section_fingerprint_change(
                last_fingerprint, new_fingerprint, SECTION_CORE
            ),
            nlu=did_section_fingerprint_change(
                last_fingerprint, new_fingerprint, SECTION_NLU
            ),
            nlg=did_section_fingerprint_change(
                last_fingerprint, new_fingerprint, SECTION_NLG
            ),
        )

        core_merge_failed = False
        if not retrain.core:
            target_path = os.path.join(train_path, DEFAULT_CORE_SUBDIRECTORY_NAME)
            core_merge_failed = not merge_model(old_core, target_path)
            retrain.core = core_merge_failed

        if not retrain.nlg:
            retrain.nlg = core_merge_failed

        if not retrain.nlu:
            target_path = os.path.join(train_path, "nlu")
            retrain.nlu = not merge_model(old_nlu, target_path)

        return retrain

def package_model(
    fingerprint: Fingerprint,
    output_directory: Text,
    train_path: Text,
    fixed_model_name: Optional[Text] = None,
    model_prefix: Text = "",
):
    """
    Compress a trained model.

    Args:
        fingerprint: fingerprint of the model
        output_directory: path to the directory in which the model should be stored
        train_path: path to uncompressed model
        fixed_model_name: name of the compressed model file
        model_prefix: prefix of the compressed model file

    Returns: path to 'tar.gz' model file
    """
    output_directory = create_output_path(
        output_directory, prefix=model_prefix, fixed_name=fixed_model_name
    )
    create_package_rasa(train_path, output_directory, fingerprint)

    print_success(
        "Your Rasa model is trained and saved at '{}'.".format(
            os.path.abspath(output_directory)
        )
    )

    return output_directory

async def update_with_new_domain(
    importer: "TrainingDataImporter", unpacked_model_path: Union[Path, Text]
) -> None:
    """Overwrites the domain of an unpacked model with a new domain.

    Args:
        importer: Importer which provides the new domain.
        unpacked_model_path: Path to the unpacked model.
    """

    model_path = Path(unpacked_model_path) / DEFAULT_CORE_SUBDIRECTORY_NAME
    domain = await importer.get_domain()

    domain.persist(model_path / DEFAULT_DOMAIN_PATH)
    domain.persist_specification(str(model_path))
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""measure_cocoapod_size.py provides size impact of a given set of cocoapods.

Usage: ./measure_cocoapod_size.py -cocoapods $POD_NAME:$POD_VERSION

"""

import argparse
import json
import os
import tempfile
from collections import OrderedDict
from xcode_project_diff import GenerateSizeDifference
from utils import shell

OBJC_APP_DIR = 'sizetestproject'
OBJC_APP_NAME = 'SizeTest'
SWIFT_APP_DIR = 'SwiftApp'
SWIFT_APP_NAME = 'SwiftApp'

MODE_SWIFT = 'swift'
MODE_OBJC = 'objc'

DEFAULT_SPEC_REPOS = ['https://cdn.cocoapods.org/']

MASTER_SOURCE = 'master'

SPEC_REPO_DICT = {
    'cpdc-internal': 'sso://cpdc-internal/spec',
    'cpdc-eap': 'sso://cpdc-eap/spec',
    'specsstaging': 'https://github.com/firebase/SpecsStaging',
    MASTER_SOURCE: 'https://cdn.cocoapods.org/'
}

def GetSampleApp(mode):
  if mode == MODE_SWIFT:
    return SWIFT_APP_DIR, SWIFT_APP_NAME
  else:
    return OBJC_APP_DIR, OBJC_APP_NAME

def InstallPods(cocoapods, target_dir, spec_repos, target_name, mode, pod_sources):
  """InstallPods installs the pods.

  Args:
    cocoapods: Mapping from pod names to pod versions.
    target_dir: The target directory.
    spec_repos: The set of spec repos.
    target_name: The name of the target.
    mode: The type of cocoapods.
    pod_sources: A dict of Pod mapping to its source.

  Returns:
    The path to the workspace.
  """
  cwd = os.getcwd()
  os.chdir(target_dir)
  shell('pod init')
  shell('touch Podfile')

  with open('Podfile', 'w') as podfile:
    for repo in spec_repos:
      podfile.write('source "{}"\n'.format(repo))
    podfile.write('\n')
    if mode == MODE_SWIFT:
      podfile.write('use_frameworks!\n')
    podfile.write('target \'{}\' do\n'.format(target_name))
    for pod, version in cocoapods.items():
      if version:
        podfile.write(' pod \'{}\', \'{}\'\n'.format(pod, version))
      elif pod_sources is not None:
        # pod_sources[pod] should have pairs like:
        # "sdk":"FirebaseDatabase" and
        # "git":"sdk/repo.git", "branch":"main" or
        # "path":"~/Documents/SDKrepo"
        for pod_config in pod_sources['pods']:
          if pod_config['sdk'] == pod:
            pod_source_config = []
            for config in pod_config.items():
              if config[0] != 'sdk':
                pod_source_config.append(":{} => \'{}\'".format(config[0], config[1]))
            podfile.write(' pod \'{}\', {}\n'.format(pod, ",".join(pod_source_config)))
            break
      else:
        podfile.write(' pod \'{}\'\n'.format(pod))
    podfile.write('end')
  shell('cat Podfile')
  shell('pod install')
  os.chdir(cwd)
  return os.path.join(target_dir, '{}.xcworkspace'.format(target_name))

def CopyProject(source_dir, target_dir):
  """CopyProject copies the project from the source to the target.

  Args:
    source_dir: The path to the source directory.
    target_dir: The path to the target directory.
  """
  shell('cp -r {} {}'.format(source_dir, target_dir))

def ValidateSourceConfig(pod_sources):
  if 'pods' not in pod_sources:
    raise ValueError(
            "The JSON config file should have 'pods' object containing pod configs.")

  for pod_config in pod_sources['pods']:
    source_keys = list(pod_config.keys())
    try:
      sdk = pod_config['sdk']
    except KeyError:
      print("SDK should be specified.")
      raise
    if sdk.strip() == "":
      raise ValueError( "SDK should not be empty or blank.")
    elif pod_config and ( source_keys[1] not in {"git", "path"} ):
      raise ValueError(
              "Pod source of SDK {} should be `git` or `path`.".format(sdk))
    elif len(source_keys) == 3:
      if source_keys[1] != "git":
        raise ValueError(
                "For multiple specs for the SDK {} ,`git` should be added with `branch`, `tag` or `commit`".format(sdk))
      if source_keys[2] not in {"branch", "tag", "commit"}:
        raise ValueError(
                "A specified version of the SDK {} should be from `branch`, `tag` or `commit`.".format(sdk))
    elif len(source_keys) > 3:
      raise ValueError(
        "Pod source of SDK {} can only specify `sdk` with `path`, `git`, or `git` and a reference (like a `branch`, `tag`, or `commit`)."
        "See --help for an example config."
        .format(sdk)
      )

def GetPodSizeImpact(parsed_args):
  """GetPodSizeImpact gets the size impact of the set of pods.

  Args:
    parsed_args: The set of arguments passed to the program.
  """
  sample_app_dir, sample_app_name = GetSampleApp(parsed_args.mode)
  cocoapods = {}
  if parsed_args.spec_repos:
    spec_repos = []
    # If cdn source is in spec_repos input, then it will be moved
    # to the end and be added as the last source.
    if MASTER_SOURCE in parsed_args.spec_repos:
        parsed_args.spec_repos.append(
            parsed_args.spec_repos.pop(
                parsed_args.spec_repos.index(
                    MASTER_SOURCE)))
    for repo in parsed_args.spec_repos:
      if repo in SPEC_REPO_DICT:
        spec_repos.append(SPEC_REPO_DICT[repo])
      else:
        spec_repos.append(repo)
  else:
    spec_repos = DEFAULT_SPEC_REPOS
  for pod in parsed_args.cocoapods:
    pod_info = pod.split(':')
    pod_name = pod_info[0].strip()
    if len(pod_info) > 1:
      pod_version = pod_info[1].strip()
    else:
      pod_version = ''
    cocoapods[pod_name] = pod_version
  # Load JSON in order since in bleeding edge version of a Pod, `git` and
  # `branch`/`tag`/`commit` are required and should be in order. e.g.
  # pod 'Alamofire', :git => 'https://github.com/Alamofire/Alamofire.git', :branch => 'dev'
  try:
    if pod_version and parsed_args.cocoapods_source_config:
      print("Since a version for the pod {} is specified, The config file {} \
              will be validated but not used for binary measurement.".format(
                  pod_name, parsed_args.cocoapods_source_config.name))
    pod_sources = json.load(parsed_args.cocoapods_source_config, \
            object_pairs_hook=OrderedDict) if parsed_args.cocoapods_source_config else None
    if pod_sources: ValidateSourceConfig(pod_sources)
  except ValueError as e:
    raise ValueError("could not decode JSON value %s: %s" % (parsed_args.cocoapods_source_config.name, e))
  base_project = tempfile.mkdtemp()
  target_project = tempfile.mkdtemp()
  target_dir = os.path.join(target_project, sample_app_dir)
  CopyProject(sample_app_dir, base_project)
  CopyProject(sample_app_dir, target_project)

  target_project = InstallPods(cocoapods,
                               target_dir,
                               spec_repos, sample_app_name, parsed_args.mode,
                               pod_sources)
  source_project = os.path.join(base_project,
                                '{}/{}.xcodeproj'.format(sample_app_dir, sample_app_name))

  source_size, target_size = GenerateSizeDifference(
      source_project, sample_app_name, target_project, sample_app_name, parsed_args.build_timeout)
  if parsed_args.json:
    # Transfer Podfile to JSON format.
    podfile = shell('pod ipc podfile-json {}/Podfile'.format(target_dir), capture_stdout=True)
    podfile_dict = json.loads(podfile)
    podfile_dict['combined_pods_extra_size'] = target_size - source_size
    with open(parsed_args.json, 'w') as json_file:
      json.dump(podfile_dict, json_file)
  print('The pods combined add an extra size of {} bytes'.format(
      target_size - source_size))

def Main():
  """Main generates the PodSize impact.
  """
  parser = argparse.ArgumentParser(description='The size impact of a cocoapod',
          formatter_class=argparse.RawTextHelpFormatter)
  parser.add_argument(
      '--cocoapods',
      metavar='N',
      type=str,
      nargs='+',
      required=True,
      help='The set of cocoapods')
  parser.add_argument(
      '--mode',
      type=str,
      choices=[MODE_SWIFT, MODE_OBJC],
      default=MODE_OBJC,
      help='Type of cocoapod'
  )
  parser.add_argument(
      '--spec_repos',
      metavar='N',
      type=str,
      nargs='+',
      required=False,
      help='The set of spec_repos')
  parser.add_argument(
      '--cocoapods_source_config',
      metavar='CONFIG_JSON',
      type=argparse.FileType('r'),
      nargs='?',
      required=False,
      default=None,
      help=''' A JSON file with customized pod source.E.g.
        {
          "pods":[
                {
                  "sdk":"FirebaseDatabase",
                  "git":"https://github.com/firebase/firebase-ios-sdk",
                  "branch":"master"
                }
            ]
        }
      If versions are specified in the `cocoapods` arg, config here will be skipped.
      ''')
  parser.add_argument(
      '--build_timeout',
      metavar='SECONDS',
      nargs='?',
      required=False,
      default=None,
      help='Timeout to build testapps.')
  parser.add_argument(
      '--json',
      metavar='OUTPUT_FILE_NAME',
      nargs='?',
      required=False,
      default=None,
      help='Output JSON file.')

  args = parser.parse_args()

  GetPodSizeImpact(args)

if __name__ == '__main__':
  Main()
<commit_msg>Throw an error for a failed build.<commit_after>#!/usr/bin/python
#
# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""measure_cocoapod_size.py provides size impact of a given set of cocoapods.

Usage: ./measure_cocoapod_size.py -cocoapods $POD_NAME:$POD_VERSION

"""

import argparse
import json
import os
import tempfile
from collections import OrderedDict
from xcode_project_diff import GenerateSizeDifference
from utils import shell

OBJC_APP_DIR = 'sizetestproject'
OBJC_APP_NAME = 'SizeTest'
SWIFT_APP_DIR = 'SwiftApp'
SWIFT_APP_NAME = 'SwiftApp'

MODE_SWIFT = 'swift'
MODE_OBJC = 'objc'

DEFAULT_SPEC_REPOS = ['https://cdn.cocoapods.org/']

MASTER_SOURCE = 'master'

SPEC_REPO_DICT = {
    'cpdc-internal': 'sso://cpdc-internal/spec',
    'cpdc-eap': 'sso://cpdc-eap/spec',
    'specsstaging': 'https://github.com/firebase/SpecsStaging',
    MASTER_SOURCE: 'https://cdn.cocoapods.org/'
}

def GetSampleApp(mode):
  if mode == MODE_SWIFT:
    return SWIFT_APP_DIR, SWIFT_APP_NAME
  else:
    return OBJC_APP_DIR, OBJC_APP_NAME

def InstallPods(cocoapods, target_dir, spec_repos, target_name, mode, pod_sources):
  """InstallPods installs the pods.

  Args:
    cocoapods: Mapping from pod names to pod versions.
    target_dir: The target directory.
    spec_repos: The set of spec repos.
    target_name: The name of the target.
    mode: The type of cocoapods.
    pod_sources: A dict of Pod mapping to its source.

  Returns:
    The path to the workspace.
  """
  cwd = os.getcwd()
  os.chdir(target_dir)
  shell('pod init')
  shell('touch Podfile')

  with open('Podfile', 'w') as podfile:
    for repo in spec_repos:
      podfile.write('source "{}"\n'.format(repo))
    podfile.write('\n')
    if mode == MODE_SWIFT:
      podfile.write('use_frameworks!\n')
    podfile.write('target \'{}\' do\n'.format(target_name))
    for pod, version in cocoapods.items():
      if version:
        podfile.write(' pod \'{}\', \'{}\'\n'.format(pod, version))
      elif pod_sources is not None:
        # pod_sources[pod] should have pairs like:
        # "sdk":"FirebaseDatabase" and
        # "git":"sdk/repo.git", "branch":"main" or
        # "path":"~/Documents/SDKrepo"
        for pod_config in pod_sources['pods']:
          if pod_config['sdk'] == pod:
            pod_source_config = []
            for config in pod_config.items():
              if config[0] != 'sdk':
                pod_source_config.append(":{} => \'{}\'".format(config[0], config[1]))
            podfile.write(' pod \'{}\', {}\n'.format(pod, ",".join(pod_source_config)))
            break
      else:
        podfile.write(' pod \'{}\'\n'.format(pod))
    podfile.write('end')
  shell('cat Podfile')
  shell('pod install')
  os.chdir(cwd)
  return os.path.join(target_dir, '{}.xcworkspace'.format(target_name))

def CopyProject(source_dir, target_dir):
  """CopyProject copies the project from the source to the target.

  Args:
    source_dir: The path to the source directory.
    target_dir: The path to the target directory.
  """
  shell('cp -r {} {}'.format(source_dir, target_dir))

def ValidateSourceConfig(pod_sources):
  if 'pods' not in pod_sources:
    raise ValueError(
            "The JSON config file should have 'pods' object containing pod configs.")

  for pod_config in pod_sources['pods']:
    source_keys = list(pod_config.keys())
    try:
      sdk = pod_config['sdk']
    except KeyError:
      print("SDK should be specified.")
      raise
    if sdk.strip() == "":
      raise ValueError( "SDK should not be empty or blank.")
    elif pod_config and ( source_keys[1] not in {"git", "path"} ):
      raise ValueError(
              "Pod source of SDK {} should be `git` or `path`.".format(sdk))
    elif len(source_keys) == 3:
      if source_keys[1] != "git":
        raise ValueError(
                "For multiple specs for the SDK {} ,`git` should be added with `branch`, `tag` or `commit`".format(sdk))
      if source_keys[2] not in {"branch", "tag", "commit"}:
        raise ValueError(
                "A specified version of the SDK {} should be from `branch`, `tag` or `commit`.".format(sdk))
    elif len(source_keys) > 3:
      raise ValueError(
        "Pod source of SDK {} can only specify `sdk` with `path`, `git`, or `git` and a reference (like a `branch`, `tag`, or `commit`)."
        "See --help for an example config."
        .format(sdk)
      )

def GetPodSizeImpact(parsed_args):
  """GetPodSizeImpact gets the size impact of the set of pods.

  Args:
    parsed_args: The set of arguments passed to the program.
  """
  sample_app_dir, sample_app_name = GetSampleApp(parsed_args.mode)
  cocoapods = {}
  if parsed_args.spec_repos:
    spec_repos = []
    # If cdn source is in spec_repos input, then it will be moved
    # to the end and be added as the last source.
    if MASTER_SOURCE in parsed_args.spec_repos:
        parsed_args.spec_repos.append(
            parsed_args.spec_repos.pop(
                parsed_args.spec_repos.index(
                    MASTER_SOURCE)))
    for repo in parsed_args.spec_repos:
      if repo in SPEC_REPO_DICT:
        spec_repos.append(SPEC_REPO_DICT[repo])
      else:
        spec_repos.append(repo)
  else:
    spec_repos = DEFAULT_SPEC_REPOS
  for pod in parsed_args.cocoapods:
    pod_info = pod.split(':')
    pod_name = pod_info[0].strip()
    if len(pod_info) > 1:
      pod_version = pod_info[1].strip()
    else:
      pod_version = ''
    cocoapods[pod_name] = pod_version
  # Load JSON in order since in bleeding edge version of a Pod, `git` and
  # `branch`/`tag`/`commit` are required and should be in order. e.g.
  # pod 'Alamofire', :git => 'https://github.com/Alamofire/Alamofire.git', :branch => 'dev'
  try:
    if pod_version and parsed_args.cocoapods_source_config:
      print("Since a version for the pod {} is specified, The config file {} \
              will be validated but not used for binary measurement.".format(
                  pod_name, parsed_args.cocoapods_source_config.name))
    pod_sources = json.load(parsed_args.cocoapods_source_config, \
            object_pairs_hook=OrderedDict) if parsed_args.cocoapods_source_config else None
    if pod_sources: ValidateSourceConfig(pod_sources)
  except ValueError as e:
    raise ValueError("could not decode JSON value %s: %s" % (parsed_args.cocoapods_source_config.name, e))
  base_project = tempfile.mkdtemp()
  target_project = tempfile.mkdtemp()
  target_dir = os.path.join(target_project, sample_app_dir)
  CopyProject(sample_app_dir, base_project)
  CopyProject(sample_app_dir, target_project)

  target_project = InstallPods(cocoapods,
                               target_dir,
                               spec_repos, sample_app_name, parsed_args.mode,
                               pod_sources)
  source_project = os.path.join(base_project,
                                '{}/{}.xcodeproj'.format(sample_app_dir, sample_app_name))

  source_size, target_size = GenerateSizeDifference(
      source_project, sample_app_name, target_project, sample_app_name, parsed_args.build_timeout)
  if parsed_args.json:
    # Transfer Podfile to JSON format.
    podfile = shell('pod ipc podfile-json {}/Podfile'.format(target_dir), capture_stdout=True)
    podfile_dict = json.loads(podfile)
    podfile_dict['combined_pods_extra_size'] = target_size - source_size
    with open(parsed_args.json, 'w') as json_file:
      json.dump(podfile_dict, json_file)
  # Throw an error if the target size is 0, an example for command
  # ./measure_cocoapod_size.py --cocoapods FirebaseABTesting FirebaseAut:8.0.0
  # This command will throw the following error:
  # ValueError: The size of the following pod combination is 0 and this could be caused by a failed build.
  # FirebaseABTesting
  # FirebaseAut:8.0.0
  if target_size == 0:
    target_pods = "\n".join(
            ["{}:{}".format(pod,version) if version != "" else pod
                for pod, version in  cocoapods.items()])
    raise ValueError(
            "The size of the following pod combination is 0 and this could be caused by a failed build.\n{}".format(target_pods))
  print('The pods combined add an extra size of {} bytes'.format(
      target_size - source_size))

def Main():
  """Main generates the PodSize impact.
  """
  parser = argparse.ArgumentParser(description='The size impact of a cocoapod',
          formatter_class=argparse.RawTextHelpFormatter)
  parser.add_argument(
      '--cocoapods',
      metavar='N',
      type=str,
      nargs='+',
      required=True,
      help='The set of cocoapods')
  parser.add_argument(
      '--mode',
      type=str,
      choices=[MODE_SWIFT, MODE_OBJC],
      default=MODE_OBJC,
      help='Type of cocoapod'
  )
  parser.add_argument(
      '--spec_repos',
      metavar='N',
      type=str,
      nargs='+',
      required=False,
      help='The set of spec_repos')
  parser.add_argument(
      '--cocoapods_source_config',
      metavar='CONFIG_JSON',
      type=argparse.FileType('r'),
      nargs='?',
      required=False,
      default=None,
      help=''' A JSON file with customized pod source.E.g.
        {
          "pods":[
                {
                  "sdk":"FirebaseDatabase",
                  "git":"https://github.com/firebase/firebase-ios-sdk",
                  "branch":"master"
                }
            ]
        }
      If versions are specified in the `cocoapods` arg, config here will be skipped.
      ''')
  parser.add_argument(
      '--build_timeout',
      metavar='SECONDS',
      nargs='?',
      required=False,
      default=None,
      help='Timeout to build testapps.')
  parser.add_argument(
      '--json',
      metavar='OUTPUT_FILE_NAME',
      nargs='?',
      required=False,
      default=None,
      help='Output JSON file.')

  args = parser.parse_args()

  GetPodSizeImpact(args)

if __name__ == '__main__':
  Main()
<|endoftext|>


<commit_before>from django.test import TestCase
from django.contrib.auth.models import User
from models import Item, Bill, BillItem, Category
from dbmanager import commit_bill

class BillTestCase(TestCase):
    def setUp(self):
        self.output = {'errors': [],
                       'bill_id': None,
                       'customer_id': 'LOL',
                       'date': None,
                       'total': 0
                      }
        bibite = Category.objects.create(name='Bibite')
        panini = Category.objects.create(name='Panini')
        piatti = Category.objects.create(name='Piatti')
        Item.objects.create(name='Coca Cola', category=bibite, quantity=11,
                            price=3.50)
        Item.objects.create(name='Panino Salsiccia', category=panini,
                            quantity=0, price=6.50)
        Item.objects.create(name='Pasta al ragu', category=piatti, quantity=3,
                            price=8.50)
        Item.objects.create(name='Acqua', category=bibite, quantity=5,
                            price=1.00)
        User.objects.create(username='Lonfo')

    def tearDown(self):
        Item.objects.all().delete()
        Category.objects.all().delete()
        User.objects.get(username='Lonfo').delete()

    def test_commit_bill_success(self):
        reqdata = {'customer_name': 'Darozzo',
                   'items': {
                       'Coca Cola': {
                           'qty': 2,
                           'notes': '',
                       },
                       'Pasta al ragu': {
                           'qty': 3,
                           'notes': 'Scotta',
                       },
                       'Acqua': {
                            'qty': 1,
                            'notes': 'Fredda'
                           }
                   }
        }
        user = User.objects.get(username='Lonfo')
        result = commit_bill(self.output, reqdata, user)
        billhd = Bill.objects.get(pk=self.output['bill_id'])
        items = Item.objects.filter(name__in=reqdata['items'].keys())
        billitems = billhd.billitem_set.filter(item__in=items)

        for itm in billitems:
            self.assertEqual(itm.quantity,
                             reqdata['items'][itm.item.name]['qty'])
            self.assertEqual(itm.item_price, itm.item.price)

        self.assertTrue(len(billitems) == 3)
        self.assertEqual(billhd.customer_name, 'Darozzo')
        self.assertEqual(billhd.server, 'Lonfo')
        self.assertEqual(result['errors'], {})
        self.assertEqual(result['customer_id'], 'LOL')
        self.assertEqual(result['total'], 33.5)

    def test_commit_bill_failure(self):
        reqdata = {'customer_name': 'Darozzo',
                   'items': {
                       'Coca Cola': {
                           'qty': 2,
                           'notes': '',
                       },
                       'Pasta al ragu': {
                           'qty': 4,
                           'notes': 'Scotta',
                       },
                       'Acqua': {
                            'qty': 6,
                            'notes': 'Fredda'
                           }
                   }
        }
        user = User.objects.get(username='Lonfo')
        result = commit_bill(self.output, reqdata, user)

        self.assertTrue(len(result['errors']) == 2)
        self.assertTrue(result['errors']['Acqua'] == 5)
        self.assertTrue(result['errors']['Pasta al ragu'] == 3)

<commit_msg>Fixed test.py (#3)<commit_after>from django.test import TestCase
from django.contrib.auth.models import User
from models import Item, Bill, BillItem, Category
from dbmanager import commit_bill

class BillTestCase(TestCase):
    def setUp(self):
        self.output = {'errors': [],
                       'bill_id': None,
                       'customer_id': 'LOL',
                       'date': None,
                       'total': 0
                      }
        bibite = Category.objects.create(name='Bibite')
        panini = Category.objects.create(name='Panini')
        piatti = Category.objects.create(name='Piatti')
        Item.objects.create(name='Coca Cola', category=bibite, quantity=11,
                            price=3.50)
        Item.objects.create(name='Panino Salsiccia', category=panini,
                            quantity=0, price=6.50)
        Item.objects.create(name='Pasta al ragu', category=piatti, quantity=3,
                            price=8.50)
        Item.objects.create(name='Acqua', category=bibite, quantity=5,
                            price=1.00)
        User.objects.create(username='Lonfo')

    def tearDown(self):
        Item.objects.all().delete()
        Category.objects.all().delete()
        User.objects.get(username='Lonfo').delete()

    def test_commit_bill_success(self):
        reqdata = {'customer_name': 'Darozzo',
                   'items': {
                       'Coca Cola': {
                           'qty': 2,
                           'notes': '',
                       },
                       'Pasta al ragu': {
                           'qty': 3,
                           'notes': 'Scotta',
                       },
                       'Acqua': {
                            'qty': 1,
                            'notes': 'Fredda'
                           }
                   }
        }
        user = User.objects.get(username='Lonfo')
        result, billhd = commit_bill(self.output, reqdata, user)
        items = Item.objects.filter(name__in=reqdata['items'].keys())
        billitems = billhd.billitem_set.filter(item__in=items)

        for itm in billitems:
            self.assertEqual(itm.quantity,
                             reqdata['items'][itm.item.name]['qty'])
            self.assertEqual(itm.item_price, itm.item.price)

        self.assertTrue(len(billitems) == 3)
        self.assertEqual(billhd.customer_name, 'Darozzo')
        self.assertEqual(billhd.server, 'Lonfo')
        self.assertEqual(result['errors'], {})
        self.assertEqual(result['customer_id'], 'LOL')
        self.assertEqual(result['total'], 33.5)

    def test_commit_bill_failure(self):
        reqdata = {'customer_name': 'Darozzo',
                   'items': {
                       'Coca Cola': {
                           'qty': 2,
                           'notes': '',
                       },
                       'Pasta al ragu': {
                           'qty': 4,
                           'notes': 'Scotta',
                       },
                       'Acqua': {
                            'qty': 6,
                            'notes': 'Fredda'
                           }
                   }
        }
        user = User.objects.get(username='Lonfo')
        result, billhd = commit_bill(self.output, reqdata, user)

        self.assertTrue(len(result['errors']) == 2)
        self.assertTrue(result['errors']['Acqua'] == 5)
        self.assertTrue(result['errors']['Pasta al ragu'] == 3)

<|endoftext|>