<commit_before>'''Example of how to use sklearn wrapper

Builds simple CNN models on MNIST and uses sklearn's GridSearchCV to find best model
'''

from __future__ import print_function
import numpy as np
np.random.seed(1337)  # for reproducibility

from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Convolution2D, MaxPooling2D
from keras.utils import np_utils
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.grid_search import GridSearchCV

nb_classes = 10

# input image dimensions
img_rows, img_cols = 28, 28

# load training data and do basic data normalization
(X_train, y_train), (X_test, y_test) = mnist.load_data()
X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)
X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255

# convert class vectors to binary class matrices
y_train = np_utils.to_categorical(y_train, nb_classes)
y_test = np_utils.to_categorical(y_test, nb_classes)

def make_model(dense_layer_sizes, nb_filters, nb_conv, nb_pool):
    '''Creates model comprised of 2 convolutional layers followed by dense layers

    dense_layer_sizes: List of layer sizes. This list has one number for each layer
    nb_filters: Number of convolutional filters in each convolutional layer
    nb_conv: Convolutional kernel size
    nb_pool: Size of pooling area for max pooling
    '''

    model = Sequential()

    model.add(Convolution2D(nb_filters, nb_conv, nb_conv,
                            border_mode='valid',
                            input_shape=(1, img_rows, img_cols)))
    model.add(Activation('relu'))
    model.add(Convolution2D(nb_filters, nb_conv, nb_conv))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
    model.add(Dropout(0.25))

    model.add(Flatten())
    for layer_size in dense_layer_sizes:
        model.add(Dense(layer_size))
    model.add(Activation('relu'))
    model.add(Dropout(0.5))
    model.add(Dense(nb_classes))
    model.add(Activation('softmax'))

    model.compile(loss='categorical_crossentropy',
                  optimizer='adadelta',
                  metrics=['accuracy'])

    return model

dense_size_candidates = [[32], [64], [32, 32], [64, 64]]
my_classifier = KerasClassifier(make_model, batch_size=32)
validator = GridSearchCV(my_classifier,
                         param_grid={'dense_layer_sizes': dense_size_candidates,
                                     # nb_epoch is avail for tuning even when not
                                     # an argument to model building function
                                     'nb_epoch': [3, 6],
                                     'nb_filters': [8],
                                     'nb_conv': [3],
                                     'nb_pool': [2]},
                         scoring='log_loss',
                         n_jobs=1)
validator.fit(X_train, y_train)

print('The parameters of the best model are: ')
print(validator.best_params_)

# validator.best_estimator_ returns sklearn-wrapped version of best model.
# validator.best_estimator_.model returns the (unwrapped) keras model
best_model = validator.best_estimator_.model
metric_names = best_model.metrics_names
metric_values = best_model.evaluate(X_test, y_test)
for metric, value in zip(metric_names, metric_values):
    print(metric, ': ', value)
<commit_msg>fix mnist_sklearn_wrapper.py error when default image_dim_ordering=tf (#5130)<commit_after>'''Example of how to use sklearn wrapper

Builds simple CNN models on MNIST and uses sklearn's GridSearchCV to find best model
'''

from __future__ import print_function
import numpy as np
np.random.seed(1337)  # for reproducibility

from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Convolution2D, MaxPooling2D
from keras.utils import np_utils
from keras.wrappers.scikit_learn import KerasClassifier
from keras import backend as K
from sklearn.grid_search import GridSearchCV

nb_classes = 10

# input image dimensions
img_rows, img_cols = 28, 28

# load training data and do basic data normalization
(X_train, y_train), (X_test, y_test) = mnist.load_data()

if K.image_dim_ordering() == 'th':
    X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)
    X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)
    input_shape = (1, img_rows, img_cols)
else:
    X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1)
    X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)
    input_shape = (img_rows, img_cols, 1)

X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255

# convert class vectors to binary class matrices
y_train = np_utils.to_categorical(y_train, nb_classes)
y_test = np_utils.to_categorical(y_test, nb_classes)

def make_model(dense_layer_sizes, nb_filters, nb_conv, nb_pool):
    '''Creates model comprised of 2 convolutional layers followed by dense layers

    dense_layer_sizes: List of layer sizes. This list has one number for each layer
    nb_filters: Number of convolutional filters in each convolutional layer
    nb_conv: Convolutional kernel size
    nb_pool: Size of pooling area for max pooling
    '''

    model = Sequential()

    model.add(Convolution2D(nb_filters, nb_conv, nb_conv,
                            border_mode='valid',
                            input_shape=input_shape))
    model.add(Activation('relu'))
    model.add(Convolution2D(nb_filters, nb_conv, nb_conv))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
    model.add(Dropout(0.25))

    model.add(Flatten())
    for layer_size in dense_layer_sizes:
        model.add(Dense(layer_size))
    model.add(Activation('relu'))
    model.add(Dropout(0.5))
    model.add(Dense(nb_classes))
    model.add(Activation('softmax'))

    model.compile(loss='categorical_crossentropy',
                  optimizer='adadelta',
                  metrics=['accuracy'])

    return model

dense_size_candidates = [[32], [64], [32, 32], [64, 64]]
my_classifier = KerasClassifier(make_model, batch_size=32)
validator = GridSearchCV(my_classifier,
                         param_grid={'dense_layer_sizes': dense_size_candidates,
                                     # nb_epoch is avail for tuning even when not
                                     # an argument to model building function
                                     'nb_epoch': [3, 6],
                                     'nb_filters': [8],
                                     'nb_conv': [3],
                                     'nb_pool': [2]},
                         scoring='log_loss',
                         n_jobs=1)
validator.fit(X_train, y_train)

print('The parameters of the best model are: ')
print(validator.best_params_)

# validator.best_estimator_ returns sklearn-wrapped version of best model.
# validator.best_estimator_.model returns the (unwrapped) keras model
best_model = validator.best_estimator_.model
metric_names = best_model.metrics_names
metric_values = best_model.evaluate(X_test, y_test)
for metric, value in zip(metric_names, metric_values):
    print(metric, ': ', value)
<|endoftext|>


<commit_before>""""
This file contains possible stages that maps_generator can run.
Some algorithms suppose a maps genration processes looks like:
    stage1, ..., stage_mwm[country_stage_1, ..., country_stage_M], ..., stageN
Only stage_mwm can contain country_
"""
import datetime
import json
import logging
import multiprocessing
import os
import shutil
import tarfile
from collections import defaultdict
from multiprocessing.pool import ThreadPool
from typing import AnyStr
from typing import Type

from descriptions.descriptions_downloader import check_and_get_checker
from descriptions.descriptions_downloader import download_from_wikidata_tags
from descriptions.descriptions_downloader import download_from_wikipedia_tags
from maps_generator.generator import coastline
from maps_generator.generator import settings
from maps_generator.generator import steps
from maps_generator.generator.env import Env
from maps_generator.generator.env import PathProvider
from maps_generator.generator.env import WORLD_COASTS_NAME
from maps_generator.generator.env import WORLD_NAME
from maps_generator.generator.exceptions import BadExitStatusError
from maps_generator.generator.gen_tool import run_gen_tool
from maps_generator.generator.stages import InternalDependency as D
from maps_generator.generator.stages import Stage
from maps_generator.generator.stages import country_stage
from maps_generator.generator.stages import depends_from_internal
from maps_generator.generator.stages import helper_stage_for
from maps_generator.generator.stages import mwm_stage
from maps_generator.generator.stages import outer_stage
from maps_generator.generator.stages import production_only
from maps_generator.generator.stages import stages
from maps_generator.generator.statistics import get_stages_info
from maps_generator.utils.file import download_files
from maps_generator.utils.file import is_verified
from post_generation.hierarchy_to_countries import hierarchy_to_countries
from post_generation.inject_promo_ids import inject_promo_ids
from post_generation.localads_mwm_to_csv import create_csv

logger = logging.getLogger("maps_generator")

def is_accepted(env: Env, stage: Type[Stage]) -> bool:
    return env.is_accepted_stage(stage)

@outer_stage
class StageDownloadAndConvertPlanet(Stage):
    def apply(self, env: Env, force_download: bool = True, **kwargs):
        if force_download or not is_verified(env.paths.planet_o5m):
            steps.step_download_and_convert_planet(
                env, force_download=force_download, **kwargs
            )

@outer_stage
class StageUpdatePlanet(Stage):
    def apply(self, env: Env, **kwargs):
        steps.step_update_planet(env, **kwargs)

@outer_stage
class StageCoastline(Stage):
    def apply(self, env: Env, use_old_if_fail=True):
        coasts_geom = "WorldCoasts.geom"
        coasts_rawgeom = "WorldCoasts.rawgeom"
        try:
            coastline.make_coastline(env)
        except BadExitStatusError as e:
            if not use_old_if_fail:
                raise e

            logger.info("Build costs failed. Try to download the costs...")
            download_files(
                {
                    settings.PLANET_COASTS_GEOM_URL: os.path.join(
                        env.paths.coastline_path, coasts_geom
                    ),
                    settings.PLANET_COASTS_RAWGEOM_URL: os.path.join(
                        env.paths.coastline_path, coasts_rawgeom
                    ),
                }
            )

        for f in [coasts_geom, coasts_rawgeom]:
            path = os.path.join(env.paths.coastline_path, f)
            shutil.copy2(path, env.paths.intermediate_data_path)

@outer_stage
class StagePreprocess(Stage):
    def apply(self, env: Env, **kwargs):
        steps.step_preprocess(env, **kwargs)

@outer_stage
@depends_from_internal(
    D(settings.HOTELS_URL, PathProvider.hotels_path, "p"),
    D(settings.PROMO_CATALOG_CITIES_URL, PathProvider.promo_catalog_cities_path, "p"),
    D(settings.POPULARITY_URL, PathProvider.popularity_path, "p"),
    D(settings.FOOD_URL, PathProvider.food_paths, "p"),
    D(settings.FOOD_TRANSLATIONS_URL, PathProvider.food_translations_path, "p"),
)
class StageFeatures(Stage):
    def apply(self, env: Env):
        extra = {}
        if is_accepted(env, StageDescriptions):
            extra.update({"idToWikidata": env.paths.id_to_wikidata_path})
        if env.production:
            extra.update(
                {
                    "booking_data": env.paths.hotels_path,
                    "promo_catalog_cities": env.paths.promo_catalog_cities_path,
                    "popular_places_data": env.paths.popularity_path,
                    "brands_data": env.paths.food_paths,
                    "brands_translations_data": env.paths.food_translations_path,
                }
            )
        if is_accepted(env, StageCoastline):
            extra.update({"emit_coasts": True})
        if is_accepted(env, StageIsolinesInfo):
            extra.update({"isolines_path": PathProvider.isolines_path()})

        steps.step_features(env, **extra)
        if os.path.exists(env.paths.packed_polygons_path):
            shutil.copy2(env.paths.packed_polygons_path, env.paths.mwm_path)

@outer_stage
@production_only
@helper_stage_for("StageDescriptions")
class StageDownloadDescriptions(Stage):
    def apply(self, env: Env):
        run_gen_tool(
            env.gen_tool,
            out=env.get_subprocess_out(),
            err=env.get_subprocess_out(),
            intermediate_data_path=env.paths.intermediate_data_path,
            user_resource_path=env.paths.user_resource_path,
            dump_wikipedia_urls=env.paths.wiki_url_path,
            idToWikidata=env.paths.id_to_wikidata_path,
            threads_count=settings.THREADS_COUNT,
        )

        langs = ("en", "ru", "es", "fr", "de")
        checker = check_and_get_checker(env.paths.popularity_path)
        download_from_wikipedia_tags(
            env.paths.wiki_url_path, env.paths.descriptions_path, langs, checker
        )
        download_from_wikidata_tags(
            env.paths.id_to_wikidata_path, env.paths.descriptions_path, langs, checker
        )

@outer_stage
@mwm_stage
class StageMwm(Stage):
    def apply(self, env: Env):
        with ThreadPool(settings.THREADS_COUNT) as pool:
            pool.map(
                lambda c: StageMwm.make_mwm(c, env),
                env.get_tmp_mwm_names(),
                chunksize=1,
            )

    @staticmethod
    def make_mwm(country: AnyStr, env: Env):
        if country == WORLD_NAME:
            StageIndex(country=country)(env)
            StageCitiesIdsWorld(country=country)(env)
        elif country == WORLD_COASTS_NAME:
            StageIndex(country=country)(env)
        else:
            StageIndex(country=country)(env)
            StageUgc(country=country)(env)
            StagePopularity(country=country)(env)
            StageSrtm(country=country)(env)
            StageDescriptions(country=country)(env)
            StageRouting(country=country)(env)
            StageRoutingTransit(country=country)(env)

        StageMwmStatistics(country=country)(env)
        env.finish_mwm(country)

@country_stage
@depends_from_internal(
    D(settings.UK_POSTCODES_URL, PathProvider.uk_postcodes_path, "p"),
    D(settings.US_POSTCODES_URL, PathProvider.us_postcodes_path, "p"),
)
class StageIndex(Stage):
    def apply(self, env: Env, country, **kwargs):
        if country == WORLD_NAME:
            steps.step_index_world(env, country, **kwargs)
        elif country == WORLD_COASTS_NAME:
            steps.step_coastline_index(env, country, **kwargs)
        else:
            if env.production:
                kwargs.update(
                    {
                        "uk_postcodes_dataset": env.paths.uk_postcodes_path,
                        "us_postcodes_dataset": env.paths.us_postcodes_path,
                    }
                )
            steps.step_index(env, country, **kwargs)

@country_stage
@production_only
class StageCitiesIdsWorld(Stage):
    def apply(self, env: Env, country, **kwargs):
        steps.step_cities_ids_world(env, country, **kwargs)

@country_stage
@depends_from_internal(D(settings.UGC_URL, PathProvider.ugc_path),)
@production_only
class StageUgc(Stage):
    def apply(self, env: Env, country, **kwargs):
        steps.step_ugc(env, country, **kwargs)

@country_stage
@production_only
class StagePopularity(Stage):
    def apply(self, env: Env, country, **kwargs):
        steps.step_popularity(env, country, **kwargs)

@country_stage
@production_only
class StageSrtm(Stage):
    def apply(self, env: Env, country, **kwargs):
        steps.step_srtm(env, country, **kwargs)

@country_stage
@production_only
class StageIsolinesInfo(Stage):
    def apply(self, env: Env, country, **kwargs):
        steps.step_isolines_info(env, country, **kwargs)

@country_stage
@production_only
class StageDescriptions(Stage):
    def apply(self, env: Env, country, **kwargs):
        steps.step_description(env, country, **kwargs)

@country_stage
class StageRouting(Stage):
    def apply(self, env: Env, country, **kwargs):
        steps.step_routing(env, country, **kwargs)

@country_stage
@depends_from_internal(D(settings.SUBWAY_URL, PathProvider.subway_path),)
class StageRoutingTransit(Stage):
    def apply(self, env: Env, country, **kwargs):
        steps.step_routing_transit(env, country, **kwargs)

@country_stage
@helper_stage_for("StageStatistics")
class StageMwmStatistics(Stage):
    def apply(self, env: Env, country, **kwargs):
        steps.step_statistics(env, country, **kwargs)

@outer_stage
@depends_from_internal(
    D(settings.PROMO_CATALOG_COUNTRIES_URL, PathProvider.promo_catalog_countries_path, "p")
)
class StageCountriesTxt(Stage):
    def apply(self, env: Env):
        countries = hierarchy_to_countries(
            env.paths.old_to_new_path,
            env.paths.borders_to_osm_path,
            env.paths.countries_synonyms_path,
            env.paths.hierarchy_path,
            env.paths.mwm_path,
            env.paths.mwm_version,
        )
        if env.production:
            countries_json = json.loads(countries)
            inject_promo_ids(
                countries_json,
                env.paths.promo_catalog_cities_path,
                env.paths.promo_catalog_countries_path,
                env.paths.mwm_path,
                env.paths.types_path,
                env.paths.mwm_path,
            )

            with open(env.paths.counties_txt_path, "w") as f:
                json.dump(countries_json, f, ensure_ascii=True, indent=1)

@outer_stage
class StageExternalResources(Stage):
    def apply(self, env: Env):
        black_list = {"00_roboto_regular.ttf"}
        resources = [
            os.path.join(env.paths.user_resource_path, file)
            for file in os.listdir(env.paths.user_resource_path)
            if file.endswith(".ttf") and file not in black_list
        ]
        for ttf_file in resources:
            shutil.copy2(ttf_file, env.paths.mwm_path)

        for file in os.listdir(env.paths.mwm_path):
            if file.startswith(WORLD_NAME) and file.endswith(".mwm"):
                resources.append(os.path.join(env.paths.mwm_path, file))

        resources.sort()
        with open(env.paths.external_resources_path, "w") as f:
            for resource in resources:
                fd = os.open(resource, os.O_RDONLY)
                f.write(f"{os.path.basename(resource)} {os.fstat(fd).st_size}\n")

@outer_stage
@production_only
class StageLocalAds(Stage):
    def apply(self, env: Env):
        create_csv(
            env.paths.localads_path,
            env.paths.mwm_path,
            env.paths.mwm_path,
            env.mwm_version,
            multiprocessing.cpu_count(),
        )
        with tarfile.open(f"{env.paths.localads_path}.tar.gz", "w:gz") as tar:
            for filename in os.listdir(env.paths.localads_path):
                tar.add(
                    os.path.join(env.paths.localads_path, filename), arcname=filename
                )

@outer_stage
class StageStatistics(Stage):
    def apply(self, env: Env):
        steps_info = get_stages_info(env.paths.log_path, {"statistics"})
        stats = defaultdict(lambda: defaultdict(dict))
        stats["steps"] = steps_info["steps"]
        for country in env.get_tmp_mwm_names():
            with open(os.path.join(env.paths.stats_path, f"{country}.json")) as f:
                stats["countries"][country] = {
                    "types": json.load(f),
                    "steps": steps_info["countries"][country]
                }

        def default(o):
            if isinstance(o, datetime.timedelta):
                return str(o)

        with open(os.path.join(env.paths.stats_path, "stats.json"), "w") as f:
            json.dump(
                stats, f, ensure_ascii=False, sort_keys=True, indent=2, default=default
            )

@outer_stage
class StageCleanup(Stage):
    def apply(self, env: Env):
        logger.info(
            f"osm2ft files will be moved from {env.paths.build_path} "
            f"to {env.paths.osm2ft_path}."
        )
        for x in os.listdir(env.paths.mwm_path):
            p = os.path.join(env.paths.mwm_path, x)
            if os.path.isfile(p) and x.endswith(".mwm.osm2ft"):
                shutil.move(p, os.path.join(env.paths.osm2ft_path, x))

        logger.info(f"{env.paths.draft_path} will be removed.")
        shutil.rmtree(env.paths.draft_path)

stages.init()
<commit_msg>[generator][python] Fixed StageIsolinesInfo.<commit_after>""""
This file contains possible stages that maps_generator can run.
Some algorithms suppose a maps genration processes looks like:
    stage1, ..., stage_mwm[country_stage_1, ..., country_stage_M], ..., stageN
Only stage_mwm can contain country_
"""
import datetime
import json
import logging
import multiprocessing
import os
import shutil
import tarfile
from collections import defaultdict
from multiprocessing.pool import ThreadPool
from typing import AnyStr
from typing import Type

from descriptions.descriptions_downloader import check_and_get_checker
from descriptions.descriptions_downloader import download_from_wikidata_tags
from descriptions.descriptions_downloader import download_from_wikipedia_tags
from maps_generator.generator import coastline
from maps_generator.generator import settings
from maps_generator.generator import steps
from maps_generator.generator.env import Env
from maps_generator.generator.env import PathProvider
from maps_generator.generator.env import WORLD_COASTS_NAME
from maps_generator.generator.env import WORLD_NAME
from maps_generator.generator.exceptions import BadExitStatusError
from maps_generator.generator.gen_tool import run_gen_tool
from maps_generator.generator.stages import InternalDependency as D
from maps_generator.generator.stages import Stage
from maps_generator.generator.stages import country_stage
from maps_generator.generator.stages import depends_from_internal
from maps_generator.generator.stages import helper_stage_for
from maps_generator.generator.stages import mwm_stage
from maps_generator.generator.stages import outer_stage
from maps_generator.generator.stages import production_only
from maps_generator.generator.stages import stages
from maps_generator.generator.statistics import get_stages_info
from maps_generator.utils.file import download_files
from maps_generator.utils.file import is_verified
from post_generation.hierarchy_to_countries import hierarchy_to_countries
from post_generation.inject_promo_ids import inject_promo_ids
from post_generation.localads_mwm_to_csv import create_csv

logger = logging.getLogger("maps_generator")

def is_accepted(env: Env, stage: Type[Stage]) -> bool:
    return env.is_accepted_stage(stage)

@outer_stage
class StageDownloadAndConvertPlanet(Stage):
    def apply(self, env: Env, force_download: bool = True, **kwargs):
        if force_download or not is_verified(env.paths.planet_o5m):
            steps.step_download_and_convert_planet(
                env, force_download=force_download, **kwargs
            )

@outer_stage
class StageUpdatePlanet(Stage):
    def apply(self, env: Env, **kwargs):
        steps.step_update_planet(env, **kwargs)

@outer_stage
class StageCoastline(Stage):
    def apply(self, env: Env, use_old_if_fail=True):
        coasts_geom = "WorldCoasts.geom"
        coasts_rawgeom = "WorldCoasts.rawgeom"
        try:
            coastline.make_coastline(env)
        except BadExitStatusError as e:
            if not use_old_if_fail:
                raise e

            logger.info("Build costs failed. Try to download the costs...")
            download_files(
                {
                    settings.PLANET_COASTS_GEOM_URL: os.path.join(
                        env.paths.coastline_path, coasts_geom
                    ),
                    settings.PLANET_COASTS_RAWGEOM_URL: os.path.join(
                        env.paths.coastline_path, coasts_rawgeom
                    ),
                }
            )

        for f in [coasts_geom, coasts_rawgeom]:
            path = os.path.join(env.paths.coastline_path, f)
            shutil.copy2(path, env.paths.intermediate_data_path)

@outer_stage
class StagePreprocess(Stage):
    def apply(self, env: Env, **kwargs):
        steps.step_preprocess(env, **kwargs)

@outer_stage
@depends_from_internal(
    D(settings.HOTELS_URL, PathProvider.hotels_path, "p"),
    D(settings.PROMO_CATALOG_CITIES_URL, PathProvider.promo_catalog_cities_path, "p"),
    D(settings.POPULARITY_URL, PathProvider.popularity_path, "p"),
    D(settings.FOOD_URL, PathProvider.food_paths, "p"),
    D(settings.FOOD_TRANSLATIONS_URL, PathProvider.food_translations_path, "p"),
)
class StageFeatures(Stage):
    def apply(self, env: Env):
        extra = {}
        if is_accepted(env, StageDescriptions):
            extra.update({"idToWikidata": env.paths.id_to_wikidata_path})
        if env.production:
            extra.update(
                {
                    "booking_data": env.paths.hotels_path,
                    "promo_catalog_cities": env.paths.promo_catalog_cities_path,
                    "popular_places_data": env.paths.popularity_path,
                    "brands_data": env.paths.food_paths,
                    "brands_translations_data": env.paths.food_translations_path,
                }
            )
        if is_accepted(env, StageCoastline):
            extra.update({"emit_coasts": True})
        if is_accepted(env, StageIsolinesInfo):
            extra.update({"isolines_path": PathProvider.isolines_path()})

        steps.step_features(env, **extra)
        if os.path.exists(env.paths.packed_polygons_path):
            shutil.copy2(env.paths.packed_polygons_path, env.paths.mwm_path)

@outer_stage
@production_only
@helper_stage_for("StageDescriptions")
class StageDownloadDescriptions(Stage):
    def apply(self, env: Env):
        run_gen_tool(
            env.gen_tool,
            out=env.get_subprocess_out(),
            err=env.get_subprocess_out(),
            intermediate_data_path=env.paths.intermediate_data_path,
            user_resource_path=env.paths.user_resource_path,
            dump_wikipedia_urls=env.paths.wiki_url_path,
            idToWikidata=env.paths.id_to_wikidata_path,
            threads_count=settings.THREADS_COUNT,
        )

        langs = ("en", "ru", "es", "fr", "de")
        checker = check_and_get_checker(env.paths.popularity_path)
        download_from_wikipedia_tags(
            env.paths.wiki_url_path, env.paths.descriptions_path, langs, checker
        )
        download_from_wikidata_tags(
            env.paths.id_to_wikidata_path, env.paths.descriptions_path, langs, checker
        )

@outer_stage
@mwm_stage
class StageMwm(Stage):
    def apply(self, env: Env):
        with ThreadPool(settings.THREADS_COUNT) as pool:
            pool.map(
                lambda c: StageMwm.make_mwm(c, env),
                env.get_tmp_mwm_names(),
                chunksize=1,
            )

    @staticmethod
    def make_mwm(country: AnyStr, env: Env):
        if country == WORLD_NAME:
            StageIndex(country=country)(env)
            StageCitiesIdsWorld(country=country)(env)
        elif country == WORLD_COASTS_NAME:
            StageIndex(country=country)(env)
        else:
            StageIndex(country=country)(env)
            StageUgc(country=country)(env)
            StagePopularity(country=country)(env)
            StageSrtm(country=country)(env)
            StageIsolinesInfo(country=country)(env)
            StageDescriptions(country=country)(env)
            StageRouting(country=country)(env)
            StageRoutingTransit(country=country)(env)

        StageMwmStatistics(country=country)(env)
        env.finish_mwm(country)

@country_stage
@depends_from_internal(
    D(settings.UK_POSTCODES_URL, PathProvider.uk_postcodes_path, "p"),
    D(settings.US_POSTCODES_URL, PathProvider.us_postcodes_path, "p"),
)
class StageIndex(Stage):
    def apply(self, env: Env, country, **kwargs):
        if country == WORLD_NAME:
            steps.step_index_world(env, country, **kwargs)
        elif country == WORLD_COASTS_NAME:
            steps.step_coastline_index(env, country, **kwargs)
        else:
            if env.production:
                kwargs.update(
                    {
                        "uk_postcodes_dataset": env.paths.uk_postcodes_path,
                        "us_postcodes_dataset": env.paths.us_postcodes_path,
                    }
                )
            steps.step_index(env, country, **kwargs)

@country_stage
@production_only
class StageCitiesIdsWorld(Stage):
    def apply(self, env: Env, country, **kwargs):
        steps.step_cities_ids_world(env, country, **kwargs)

@country_stage
@depends_from_internal(D(settings.UGC_URL, PathProvider.ugc_path),)
@production_only
class StageUgc(Stage):
    def apply(self, env: Env, country, **kwargs):
        steps.step_ugc(env, country, **kwargs)

@country_stage
@production_only
class StagePopularity(Stage):
    def apply(self, env: Env, country, **kwargs):
        steps.step_popularity(env, country, **kwargs)

@country_stage
@production_only
class StageSrtm(Stage):
    def apply(self, env: Env, country, **kwargs):
        steps.step_srtm(env, country, **kwargs)

@country_stage
@production_only
class StageIsolinesInfo(Stage):
    def apply(self, env: Env, country, **kwargs):
        steps.step_isolines_info(env, country, **kwargs)

@country_stage
@production_only
class StageDescriptions(Stage):
    def apply(self, env: Env, country, **kwargs):
        steps.step_description(env, country, **kwargs)

@country_stage
class StageRouting(Stage):
    def apply(self, env: Env, country, **kwargs):
        steps.step_routing(env, country, **kwargs)

@country_stage
@depends_from_internal(D(settings.SUBWAY_URL, PathProvider.subway_path),)
class StageRoutingTransit(Stage):
    def apply(self, env: Env, country, **kwargs):
        steps.step_routing_transit(env, country, **kwargs)

@country_stage
@helper_stage_for("StageStatistics")
class StageMwmStatistics(Stage):
    def apply(self, env: Env, country, **kwargs):
        steps.step_statistics(env, country, **kwargs)

@outer_stage
@depends_from_internal(
    D(settings.PROMO_CATALOG_COUNTRIES_URL, PathProvider.promo_catalog_countries_path, "p")
)
class StageCountriesTxt(Stage):
    def apply(self, env: Env):
        countries = hierarchy_to_countries(
            env.paths.old_to_new_path,
            env.paths.borders_to_osm_path,
            env.paths.countries_synonyms_path,
            env.paths.hierarchy_path,
            env.paths.mwm_path,
            env.paths.mwm_version,
        )
        if env.production:
            countries_json = json.loads(countries)
            inject_promo_ids(
                countries_json,
                env.paths.promo_catalog_cities_path,
                env.paths.promo_catalog_countries_path,
                env.paths.mwm_path,
                env.paths.types_path,
                env.paths.mwm_path,
            )

            with open(env.paths.counties_txt_path, "w") as f:
                json.dump(countries_json, f, ensure_ascii=True, indent=1)

@outer_stage
class StageExternalResources(Stage):
    def apply(self, env: Env):
        black_list = {"00_roboto_regular.ttf"}
        resources = [
            os.path.join(env.paths.user_resource_path, file)
            for file in os.listdir(env.paths.user_resource_path)
            if file.endswith(".ttf") and file not in black_list
        ]
        for ttf_file in resources:
            shutil.copy2(ttf_file, env.paths.mwm_path)

        for file in os.listdir(env.paths.mwm_path):
            if file.startswith(WORLD_NAME) and file.endswith(".mwm"):
                resources.append(os.path.join(env.paths.mwm_path, file))

        resources.sort()
        with open(env.paths.external_resources_path, "w") as f:
            for resource in resources:
                fd = os.open(resource, os.O_RDONLY)
                f.write(f"{os.path.basename(resource)} {os.fstat(fd).st_size}\n")

@outer_stage
@production_only
class StageLocalAds(Stage):
    def apply(self, env: Env):
        create_csv(
            env.paths.localads_path,
            env.paths.mwm_path,
            env.paths.mwm_path,
            env.mwm_version,
            multiprocessing.cpu_count(),
        )
        with tarfile.open(f"{env.paths.localads_path}.tar.gz", "w:gz") as tar:
            for filename in os.listdir(env.paths.localads_path):
                tar.add(
                    os.path.join(env.paths.localads_path, filename), arcname=filename
                )

@outer_stage
class StageStatistics(Stage):
    def apply(self, env: Env):
        steps_info = get_stages_info(env.paths.log_path, {"statistics"})
        stats = defaultdict(lambda: defaultdict(dict))
        stats["steps"] = steps_info["steps"]
        for country in env.get_tmp_mwm_names():
            with open(os.path.join(env.paths.stats_path, f"{country}.json")) as f:
                stats["countries"][country] = {
                    "types": json.load(f),
                    "steps": steps_info["countries"][country]
                }

        def default(o):
            if isinstance(o, datetime.timedelta):
                return str(o)

        with open(os.path.join(env.paths.stats_path, "stats.json"), "w") as f:
            json.dump(
                stats, f, ensure_ascii=False, sort_keys=True, indent=2, default=default
            )

@outer_stage
class StageCleanup(Stage):
    def apply(self, env: Env):
        logger.info(
            f"osm2ft files will be moved from {env.paths.build_path} "
            f"to {env.paths.osm2ft_path}."
        )
        for x in os.listdir(env.paths.mwm_path):
            p = os.path.join(env.paths.mwm_path, x)
            if os.path.isfile(p) and x.endswith(".mwm.osm2ft"):
                shutil.move(p, os.path.join(env.paths.osm2ft_path, x))

        logger.info(f"{env.paths.draft_path} will be removed.")
        shutil.rmtree(env.paths.draft_path)

stages.init()
<|endoftext|>


<commit_before># Copyright (c) 2015 SONATA-NFV, Thales Communications & Security
# ALL RIGHTS RESERVED.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Neither the name of the SONATA-NFV, Thales Communications & Security
# nor the names of its contributors may be used to endorse or promote
# products derived from this software without specific prior written
# permission.
#
# This work has been performed in the framework of the SONATA project,
# funded by the European Commission under Grant number 671517 through
# the Horizon 2020 and 5G-PPP programmes. The authors would like to
# acknowledge the contributions of their colleagues of the SONATA
# partner consortium (www.sonata-nfv.eu).

"""son-analyze command line tool"""

import sys
import os
import logging
import signal
import urllib.parse
import uuid
from argparse import ArgumentParser, Namespace, ArgumentTypeError
import typing  # noqa pylint: disable=unused-import
from typing import List
from pkg_resources import resource_filename  # type: ignore
from docker import APIClient  # type: ignore
from son_analyze import __version__
from son_analyze.cli import fetch_cmd
from son_analyze.core import types
from son_analyze.ops.fetch import Kind

_LOGGER = logging.getLogger(__name__)
_IMAGE_TAG = 'son-analyze-scikit'

def bootstrap(args: Namespace) -> None:
    """Create the images used by son-analyze in the current host"""
    cli = APIClient(base_url=args.docker_socket)
    root_context = os.path.realpath(
        resource_filename('son_analyze.cli', '../../..'))
    _LOGGER.info('The root context path is: %s', root_context)
    path = resource_filename('son_analyze.cli.resources',
                             'anaconda.Dockerfile')
    path = os.path.relpath(path, root_context)
    _LOGGER.info('The relative path to the bootstrap dockerfile is: %s', path)
    # import pdb; pdb.set_trace()
    for line in cli.build(path=root_context, tag=_IMAGE_TAG,
                          dockerfile=path, rm=True, decode=True):
        if "stream" in line:
            print('> ', line["stream"], end="")
        else:
            print(line)
            sys.exit(1)
    sys.exit(0)

def run(args: Namespace) -> None:
    """Run an analysis framework environment"""
    cli = APIClient(base_url=args.docker_socket)
    token_path = os.path.join(args.son_workspace, 'platforms', 'token.txt')
    _LOGGER.info('The path to the Sonata token file is: %s', token_path)
    if not (os.path.isfile(token_path) and os.access(token_path, os.R_OK)):
        _LOGGER.error('The Sonata token file %s is not a readable file',
                      token_path)
        sys.exit(1)
    binds = {
        '/dev/random': {
            'bind': '/dev/random'
        },
        '/dev/urandom': {
            'bind': '/dev/urandom'
        },
        token_path: {
            'bind': token_path,
            'mode': 'ro'
        }
    }  # type: typing.Dict[str, typing.Dict[str, str]]
    if args.dynamic_mount:
        field_name = os.path.realpath(
            resource_filename('son_analyze.cli', '../../..'))
        new_entry = {
            field_name: {
                'bind': '/son-analyze',
                'mode': 'rw'
            }
        }
        binds.update(new_entry)
    host_config = cli.create_host_config(
        port_bindings={8888: args.jupiter_port},
        binds=binds)
    container = cli.create_container(image=_IMAGE_TAG+':latest',
                                     labels=['com.sonata.analyze'],
                                     ports=[8888],
                                     host_config=host_config,
                                     user='root',
                                     environment=['GRANT_SUDO=yes'],
                                     command=['start-notebook.sh',
                                              "--NotebookApp.token=''",
                                              ("--NotebookApp."
                                               "iopub_data_rate_limit="
                                               "10000000000")])
    container_id = container.get('Id')
    cli.start(container=container_id)

    def cleanup():
        """Remove the container"""
        cli.remove_container(container=container_id, force=True)

    def signal_term_handler(unused1, unused2):  # noqa pylint: disable=unused-argument
        """Catch signal to clean the containers"""
        print('Interruption detected, stopping environment')
        cleanup()
        sys.exit(1)

    signal.signal(signal.SIGTERM, signal_term_handler)
    signal.signal(signal.SIGINT, signal_term_handler)

    print('Browse http://localhost:{} \n'
          'Type Ctrl-C to exit'.format(args.jupiter_port))
    exit_code = 0
    exit_code = cli.wait(container=container_id)
    cleanup()
    sys.exit(exit_code)

def version(args: Namespace) -> None:
    """Print the current version and exit"""
    msg = 'son-analyze version: {}'.format(__version__)
    if args.short:
        msg = __version__
    print(msg)
    sys.exit(0)

def resource_target(raw_target: str) -> types.ResourceTargetTuple:
    """Define the type of resource"""
    if ',' in raw_target:
        try:
            rvendor, rname, rversion = raw_target.split(',')
            return types.ResourceTargetTuple(vendor=rvendor, name=rname,
                                             version=rversion, uuid=None)
        except:
            raise ArgumentTypeError("Target must have the form: "
                                    "<vendor>,<name>,<version>")
    else:
        try:
            tid = str(uuid.UUID(raw_target))
            return types.ResourceTargetTuple(vendor=None, name=None,
                                             version=None, uuid=tid)
        except:
            raise ArgumentTypeError("Target uuid must have the form: "
                                    "<12345678-1234-1234-1234-123456789012>")

def url_type(raw_url: str) -> urllib.parse.ParseResult:
    """Define the type of a URL"""
    url = urllib.parse.urlparse(raw_url, scheme='http')
    isvalid = all(getattr(url, attr) for attr in ['scheme', 'netloc'])
    if isvalid:
        return url
    else:
        raise ArgumentTypeError("Url is not valid")

def fetch_func(args: Namespace) -> None:
    """Fetch data"""
    fetch_cmd.fetch_cmd(args.endpoint, args.dworkspace_dir, args.kind,
                        args.target)
    sys.exit(0)

def dispatch(raw_args: List) -> None:
    """Parse the raw_args and dispatch the control flow"""
    parser = ArgumentParser(description=('An analysis framework '
                                         'creation tool for Sonata'))
    parser.add_argument('-v', '--verbose', default=logging.WARNING,
                        action="store_const", dest="logLevel",
                        const=logging.INFO, help='increase verbosity')
    parser.add_argument('--docker-socket', type=str,
                        default='unix://var/run/docker.sock',
                        action='store',
                        help=('An uri to the docker socket '
                              '(default: %(default)s)'))
    dworkspace_dir = os.path.join(os.path.expanduser('~'), '.son-workspace')
    parser.add_argument('--son-workspace', default=dworkspace_dir,
                        action='store', type=str, metavar='DIR',
                        help='A path to the Sonata workspace directory')

    def no_command(_: Namespace) -> None:
        """Print the help usage and exit"""
        parser.print_help()
        sys.exit(0)
    parser.set_defaults(func=no_command)
    subparsers = parser.add_subparsers()

    parser_version = subparsers.add_parser('version', help='Show the version')
    parser_version.add_argument('--short', default=False, action='store_true',
                                help='Shows only the version')
    parser_version.set_defaults(func=version)

    parser_bootstrap = subparsers.add_parser('bootstrap',
                                             help='Bootstrap son-analyze')
    parser_bootstrap.set_defaults(func=bootstrap)

    parser_run = subparsers.add_parser('run', help='Run an environment')
    parser_run.add_argument('--dynamic-mount', default=False,
                            action='store_true',
                            help=('(Dev) Dynamically mount the R code'
                                  ' inside the environment'))
    parser_run.add_argument('--port', type=int,
                            default=8888, action='store', dest='jupiter_port',
                            help=('The listening port for the Jupiter '
                                  'server (default: %(default)d)'))
    parser_run.set_defaults(func=run)

    parser_fetch = subparsers.add_parser('fetch', help='Fetch data/metrics')
    default_val = urllib.parse.urlparse(
        'http://sp.int2.sonata-nfv.eu:9090', scheme='http')
    help_msg = 'A Gatekeeper endpoint (default: {})'.format(
        default_val.geturl())
    parser_fetch.add_argument('--endpoint', action='store', help=help_msg,
                              metavar='URL', type=url_type,
                              default=default_val)
    choices = tuple(elt.name for elt in Kind)  # type: ignore
    parser_fetch.add_argument('kind', help="The resource's type",
                              type=str, choices=choices)
    parser_fetch.add_argument('target', type=resource_target,
                              help=('A resource specified by: '
                                    '<vendor>,<name>,<version> or <uuid>'))
    parser_fetch.set_defaults(func=fetch_func)

    args = parser.parse_args(raw_args)
    logging.basicConfig(level=args.logLevel)
    _LOGGER.debug("Setting level to %s", args.logLevel)
    args.func(args)
    assert False  # this line is impossible to reach

def main() -> None:
    """Main entry point for the son-analyze command line tool"""
    dispatch(sys.argv[1:])
<commit_msg>Fix invalid field<commit_after># Copyright (c) 2015 SONATA-NFV, Thales Communications & Security
# ALL RIGHTS RESERVED.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Neither the name of the SONATA-NFV, Thales Communications & Security
# nor the names of its contributors may be used to endorse or promote
# products derived from this software without specific prior written
# permission.
#
# This work has been performed in the framework of the SONATA project,
# funded by the European Commission under Grant number 671517 through
# the Horizon 2020 and 5G-PPP programmes. The authors would like to
# acknowledge the contributions of their colleagues of the SONATA
# partner consortium (www.sonata-nfv.eu).

"""son-analyze command line tool"""

import sys
import os
import logging
import signal
import urllib.parse
import uuid
from argparse import ArgumentParser, Namespace, ArgumentTypeError
import typing  # noqa pylint: disable=unused-import
from typing import List
from pkg_resources import resource_filename  # type: ignore
from docker import APIClient  # type: ignore
from son_analyze import __version__
from son_analyze.cli import fetch_cmd
from son_analyze.core import types
from son_analyze.ops.fetch import Kind

_LOGGER = logging.getLogger(__name__)
_IMAGE_TAG = 'son-analyze-scikit'

def bootstrap(args: Namespace) -> None:
    """Create the images used by son-analyze in the current host"""
    cli = APIClient(base_url=args.docker_socket)
    root_context = os.path.realpath(
        resource_filename('son_analyze.cli', '../../..'))
    _LOGGER.info('The root context path is: %s', root_context)
    path = resource_filename('son_analyze.cli.resources',
                             'anaconda.Dockerfile')
    path = os.path.relpath(path, root_context)
    _LOGGER.info('The relative path to the bootstrap dockerfile is: %s', path)
    # import pdb; pdb.set_trace()
    for line in cli.build(path=root_context, tag=_IMAGE_TAG,
                          dockerfile=path, rm=True, decode=True):
        if "stream" in line:
            print('> ', line["stream"], end="")
        else:
            print(line)
            sys.exit(1)
    sys.exit(0)

def run(args: Namespace) -> None:
    """Run an analysis framework environment"""
    cli = APIClient(base_url=args.docker_socket)
    token_path = os.path.join(args.son_workspace, 'platforms', 'token.txt')
    _LOGGER.info('The path to the Sonata token file is: %s', token_path)
    if not (os.path.isfile(token_path) and os.access(token_path, os.R_OK)):
        _LOGGER.error('The Sonata token file %s is not a readable file',
                      token_path)
        sys.exit(1)
    binds = {
        '/dev/random': {
            'bind': '/dev/random'
        },
        '/dev/urandom': {
            'bind': '/dev/urandom'
        },
        token_path: {
            'bind': token_path,
            'mode': 'ro'
        }
    }  # type: typing.Dict[str, typing.Dict[str, str]]
    if args.dynamic_mount:
        field_name = os.path.realpath(
            resource_filename('son_analyze.cli', '../../..'))
        new_entry = {
            field_name: {
                'bind': '/son-analyze',
                'mode': 'rw'
            }
        }
        binds.update(new_entry)
    host_config = cli.create_host_config(
        port_bindings={8888: args.jupiter_port},
        binds=binds)
    container = cli.create_container(image=_IMAGE_TAG+':latest',
                                     labels=['com.sonata.analyze'],
                                     ports=[8888],
                                     host_config=host_config,
                                     user='root',
                                     environment=['GRANT_SUDO=yes'],
                                     command=['start-notebook.sh',
                                              "--NotebookApp.token=''",
                                              ("--NotebookApp."
                                               "iopub_data_rate_limit="
                                               "10000000000")])
    container_id = container.get('Id')
    cli.start(container=container_id)

    def cleanup():
        """Remove the container"""
        cli.remove_container(container=container_id, force=True)

    def signal_term_handler(unused1, unused2):  # noqa pylint: disable=unused-argument
        """Catch signal to clean the containers"""
        print('Interruption detected, stopping environment')
        cleanup()
        sys.exit(1)

    signal.signal(signal.SIGTERM, signal_term_handler)
    signal.signal(signal.SIGINT, signal_term_handler)

    print('Browse http://localhost:{} \n'
          'Type Ctrl-C to exit'.format(args.jupiter_port))
    exit_code = 0
    exit_code = cli.wait(container=container_id)
    cleanup()
    sys.exit(exit_code)

def version(args: Namespace) -> None:
    """Print the current version and exit"""
    msg = 'son-analyze version: {}'.format(__version__)
    if args.short:
        msg = __version__
    print(msg)
    sys.exit(0)

def resource_target(raw_target: str) -> types.ResourceTargetTuple:
    """Define the type of resource"""
    if ',' in raw_target:
        try:
            rvendor, rname, rversion = raw_target.split(',')
            return types.ResourceTargetTuple(vendor=rvendor, name=rname,
                                             version=rversion, uuid=None)
        except:
            raise ArgumentTypeError("Target must have the form: "
                                    "<vendor>,<name>,<version>")
    else:
        try:
            tid = str(uuid.UUID(raw_target))
            return types.ResourceTargetTuple(vendor=None, name=None,
                                             version=None, uuid=tid)
        except:
            raise ArgumentTypeError("Target uuid must have the form: "
                                    "<12345678-1234-1234-1234-123456789012>")

def url_type(raw_url: str) -> urllib.parse.ParseResult:
    """Define the type of a URL"""
    url = urllib.parse.urlparse(raw_url, scheme='http')
    isvalid = all(getattr(url, attr) for attr in ['scheme', 'netloc'])
    if isvalid:
        return url
    else:
        raise ArgumentTypeError("Url is not valid")

def fetch_func(args: Namespace) -> None:
    """Fetch data"""
    fetch_cmd.fetch_cmd(args.endpoint, args.son_workspace, args.kind,
                        args.target)
    sys.exit(0)

def dispatch(raw_args: List) -> None:
    """Parse the raw_args and dispatch the control flow"""
    parser = ArgumentParser(description=('An analysis framework '
                                         'creation tool for Sonata'))
    parser.add_argument('-v', '--verbose', default=logging.WARNING,
                        action="store_const", dest="logLevel",
                        const=logging.INFO, help='increase verbosity')
    parser.add_argument('--docker-socket', type=str,
                        default='unix://var/run/docker.sock',
                        action='store',
                        help=('An uri to the docker socket '
                              '(default: %(default)s)'))
    dworkspace_dir = os.path.join(os.path.expanduser('~'), '.son-workspace')
    parser.add_argument('--son-workspace', default=dworkspace_dir,
                        action='store', type=str, metavar='DIR',
                        help='A path to the Sonata workspace directory')

    def no_command(_: Namespace) -> None:
        """Print the help usage and exit"""
        parser.print_help()
        sys.exit(0)
    parser.set_defaults(func=no_command)
    subparsers = parser.add_subparsers()

    parser_version = subparsers.add_parser('version', help='Show the version')
    parser_version.add_argument('--short', default=False, action='store_true',
                                help='Shows only the version')
    parser_version.set_defaults(func=version)

    parser_bootstrap = subparsers.add_parser('bootstrap',
                                             help='Bootstrap son-analyze')
    parser_bootstrap.set_defaults(func=bootstrap)

    parser_run = subparsers.add_parser('run', help='Run an environment')
    parser_run.add_argument('--dynamic-mount', default=False,
                            action='store_true',
                            help=('(Dev) Dynamically mount the R code'
                                  ' inside the environment'))
    parser_run.add_argument('--port', type=int,
                            default=8888, action='store', dest='jupiter_port',
                            help=('The listening port for the Jupiter '
                                  'server (default: %(default)d)'))
    parser_run.set_defaults(func=run)

    parser_fetch = subparsers.add_parser('fetch', help='Fetch data/metrics')
    default_val = urllib.parse.urlparse(
        'http://sp.int2.sonata-nfv.eu:9090', scheme='http')
    help_msg = 'A Gatekeeper endpoint (default: {})'.format(
        default_val.geturl())
    parser_fetch.add_argument('--endpoint', action='store', help=help_msg,
                              metavar='URL', type=url_type,
                              default=default_val)
    choices = tuple(elt.name for elt in Kind)  # type: ignore
    parser_fetch.add_argument('kind', help="The resource's type",
                              type=str, choices=choices)
    parser_fetch.add_argument('target', type=resource_target,
                              help=('A resource specified by: '
                                    '<vendor>,<name>,<version> or <uuid>'))
    parser_fetch.set_defaults(func=fetch_func)

    args = parser.parse_args(raw_args)
    logging.basicConfig(level=args.logLevel)
    _LOGGER.debug("Setting level to %s", args.logLevel)
    args.func(args)
    assert False  # this line is impossible to reach

def main() -> None:
    """Main entry point for the son-analyze command line tool"""
    dispatch(sys.argv[1:])
<|endoftext|>


<commit_before>'''Neural style transfer with Keras.

Before running this script, download the weights for the VGG16 model at:
https://drive.google.com/file/d/0Bz7KyqmuGsilT0J5dmRCM0ROVHc/view?usp=sharing
(source: https://gist.github.com/baraldilorenzo/07d7802847aaad0a35d3)
and make sure the variable `weights_path` in this script matches the location of the file.

Run the script with:
```
python neural_style.py path_to_your_base_image.jpg path_to_your_reference.jpg prefix_for_results
```
e.g.:
```
python neural_style.py img/tuebingen.jpg img/starry_night.jpg results/my_result
```

It is preferrable to run this script on GPU, for speed.
If running on CPU, prefer the TensorFlow backend (much faster).

# Details

Style transfer consists in generating an image
with the same "content" as a base image, but with the
"style" of a different picture (typically artistic).

This is achieved through the optimization of a loss function
that has 3 components: "style loss", "content loss",
and "total variation loss":

 - The content loss is a L2 distance between the pixels of the base
image and the pixels of the combination image, keeping the generated image
close enough to the original one.

- The total variation loss imposes local spatial continuity between
the pixels of the combination image, giving it visual coherence.

- The style loss is where the deep learning keeps in --that one is defined
using a deep convolutional neural network. Precisely, it consists in a sum of
L2 distances betwen the Gram matrices of the representations of
the base image and the style reference image, extracted from
different layers of a convnet (trained on ImageNet). The general idea
is to capture color/texture information at different spatial
scales (fairly large scales --defined by the depth of the layer considered).

# Reference:
    - [A Neural Algorithm of Artistic Style](http://arxiv.org/abs/1508.06576)
'''

from __future__ import print_function
import cv2
import numpy as np
from scipy.optimize import fmin_l_bfgs_b
import time
import argparse
import h5py

from keras.models import Sequential
from keras.layers.convolutional import Convolution2D, ZeroPadding2D, MaxPooling2D
from keras.layers.core import Dense, Dropout, Flatten
from keras import backend as K

parser = argparse.ArgumentParser(description='Neural style transfer with Keras.')
parser.add_argument('base_image_path', metavar='base', type=str,
                    help='Path to the image to transform.')
parser.add_argument('style_reference_image_path', metavar='ref', type=str,
                    help='Path to the style reference image.')
parser.add_argument('result_prefix', metavar='res_prefix', type=str,
                    help='Prefix for the saved results.')

# base_image_path = 'tuebingen.jpg'
# style_reference_image_path = 'starry_night.jpg'
# result_prefix = 'my_result_th'
args = parser.parse_args()
base_image_path = args.base_image_path
style_reference_image_path = args.style_reference_image_path
result_prefix = args.result_prefix
weights_path = 'vgg16_weights.h5'

# these are the weights of the different loss components
total_variation_weight = 1.
style_weight = 1.
content_weight = 0.025

# dimensions of the generated picture.
img_width = 400
img_height = 400
assert img_height == img_width, 'Due to the use of the Gram matrix, width and height must match.'

# util function to open, resize and format pictures into appropriate tensors
def preprocess_image(image_path):
    im = cv2.resize(cv2.imread(image_path), (img_width, img_height))
    im = im.transpose((2, 0, 1))
    im = np.expand_dims(im, axis=0)
    return im

# util function to convert a tensor into a valid image
def deprocess_image(x):
    x = x.transpose((1, 2, 0))
    x = np.clip(x, 0, 255).astype('uint8')
    return x

# get tensor representations of our images
base_image = K.variable(preprocess_image(base_image_path))
style_reference_image = K.variable(preprocess_image(style_reference_image_path))

# this will contain our generated image
combination_image = K.placeholder((1, 3, img_width, img_height))

# combine the 3 images into a single Keras tensor
input_tensor = K.concatenate([base_image,
                              style_reference_image,
                              combination_image], axis=0)

# build the VGG16 network with our 3 images as input
first_layer = ZeroPadding2D((1, 1), input_shape=(3, img_width, img_height))
first_layer.input = input_tensor

model = Sequential()
model.add(first_layer)
model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(64, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_2'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

# load the weights of the VGG16 networks
# (trained on ImageNet, won the ILSVRC competition in 2014)
# note: when there is a complete match between your model definition
# and your weight savefile, you can simply call model.load_weights(filename)
f = h5py.File(weights_path)
for k in range(f.attrs['nb_layers']):
    if k >= len(model.layers):
        # we don't look at the last (fully-connected) layers in the savefile
        break
    g = f['layer_{}'.format(k)]
    weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]
    model.layers[k].set_weights(weights)
f.close()
print('Model loaded.')

# get the symbolic outputs of each "key" layer (we gave them unique names).
outputs_dict = dict([(layer.name, layer.get_output()) for layer in model.layers])

# compute the neural style loss
# first we need to define 4 util functions

# the gram matrix of an image tensor (feature-wise outer product)
def gram_matrix(x):
    assert K.ndim(x) == 3
    features = K.batch_flatten(x)
    gram = K.dot(features, K.transpose(features))
    return gram

# the "style loss" is designed to maintain
# the style of the reference image in the generated image.
# It is based on the gram matrices (which capture style) of
# feature maps from the style reference image
# and from the generated image
def style_loss(style, combination):
    assert K.ndim(style) == 3
    assert K.ndim(combination) == 3
    S = gram_matrix(style)
    C = gram_matrix(combination)
    channels = 3
    size = img_width * img_height
    return K.sum(K.square(S - C)) / (4. * (channels ** 2) * (size ** 2))

# an auxiliary loss function
# designed to maintain the "content" of the
# base image in the generated image
def content_loss(base, combination):
    return K.sum(K.square(combination - base))

# the 3rd loss function, total variation loss,
# designed to keep the generated image locally coherent
def total_variation_loss(x):
    assert K.ndim(x) == 4
    a = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, 1:, :img_height-1])
    b = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, :img_width-1, 1:])
    return K.sum(K.pow(a + b, 1.25))

# combine these loss functions into a single scalar
loss = K.variable(0.)
layer_features = outputs_dict['conv4_2']
base_image_features = layer_features[0, :, :, :]
combination_features = layer_features[2, :, :, :]
loss += content_weight * content_loss(base_image_features,
                                      combination_features)

feature_layers = ['conv1_1', 'conv2_1', 'conv3_1', 'conv4_1', 'conv5_1']
for layer_name in feature_layers:
    layer_features = outputs_dict[layer_name]
    style_reference_features = layer_features[1, :, :, :]
    combination_features = layer_features[2, :, :, :]
    sl = style_loss(style_reference_features, combination_features)
    loss += (style_weight / len(feature_layers)) * sl
loss += total_variation_weight * total_variation_loss(combination_image)

# get the gradients of the generated image wrt the loss
grads = K.gradients(loss, combination_image)

# set up helper functions to extract the loss and gradients
# from the computational graph as Numpy arrays
f_grads = K.function([combination_image], grads)
def eval_grads(x):
    x = x.reshape((1, 3, img_width, img_height))
    return np.array(f_grads([x])).flatten().astype('float64')

f_loss = K.function([combination_image], [loss])
def eval_loss(x):
    x = x.reshape((1, 3, img_width, img_height))
    return f_loss([x])[0].astype('float64')

# run scipy-based optimization (L-BFGS) over the pixels of the generated image
# so as to minimize the neural style loss
x = np.random.uniform(0, 255, (1, 3, img_width, img_height))
for i in range(10):
    print('Start of iteration', i)
    eval_loss_calls = 0
    start_time = time.time()
    x, min_val, info = fmin_l_bfgs_b(eval_loss, x.flatten(),
                                     fprime=eval_grads, maxfun=20)
    print('Current loss value:', min_val)
    # save current generated image
    im = deprocess_image(x.reshape((3, img_width, img_height)))
    fname = result_prefix + '_at_iteration_%d.png' % i
    cv2.imwrite(fname, im)
    end_time = time.time()
    print('Image saved as', fname)
    print('Iteration %d completed in %ds' % (i, end_time - start_time))
<commit_msg>Remove unnecessary commented code.<commit_after>'''Neural style transfer with Keras.

Before running this script, download the weights for the VGG16 model at:
https://drive.google.com/file/d/0Bz7KyqmuGsilT0J5dmRCM0ROVHc/view?usp=sharing
(source: https://gist.github.com/baraldilorenzo/07d7802847aaad0a35d3)
and make sure the variable `weights_path` in this script matches the location of the file.

Run the script with:
```
python neural_style.py path_to_your_base_image.jpg path_to_your_reference.jpg prefix_for_results
```
e.g.:
```
python neural_style.py img/tuebingen.jpg img/starry_night.jpg results/my_result
```

It is preferrable to run this script on GPU, for speed.
If running on CPU, prefer the TensorFlow backend (much faster).

Example result: https://twitter.com/fchollet/status/686631033085677568

# Details

Style transfer consists in generating an image
with the same "content" as a base image, but with the
"style" of a different picture (typically artistic).

This is achieved through the optimization of a loss function
that has 3 components: "style loss", "content loss",
and "total variation loss":

 - The content loss is a L2 distance between the pixels of the base
image and the pixels of the combination image, keeping the generated image
close enough to the original one.

- The total variation loss imposes local spatial continuity between
the pixels of the combination image, giving it visual coherence.

- The style loss is where the deep learning keeps in --that one is defined
using a deep convolutional neural network. Precisely, it consists in a sum of
L2 distances betwen the Gram matrices of the representations of
the base image and the style reference image, extracted from
different layers of a convnet (trained on ImageNet). The general idea
is to capture color/texture information at different spatial
scales (fairly large scales --defined by the depth of the layer considered).

# Reference:
    - [A Neural Algorithm of Artistic Style](http://arxiv.org/abs/1508.06576)
'''

from __future__ import print_function
import cv2
import numpy as np
from scipy.optimize import fmin_l_bfgs_b
import time
import argparse
import h5py

from keras.models import Sequential
from keras.layers.convolutional import Convolution2D, ZeroPadding2D, MaxPooling2D
from keras.layers.core import Dense, Dropout, Flatten
from keras import backend as K

parser = argparse.ArgumentParser(description='Neural style transfer with Keras.')
parser.add_argument('base_image_path', metavar='base', type=str,
                    help='Path to the image to transform.')
parser.add_argument('style_reference_image_path', metavar='ref', type=str,
                    help='Path to the style reference image.')
parser.add_argument('result_prefix', metavar='res_prefix', type=str,
                    help='Prefix for the saved results.')

args = parser.parse_args()
base_image_path = args.base_image_path
style_reference_image_path = args.style_reference_image_path
result_prefix = args.result_prefix
weights_path = 'vgg16_weights.h5'

# these are the weights of the different loss components
total_variation_weight = 1.
style_weight = 1.
content_weight = 0.025

# dimensions of the generated picture.
img_width = 400
img_height = 400
assert img_height == img_width, 'Due to the use of the Gram matrix, width and height must match.'

# util function to open, resize and format pictures into appropriate tensors
def preprocess_image(image_path):
    im = cv2.resize(cv2.imread(image_path), (img_width, img_height))
    im = im.transpose((2, 0, 1))
    im = np.expand_dims(im, axis=0)
    return im

# util function to convert a tensor into a valid image
def deprocess_image(x):
    x = x.transpose((1, 2, 0))
    x = np.clip(x, 0, 255).astype('uint8')
    return x

# get tensor representations of our images
base_image = K.variable(preprocess_image(base_image_path))
style_reference_image = K.variable(preprocess_image(style_reference_image_path))

# this will contain our generated image
combination_image = K.placeholder((1, 3, img_width, img_height))

# combine the 3 images into a single Keras tensor
input_tensor = K.concatenate([base_image,
                              style_reference_image,
                              combination_image], axis=0)

# build the VGG16 network with our 3 images as input
first_layer = ZeroPadding2D((1, 1), input_shape=(3, img_width, img_height))
first_layer.input = input_tensor

model = Sequential()
model.add(first_layer)
model.add(Convolution2D(64, 3, 3, activation='relu', name='conv1_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(64, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, 3, 3, activation='relu', name='conv2_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(128, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu', name='conv3_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(256, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv4_2'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu', name='conv5_1'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(512, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))

# load the weights of the VGG16 networks
# (trained on ImageNet, won the ILSVRC competition in 2014)
# note: when there is a complete match between your model definition
# and your weight savefile, you can simply call model.load_weights(filename)
f = h5py.File(weights_path)
for k in range(f.attrs['nb_layers']):
    if k >= len(model.layers):
        # we don't look at the last (fully-connected) layers in the savefile
        break
    g = f['layer_{}'.format(k)]
    weights = [g['param_{}'.format(p)] for p in range(g.attrs['nb_params'])]
    model.layers[k].set_weights(weights)
f.close()
print('Model loaded.')

# get the symbolic outputs of each "key" layer (we gave them unique names).
outputs_dict = dict([(layer.name, layer.get_output()) for layer in model.layers])

# compute the neural style loss
# first we need to define 4 util functions

# the gram matrix of an image tensor (feature-wise outer product)
def gram_matrix(x):
    assert K.ndim(x) == 3
    features = K.batch_flatten(x)
    gram = K.dot(features, K.transpose(features))
    return gram

# the "style loss" is designed to maintain
# the style of the reference image in the generated image.
# It is based on the gram matrices (which capture style) of
# feature maps from the style reference image
# and from the generated image
def style_loss(style, combination):
    assert K.ndim(style) == 3
    assert K.ndim(combination) == 3
    S = gram_matrix(style)
    C = gram_matrix(combination)
    channels = 3
    size = img_width * img_height
    return K.sum(K.square(S - C)) / (4. * (channels ** 2) * (size ** 2))

# an auxiliary loss function
# designed to maintain the "content" of the
# base image in the generated image
def content_loss(base, combination):
    return K.sum(K.square(combination - base))

# the 3rd loss function, total variation loss,
# designed to keep the generated image locally coherent
def total_variation_loss(x):
    assert K.ndim(x) == 4
    a = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, 1:, :img_height-1])
    b = K.square(x[:, :, :img_width-1, :img_height-1] - x[:, :, :img_width-1, 1:])
    return K.sum(K.pow(a + b, 1.25))

# combine these loss functions into a single scalar
loss = K.variable(0.)
layer_features = outputs_dict['conv4_2']
base_image_features = layer_features[0, :, :, :]
combination_features = layer_features[2, :, :, :]
loss += content_weight * content_loss(base_image_features,
                                      combination_features)

feature_layers = ['conv1_1', 'conv2_1', 'conv3_1', 'conv4_1', 'conv5_1']
for layer_name in feature_layers:
    layer_features = outputs_dict[layer_name]
    style_reference_features = layer_features[1, :, :, :]
    combination_features = layer_features[2, :, :, :]
    sl = style_loss(style_reference_features, combination_features)
    loss += (style_weight / len(feature_layers)) * sl
loss += total_variation_weight * total_variation_loss(combination_image)

# get the gradients of the generated image wrt the loss
grads = K.gradients(loss, combination_image)

# set up helper functions to extract the loss and gradients
# from the computational graph as Numpy arrays
f_grads = K.function([combination_image], grads)
def eval_grads(x):
    x = x.reshape((1, 3, img_width, img_height))
    return np.array(f_grads([x])).flatten().astype('float64')

f_loss = K.function([combination_image], [loss])
def eval_loss(x):
    x = x.reshape((1, 3, img_width, img_height))
    return f_loss([x])[0].astype('float64')

# run scipy-based optimization (L-BFGS) over the pixels of the generated image
# so as to minimize the neural style loss
x = np.random.uniform(0, 255, (1, 3, img_width, img_height))
for i in range(10):
    print('Start of iteration', i)
    eval_loss_calls = 0
    start_time = time.time()
    x, min_val, info = fmin_l_bfgs_b(eval_loss, x.flatten(),
                                     fprime=eval_grads, maxfun=20)
    print('Current loss value:', min_val)
    # save current generated image
    im = deprocess_image(x.reshape((3, img_width, img_height)))
    fname = result_prefix + '_at_iteration_%d.png' % i
    cv2.imwrite(fname, im)
    end_time = time.time()
    print('Image saved as', fname)
    print('Iteration %d completed in %ds' % (i, end_time - start_time))
<|endoftext|>


<commit_before>from data_collection.management.commands import BaseXpressDemocracyClubCsvImporter

class Command(BaseXpressDemocracyClubCsvImporter):
    council_id = "E07000156"
    addresses_name = (
        "europarl.2019-05-23/Version 1/DC Polling Station Data Wellingborough.CSV"
    )
    stations_name = (
        "europarl.2019-05-23/Version 1/DC Polling Station Data Wellingborough.CSV"
    )
    elections = ["europarl.2019-05-23"]
<commit_msg>Import script for Wellingborough (closes #2402)<commit_after>from data_collection.management.commands import BaseXpressDemocracyClubCsvImporter

class Command(BaseXpressDemocracyClubCsvImporter):
    council_id = "E07000156"
    addresses_name = (
        "parl.2019-12-12/Version 1/Democracy_Club__12December2019welling.csv"
    )
    stations_name = (
        "parl.2019-12-12/Version 1/Democracy_Club__12December2019welling.csv"
    )
    elections = ["parl.2019-12-12"]
    csv_delimiter = "\t"
    allow_station_point_from_postcode = False
<|endoftext|>


<commit_before>import argparse
import logging

from sanic import Sanic
from sanic import response

from rasa_core.domain import Domain
from rasa_core.nlg import TemplatedNaturalLanguageGenerator
from rasa_core.trackers import DialogueStateTracker

logger = logging.getLogger(__name__)

DEFAULT_SERVER_PORT = 5056

DEFAULT_SANIC_WORKERS = 1

def create_argument_parser():
    """Parse all the command line arguments for the nlg server script."""

    parser = argparse.ArgumentParser(
        description='starts the nlg endpoint')
    parser.add_argument(
        '-p', '--port',
        default=DEFAULT_SERVER_PORT,
        type=int,
        help="port to run the server at")
    parser.add_argument(
        '--workers',
        default=DEFAULT_SANIC_WORKERS,
        type=int,
        help="Number of processes to spin up")
    parser.add_argument(
        '-d', '--domain',
        type=str,
        default=None,
        help="path of the domain file to load utterances from"
    )

    return parser

async def generate_response(nlg_call, domain):
    """Mock response generator which generates the responses from the
    bot's domain file.
    """
    kwargs = nlg_call.get("arguments", {})
    template = nlg_call.get("template")
    sender_id = nlg_call.get("tracker", {}).get("sender_id")
    events = nlg_call.get("tracker", {}).get("events")
    tracker = DialogueStateTracker.from_dict(
        sender_id, events, domain.slots)
    channel_name = nlg_call.get("channel")

    return await TemplatedNaturalLanguageGenerator(domain.templates).generate(
        template, tracker, channel_name, **kwargs)

def run_server(domain, port, workers):
    app = Sanic(__name__)

    @app.route("/nlg", methods=['POST', 'OPTIONS'])
    async def nlg(request):
        """Endpoints which processes the Core request for a bot response."""
        nlg_call = request.json
        bot_response = await generate_response(nlg_call, domain)

        return response.json(bot_response)

    app.run(host='0.0.0.0',
            port=port,
            workers=sanic_workers)

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)

    # Running as standalone python application
    arg_parser = create_argument_parser()
    cmdline_args = arg_parser.parse_args()
    domain = Domain.load(cmdline_args.domain)

    run_server(domain, cmdline_args.port, cmdline_args.sanic_workers)
<commit_msg>Update examples/nlg_server/nlg_server.py<commit_after>import argparse
import logging

from sanic import Sanic
from sanic import response

from rasa_core.domain import Domain
from rasa_core.nlg import TemplatedNaturalLanguageGenerator
from rasa_core.trackers import DialogueStateTracker

logger = logging.getLogger(__name__)

DEFAULT_SERVER_PORT = 5056

DEFAULT_SANIC_WORKERS = 1

def create_argument_parser():
    """Parse all the command line arguments for the nlg server script."""

    parser = argparse.ArgumentParser(
        description='starts the nlg endpoint')
    parser.add_argument(
        '-p', '--port',
        default=DEFAULT_SERVER_PORT,
        type=int,
        help="port to run the server at")
    parser.add_argument(
        '--workers',
        default=DEFAULT_SANIC_WORKERS,
        type=int,
        help="Number of processes to spin up")
    parser.add_argument(
        '-d', '--domain',
        type=str,
        default=None,
        help="path of the domain file to load utterances from"
    )

    return parser

async def generate_response(nlg_call, domain):
    """Mock response generator which generates the responses from the
    bot's domain file.
    """
    kwargs = nlg_call.get("arguments", {})
    template = nlg_call.get("template")
    sender_id = nlg_call.get("tracker", {}).get("sender_id")
    events = nlg_call.get("tracker", {}).get("events")
    tracker = DialogueStateTracker.from_dict(
        sender_id, events, domain.slots)
    channel_name = nlg_call.get("channel")

    return await TemplatedNaturalLanguageGenerator(domain.templates).generate(
        template, tracker, channel_name, **kwargs)

def run_server(domain, port, workers):
    app = Sanic(__name__)

    @app.route("/nlg", methods=['POST', 'OPTIONS'])
    async def nlg(request):
        """Endpoints which processes the Core request for a bot response."""
        nlg_call = request.json
        bot_response = await generate_response(nlg_call, domain)

        return response.json(bot_response)

    app.run(host='0.0.0.0',
            port=port,
            workers=sanic_workers)

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)

    # Running as standalone python application
    arg_parser = create_argument_parser()
    cmdline_args = arg_parser.parse_args()
    domain = Domain.load(cmdline_args.domain)

    run_server(domain, cmdline_args.port, cmdline_args.workers)
<|endoftext|>