<commit_before>#!/usr/bin/env python3
#
# Tests the Pints plot methods.
#
# This file is part of PINTS.
#  Copyright (c) 2017-2018, University of Oxford.
#  For licensing information, see the LICENSE file distributed with the PINTS
#  software package.
#
import pints
import pints.toy as toy
import pints.plot
import unittest
import numpy as np
import matplotlib

# Select matplotlib backend that doesn't require a screen
matplotlib.use('Agg')

debug = False

class TestPlot(unittest.TestCase):
    """
    Tests Pints plot methods.
    """
    def __init__(self, name):
        super(TestPlot, self).__init__(name)

        # Create toy model (single output)
        self.model = toy.LogisticModel()
        self.real_parameters = [0.015, 500]
        self.times = np.linspace(0, 1000, 100)  # small problem
        self.values = self.model.simulate(self.real_parameters, self.times)

        # Add noise
        self.noise = 10
        self.values += np.random.normal(0, self.noise, self.values.shape)
        self.real_parameters.append(self.noise)
        self.real_parameters = np.array(self.real_parameters)

        # Create an object with links to the model and time series
        self.problem = pints.SingleOutputProblem(
            self.model, self.times, self.values)

        # Create a uniform prior over both the parameters and the new noise
        # variable
        self.lower = [0.01, 400, self.noise * 0.1]
        self.upper = [0.02, 600, self.noise * 100]
        self.log_prior = pints.UniformLogPrior(
            self.lower,
            self.upper
        )

        # Create a log likelihood
        self.log_likelihood = pints.UnknownNoiseLogLikelihood(self.problem)

        # Create an un-normalised log-posterior (log-likelihood + log-prior)
        self.log_posterior = pints.LogPosterior(
            self.log_likelihood, self.log_prior)

        # Run MCMC
        self.x0 = [
            self.real_parameters * 1.1,
            self.real_parameters * 0.9,
            self.real_parameters * 1.05
        ]
        mcmc = pints.MCMCSampling(self.log_posterior, 3, self.x0,
                                  method=pints.AdaptiveCovarianceMCMC)
        mcmc.set_max_iterations(300)  # make it as small as possible
        mcmc.set_log_to_screen(False)
        self.samples = mcmc.run()

        # Create toy model (multi-output)
        self.model2 = toy.LotkaVolterraModel()
        self.real_parameters2 = self.model2.suggested_parameters()
        self.times2 = self.model2.suggested_times()[::10]  # down sample it
        self.values2 = self.model2.simulate(self.real_parameters2, self.times2)

        # Add noise
        self.noise2 = 0.05
        self.values2 += np.random.normal(0, self.noise2, self.values2.shape)

        # Create an object with links to the model and time series
        self.problem2 = pints.MultiOutputProblem(
            self.model2, self.times2, self.values2)

        # Create a uniform prior over both the parameters and the new noise
        # variable
        self.log_prior2 = pints.UniformLogPrior([1, 1, 1, 1], [6, 6, 6, 6])
        # Create a log likelihood
        self.log_likelihood2 = pints.KnownNoiseLogLikelihood(self.problem2,
                                                             self.noise2)

        # Create an un-normalised log-posterior (log-likelihood + log-prior)
        self.log_posterior2 = pints.LogPosterior(
            self.log_likelihood2, self.log_prior2)

        # Run MCMC
        self.x02 = [
            self.real_parameters2 * 1.1,
            self.real_parameters2 * 0.9,
            self.real_parameters2 * 1.05
        ]
        mcmc = pints.MCMCSampling(self.log_posterior2, 3, self.x02,
                                  method=pints.AdaptiveCovarianceMCMC)
        mcmc.set_max_iterations(300)  # make it as small as possible
        mcmc.set_log_to_screen(False)
        self.samples2 = mcmc.run()

    def test_function(self):
        """
        Tests the function function.
        """
        # Test it can plot without error
        pints.plot.function(self.log_posterior, self.real_parameters)

        # Check lower and upper bounds input gives no error
        pints.plot.function(self.log_posterior, self.real_parameters,
                            self.lower, self.upper)
        # Check invalid lower bound
        self.assertRaises(
            ValueError, pints.plot.function, self.log_posterior,
            self.real_parameters, self.lower[:-1], self.upper
        )
        # Check invalid upper bound
        self.assertRaises(
            ValueError, pints.plot.function, self.log_posterior,
            self.real_parameters, self.lower, self.upper[:-1]
        )

        # Check evaluations gives no error
        pints.plot.function(self.log_posterior, self.real_parameters,
                            evaluations=5)

        # Check invalid function input
        self.assertRaises(
            ValueError, pints.plot.function, self.real_parameters,
            self.real_parameters
        )

        # Check invalid dimension input
        self.assertRaises(
            ValueError, pints.plot.function, self.log_posterior,
            list(self.real_parameters) + [0]
        )

        # Check invalid evaluations input
        self.assertRaises(
            ValueError, pints.plot.function, self.log_posterior,
            self.real_parameters, evaluations=-1
        )

    def test_function_between_points(self):
        """
        Tests the function_between_points function.
        """
        # Test it can plot without error
        pints.plot.function_between_points(self.log_posterior,
                                           self.real_parameters * 0.8,
                                           self.real_parameters * 1.2)

        # Check the two points are reversible
        pints.plot.function_between_points(self.log_posterior,
                                           self.real_parameters * 1.2,
                                           self.real_parameters * 0.8)

        # Check padding gives no error
        pints.plot.function_between_points(self.log_posterior,
                                           self.real_parameters * 0.8,
                                           self.real_parameters * 1.2,
                                           padding=0.5)

        # Check evaluations gives no error
        pints.plot.function_between_points(self.log_posterior,
                                           self.real_parameters * 0.8,
                                           self.real_parameters * 1.2,
                                           evaluations=5)

        # Check invalid function input
        self.assertRaises(
            ValueError, pints.plot.function_between_points,
            self.real_parameters,
            self.real_parameters * 1.2,
            self.real_parameters * 0.8
        )

        # Check invalid dimension input
        self.assertRaises(
            ValueError, pints.plot.function_between_points,
            self.log_posterior,
            list(self.real_parameters) + [0],
            self.real_parameters * 0.8
        )

        # Check invalid padding input
        self.assertRaises(
            ValueError, pints.plot.function_between_points,
            self.log_posterior,
            self.real_parameters * 1.2,
            self.real_parameters * 0.8,
            padding=-1
        )

        # Check invalid evaluations input
        self.assertRaises(
            ValueError, pints.plot.function_between_points,
            self.log_posterior,
            self.real_parameters * 1.2,
            self.real_parameters * 0.8,
            evaluations=-1
        )

    def test_histogram(self):
        """
        Tests the histogram function.
        """
        few_samples = self.samples[:, ::10, :]
        # Test it can plot without error
        fig, axes = pints.plot.histogram(self.samples,
                                         ref_parameters=self.real_parameters)
        # Check it returns matplotlib figure and axes
        # self.assertIsInstance(axes, self.matplotlibAxesClass)

        # Test compatiblity with one chain only
        pints.plot.histogram([self.samples[0]])

        # Check n_percentiles gives no error
        pints.plot.histogram(few_samples, n_percentiles=50)

        # Check invalid samples input
        self.assertRaises(
            ValueError, pints.plot.histogram,
            [self.samples[0, :, :], self.samples[1:, :, :-1]]
        )

        # Check invalid ref_parameter input
        self.assertRaises(
            ValueError, pints.plot.histogram,
            self.samples, [self.real_parameters[0]]
        )

    def test_trace(self):
        """
        Tests the trace function.
        """
        few_samples = self.samples[:, ::10, :]
        # Test it can plot without error
        fig, axes = pints.plot.trace(self.samples,
                                     ref_parameters=self.real_parameters)

        # Test compatiblity with one chain only
        pints.plot.trace([self.samples[0]])

        # Check n_percentiles gives no error
        pints.plot.trace(few_samples, n_percentiles=50)

        # Check invalid samples input
        self.assertRaises(
            ValueError, pints.plot.trace,
            [self.samples[0, :, :], self.samples[1:, :, :-1]]
        )

        # Check invalid ref_parameter input
        self.assertRaises(
            ValueError, pints.plot.trace,
            self.samples, [self.real_parameters[0]]
        )

    def test_autocorrelation(self):
        """
        Tests the autocorrelation function.
        """
        # Test it can plot without error
        pints.plot.autocorrelation(self.samples[0], max_lags=20)

        # Check invalid input of samples
        self.assertRaises(
            ValueError, pints.plot.autocorrelation, self.samples
        )

    def test_series(self):
        """
        Tests the series function.
        """
        few_samples = self.samples[0][::30, :]
        # Test it can plot without error
        pints.plot.series(self.samples[0], self.problem)

        # Test thinning gives no error
        pints.plot.series(few_samples, self.problem, thinning=1)
        # Test invalid thinning input
        self.assertRaises(
            ValueError, pints.plot.series, few_samples, self.problem,
            thinning=0
        )

        # Check invalid input of samples
        self.assertRaises(
            ValueError, pints.plot.series, self.samples, self.problem
        )

        # Check reference parameters gives no error
        pints.plot.series(few_samples, self.problem,
                          ref_parameters=self.real_parameters)
        # Check invalid reference parameters input
        self.assertRaises(
            ValueError, pints.plot.series, few_samples, self.problem,
            self.real_parameters[:-2]
        )

        # Test mutli-output
        few_samples2 = self.samples2[0][::30, :]
        # Test it can plot without error
        pints.plot.series(self.samples2[0], self.problem2)

        # Test thinning gives no error
        pints.plot.series(few_samples2, self.problem2, thinning=1)
        # Test invalid thinning input
        self.assertRaises(
            ValueError, pints.plot.series, few_samples2, self.problem2,
            thinning=0
        )

        # Check invalid input of samples
        self.assertRaises(
            ValueError, pints.plot.series, self.samples2, self.problem2
        )

        # Check reference parameters gives no error
        pints.plot.series(few_samples2, self.problem2,
                          ref_parameters=self.real_parameters2)
        # Check invalid reference parameters input
        self.assertRaises(
            ValueError, pints.plot.series, few_samples2, self.problem2,
            self.real_parameters2[:-2]
        )

    def test_pairwise(self):
        """
        Tests the pairwise function.
        """
        few_samples = self.samples[0][::30, :]
        # Test it can plot without error
        pints.plot.pairwise(self.samples[0],
                            ref_parameters=self.real_parameters)

        # Test kde gives no error
        pints.plot.pairwise(few_samples, kde=True,
                            ref_parameters=self.real_parameters)

        # Test opacity gives no error
        pints.plot.pairwise(few_samples, opacity=0.2)

        # Test opacity auto setting gives no error
        pints.plot.pairwise(few_samples[:5, :])

        # Test n_percentiles gives no error
        pints.plot.pairwise(few_samples, n_percentiles=50)

        # Check invalid input of samples
        self.assertRaises(
            ValueError, pints.plot.pairwise, self.samples
        )

        # Check invalid ref_parameter input
        self.assertRaises(
            ValueError, pints.plot.pairwise,
            self.samples, ref_parameters=[self.real_parameters[0]]
        )

if __name__ == '__main__':
    print('Add -v for more debug output')
    import sys
    if '-v' in sys.argv:
        debug = True
    unittest.main()
<commit_msg>Add tests to pints.plot<commit_after>#!/usr/bin/env python3
#
# Tests the Pints plot methods.
#
# This file is part of PINTS.
#  Copyright (c) 2017-2018, University of Oxford.
#  For licensing information, see the LICENSE file distributed with the PINTS
#  software package.
#
import pints
import pints.toy as toy
import pints.plot
import unittest
import numpy as np
import matplotlib

# Select matplotlib backend that doesn't require a screen
matplotlib.use('Agg')

debug = False

class TestPlot(unittest.TestCase):
    """
    Tests Pints plot methods.
    """
    def __init__(self, name):
        super(TestPlot, self).__init__(name)

        # Create toy model (single output)
        self.model = toy.LogisticModel()
        self.real_parameters = [0.015, 500]
        self.times = np.linspace(0, 1000, 100)  # small problem
        self.values = self.model.simulate(self.real_parameters, self.times)

        # Add noise
        self.noise = 10
        self.values += np.random.normal(0, self.noise, self.values.shape)
        self.real_parameters.append(self.noise)
        self.real_parameters = np.array(self.real_parameters)

        # Create an object with links to the model and time series
        self.problem = pints.SingleOutputProblem(
            self.model, self.times, self.values)

        # Create a uniform prior over both the parameters and the new noise
        # variable
        self.lower = [0.01, 400, self.noise * 0.1]
        self.upper = [0.02, 600, self.noise * 100]
        self.log_prior = pints.UniformLogPrior(
            self.lower,
            self.upper
        )

        # Create a log likelihood
        self.log_likelihood = pints.UnknownNoiseLogLikelihood(self.problem)

        # Create an un-normalised log-posterior (log-likelihood + log-prior)
        self.log_posterior = pints.LogPosterior(
            self.log_likelihood, self.log_prior)

        # Run MCMC
        self.x0 = [
            self.real_parameters * 1.1,
            self.real_parameters * 0.9,
            self.real_parameters * 1.05
        ]
        mcmc = pints.MCMCSampling(self.log_posterior, 3, self.x0,
                                  method=pints.AdaptiveCovarianceMCMC)
        mcmc.set_max_iterations(300)  # make it as small as possible
        mcmc.set_log_to_screen(False)
        self.samples = mcmc.run()

        # Create toy model (multi-output)
        self.model2 = toy.LotkaVolterraModel()
        self.real_parameters2 = self.model2.suggested_parameters()
        self.times2 = self.model2.suggested_times()[::10]  # down sample it
        self.values2 = self.model2.simulate(self.real_parameters2, self.times2)

        # Add noise
        self.noise2 = 0.05
        self.values2 += np.random.normal(0, self.noise2, self.values2.shape)

        # Create an object with links to the model and time series
        self.problem2 = pints.MultiOutputProblem(
            self.model2, self.times2, self.values2)

        # Create a uniform prior over both the parameters and the new noise
        # variable
        self.log_prior2 = pints.UniformLogPrior([1, 1, 1, 1], [6, 6, 6, 6])
        # Create a log likelihood
        self.log_likelihood2 = pints.KnownNoiseLogLikelihood(self.problem2,
                                                             self.noise2)

        # Create an un-normalised log-posterior (log-likelihood + log-prior)
        self.log_posterior2 = pints.LogPosterior(
            self.log_likelihood2, self.log_prior2)

        # Run MCMC
        self.x02 = [
            self.real_parameters2 * 1.1,
            self.real_parameters2 * 0.9,
            self.real_parameters2 * 1.05
        ]
        mcmc = pints.MCMCSampling(self.log_posterior2, 3, self.x02,
                                  method=pints.AdaptiveCovarianceMCMC)
        mcmc.set_max_iterations(300)  # make it as small as possible
        mcmc.set_log_to_screen(False)
        self.samples2 = mcmc.run()

    def test_function(self):
        """
        Tests the function function.
        """
        # Test it can plot without error
        pints.plot.function(self.log_posterior, self.real_parameters)

        # Check lower and upper bounds input gives no error
        pints.plot.function(self.log_posterior, self.real_parameters,
                            self.lower, self.upper)
        # Check invalid lower bound
        self.assertRaises(
            ValueError, pints.plot.function, self.log_posterior,
            self.real_parameters, self.lower[:-1], self.upper
        )
        # Check invalid upper bound
        self.assertRaises(
            ValueError, pints.plot.function, self.log_posterior,
            self.real_parameters, self.lower, self.upper[:-1]
        )

        # Check evaluations gives no error
        pints.plot.function(self.log_posterior, self.real_parameters,
                            evaluations=5)

        # Check invalid function input
        self.assertRaises(
            ValueError, pints.plot.function, self.real_parameters,
            self.real_parameters
        )

        # Check invalid dimension input
        self.assertRaises(
            ValueError, pints.plot.function, self.log_posterior,
            list(self.real_parameters) + [0]
        )

        # Check invalid evaluations input
        self.assertRaises(
            ValueError, pints.plot.function, self.log_posterior,
            self.real_parameters, evaluations=-1
        )

    def test_function_between_points(self):
        """
        Tests the function_between_points function.
        """
        # Test it can plot without error
        pints.plot.function_between_points(self.log_posterior,
                                           self.real_parameters * 0.8,
                                           self.real_parameters * 1.2)

        # Check the two points are reversible
        pints.plot.function_between_points(self.log_posterior,
                                           self.real_parameters * 1.2,
                                           self.real_parameters * 0.8)

        # Check padding gives no error
        pints.plot.function_between_points(self.log_posterior,
                                           self.real_parameters * 0.8,
                                           self.real_parameters * 1.2,
                                           padding=0.5)

        # Check evaluations gives no error
        pints.plot.function_between_points(self.log_posterior,
                                           self.real_parameters * 0.8,
                                           self.real_parameters * 1.2,
                                           evaluations=5)

        # Check invalid function input
        self.assertRaises(
            ValueError, pints.plot.function_between_points,
            self.real_parameters,
            self.real_parameters * 1.2,
            self.real_parameters * 0.8
        )

        # Check invalid dimension input
        self.assertRaises(
            ValueError, pints.plot.function_between_points,
            self.log_posterior,
            list(self.real_parameters) + [0],
            self.real_parameters * 0.8
        )

        # Check invalid padding input
        self.assertRaises(
            ValueError, pints.plot.function_between_points,
            self.log_posterior,
            self.real_parameters * 1.2,
            self.real_parameters * 0.8,
            padding=-1
        )

        # Check invalid evaluations input
        self.assertRaises(
            ValueError, pints.plot.function_between_points,
            self.log_posterior,
            self.real_parameters * 1.2,
            self.real_parameters * 0.8,
            evaluations=-1
        )

    def test_histogram(self):
        """
        Tests the histogram function.
        """
        few_samples = self.samples[:, ::10, :]
        # Test it can plot without error
        fig, axes = pints.plot.histogram(self.samples,
                                         ref_parameters=self.real_parameters)
        # Check it returns matplotlib figure and axes
        # self.assertIsInstance(axes, self.matplotlibAxesClass)

        # Test compatiblity with one chain only
        pints.plot.histogram([self.samples[0]])

        # Check n_percentiles gives no error
        pints.plot.histogram(few_samples, n_percentiles=50)

        # Check invalid samples input
        self.assertRaises(
            ValueError, pints.plot.histogram,
            [self.samples[0, :, :], self.samples[1:, :, :-1]]
        )

        # Check invalid ref_parameter input
        self.assertRaises(
            ValueError, pints.plot.histogram,
            self.samples, [self.real_parameters[0]]
        )

    def test_trace(self):
        """
        Tests the trace function.
        """
        few_samples = self.samples[:, ::10, :]
        # Test it can plot without error
        fig, axes = pints.plot.trace(self.samples,
                                     ref_parameters=self.real_parameters)

        # Test compatiblity with one chain only
        pints.plot.trace([self.samples[0]])

        # Check n_percentiles gives no error
        pints.plot.trace(few_samples, n_percentiles=50)

        # Check invalid samples input
        self.assertRaises(
            ValueError, pints.plot.trace,
            [self.samples[0, :, :], self.samples[1:, :, :-1]]
        )

        # Check invalid ref_parameter input
        self.assertRaises(
            ValueError, pints.plot.trace,
            self.samples, [self.real_parameters[0]]
        )

    def test_autocorrelation(self):
        """
        Tests the autocorrelation function.
        """
        # Test it can plot without error
        pints.plot.autocorrelation(self.samples[0], max_lags=20)

        # Check invalid input of samples
        self.assertRaises(
            ValueError, pints.plot.autocorrelation, self.samples
        )

    def test_series(self):
        """
        Tests the series function.
        """
        few_samples = self.samples[0][::30, :]
        # Test it can plot without error
        pints.plot.series(self.samples[0], self.problem)

        # Test thinning gives no error
        pints.plot.series(few_samples, self.problem, thinning=1)
        # Test invalid thinning input
        self.assertRaises(
            ValueError, pints.plot.series, few_samples, self.problem,
            thinning=0
        )

        # Check invalid input of samples
        self.assertRaises(
            ValueError, pints.plot.series, self.samples, self.problem
        )

        # Check reference parameters gives no error
        pints.plot.series(few_samples, self.problem,
                          ref_parameters=self.real_parameters)
        # Check invalid reference parameters input
        self.assertRaises(
            ValueError, pints.plot.series, few_samples, self.problem,
            self.real_parameters[:-2]
        )

        # Test mutli-output
        few_samples2 = self.samples2[0][::30, :]
        # Test it can plot without error
        pints.plot.series(self.samples2[0], self.problem2)

        # Test thinning gives no error
        pints.plot.series(few_samples2, self.problem2, thinning=1)
        # Test invalid thinning input
        self.assertRaises(
            ValueError, pints.plot.series, few_samples2, self.problem2,
            thinning=0
        )

        # Check invalid input of samples
        self.assertRaises(
            ValueError, pints.plot.series, self.samples2, self.problem2
        )

        # Check reference parameters gives no error
        pints.plot.series(few_samples2, self.problem2,
                          ref_parameters=self.real_parameters2)
        # Check invalid reference parameters input
        self.assertRaises(
            ValueError, pints.plot.series, few_samples2, self.problem2,
            self.real_parameters2[:-2]
        )

    def test_pairwise(self):
        """
        Tests the pairwise function.
        """
        few_samples = self.samples[0][::30, :]
        # Test it can plot without error
        pints.plot.pairwise(self.samples[0],
                            ref_parameters=self.real_parameters)

        # Test kde gives no error
        pints.plot.pairwise(few_samples, kde=True,
                            ref_parameters=self.real_parameters)

        # Test opacity gives no error
        pints.plot.pairwise(few_samples, opacity=0.2)

        # Test opacity auto setting gives no error
        pints.plot.pairwise(few_samples[:5, :])

        # Test n_percentiles gives no error
        pints.plot.pairwise(few_samples, n_percentiles=50)

        # Check invalid input of samples
        self.assertRaises(
            ValueError, pints.plot.pairwise, self.samples
        )

        # Check invalid ref_parameter input
        self.assertRaises(
            ValueError, pints.plot.pairwise,
            few_samples, ref_parameters=[self.real_parameters[0]]
        )

if __name__ == '__main__':
    print('Add -v for more debug output')
    import sys
    if '-v' in sys.argv:
        debug = True
    unittest.main()
<|endoftext|>


<commit_before>import os
import argparse
from typing import List, Text

import questionary
from rasa.cli import train
from rasa.cli.shell import shell
from rasa.cli.utils import create_output_path
from rasa_core.utils import print_success

def add_subparser(subparsers: argparse._SubParsersAction,
                  parents: List[argparse.ArgumentParser]):
    scaffold_parser = subparsers.add_parser(
        "init",
        parents=parents,
        help="Create a new project from a initial_project")
    scaffold_parser.set_defaults(func=run)

def print_train_or_instructions(args: argparse.Namespace, path: Text) -> None:
    print_success("Your bot is ready to go!")
    should_train = questionary.confirm("Do you want me to train an initial "
                                       "model for the bot? 💪🏽").ask()
    if should_train:
        args.config = os.path.join(path, "config.yml")
        args.stories = os.path.join(path, "data/core")
        args.domain = os.path.join(path, "domain.yml")
        args.nlu = os.path.join(path, "data/nlu")
        args.out = os.path.join(path, create_output_path())

        args.model = train.train(args)

        print_run_or_instructions(args, path)

    else:
        print_success("No problem 👍🏼. You can also train me later by going "
                      "to the project directory and running 'rasa train'."
                      "".format(path))

def print_run_or_instructions(args: argparse.Namespace, path: Text) -> None:
    from rasa_core import constants

    should_run = questionary.confirm("Do you want to speak to the trained bot "
                                     "on the command line? 🤖").ask()

    if should_run:
        # provide defaults for command line arguments
        attributes = ["endpoints", "credentials", "cors", "auth_token",
                      "jwt_secret", "jwt_method", "enable_api"]
        for a in attributes:
            setattr(args, a, None)

        args.port = constants.DEFAULT_SERVER_PORT

        shell(args)
    else:
        print_success("Ok 👍🏼. If you want to speak to the bot later, "
                      "change into the project directory and run 'rasa shell'."
                      "".format(path))

def init_project(args: argparse.Namespace, path: Text) -> None:
    _create_initial_project(path)
    print("Created project directory at '{}'.".format(os.path.abspath(path)))
    print_train_or_instructions(args, path)

def _create_initial_project(path: Text) -> None:
    from distutils.dir_util import copy_tree

    copy_tree(scaffold_path(), path)

def scaffold_path() -> Text:
    import pkg_resources
    return pkg_resources.resource_filename(__name__, "initial_project")

def print_cancel() -> None:
    print_success("Ok. Then I stop here. If you need me again, simply type "
                  "'rasa init' 🙋🏽♀")
    exit(0)

def _ask_create_path(path: Text) -> None:
    should_create = questionary.confirm("Path '{}' does not exist 🧐. "
                                        "Should I create it?"
                                        "".format(path)).ask()
    if should_create:
        os.makedirs(path)
    else:
        print_success("Ok. Then I stop here. If you need me again, simply type "
                      "'rasa init' 🙋🏽♀")
        exit(0)

def _ask_overwrite(path: Text) -> None:
    overwrite = questionary.confirm("Directory '{}' is not empty. Continue?"
                                    "".format(os.path.abspath(path))).ask()
    if not overwrite:
        print_cancel()

def run(args: argparse.Namespace) -> None:
    from rasa_core.utils import print_success

    print_success("Welcome to Rasa! 🤖\n")
    print("To get started quickly, I can assist you to create an "
          "initial project.\n"
          "If you need some help to get from this template to a "
          "bad ass contextual assistant, checkout our quickstart guide"
          "here: https://rasa.com/docs/core/quickstart \n\n"
          "Now let's start! 👇🏽\n")
    path = questionary.text("Please enter a folder path where I should create "
                            "the initial project [default: current directory]",
                            default=".").ask()

    if not os.path.isdir(path):
        _ask_create_path(path)

    if path is None or not os.path.isdir(path):
        print_cancel()

    if len(os.listdir(path)) > 0:
        _ask_overwrite(path)

    init_project(args, path)
<commit_msg>Update rasa/cli/scaffold.py<commit_after>import os
import argparse
from typing import List, Text

import questionary
from rasa.cli import train
from rasa.cli.shell import shell
from rasa.cli.utils import create_output_path
from rasa_core.utils import print_success

def add_subparser(subparsers: argparse._SubParsersAction,
                  parents: List[argparse.ArgumentParser]):
    scaffold_parser = subparsers.add_parser(
        "init",
        parents=parents,
        help="Create a new project from a initial_project")
    scaffold_parser.set_defaults(func=run)

def print_train_or_instructions(args: argparse.Namespace, path: Text) -> None:
    print_success("Your bot is ready to go!")
    should_train = questionary.confirm("Do you want me to train an initial "
                                       "model for the bot? 💪🏽").ask()
    if should_train:
        args.config = os.path.join(path, "config.yml")
        args.stories = os.path.join(path, "data", "core")
        args.domain = os.path.join(path, "domain.yml")
        args.nlu = os.path.join(path, "data/nlu")
        args.out = os.path.join(path, create_output_path())

        args.model = train.train(args)

        print_run_or_instructions(args, path)

    else:
        print_success("No problem 👍🏼. You can also train me later by going "
                      "to the project directory and running 'rasa train'."
                      "".format(path))

def print_run_or_instructions(args: argparse.Namespace, path: Text) -> None:
    from rasa_core import constants

    should_run = questionary.confirm("Do you want to speak to the trained bot "
                                     "on the command line? 🤖").ask()

    if should_run:
        # provide defaults for command line arguments
        attributes = ["endpoints", "credentials", "cors", "auth_token",
                      "jwt_secret", "jwt_method", "enable_api"]
        for a in attributes:
            setattr(args, a, None)

        args.port = constants.DEFAULT_SERVER_PORT

        shell(args)
    else:
        print_success("Ok 👍🏼. If you want to speak to the bot later, "
                      "change into the project directory and run 'rasa shell'."
                      "".format(path))

def init_project(args: argparse.Namespace, path: Text) -> None:
    _create_initial_project(path)
    print("Created project directory at '{}'.".format(os.path.abspath(path)))
    print_train_or_instructions(args, path)

def _create_initial_project(path: Text) -> None:
    from distutils.dir_util import copy_tree

    copy_tree(scaffold_path(), path)

def scaffold_path() -> Text:
    import pkg_resources
    return pkg_resources.resource_filename(__name__, "initial_project")

def print_cancel() -> None:
    print_success("Ok. Then I stop here. If you need me again, simply type "
                  "'rasa init' 🙋🏽♀")
    exit(0)

def _ask_create_path(path: Text) -> None:
    should_create = questionary.confirm("Path '{}' does not exist 🧐. "
                                        "Should I create it?"
                                        "".format(path)).ask()
    if should_create:
        os.makedirs(path)
    else:
        print_success("Ok. Then I stop here. If you need me again, simply type "
                      "'rasa init' 🙋🏽♀")
        exit(0)

def _ask_overwrite(path: Text) -> None:
    overwrite = questionary.confirm("Directory '{}' is not empty. Continue?"
                                    "".format(os.path.abspath(path))).ask()
    if not overwrite:
        print_cancel()

def run(args: argparse.Namespace) -> None:
    from rasa_core.utils import print_success

    print_success("Welcome to Rasa! 🤖\n")
    print("To get started quickly, I can assist you to create an "
          "initial project.\n"
          "If you need some help to get from this template to a "
          "bad ass contextual assistant, checkout our quickstart guide"
          "here: https://rasa.com/docs/core/quickstart \n\n"
          "Now let's start! 👇🏽\n")
    path = questionary.text("Please enter a folder path where I should create "
                            "the initial project [default: current directory]",
                            default=".").ask()

    if not os.path.isdir(path):
        _ask_create_path(path)

    if path is None or not os.path.isdir(path):
        print_cancel()

    if len(os.listdir(path)) > 0:
        _ask_overwrite(path)

    init_project(args, path)
<|endoftext|>


<commit_before>import os
from distutils.dir_util import copy_tree

import questionary
from rasa.cli import train
from rasa_core.utils import print_success
from rasa_core import constants

from rasa.cli.shell import shell
from rasa.cli.train import create_default_output_path

def add_subparser(subparsers, parents):
    scaffold_parser = subparsers.add_parser(
        "init",
        parents=parents,
        help="Create a new project from a initial_project")
    scaffold_parser.set_defaults(func=run)

def scaffold_path():
    import pkg_resources
    return pkg_resources.resource_filename(__name__, "initial_project")

def print_train_or_instructions(args, path):
    print_success("Your bot initial_project is ready to go!")
    should_train = questionary.confirm("Do you want me to train an initial "
                                       "model for the bot?").ask()
    if should_train:
        args.config = os.path.join(path, "config.yml")
        args.stories = os.path.join(path, "data/core")
        args.domain = os.path.join(path, "domain.yml")
        args.nlu = os.path.join(path, "data/nlu")
        args.out = os.path.join(path, create_default_output_path())

        args.model = train.train(args)

        print_run_or_instructions(args)

    else:
        print("Great. To train your bot, run `cd {} && rasa train`."
              "".format(path))

def print_run_or_instructions(args):
    should_run = questionary.confirm("Do you want me to run the trained model "
                                     "in the command line?").ask()

    if should_run:
        # provide defaults for command line arguments
        attributes = ["endpoints", "credentials", "cors", "auth_token",
                      "jwt_secret", "jwt_method", "enable_api"]
        for a in attributes:
            setattr(args, a, None)

        args.port = constants.DEFAULT_SERVER_PORT

        shell(args)
    else:
        print("Great. To run your bot, run `cd {} && rasa shell`."
              "".format(args.model))

def init_project(args, path):
    if not os.path.isdir(path):
        os.makedirs(path)

    copy_tree(scaffold_path(), path)

    print("Created project directory at '{}'.".format(os.path.abspath(path)))
    print_train_or_instructions(args, path)

def run(args):
    path = questionary.text("Please enter a folder path for the bot "
                            "[default: current directory]", ".").ask()
    if path is None or not os.path.isdir(path):
        print("Path '{}' is not a valid directory. Aborted creation."
              "".format(path))
        exit(1)
    else:
        if len(os.listdir(path)) > 0:
            overwrite = questionary.confirm(
                "Directory '{}' is not empty. Continue?"
                "".format(os.path.abspath(path))).ask()
            if not overwrite:
                print("Aborted creation.")
                exit(1)
        init_project(args, path)
<commit_msg>Ask whether directory should be created<commit_after>import os
from distutils.dir_util import copy_tree

import questionary
from rasa.cli import train
from rasa_core.utils import print_success
from rasa_core import constants

from rasa.cli.shell import shell
from rasa.cli.train import create_default_output_path

def add_subparser(subparsers, parents):
    scaffold_parser = subparsers.add_parser(
        "init",
        parents=parents,
        help="Create a new project from a initial_project")
    scaffold_parser.set_defaults(func=run)

def scaffold_path():
    import pkg_resources
    return pkg_resources.resource_filename(__name__, "initial_project")

def print_train_or_instructions(args, path):
    print_success("Your bot initial_project is ready to go!")
    should_train = questionary.confirm("Do you want me to train an initial "
                                       "model for the bot?").ask()
    if should_train:
        args.config = os.path.join(path, "config.yml")
        args.stories = os.path.join(path, "data/core")
        args.domain = os.path.join(path, "domain.yml")
        args.nlu = os.path.join(path, "data/nlu")
        args.out = os.path.join(path, create_default_output_path())

        args.model = train.train(args)

        print_run_or_instructions(args)

    else:
        print("Great. To train your bot, run `cd {} && rasa train`."
              "".format(path))

def print_run_or_instructions(args):
    should_run = questionary.confirm("Do you want me to run the trained model "
                                     "in the command line?").ask()

    if should_run:
        # provide defaults for command line arguments
        attributes = ["endpoints", "credentials", "cors", "auth_token",
                      "jwt_secret", "jwt_method", "enable_api"]
        for a in attributes:
            setattr(args, a, None)

        args.port = constants.DEFAULT_SERVER_PORT

        shell(args)
    else:
        print("Great. To run your bot, run `cd {} && rasa shell`."
              "".format(args.model))

def init_project(args, path):
    copy_tree(scaffold_path(), path)

    print("Created project directory at '{}'.".format(os.path.abspath(path)))
    print_train_or_instructions(args, path)

def run(args):
    path = questionary.text("Please enter a folder path for the bot "
                            "[default: current directory]", ".").ask()

    if not os.path.isdir(path):
        should_create = questionary.confirm("Path '{}' does not exist. Should "
                                            "I create it?".format(path)).ask()
        if should_create:
            os.makedirs(path)
        else:
            print("Ok. Then I stop here.")
            exit(0)

    if path is None or not os.path.isdir(path):
        print("Path '{}' is not a valid directory. Aborted creation."
              "".format(path))
        exit(1)

    if len(os.listdir(path)) > 0:
        overwrite = questionary.confirm(
            "Directory '{}' is not empty. Continue?"
            "".format(os.path.abspath(path))).ask()
        if not overwrite:
            print("Aborted creation.")
            exit(1)

    init_project(args, path)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

# rss is the interface to rss parsing,
# processing and storing.
#
# @author Jin Yuan
# @contact jinyuan@baidu.com
# @created Jul. 20, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

from administration import database as db_feeds
import calendar
import database as db_news
from datetime import datetime, timedelta
from data_processor import summarizer
from data_processor import transcoder
from data_processor import tts_provider
from image_processor import image_helper
from image_processor import thumbnail
import memory
import random
import rss_parser
import time

from administration.config import CATEGORY_IMAGE_SIZE
from administration.config import DATABASE_REMOVAL_DAYS
from administration.config import HOT_IMAGE_SIZE
from administration.config import LANGUAGES
from administration.config import MEMORY_EXPIRATION_DAYS
from administration.config import MIN_IMAGE_SIZE
from administration.config import THUMBNAIL_LANDSCAPE_SIZE
from administration.config import THUMBNAIL_PORTRAIT_SIZE
from administration.config import THUMBNAIL_STYLE

def _value_added_process(entries=None, language=None, transcoder_type='chengdujin'):
    """
    add more value to an entry
    tts, transcode, images, redis_entry_expiration, database_entry_expiration
    """
    if not entries:
        return None
    if not language or language not in LANGUAGES:
        raise Exception("ERROR: language not found or not supported!")

    entries_new = []
    for entry in entries:
        # get a random int from 100 million possibilities
        try:
            # [MUST-HAVE] transcoding
            rand = random.randint(0, 100000000)
            transcoded_relative_path = '%s_%s_%s_%i' % (
                entry['language'], entry['feed_id'], entry['updated'], rand)
            # high chances transcoder cannot work properly
            entry['transcoded'], entry['transcoded_local'], raw_transcoded_content, images_from_transcoded = transcoder.convert(
                entry['language'], entry['title'], entry['link'], transcoder_type, transcoded_relative_path)

            # [MUST-HAVE] summary
            entry['summary'] = summarizer.extract(
                entry['summary'], raw_transcoded_content, entry['language'])

            # process images found in the transcoded data
            if images_from_transcoded:
                # images from transcoded are already normalized
                entry['images'].extend(images_from_transcoded)
                # remove duplicated images
                entry['images'] = image_helper.dedupe_images(
                    entry['images']) if entry.has_key('images') and entry['images'] else None
            # make images none if nothing's there, instead of a []
            entry['images'] = entry['images'] if entry.has_key(
                'images') and entry['images'] else None

            # [OPTIONAL] generate 3 types of images: thumbnail, category image and hot news image
            if entry.has_key('images') and entry['images']:
                biggest = image_helper.find_biggest_image(entry['images'])
                if biggest:
                    try:
                        rand = random.randint(0, 100000000)
                        image_relative_path = '%s_%s_%s_%i' % (
                            entry['language'], entry['feed_id'], entry['updated'], rand)

                        # hot news image
                        hot_web, hot_local = image_helper.scale_image(
                            image=biggest, size_expected=HOT_IMAGE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_hotnews' % image_relative_path)
                        entry['hot_news_image'] = hot_web if hot_web else None
                        entry[
                            'hot_news_image_local'] = hot_local if hot_local else None

                        # category image
                        category_web, category_local = image_helper.scale_image(
                            image=biggest, size_expected=CATEGORY_IMAGE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_category' % image_relative_path)
                        entry[
                            'category_image'] = category_web if category_web else None
                        entry[
                            'category_image_local'] = category_local if category_local else None

                        # thumbnail image
                        # thumbnail_landscape_size
                        if float(biggest['width']) / float(biggest['height']) >= THUMBNAIL_STYLE:
                            thumbnail_web, thumbnail_local = image_helper.scale_image(
                                image=biggest, size_expected=THUMBNAIL_LANDSCAPE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_thumbnail' % image_relative_path)
                        else:  # else thumbnail_portrait_size
                            thumbnail_web, thumbnail_local = image_helper.scale_image(
                                image=biggest, size_expected=THUMBNAIL_PORTRAIT_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_thumbnail' % image_relative_path)
                        entry[
                            'thumbnail_image'] = thumbnail_web if thumbnail_web else None
                        entry[
                            'thumbnail_image_local'] = thumbnail_local if thumbnail_local else None

                        # for older version users
                        entry['image'] = entry['thumbnail_image'][
                            'url'] if thumbnail_web else None
                    except IOError as k:
                        entry['error'].append(str(k) + '\n')

            # [OPTIONAL] google tts not for indonesian
            """
            if entry['language'] != 'ind':
                try:
                    tts_relative_path = '%s_%s_%s_%i.mp3' % (
                        entry['language'], entry['feed_id'], entry['updated'], rand)
                    entry['mp3'], entry['mp3_local'] = tts_provider.google(
                        entry['language'], entry['title'], tts_relative_path)
                except Exception as k:
                    print k, '... cannot generate TTS for %s' % entry['link']
                    entry['error'].append(k + '\n')
                    entry['mp3'] = None
                    entry['mp3_local'] = None
            """

            # [MUST-HAVE] add expiration data
            def _expired(updated, days_to_deadline):
                """
                compute expiration information
                return time string and unix time
                """
                deadline = datetime.utcfromtimestamp(
                    updated) + timedelta(days=days_to_deadline)
                return time.asctime(time.gmtime(calendar.timegm(deadline.timetuple())))

            entry['memory_expired'] = _expired(
                entry['updated'], MEMORY_EXPIRATION_DAYS)
            entry['database_expired'] = _expired(
                entry['updated'], DATABASE_REMOVAL_DAYS)

            entry['error'] = entry['error'] if entry['error'] else None
            entries_new.append(entry)
            print
            print
        except Exception as k:
            print k
    return entries_new

# TODO: code to remove added items if things suck at database/memory
def update(feed_link=None, feed_id=None, language=None, categories=None, transcoder_type='chengdujin'):
    """
    update could be called
    1. from task procedure: feed_id
    2. after an rss is added: feed_id
    3. manually for testing purpose: feed_link, language
    Note. categories are ids of category item
    Notel categories are kept for manual testing
    """
    if not feed_id and not (feed_link and language):
        raise Exception(
            "ERROR: Method signature not well formed!")

    if feed_id:
        feed = db_feeds.get(feed_id=feed_id)
    else:
        feed = db_feeds.get(feed_link=feed_link, language=language)

    feed_id = str(feed['_id']) if feed else feed_id
    feed_link = feed['feed_link'] if feed else feed_link
    language = feed['language'] if feed else language
    categories = feed['categories'] if feed else categories
    feed_title = feed['feed_title'] if feed and 'feed_title' in feed else None
    transcoder_type = feed['transcoder'] if feed else transcoder_type
    etag = feed['etag'] if feed and 'etag' in feed else None
    modified = feed['modified'] if feed and 'modified' in feed else None

    # parse rss reading from remote rss servers
    entries, status_new, feed_title_new, etag_new, modified_new = rss_parser.parse(
        feed_link, feed_id, feed_title, language, categories, etag, modified)

    # filter out existing entries in db_news
    # there are some possible exceptions -- yet let it be
    entries = db_news.dedup(entries, language)
    # and do tts, big_images, image as well as transcode.
    entries = _value_added_process(entries, language, transcoder_type)

    # update new entries to db_news
    # each entry is added with _id
    entries = db_news.update(entries, language)
    # and some data, like feed_title, etag and modified to db_feeds
    # only feed_id is necessary, others are optional **kwargs
    db_feeds.update(feed_id=feed_id, status=status_new,
                    feed_title=feed_title_new, etag=etag_new, modified=modified_new)

    # store in memory
    memory.update(entries, language, categories)
    return entries
<commit_msg>renamed admin....config to config<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

# rss is the interface to rss parsing,
# processing and storing.
#
# @author Jin Yuan
# @contact jinyuan@baidu.com
# @created Jul. 20, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

from administration import database as db_feeds
import calendar
import database as db_news
from datetime import datetime, timedelta
from data_processor import summarizer
from data_processor import transcoder
from data_processor import tts_provider
from image_processor import image_helper
from image_processor import thumbnail
import memory
import random
import rss_parser
import time

from config import CATEGORY_IMAGE_SIZE
from config import DATABASE_REMOVAL_DAYS
from config import HOT_IMAGE_SIZE
from config import LANGUAGES
from config import MEMORY_EXPIRATION_DAYS
from config import MIN_IMAGE_SIZE
from config import THUMBNAIL_LANDSCAPE_SIZE
from config import THUMBNAIL_PORTRAIT_SIZE
from config import THUMBNAIL_STYLE

def _value_added_process(entries=None, language=None, transcoder_type='chengdujin'):
    """
    add more value to an entry
    tts, transcode, images, redis_entry_expiration, database_entry_expiration
    """
    if not entries:
        return None
    if not language or language not in LANGUAGES:
        raise Exception("ERROR: language not found or not supported!")

    entries_new = []
    for entry in entries:
        # get a random int from 100 million possibilities
        try:
            # [MUST-HAVE] transcoding
            rand = random.randint(0, 100000000)
            transcoded_relative_path = '%s_%s_%s_%i' % (
                entry['language'], entry['feed_id'], entry['updated'], rand)
            # high chances transcoder cannot work properly
            entry['transcoded'], entry['transcoded_local'], raw_transcoded_content, images_from_transcoded = transcoder.convert(
                entry['language'], entry['title'], entry['link'], transcoder_type, transcoded_relative_path)

            # [MUST-HAVE] summary
            entry['summary'] = summarizer.extract(
                entry['summary'], raw_transcoded_content, entry['language'])

            # process images found in the transcoded data
            if images_from_transcoded:
                # images from transcoded are already normalized
                entry['images'].extend(images_from_transcoded)
                # remove duplicated images
                entry['images'] = image_helper.dedupe_images(
                    entry['images']) if entry.has_key('images') and entry['images'] else None
            # make images none if nothing's there, instead of a []
            entry['images'] = entry['images'] if entry.has_key(
                'images') and entry['images'] else None

            # [OPTIONAL] generate 3 types of images: thumbnail, category image and hot news image
            if entry.has_key('images') and entry['images']:
                biggest = image_helper.find_biggest_image(entry['images'])
                if biggest:
                    try:
                        rand = random.randint(0, 100000000)
                        image_relative_path = '%s_%s_%s_%i' % (
                            entry['language'], entry['feed_id'], entry['updated'], rand)

                        # hot news image
                        hot_web, hot_local = image_helper.scale_image(
                            image=biggest, size_expected=HOT_IMAGE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_hotnews' % image_relative_path)
                        entry['hot_news_image'] = hot_web if hot_web else None
                        entry[
                            'hot_news_image_local'] = hot_local if hot_local else None

                        # category image
                        category_web, category_local = image_helper.scale_image(
                            image=biggest, size_expected=CATEGORY_IMAGE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_category' % image_relative_path)
                        entry[
                            'category_image'] = category_web if category_web else None
                        entry[
                            'category_image_local'] = category_local if category_local else None

                        # thumbnail image
                        # thumbnail_landscape_size
                        if float(biggest['width']) / float(biggest['height']) >= THUMBNAIL_STYLE:
                            thumbnail_web, thumbnail_local = image_helper.scale_image(
                                image=biggest, size_expected=THUMBNAIL_LANDSCAPE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_thumbnail' % image_relative_path)
                        else:  # else thumbnail_portrait_size
                            thumbnail_web, thumbnail_local = image_helper.scale_image(
                                image=biggest, size_expected=THUMBNAIL_PORTRAIT_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_thumbnail' % image_relative_path)
                        entry[
                            'thumbnail_image'] = thumbnail_web if thumbnail_web else None
                        entry[
                            'thumbnail_image_local'] = thumbnail_local if thumbnail_local else None

                        # for older version users
                        entry['image'] = entry['thumbnail_image'][
                            'url'] if thumbnail_web else None
                    except IOError as k:
                        entry['error'].append(str(k) + '\n')

            # [OPTIONAL] google tts not for indonesian
            """
            if entry['language'] != 'ind':
                try:
                    tts_relative_path = '%s_%s_%s_%i.mp3' % (
                        entry['language'], entry['feed_id'], entry['updated'], rand)
                    entry['mp3'], entry['mp3_local'] = tts_provider.google(
                        entry['language'], entry['title'], tts_relative_path)
                except Exception as k:
                    print k, '... cannot generate TTS for %s' % entry['link']
                    entry['error'].append(k + '\n')
                    entry['mp3'] = None
                    entry['mp3_local'] = None
            """

            # [MUST-HAVE] add expiration data
            def _expired(updated, days_to_deadline):
                """
                compute expiration information
                return time string and unix time
                """
                deadline = datetime.utcfromtimestamp(
                    updated) + timedelta(days=days_to_deadline)
                return time.asctime(time.gmtime(calendar.timegm(deadline.timetuple())))

            entry['memory_expired'] = _expired(
                entry['updated'], MEMORY_EXPIRATION_DAYS)
            entry['database_expired'] = _expired(
                entry['updated'], DATABASE_REMOVAL_DAYS)

            entry['error'] = entry['error'] if entry['error'] else None
            entries_new.append(entry)
            print
            print
        except Exception as k:
            print k
    return entries_new

# TODO: code to remove added items if things suck at database/memory
def update(feed_link=None, feed_id=None, language=None, categories=None, transcoder_type='chengdujin'):
    """
    update could be called
    1. from task procedure: feed_id
    2. after an rss is added: feed_id
    3. manually for testing purpose: feed_link, language
    Note. categories are ids of category item
    Notel categories are kept for manual testing
    """
    if not feed_id and not (feed_link and language):
        raise Exception(
            "ERROR: Method signature not well formed!")

    if feed_id:
        feed = db_feeds.get(feed_id=feed_id)
    else:
        feed = db_feeds.get(feed_link=feed_link, language=language)

    feed_id = str(feed['_id']) if feed else feed_id
    feed_link = feed['feed_link'] if feed else feed_link
    language = feed['language'] if feed else language
    categories = feed['categories'] if feed else categories
    feed_title = feed['feed_title'] if feed and 'feed_title' in feed else None
    transcoder_type = feed['transcoder'] if feed else transcoder_type
    etag = feed['etag'] if feed and 'etag' in feed else None
    modified = feed['modified'] if feed and 'modified' in feed else None

    # parse rss reading from remote rss servers
    entries, status_new, feed_title_new, etag_new, modified_new = rss_parser.parse(
        feed_link, feed_id, feed_title, language, categories, etag, modified)

    # filter out existing entries in db_news
    # there are some possible exceptions -- yet let it be
    entries = db_news.dedup(entries, language)
    # and do tts, big_images, image as well as transcode.
    entries = _value_added_process(entries, language, transcoder_type)

    # update new entries to db_news
    # each entry is added with _id
    entries = db_news.update(entries, language)
    # and some data, like feed_title, etag and modified to db_feeds
    # only feed_id is necessary, others are optional **kwargs
    db_feeds.update(feed_id=feed_id, status=status_new,
                    feed_title=feed_title_new, etag=etag_new, modified=modified_new)

    # store in memory
    memory.update(entries, language, categories)
    return entries
<|endoftext|>


<commit_before>import asyncio
import logging
import os
import shutil
import warnings
from types import TracebackType
from typing import Any, Coroutine, Dict, List, Optional, Text, Type, TypeVar

import rasa.core.utils
import rasa.utils.io
from rasa.constants import (
    DEFAULT_LOG_LEVEL_LIBRARIES,
    ENV_LOG_LEVEL_LIBRARIES,
)
from rasa.shared.constants import DEFAULT_LOG_LEVEL, ENV_LOG_LEVEL
import rasa.shared.utils.io

logger = logging.getLogger(__name__)

T = TypeVar("T")

class TempDirectoryPath(str):
    """Represents a path to an temporary directory. When used as a context
    manager, it erases the contents of the directory on exit.

    """

    def __enter__(self) -> "TempDirectoryPath":
        return self

    def __exit__(
        self,
        _exc: Optional[Type[BaseException]],
        _value: Optional[Exception],
        _tb: Optional[TracebackType],
    ) -> bool:
        if os.path.exists(self):
            shutil.rmtree(self)

def read_global_config(path: Text) -> Dict[Text, Any]:
    """Read global Rasa configuration.

    Args:
        path: Path to the configuration
    Returns:
        The global configuration
    """
    # noinspection PyBroadException
    try:
        return rasa.shared.utils.io.read_config_file(path)
    except Exception:
        # if things go south we pretend there is no config
        return {}

def set_log_level(log_level: Optional[int] = None):
    """Set log level of Rasa and Tensorflow either to the provided log level or
    to the log level specified in the environment variable 'LOG_LEVEL'. If none is set
    a default log level will be used."""

    if not log_level:
        log_level = os.environ.get(ENV_LOG_LEVEL, DEFAULT_LOG_LEVEL)
        log_level = logging.getLevelName(log_level)

    logging.getLogger("rasa").setLevel(log_level)

    update_tensorflow_log_level()
    update_asyncio_log_level()
    update_apscheduler_log_level()
    update_socketio_log_level()

    os.environ[ENV_LOG_LEVEL] = logging.getLevelName(log_level)

def update_apscheduler_log_level() -> None:
    log_level = os.environ.get(ENV_LOG_LEVEL_LIBRARIES, DEFAULT_LOG_LEVEL_LIBRARIES)

    apscheduler_loggers = [
        "apscheduler",
        "apscheduler.scheduler",
        "apscheduler.executors",
        "apscheduler.executors.default",
    ]

    for logger_name in apscheduler_loggers:
        logging.getLogger(logger_name).setLevel(log_level)
        logging.getLogger(logger_name).propagate = False

def update_socketio_log_level() -> None:
    log_level = os.environ.get(ENV_LOG_LEVEL_LIBRARIES, DEFAULT_LOG_LEVEL_LIBRARIES)

    socketio_loggers = ["websockets.protocol", "engineio.server", "socketio.server"]

    for logger_name in socketio_loggers:
        logging.getLogger(logger_name).setLevel(log_level)
        logging.getLogger(logger_name).propagate = False

def update_tensorflow_log_level() -> None:
    """Set the log level of Tensorflow to the log level specified in the environment
    variable 'LOG_LEVEL_LIBRARIES'."""

    # Disables libvinfer, tensorRT, cuda, AVX2 and FMA warnings (CPU support). This variable needs to be set before the
    # first import since some warnings are raised on the first import.
    os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"

    import tensorflow as tf

    log_level = os.environ.get(ENV_LOG_LEVEL_LIBRARIES, DEFAULT_LOG_LEVEL_LIBRARIES)

    if log_level == "DEBUG":
        tf_log_level = tf.compat.v1.logging.DEBUG
    elif log_level == "INFO":
        tf_log_level = tf.compat.v1.logging.INFO
    elif log_level == "WARNING":
        tf_log_level = tf.compat.v1.logging.WARN
    else:
        tf_log_level = tf.compat.v1.logging.ERROR

    tf.compat.v1.logging.set_verbosity(tf_log_level)
    logging.getLogger("tensorflow").propagate = False

def update_sanic_log_level(log_file: Optional[Text] = None):
    """Set the log level of sanic loggers to the log level specified in the environment
    variable 'LOG_LEVEL_LIBRARIES'."""
    from sanic.log import logger, error_logger, access_logger

    log_level = os.environ.get(ENV_LOG_LEVEL_LIBRARIES, DEFAULT_LOG_LEVEL_LIBRARIES)

    logger.setLevel(log_level)
    error_logger.setLevel(log_level)
    access_logger.setLevel(log_level)

    logger.propagate = False
    error_logger.propagate = False
    access_logger.propagate = False

    if log_file is not None:
        formatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s]  %(message)s")
        file_handler = logging.FileHandler(log_file)
        file_handler.setFormatter(formatter)

        logger.addHandler(file_handler)
        error_logger.addHandler(file_handler)
        access_logger.addHandler(file_handler)

def update_asyncio_log_level() -> None:
    """Set the log level of asyncio to the log level specified in the environment
    variable 'LOG_LEVEL_LIBRARIES'."""
    log_level = os.environ.get(ENV_LOG_LEVEL_LIBRARIES, DEFAULT_LOG_LEVEL_LIBRARIES)
    logging.getLogger("asyncio").setLevel(log_level)

def set_log_and_warnings_filters() -> None:
    """
    Set log filters on the root logger, and duplicate filters for warnings.

    Filters only propagate on handlers, not loggers.
    """
    for handler in logging.getLogger().handlers:
        handler.addFilter(RepeatedLogFilter())

    warnings.filterwarnings("once", category=UserWarning)

def obtain_verbosity() -> int:
    """Returns a verbosity level according to the set log level."""
    log_level = os.environ.get(ENV_LOG_LEVEL, DEFAULT_LOG_LEVEL)

    verbosity = 0
    if log_level == "DEBUG":
        verbosity = 2
    if log_level == "INFO":
        verbosity = 1

    return verbosity

def sort_list_of_dicts_by_first_key(dicts: List[Dict]) -> List[Dict]:
    """Sorts a list of dictionaries by their first key."""
    return sorted(dicts, key=lambda d: list(d.keys())[0])

def write_global_config_value(name: Text, value: Any) -> None:
    """Read global Rasa configuration."""

    # need to use `rasa.constants.GLOBAL_USER_CONFIG_PATH` to allow patching
    # in tests
    config_path = rasa.constants.GLOBAL_USER_CONFIG_PATH
    try:
        os.makedirs(os.path.dirname(config_path), exist_ok=True)

        c = read_global_config(config_path)
        c[name] = value
        rasa.core.utils.dump_obj_as_yaml_to_file(
            rasa.constants.GLOBAL_USER_CONFIG_PATH, c
        )
    except Exception as e:
        logger.warning(f"Failed to write global config. Error: {e}. Skipping.")

def read_global_config_value(name: Text, unavailable_ok: bool = True) -> Any:
    """Read a value from the global Rasa configuration."""

    def not_found():
        if unavailable_ok:
            return None
        else:
            raise ValueError(f"Configuration '{name}' key not found.")

    # need to use `rasa.constants.GLOBAL_USER_CONFIG_PATH` to allow patching
    # in tests
    config_path = rasa.constants.GLOBAL_USER_CONFIG_PATH

    if not os.path.exists(config_path):
        return not_found()

    c = read_global_config(config_path)

    if name in c:
        return c[name]
    else:
        return not_found()

def update_existing_keys(
    original: Dict[Any, Any], updates: Dict[Any, Any]
) -> Dict[Any, Any]:
    """Iterate through all the updates and update a value in the original dictionary.

    If the updates contain a key that is not present in the original dict, it will
    be ignored."""

    updated = original.copy()
    for k, v in updates.items():
        if k in updated:
            updated[k] = v
    return updated

class RepeatedLogFilter(logging.Filter):
    """Filter repeated log records."""

    last_log = None

    def filter(self, record):
        current_log = (
            record.levelno,
            record.pathname,
            record.lineno,
            record.msg,
            record.args,
        )
        if current_log != self.last_log:
            self.last_log = current_log
            return True
        return False

def run_in_loop(
    f: Coroutine[Any, Any, T], loop: Optional[asyncio.AbstractEventLoop] = None
) -> T:
    """Execute the awaitable in the passed loop.

    If no loop is passed, the currently existing one is used or a new one is created
    if no loop has been started in the current context.

    After the awaitable is finished, all remaining tasks on the loop will be
    awaited as well (background tasks).

    WARNING: don't use this if there are never ending background tasks scheduled.
        in this case, this function will never return.

    Args:
       f: function to execute
       loop: loop to use for the execution

    Returns:
        return value from the function
    """

    if loop is None:
        try:
            loop = asyncio.get_event_loop()
        except RuntimeError:
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)
    result = loop.run_until_complete(f)

    # Let's also finish all running tasks:
    pending = asyncio.Task.all_tasks()
    loop.run_until_complete(asyncio.gather(*pending))

    return result
<commit_msg>fixed global config writing (#6761)<commit_after>import asyncio
import logging
import os
import shutil
import warnings
from types import TracebackType
from typing import Any, Coroutine, Dict, List, Optional, Text, Type, TypeVar

import rasa.core.utils
import rasa.utils.io
from rasa.constants import (
    DEFAULT_LOG_LEVEL_LIBRARIES,
    ENV_LOG_LEVEL_LIBRARIES,
)
from rasa.shared.constants import DEFAULT_LOG_LEVEL, ENV_LOG_LEVEL
import rasa.shared.utils.io

logger = logging.getLogger(__name__)

T = TypeVar("T")

class TempDirectoryPath(str):
    """Represents a path to an temporary directory. When used as a context
    manager, it erases the contents of the directory on exit.

    """

    def __enter__(self) -> "TempDirectoryPath":
        return self

    def __exit__(
        self,
        _exc: Optional[Type[BaseException]],
        _value: Optional[Exception],
        _tb: Optional[TracebackType],
    ) -> bool:
        if os.path.exists(self):
            shutil.rmtree(self)

def read_global_config(path: Text) -> Dict[Text, Any]:
    """Read global Rasa configuration.

    Args:
        path: Path to the configuration
    Returns:
        The global configuration
    """
    # noinspection PyBroadException
    try:
        return rasa.shared.utils.io.read_config_file(path)
    except Exception:
        # if things go south we pretend there is no config
        return {}

def set_log_level(log_level: Optional[int] = None):
    """Set log level of Rasa and Tensorflow either to the provided log level or
    to the log level specified in the environment variable 'LOG_LEVEL'. If none is set
    a default log level will be used."""

    if not log_level:
        log_level = os.environ.get(ENV_LOG_LEVEL, DEFAULT_LOG_LEVEL)
        log_level = logging.getLevelName(log_level)

    logging.getLogger("rasa").setLevel(log_level)

    update_tensorflow_log_level()
    update_asyncio_log_level()
    update_apscheduler_log_level()
    update_socketio_log_level()

    os.environ[ENV_LOG_LEVEL] = logging.getLevelName(log_level)

def update_apscheduler_log_level() -> None:
    log_level = os.environ.get(ENV_LOG_LEVEL_LIBRARIES, DEFAULT_LOG_LEVEL_LIBRARIES)

    apscheduler_loggers = [
        "apscheduler",
        "apscheduler.scheduler",
        "apscheduler.executors",
        "apscheduler.executors.default",
    ]

    for logger_name in apscheduler_loggers:
        logging.getLogger(logger_name).setLevel(log_level)
        logging.getLogger(logger_name).propagate = False

def update_socketio_log_level() -> None:
    log_level = os.environ.get(ENV_LOG_LEVEL_LIBRARIES, DEFAULT_LOG_LEVEL_LIBRARIES)

    socketio_loggers = ["websockets.protocol", "engineio.server", "socketio.server"]

    for logger_name in socketio_loggers:
        logging.getLogger(logger_name).setLevel(log_level)
        logging.getLogger(logger_name).propagate = False

def update_tensorflow_log_level() -> None:
    """Set the log level of Tensorflow to the log level specified in the environment
    variable 'LOG_LEVEL_LIBRARIES'."""

    # Disables libvinfer, tensorRT, cuda, AVX2 and FMA warnings (CPU support). This variable needs to be set before the
    # first import since some warnings are raised on the first import.
    os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"

    import tensorflow as tf

    log_level = os.environ.get(ENV_LOG_LEVEL_LIBRARIES, DEFAULT_LOG_LEVEL_LIBRARIES)

    if log_level == "DEBUG":
        tf_log_level = tf.compat.v1.logging.DEBUG
    elif log_level == "INFO":
        tf_log_level = tf.compat.v1.logging.INFO
    elif log_level == "WARNING":
        tf_log_level = tf.compat.v1.logging.WARN
    else:
        tf_log_level = tf.compat.v1.logging.ERROR

    tf.compat.v1.logging.set_verbosity(tf_log_level)
    logging.getLogger("tensorflow").propagate = False

def update_sanic_log_level(log_file: Optional[Text] = None):
    """Set the log level of sanic loggers to the log level specified in the environment
    variable 'LOG_LEVEL_LIBRARIES'."""
    from sanic.log import logger, error_logger, access_logger

    log_level = os.environ.get(ENV_LOG_LEVEL_LIBRARIES, DEFAULT_LOG_LEVEL_LIBRARIES)

    logger.setLevel(log_level)
    error_logger.setLevel(log_level)
    access_logger.setLevel(log_level)

    logger.propagate = False
    error_logger.propagate = False
    access_logger.propagate = False

    if log_file is not None:
        formatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s]  %(message)s")
        file_handler = logging.FileHandler(log_file)
        file_handler.setFormatter(formatter)

        logger.addHandler(file_handler)
        error_logger.addHandler(file_handler)
        access_logger.addHandler(file_handler)

def update_asyncio_log_level() -> None:
    """Set the log level of asyncio to the log level specified in the environment
    variable 'LOG_LEVEL_LIBRARIES'."""
    log_level = os.environ.get(ENV_LOG_LEVEL_LIBRARIES, DEFAULT_LOG_LEVEL_LIBRARIES)
    logging.getLogger("asyncio").setLevel(log_level)

def set_log_and_warnings_filters() -> None:
    """
    Set log filters on the root logger, and duplicate filters for warnings.

    Filters only propagate on handlers, not loggers.
    """
    for handler in logging.getLogger().handlers:
        handler.addFilter(RepeatedLogFilter())

    warnings.filterwarnings("once", category=UserWarning)

def obtain_verbosity() -> int:
    """Returns a verbosity level according to the set log level."""
    log_level = os.environ.get(ENV_LOG_LEVEL, DEFAULT_LOG_LEVEL)

    verbosity = 0
    if log_level == "DEBUG":
        verbosity = 2
    if log_level == "INFO":
        verbosity = 1

    return verbosity

def sort_list_of_dicts_by_first_key(dicts: List[Dict]) -> List[Dict]:
    """Sorts a list of dictionaries by their first key."""
    return sorted(dicts, key=lambda d: list(d.keys())[0])

def write_global_config_value(name: Text, value: Any) -> None:
    """Read global Rasa configuration."""

    # need to use `rasa.constants.GLOBAL_USER_CONFIG_PATH` to allow patching
    # in tests
    config_path = rasa.constants.GLOBAL_USER_CONFIG_PATH
    try:
        os.makedirs(os.path.dirname(config_path), exist_ok=True)

        c = read_global_config(config_path)
        c[name] = value
        rasa.shared.utils.io.write_yaml(c, rasa.constants.GLOBAL_USER_CONFIG_PATH)
    except Exception as e:
        logger.warning(f"Failed to write global config. Error: {e}. Skipping.")

def read_global_config_value(name: Text, unavailable_ok: bool = True) -> Any:
    """Read a value from the global Rasa configuration."""

    def not_found():
        if unavailable_ok:
            return None
        else:
            raise ValueError(f"Configuration '{name}' key not found.")

    # need to use `rasa.constants.GLOBAL_USER_CONFIG_PATH` to allow patching
    # in tests
    config_path = rasa.constants.GLOBAL_USER_CONFIG_PATH

    if not os.path.exists(config_path):
        return not_found()

    c = read_global_config(config_path)

    if name in c:
        return c[name]
    else:
        return not_found()

def update_existing_keys(
    original: Dict[Any, Any], updates: Dict[Any, Any]
) -> Dict[Any, Any]:
    """Iterate through all the updates and update a value in the original dictionary.

    If the updates contain a key that is not present in the original dict, it will
    be ignored."""

    updated = original.copy()
    for k, v in updates.items():
        if k in updated:
            updated[k] = v
    return updated

class RepeatedLogFilter(logging.Filter):
    """Filter repeated log records."""

    last_log = None

    def filter(self, record):
        current_log = (
            record.levelno,
            record.pathname,
            record.lineno,
            record.msg,
            record.args,
        )
        if current_log != self.last_log:
            self.last_log = current_log
            return True
        return False

def run_in_loop(
    f: Coroutine[Any, Any, T], loop: Optional[asyncio.AbstractEventLoop] = None
) -> T:
    """Execute the awaitable in the passed loop.

    If no loop is passed, the currently existing one is used or a new one is created
    if no loop has been started in the current context.

    After the awaitable is finished, all remaining tasks on the loop will be
    awaited as well (background tasks).

    WARNING: don't use this if there are never ending background tasks scheduled.
        in this case, this function will never return.

    Args:
       f: function to execute
       loop: loop to use for the execution

    Returns:
        return value from the function
    """

    if loop is None:
        try:
            loop = asyncio.get_event_loop()
        except RuntimeError:
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)
    result = loop.run_until_complete(f)

    # Let's also finish all running tasks:
    pending = asyncio.Task.all_tasks()
    loop.run_until_complete(asyncio.gather(*pending))

    return result
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

# rss is the interface to rss parsing,
# processing and storing.
#
# @author Jin Yuan
# @contact jinyuan@baidu.com
# @created Jul. 20, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

from administration import database as db_feeds
import calendar
import database as db_news
from datetime import datetime, timedelta
from data_processor import transcoder
from data_processor import tts_provider
import Image
from image_processor import image_helper
from image_processor import thumbnail
import memory
import random
import rss_parser
from cStringIO import StringIO
import time
import urllib2

from administration.config import CATEGORY_IMAGE_SIZE
from administration.config import DATABASE_REMOVAL_DAYS
from administration.config import HOT_IMAGE_SIZE
from administration.config import LANGUAGES
from administration.config import MEMORY_EXPIRATION_DAYS
from administration.config import MIN_IMAGE_SIZE
from administration.config import THUMBNAIL_IMAGE_SIZE

def _value_added_process(entries=None, language=None, transcoder_type='chengdujin'):
    """
    add more value to an entry
    tts, transcode, images, redis_entry_expiration, database_entry_expiration
    """
    if not entries:
        return None
    if not language or language not in LANGUAGES:
        raise Exception("ERROR: language not found or not supported!")

    entries_new = []
    for entry in entries:
        # get a random int from 100 million possibilities
        try:
            # [MUST-HAVE] transcoding
            rand = random.randint(0, 100000000)
            transcoded_relative_path = '%s_%s_%s_%i' % (entry['language'], entry['feed_id'], entry['updated'], rand)
            # high chances transcoder cannot work properly
            entry['transcoded'], entry['transcoded_local'], images_from_transcoded = transcoder.convert(entry['language'], entry['title'], entry['link'], transcoder_type, transcoded_relative_path)

            # [MUST-HAVE] summary

            # process images found in the transcoded data
            if images_from_transcoded:
                # images from transcoded are already normalized
                entry['images'].extend(images_from_transcoded)
                # remove duplicated images
                entry['images'] = image_helper.dedupe_images(entry['images']) if entry.has_key('images') and entry['images'] else None
            # make images none if nothing's there, instead of a []
            entry['images'] = entry['images'] if entry.has_key('images') and entry['images'] else None

            # [OPTIONAL] generate 3 types of images: thumbnail, category image and hot news image
            if entry.has_key('images') and entry['images']:
                biggest = image_helper.find_biggest_image(entry['images'])
                if biggest:
                    try:
                        rand = random.randint(0, 100000000)
                        image_relative_path = '%s_%s_%s_%i' % (entry['language'], entry['feed_id'], entry['updated'], rand)
                        image_downloaded = StringIO(urllib2.urlopen(biggest['url']).read())
                        image_data = Image.open(image_downloaded)
                        image_data_category = image_data
                        image_data_thumbnail = image_data

                        # hot news image
                        hot_web, hot_local = image_helper.scale_image(image=biggest, image_data=image_data, size_expected=HOT_IMAGE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_hotnews' % image_relative_path)
                        entry['hot_news_image'] = hot_web if hot_web else None
                        entry['hot_news_image_local'] = hot_local if hot_local else None
                        # category image

                        category_web, category_local = image_helper.scale_image(image=biggest, image_data=image_data_category, size_expected=CATEGORY_IMAGE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_category' % image_relative_path)
                        entry['category_image'] = category_web if category_web else None
                        entry['category_image_local'] = category_local if category_local else None
                        # new item thumbnail image
                        thumbnail_web, thumbnail_local = image_helper.scale_image(image=biggest, image_data=image_data_thumbnail, size_expected=THUMBNAIL_IMAGE_SIZE, resize_by_width=True, crop_by_center=True, relative_path='%s_thumbnail' % image_relative_path)
                        entry['thumbnail_image'] = thumbnail_web if thumbnail_web else None
                        entry['thumbnail_image_local'] = thumbnail_local if thumbnail_local else None
                        # for older version users
                        entry['image'] = entry['thumbnail_image']['url'] if thumbnail_web else None
                    except IOError as k:
                        entry['error'].append(str(k) + '\n')

            # [OPTIONAL] google tts not for indonesian
            if entry['language'] != 'ind':
                try:
                    tts_relative_path = '%s_%s_%s_%i.mp3' % (
                        entry['language'], entry['feed_id'], entry['updated'], rand)
                    entry['mp3'], entry['mp3_local'] = tts_provider.google(
                        entry['language'], entry['title'], tts_relative_path)
                except Exception as k:
                    print k, '... cannot generate TTS for %s' % entry['link']
                    entry['error'].append(k + '\n')
                    entry['mp3'] = None
                    entry['mp3_local'] = None

            # [MUST-HAVE] add expiration data
            def _expired(updated, days_to_deadline):
                """
                compute expiration information
                return time string and unix time
                """
                deadline = datetime.utcfromtimestamp(
                    updated) + timedelta(days=days_to_deadline)
                return time.asctime(time.gmtime(calendar.timegm(deadline.timetuple())))

            entry['memory_expired'] = _expired(
                entry['updated'], MEMORY_EXPIRATION_DAYS)
            entry['database_expired'] = _expired(
                entry['updated'], DATABASE_REMOVAL_DAYS)

            entry['error'] = entry['error'] if entry['error'] else None
            entries_new.append(entry)
            print
            print
        except Exception as k:
            print k
    return entries_new

# TODO: code to remove added items if things suck at database/memory
def update(feed_link=None, feed_id=None, language=None, categories=None, transcoder_type='chengdujin'):
    """
    update could be called
    1. from task procedure: feed_id
    2. after an rss is added: feed_id
    3. manually for testing purpose: feed_link, language
    Note. categories are ids of category item
    Notel categories are kept for manual testing
    """
    if not feed_id and not (feed_link and language) :
        raise Exception(
            "ERROR: Method signature not well formed!")

    if feed_id:
        feed = db_feeds.get(feed_id=feed_id)
    else:
        feed = db_feeds.get(feed_link=feed_link, language=language)

    feed_id = str(feed['_id']) if feed else feed_id
    feed_link = feed['feed_link'] if feed else feed_link
    language = feed['language'] if feed else language
    categories = feed['categories'] if feed else categories
    feed_title = feed['feed_title'] if feed and 'feed_title' in feed else None
    transcoder_type = feed['transcoder'] if feed else transcoder_type
    etag = feed['etag'] if feed and 'etag' in feed else None
    modified = feed['modified'] if feed and 'modified' in feed else None

    # parse rss reading from remote rss servers
    entries, status_new, feed_title_new, etag_new, modified_new = rss_parser.parse(
        feed_link, feed_id, feed_title, language, categories, etag, modified)

    # filter out existing entries in db_news
    # there are some possible exceptions -- yet let it be
    entries = db_news.dedup(entries, language)
    # and do tts, big_images, image as well as transcode.
    entries = _value_added_process(entries, language, transcoder_type)

    # update new entries to db_news
    # each entry is added with _id
    entries = db_news.update(entries, language)
    # and some data, like feed_title, etag and modified to db_feeds
    # only feed_id is necessary, others are optional **kwargs
    db_feeds.update(feed_id=feed_id, status=status_new, feed_title=feed_title_new, etag=etag_new, modified=modified_new)

    # store in memory
    memory.update(entries, language, categories)
    return entries
<commit_msg>paused tts for testing<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

# rss is the interface to rss parsing,
# processing and storing.
#
# @author Jin Yuan
# @contact jinyuan@baidu.com
# @created Jul. 20, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

from administration import database as db_feeds
import calendar
import database as db_news
from datetime import datetime, timedelta
from data_processor import transcoder
from data_processor import tts_provider
import Image
from image_processor import image_helper
from image_processor import thumbnail
import memory
import random
import rss_parser
from cStringIO import StringIO
import time
import urllib2

from administration.config import CATEGORY_IMAGE_SIZE
from administration.config import DATABASE_REMOVAL_DAYS
from administration.config import HOT_IMAGE_SIZE
from administration.config import LANGUAGES
from administration.config import MEMORY_EXPIRATION_DAYS
from administration.config import MIN_IMAGE_SIZE
from administration.config import THUMBNAIL_IMAGE_SIZE

def _value_added_process(entries=None, language=None, transcoder_type='chengdujin'):
    """
    add more value to an entry
    tts, transcode, images, redis_entry_expiration, database_entry_expiration
    """
    if not entries:
        return None
    if not language or language not in LANGUAGES:
        raise Exception("ERROR: language not found or not supported!")

    entries_new = []
    for entry in entries:
        # get a random int from 100 million possibilities
        try:
            # [MUST-HAVE] transcoding
            rand = random.randint(0, 100000000)
            transcoded_relative_path = '%s_%s_%s_%i' % (entry['language'], entry['feed_id'], entry['updated'], rand)
            # high chances transcoder cannot work properly
            entry['transcoded'], entry['transcoded_local'], images_from_transcoded = transcoder.convert(entry['language'], entry['title'], entry['link'], transcoder_type, transcoded_relative_path)

            # [MUST-HAVE] summary

            # process images found in the transcoded data
            if images_from_transcoded:
                # images from transcoded are already normalized
                entry['images'].extend(images_from_transcoded)
                # remove duplicated images
                entry['images'] = image_helper.dedupe_images(entry['images']) if entry.has_key('images') and entry['images'] else None
            # make images none if nothing's there, instead of a []
            entry['images'] = entry['images'] if entry.has_key('images') and entry['images'] else None

            # [OPTIONAL] generate 3 types of images: thumbnail, category image and hot news image
            if entry.has_key('images') and entry['images']:
                biggest = image_helper.find_biggest_image(entry['images'])
                if biggest:
                    try:
                        rand = random.randint(0, 100000000)
                        image_relative_path = '%s_%s_%s_%i' % (entry['language'], entry['feed_id'], entry['updated'], rand)
                        image_downloaded = StringIO(urllib2.urlopen(biggest['url']).read())
                        image_data = Image.open(image_downloaded)
                        image_data_category = image_data
                        image_data_thumbnail = image_data

                        # hot news image
                        hot_web, hot_local = image_helper.scale_image(image=biggest, image_data=image_data, size_expected=HOT_IMAGE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_hotnews' % image_relative_path)
                        entry['hot_news_image'] = hot_web if hot_web else None
                        entry['hot_news_image_local'] = hot_local if hot_local else None
                        # category image

                        category_web, category_local = image_helper.scale_image(image=biggest, image_data=image_data_category, size_expected=CATEGORY_IMAGE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_category' % image_relative_path)
                        entry['category_image'] = category_web if category_web else None
                        entry['category_image_local'] = category_local if category_local else None
                        # new item thumbnail image
                        thumbnail_web, thumbnail_local = image_helper.scale_image(image=biggest, image_data=image_data_thumbnail, size_expected=THUMBNAIL_IMAGE_SIZE, resize_by_width=True, crop_by_center=True, relative_path='%s_thumbnail' % image_relative_path)
                        entry['thumbnail_image'] = thumbnail_web if thumbnail_web else None
                        entry['thumbnail_image_local'] = thumbnail_local if thumbnail_local else None
                        # for older version users
                        entry['image'] = entry['thumbnail_image']['url'] if thumbnail_web else None
                    except IOError as k:
                        entry['error'].append(str(k) + '\n')

            # [OPTIONAL] google tts not for indonesian
            """
            if entry['language'] != 'ind':
                try:
                    tts_relative_path = '%s_%s_%s_%i.mp3' % (
                        entry['language'], entry['feed_id'], entry['updated'], rand)
                    entry['mp3'], entry['mp3_local'] = tts_provider.google(
                        entry['language'], entry['title'], tts_relative_path)
                except Exception as k:
                    print k, '... cannot generate TTS for %s' % entry['link']
                    entry['error'].append(k + '\n')
                    entry['mp3'] = None
                    entry['mp3_local'] = None
            """

            # [MUST-HAVE] add expiration data
            def _expired(updated, days_to_deadline):
                """
                compute expiration information
                return time string and unix time
                """
                deadline = datetime.utcfromtimestamp(
                    updated) + timedelta(days=days_to_deadline)
                return time.asctime(time.gmtime(calendar.timegm(deadline.timetuple())))

            entry['memory_expired'] = _expired(
                entry['updated'], MEMORY_EXPIRATION_DAYS)
            entry['database_expired'] = _expired(
                entry['updated'], DATABASE_REMOVAL_DAYS)

            entry['error'] = entry['error'] if entry['error'] else None
            entries_new.append(entry)
            print
            print
        except Exception as k:
            print k
    return entries_new

# TODO: code to remove added items if things suck at database/memory
def update(feed_link=None, feed_id=None, language=None, categories=None, transcoder_type='chengdujin'):
    """
    update could be called
    1. from task procedure: feed_id
    2. after an rss is added: feed_id
    3. manually for testing purpose: feed_link, language
    Note. categories are ids of category item
    Notel categories are kept for manual testing
    """
    if not feed_id and not (feed_link and language) :
        raise Exception(
            "ERROR: Method signature not well formed!")

    if feed_id:
        feed = db_feeds.get(feed_id=feed_id)
    else:
        feed = db_feeds.get(feed_link=feed_link, language=language)

    feed_id = str(feed['_id']) if feed else feed_id
    feed_link = feed['feed_link'] if feed else feed_link
    language = feed['language'] if feed else language
    categories = feed['categories'] if feed else categories
    feed_title = feed['feed_title'] if feed and 'feed_title' in feed else None
    transcoder_type = feed['transcoder'] if feed else transcoder_type
    etag = feed['etag'] if feed and 'etag' in feed else None
    modified = feed['modified'] if feed and 'modified' in feed else None

    # parse rss reading from remote rss servers
    entries, status_new, feed_title_new, etag_new, modified_new = rss_parser.parse(
        feed_link, feed_id, feed_title, language, categories, etag, modified)

    # filter out existing entries in db_news
    # there are some possible exceptions -- yet let it be
    entries = db_news.dedup(entries, language)
    # and do tts, big_images, image as well as transcode.
    entries = _value_added_process(entries, language, transcoder_type)

    # update new entries to db_news
    # each entry is added with _id
    entries = db_news.update(entries, language)
    # and some data, like feed_title, etag and modified to db_feeds
    # only feed_id is necessary, others are optional **kwargs
    db_feeds.update(feed_id=feed_id, status=status_new, feed_title=feed_title_new, etag=etag_new, modified=modified_new)

    # store in memory
    memory.update(entries, language, categories)
    return entries
<|endoftext|>