<commit_before>import numpy as np
import sympy as sp
import multiprocessing
from means.simulation.trajectory import Trajectory
from means.io.serialise import SerialisableObject
from means.util.sympyhelpers import substitute_all
from means.model import Model

class StochasticProblem(Model):
    """
    The formulation of a model for stochastic simulations such as GSSA.
    """
    def __init__(self, model):
        super(StochasticProblem, self).__init__(model.constants, model.species,
                                                model.propensities, model.stoichiometry_matrix)
        self.__change = np.array(model.stoichiometry_matrix.T).astype("int")
    @property
    def change(self):
        return self.__change

class SSASimulation(SerialisableObject):
    """
        A class providing an implementation of the exact Gillespie Stochastic Simulation Algorithm [Gillespie77].

            >>> PROBLEM = StochasticProblem(MODEL_P53)
            >>> RATES = [90, 0.002, 1.7, 1.1, 0.93, 0.96, 0.01]
            >>> INITIAL_CONDITIONS = [70, 30, 60]
            >>> TIME_RANGE = np.arange(0, 40, .1)
            >>> N_SSA = 10
            >>> ssas = SSASimulation(PROBLEM)
            >>> mean_trajectories = ssas.simulate_system(RATES, INITIAL_CONDITIONS, TIME_RANGE, N_SSA)
            >>> print  mean_trajectories

    .. [Gillespie77]Gillespie, Daniel T. "Exact stochastic simulation of coupled chemical reactions."\
         The journal of physical chemistry 81.25 (1977): 2340-2361.
    """
    def __init__(self, stochastic_problem, random_seed=None):
        self.__random_seed = random_seed
        self.__problem = stochastic_problem

    def _validate_parameters(self, parameters, initial_conditions):

        if len(self.__problem.species) != len(initial_conditions):
            exception_str = "The number of initial conditions and the number of species are different. ({0} != {1})"
            raise Exception(exception_str.format(len(self.__problem.species), len(initial_conditions)))

        elif len(self.__problem.constants) != len(parameters):
            exception_str = "The number of parameters and the number of constants are different. ({0} != {1})"
            raise Exception(exception_str.format(len(self.__problem.constants), len(parameters)))

    def simulate_system(self, parameters, initial_conditions, timepoints, n_simulations, number_of_processes=1,
                        return_average=True):
        """
        Perform a given number of Gillespie SSA simulations and returns trajectories for of each species.
        Each trajectory is interpolated at the given time points.
        By default, the average amounts of species for all simulations is returned.

        :param parameters: list of the initial values for the constants in the model.
                                  Must be in the same order as in the model
        :param initial_conditions: List of the initial values for the equations in the problem.
                        Must be in the same order as these equations occur.

        :param timepoints: A list of time points to simulate the system for

        :param number_of_processes: the number of parallel process to be run
        :param return_average: whether the average of all simulations should be returned

        :return: a list of :class:`~means.simulation.simulate.Trajectory` one per species in the problem,
            or a list of lists of trajectories (one per simulation) if `return_average == False`.
        :rtype: list[:class:`~means.simulation.simulate.Trajectory`]
        """
        self._validate_parameters(parameters, initial_conditions)
        t_max= max(timepoints)

        substitution_pairs = dict(zip(self.__problem.constants, parameters))
        propensities = substitute_all(self.__problem.propensities, substitution_pairs)
        # lambdify the propensities for fast evaluation
        population_rates_as_function = sp.lambdify(tuple(self.__problem.species),
                                                propensities, modules="numpy")

        if not self.__random_seed:
            seed_for_processes = [None] * n_simulations
        else:
            seed_for_processes = [i for i in range(self.__random_seed, n_simulations + self.__random_seed)]

        if number_of_processes ==1:
            ssa_generator = _SSAGenerator(population_rates_as_function,
                                        self.__problem.__change, initial_conditions, t_max, seed=self.__random_seed)

            results = map(ssa_generator.generate_single_simulation, seed_for_processes)

        else:
            p = multiprocessing.Pool(number_of_processes,
                    initializer=multiprocessing_pool_initialiser,
                    initargs=[population_rates_as_function, self.__problem.change,
                              initial_conditions, t_max, self.__random_seed])

            results = p.map(multiprocessing_apply_ssa, seed_for_processes)
            p.close()

        resampled_results = [[traj.resample(timepoints) for traj in res] for res in results]
        if not return_average:
            return resampled_results

        mean_trajectories = [sum(trajs)/float(len(trajs)) for trajs in zip(*resampled_results)]
        return mean_trajectories

def multiprocessing_pool_initialiser(population_rates_as_function, change, initial_conditions, t_max, seed):
    global ssa_generator
    current = multiprocessing.current_process()
    #increment the random seed inside each process at creation, so the result should be reproducible
    if seed:
        seed += current._identity[0]
    else:
        seed = current._identity[0]
    ssa_generator = _SSAGenerator(population_rates_as_function, change,initial_conditions, t_max, seed)

def multiprocessing_apply_ssa(x):
    """
    Used in the SSASimulation class.
    Needs to be in global scope for multiprocessing module to pick it up
    """
    result = ssa_generator.generate_single_simulation(x)
    return result

class _SSAGenerator(object):
    def __init__(self, population_rates_as_function, change, initial_conditions, t_max, seed):
        """
        :param population_rates_as_function: function to evaluate propensities given the amount of species
        :param change: the change matrix (transpose of the stoichiometry matrix) as an numpy in array
        :param initial_conditions: the initial conditions of the system
        :param t_max: the time when the simulation should stop
        :param seed: an integer to initialise the random seed. If `None`, the random seed will be set
                automatically (e.g. from /dev/random) once for all.
        """
        self.__rng = np.random.RandomState(seed)
        self.__population_rates_as_function = population_rates_as_function
        self.__change = change
        self.__initial_conditions = initial_conditions
        self.__t_max = t_max

    def _gssa(self, initial_conditions, t_max):
        """
        This function is inspired from Yoav Ram's code available at:
        http://nbviewer.ipython.org/github/yoavram/ipython-notebooks/blob/master/GSSA.ipynb

        :param initial_conditions: the initial conditions of the system
        :param t_max:  the time when the simulation should stop
        :return:
        """
        # set the initial conditions and t0 = 0.
        species_over_time = [np.array(initial_conditions).astype("int16")]
        t = 0
        time_points = [t]
        while t < t_max and species_over_time[-1].sum() > 0:
            last = species_over_time[-1]
            e, dt = self._draw(last)
            t += dt
            species_over_time.append(last + self.__change[e,:])
            time_points.append(t)
        return time_points, np.array(species_over_time).T

    def _draw(self, population):
        population_rates = np.array(self.__population_rates_as_function(*population))
        sum_rates = population_rates.sum()
        time = self.__rng.exponential(1.0/sum_rates)
        event_distribution = population_rates.flatten()
        event_distribution /= event_distribution.sum()
        event = self.__rng.multinomial(1, event_distribution).argmax()
        return event, time

    def generate_single_simulation(self, x):
        """
        Generate a single SSA simulation
        :param x: an integer to reset the random seed. If None, the initial random number generator is used
        :return: a list of :class:`~means.simulation.simulate.Trajectory` one per species in the problem
        :rtype: list[:class:`~means.simulation.simulate.Trajectory`]
        """
        #reset random seed
        if x:
            self.__rng = np.random.RandomState(x)

        # perform one stochastic simulation
        time_points, species_over_time = self._gssa(self.__initial_conditions, self.__t_max)

        n_species = len([_ for _ in species_over_time])
        # build trajectories
        trajectories = [Trajectory(time_points, spot, desc) for
                        spot, desc in zip(species_over_time, range(n_species))]

        return trajectories

<commit_msg>add propper description to trajectories<commit_after>import numpy as np
import sympy as sp
import multiprocessing
from means.simulation.trajectory import Trajectory
from means.io.serialise import SerialisableObject
from means.util.sympyhelpers import substitute_all
from means.approximation.ode_problem import Moment
from means.model import Model

class StochasticProblem(Model):
    """
    The formulation of a model for stochastic simulations such as GSSA.
    """
    def __init__(self, model):
        super(StochasticProblem, self).__init__(model.constants, model.species,
                                                model.propensities, model.stoichiometry_matrix)
        self.__change = np.array(model.stoichiometry_matrix.T).astype("int")
    @property
    def change(self):
        return self.__change

class SSASimulation(SerialisableObject):
    """
        A class providing an implementation of the exact Gillespie Stochastic Simulation Algorithm [Gillespie77].

            >>> PROBLEM = StochasticProblem(MODEL_P53)
            >>> RATES = [90, 0.002, 1.7, 1.1, 0.93, 0.96, 0.01]
            >>> INITIAL_CONDITIONS = [70, 30, 60]
            >>> TIME_RANGE = np.arange(0, 40, .1)
            >>> N_SSA = 10
            >>> ssas = SSASimulation(PROBLEM)
            >>> mean_trajectories = ssas.simulate_system(RATES, INITIAL_CONDITIONS, TIME_RANGE, N_SSA)
            >>> print  mean_trajectories

    .. [Gillespie77]Gillespie, Daniel T. "Exact stochastic simulation of coupled chemical reactions."\
         The journal of physical chemistry 81.25 (1977): 2340-2361.
    """
    def __init__(self, stochastic_problem, random_seed=None):
        self.__random_seed = random_seed
        self.__problem = stochastic_problem

    def _validate_parameters(self, parameters, initial_conditions):

        if len(self.__problem.species) != len(initial_conditions):
            exception_str = "The number of initial conditions and the number of species are different. ({0} != {1})"
            raise Exception(exception_str.format(len(self.__problem.species), len(initial_conditions)))

        elif len(self.__problem.constants) != len(parameters):
            exception_str = "The number of parameters and the number of constants are different. ({0} != {1})"
            raise Exception(exception_str.format(len(self.__problem.constants), len(parameters)))

    def simulate_system(self, parameters, initial_conditions, timepoints, n_simulations, number_of_processes=1,
                        return_average=True):
        """
        Perform a given number of Gillespie SSA simulations and returns trajectories for of each species.
        Each trajectory is interpolated at the given time points.
        By default, the average amounts of species for all simulations is returned.

        :param parameters: list of the initial values for the constants in the model.
                                  Must be in the same order as in the model
        :param initial_conditions: List of the initial values for the equations in the problem.
                        Must be in the same order as these equations occur.

        :param timepoints: A list of time points to simulate the system for

        :param number_of_processes: the number of parallel process to be run
        :param return_average: whether the average of all simulations should be returned

        :return: a list of :class:`~means.simulation.simulate.Trajectory` one per species in the problem,
            or a list of lists of trajectories (one per simulation) if `return_average == False`.
        :rtype: list[:class:`~means.simulation.simulate.Trajectory`]
        """
        self._validate_parameters(parameters, initial_conditions)
        t_max= max(timepoints)

        substitution_pairs = dict(zip(self.__problem.constants, parameters))
        propensities = substitute_all(self.__problem.propensities, substitution_pairs)
        # lambdify the propensities for fast evaluation
        population_rates_as_function = sp.lambdify(tuple(self.__problem.species),
                                                propensities, modules="numpy")

        if not self.__random_seed:
            seed_for_processes = [None] * n_simulations
        else:
            seed_for_processes = [i for i in range(self.__random_seed, n_simulations + self.__random_seed)]

        if number_of_processes ==1:
            ssa_generator = _SSAGenerator(population_rates_as_function,
                                        self.__problem.change, self.__problem.species,
                                        initial_conditions, t_max, seed=self.__random_seed)

            results = map(ssa_generator.generate_single_simulation, seed_for_processes)

        else:
            p = multiprocessing.Pool(number_of_processes,
                    initializer=multiprocessing_pool_initialiser,
                    initargs=[population_rates_as_function, self.__problem.change,
                              self.__problem.species,
                              initial_conditions, t_max, self.__random_seed])

            results = p.map(multiprocessing_apply_ssa, seed_for_processes)
            p.close()

        resampled_results = [[traj.resample(timepoints) for traj in res] for res in results]
        if not return_average:
            return resampled_results

        mean_trajectories = [sum(trajs)/float(len(trajs)) for trajs in zip(*resampled_results)]
        return mean_trajectories

def multiprocessing_pool_initialiser(population_rates_as_function, change, species,
                                     initial_conditions, t_max, seed):
    global ssa_generator
    current = multiprocessing.current_process()
    #increment the random seed inside each process at creation, so the result should be reproducible
    if seed:
        seed += current._identity[0]
    else:
        seed = current._identity[0]
    ssa_generator = _SSAGenerator(population_rates_as_function, change, species, initial_conditions, t_max, seed)

def multiprocessing_apply_ssa(x):
    """
    Used in the SSASimulation class.
    Needs to be in global scope for multiprocessing module to pick it up
    """
    result = ssa_generator.generate_single_simulation(x)
    return result

class _SSAGenerator(object):
    def __init__(self, population_rates_as_function, change, species, initial_conditions, t_max, seed):
        """
        :param population_rates_as_function: function to evaluate propensities given the amount of species
        :param change: the change matrix (transpose of the stoichiometry matrix) as an numpy in array
        :param initial_conditions: the initial conditions of the system
        :param t_max: the time when the simulation should stop
        :param seed: an integer to initialise the random seed. If `None`, the random seed will be set
                automatically (e.g. from /dev/random) once for all.
        """
        self.__rng = np.random.RandomState(seed)
        self.__population_rates_as_function = population_rates_as_function
        self.__change = change
        self.__initial_conditions = initial_conditions
        self.__t_max = t_max
        self.__species = species

    def _gssa(self, initial_conditions, t_max):
        """
        This function is inspired from Yoav Ram's code available at:
        http://nbviewer.ipython.org/github/yoavram/ipython-notebooks/blob/master/GSSA.ipynb

        :param initial_conditions: the initial conditions of the system
        :param t_max:  the time when the simulation should stop
        :return:
        """
        # set the initial conditions and t0 = 0.
        species_over_time = [np.array(initial_conditions).astype("int16")]
        t = 0
        time_points = [t]
        while t < t_max and species_over_time[-1].sum() > 0:
            last = species_over_time[-1]
            e, dt = self._draw(last)
            t += dt
            species_over_time.append(last + self.__change[e,:])
            time_points.append(t)
        return time_points, np.array(species_over_time).T

    def _draw(self, population):
        population_rates = np.array(self.__population_rates_as_function(*population))
        sum_rates = population_rates.sum()
        time = self.__rng.exponential(1.0/sum_rates)
        event_distribution = population_rates.flatten()
        event_distribution /= event_distribution.sum()
        event = self.__rng.multinomial(1, event_distribution).argmax()
        return event, time

    def generate_single_simulation(self, x):
        """
        Generate a single SSA simulation
        :param x: an integer to reset the random seed. If None, the initial random number generator is used
        :return: a list of :class:`~means.simulation.simulate.Trajectory` one per species in the problem
        :rtype: list[:class:`~means.simulation.simulate.Trajectory`]
        """
        #reset random seed
        if x:
            self.__rng = np.random.RandomState(x)

        # perform one stochastic simulation
        time_points, species_over_time = self._gssa(self.__initial_conditions, self.__t_max)

        # build descriptors for first order raw moments aka expectations (e.g. [1, 0, 0], [0, 1, 0] and [0, 0, 1])
        descriptors = []
        for i, s in enumerate(self.__species):
            row = [0] * len(self.__species)
            row[i] = 1
            descriptors.append(Moment(row, s))

        # build trajectories
        trajectories = [Trajectory(time_points, spot, desc) for
                        spot, desc in zip(species_over_time, descriptors)]

        return trajectories

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import os
import subprocess

from click.testing import CliRunner

from great_expectations.cli import cli
from tests.cli.utils import assert_no_logging_messages_or_tracebacks

def test_tap_help_output(caplog,):
    runner = CliRunner(mix_stderr=False)
    result = runner.invoke(cli, ["tap"], catch_exceptions=False)
    assert result.exit_code == 0
    assert (
        """Commands:
  new  BETA! Create a new tap file for easy deployments"""
        in result.stdout
    )
    assert_no_logging_messages_or_tracebacks(caplog, result)

def test_tap_new_with_filename_not_ending_in_py_raises_helpful_error(
    caplog, empty_data_context
):
    """
    We call the "tap new" command with a bogus filename

    The command should:
    - exit with a clear error message
    """
    context = empty_data_context
    root_dir = context.root_directory
    runner = CliRunner(mix_stderr=False)
    result = runner.invoke(
        cli, f"tap new sweet_suite tap -d {root_dir}", catch_exceptions=False,
    )
    stdout = result.stdout

    assert result.exit_code == 1
    assert "This is a BETA feature which may change" in stdout
    assert "Tap filename must end in .py. Please correct and re-run" in stdout

    assert_no_logging_messages_or_tracebacks(caplog, result)

def test_tap_new_on_context_with_no_datasources(caplog, empty_data_context):
    """
    We call the "tap new" command on a data context that has no datasources
    configured.

    The command should:
    - exit with a clear error message
    """
    root_dir = empty_data_context.root_directory
    runner = CliRunner(mix_stderr=False)
    result = runner.invoke(
        cli, f"tap new not_a_suite tap.py -d {root_dir}", catch_exceptions=False,
    )
    stdout = result.stdout

    assert result.exit_code == 1
    assert "This is a BETA feature which may change" in stdout
    assert "No datasources found in the context" in stdout

    assert_no_logging_messages_or_tracebacks(caplog, result)

def test_tap_new_with_non_existant_suite(caplog, empty_data_context):
    """
    We call the "tap new" command on a data context that has a datasource
    configured and no suites.

    The command should:
    - exit with a clear error message
    """
    empty_data_context.add_datasource(
        "my_datasource",
        module_name="great_expectations.datasource",
        class_name="PandasDatasource",
    )
    root_dir = empty_data_context.root_directory
    runner = CliRunner(mix_stderr=False)
    result = runner.invoke(
        cli, f"tap new not_a_suite tap.py -d {root_dir}", catch_exceptions=False,
    )
    stdout = result.stdout

    assert result.exit_code == 1
    assert "This is a BETA feature which may change" in stdout
    assert "Could not find a suite named `not_a_suite`" in stdout

    assert_no_logging_messages_or_tracebacks(caplog, result)

def test_tap_new_on_context_with_2_datasources_with_no_datasource_option_prompts_user(
    caplog, empty_data_context
):
    """
    We call the "tap new" command on a data context that has 2 datasources
    configured.

    The command should:
    - prompt the user to choose a datasource
    """
    empty_data_context.add_datasource(
        "1_datasource",
        module_name="great_expectations.datasource",
        class_name="PandasDatasource",
    )
    empty_data_context.add_datasource(
        "2_my_datasource",
        module_name="great_expectations.datasource",
        class_name="PandasDatasource",
    )
    root_dir = empty_data_context.root_directory
    runner = CliRunner(mix_stderr=False)
    result = runner.invoke(
        cli,
        f"tap new not_a_suite tap.py -d {root_dir}",
        input="1\n",
        catch_exceptions=False,
    )
    stdout = result.stdout

    assert "This is a BETA feature which may change" in stdout
    assert "Select a datasource" in stdout
    assert result.exit_code == 1

    assert_no_logging_messages_or_tracebacks(caplog, result)

def test_tap_new_on_context_builds_runnable_tap_file(
    caplog, empty_data_context, filesystem_csv
):
    """
    We call the "tap new" command on a data context that has 2 datasources
    configured.

    The command should:
    - prompt the user to choose a datasource
    - create the tap file

    This test then runs the tap file to verify it is runnable.
    """
    context = empty_data_context
    root_dir = context.root_directory
    csv = os.path.join(filesystem_csv, "f1.csv")

    context.add_datasource(
        "1_datasource",
        module_name="great_expectations.datasource",
        class_name="PandasDatasource",
    )
    context.create_expectation_suite("sweet_suite")
    suite = context.get_expectation_suite("sweet_suite")
    context.save_expectation_suite(suite)
    runner = CliRunner(mix_stderr=False)
    result = runner.invoke(
        cli,
        f"tap new sweet_suite tap.py -d {root_dir}",
        input=f"{csv}\n",
        catch_exceptions=False,
    )
    stdout = result.stdout

    assert "This is a BETA feature which may change" in stdout
    assert "Enter the path (relative or absolute) of a data file" in stdout
    assert "A new tap has been generated" in stdout
    assert result.exit_code == 0

    assert_no_logging_messages_or_tracebacks(caplog, result)

    tap_file = os.path.abspath(os.path.join(root_dir, "..", "tap.py"))
    # In travis on osx, python may not execute from the build dir
    curdir = os.path.abspath(os.getcwd())
    build_dir = os.environ.get("TRAVIS_BUILD_DIR", None) or curdir
    try:
        os.chdir(build_dir)
        status, output = subprocess.getstatusoutput(f"python {tap_file}")
        assert status == 0
        assert output == "Validation Succeeded!"
    except:
        raise
    finally:
        os.chdir(curdir)

def test_tap_new_on_context_builds_runnable_tap_file_that_fails_validation(
    caplog, empty_data_context, filesystem_csv
):
    """
    We call the "tap new" command on a data context that has 1 datasource
    configured with a suite that will fail.

    The command should:
    - prompt the user to choose a datasource
    - create the tap file

    This test then runs the tap file to verify it is runnable and fails
    correctly.
    """
    context = empty_data_context
    root_dir = context.root_directory
    csv = os.path.join(filesystem_csv, "f1.csv")

    context.add_datasource(
        "1_datasource",
        module_name="great_expectations.datasource",
        class_name="PandasDatasource",
    )
    context.create_expectation_suite("sweet_suite")
    suite = context.get_expectation_suite("sweet_suite")
    context.save_expectation_suite(suite)
    batch_kwargs = {"datasource": "1_datasource", "path": csv}
    batch = context.get_batch(batch_kwargs, suite)
    # Make an expectation that will fail and save it.
    batch.expect_table_column_count_to_equal(9999)
    batch.save_expectation_suite(discard_failed_expectations=False)

    runner = CliRunner(mix_stderr=False)
    result = runner.invoke(
        cli,
        f"tap new sweet_suite tap.py -d {root_dir}",
        input=f"{csv}\n",
        catch_exceptions=False,
    )
    stdout = result.stdout

    assert "This is a BETA feature which may change" in stdout
    assert "Enter the path (relative or absolute) of a data file" in stdout
    assert "A new tap has been generated" in stdout
    assert result.exit_code == 0

    assert_no_logging_messages_or_tracebacks(caplog, result)
    # In travis on osx, python may not execute from the build dir
    curdir = os.path.abspath(os.getcwd())
    build_dir = os.environ.get("TRAVIS_BUILD_DIR", None) or curdir
    print(os.environ.get("TRAVIS_BUILD_DIR"))
    print(curdir)
    print(build_dir)
    print(os.listdir(build_dir))
    try:
        os.chdir(build_dir)
        tap_file = os.path.abspath(os.path.join(root_dir, "..", "tap.py"))
        status, output = subprocess.getstatusoutput(f"python {tap_file}")
        assert status == 1
        assert output == "Validation Failed!"
    except:
        raise
    finally:
        os.chdir(curdir)

def test_tap_new_on_context_with_1_datasources_with_no_datasource_option_prompts_user_and_generates_runnable_tap_file(
    caplog, empty_data_context, filesystem_csv
):
    """
    We call the "tap new" command on a data context that has 1 datasources
    configured.

    The command should:
    - NOT prompt the user to choose a datasource
    - create the tap file

    This test then runs the tap file to verify it is runnable.
    """
    context = empty_data_context
    root_dir = context.root_directory
    csv = os.path.join(filesystem_csv, "f1.csv")

    context.add_datasource(
        "1_datasource",
        module_name="great_expectations.datasource",
        class_name="PandasDatasource",
    )
    context.create_expectation_suite("sweet_suite")
    suite = context.get_expectation_suite("sweet_suite")
    context.save_expectation_suite(suite)
    runner = CliRunner(mix_stderr=False)
    result = runner.invoke(
        cli,
        f"tap new sweet_suite tap.py -d {root_dir}",
        input=f"{csv}\n",
        catch_exceptions=False,
    )
    stdout = result.stdout

    assert "This is a BETA feature which may change" in stdout
    assert "Select a datasource" not in stdout
    assert "A new tap has been generated" in stdout
    assert result.exit_code == 0

    assert_no_logging_messages_or_tracebacks(caplog, result)

    tap_file = os.path.abspath(os.path.join(root_dir, "..", "tap.py"))
    status, output = subprocess.getstatusoutput(f"python {tap_file}")
    assert status == 0
    assert output == "Validation Succeeded!"
<commit_msg>Try build-dir alternative in travis<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import os
import subprocess

from click.testing import CliRunner

from great_expectations.cli import cli
from tests.cli.utils import assert_no_logging_messages_or_tracebacks

def test_tap_help_output(caplog,):
    runner = CliRunner(mix_stderr=False)
    result = runner.invoke(cli, ["tap"], catch_exceptions=False)
    assert result.exit_code == 0
    assert (
        """Commands:
  new  BETA! Create a new tap file for easy deployments"""
        in result.stdout
    )
    assert_no_logging_messages_or_tracebacks(caplog, result)

def test_tap_new_with_filename_not_ending_in_py_raises_helpful_error(
    caplog, empty_data_context
):
    """
    We call the "tap new" command with a bogus filename

    The command should:
    - exit with a clear error message
    """
    context = empty_data_context
    root_dir = context.root_directory
    runner = CliRunner(mix_stderr=False)
    result = runner.invoke(
        cli, f"tap new sweet_suite tap -d {root_dir}", catch_exceptions=False,
    )
    stdout = result.stdout

    assert result.exit_code == 1
    assert "This is a BETA feature which may change" in stdout
    assert "Tap filename must end in .py. Please correct and re-run" in stdout

    assert_no_logging_messages_or_tracebacks(caplog, result)

def test_tap_new_on_context_with_no_datasources(caplog, empty_data_context):
    """
    We call the "tap new" command on a data context that has no datasources
    configured.

    The command should:
    - exit with a clear error message
    """
    root_dir = empty_data_context.root_directory
    runner = CliRunner(mix_stderr=False)
    result = runner.invoke(
        cli, f"tap new not_a_suite tap.py -d {root_dir}", catch_exceptions=False,
    )
    stdout = result.stdout

    assert result.exit_code == 1
    assert "This is a BETA feature which may change" in stdout
    assert "No datasources found in the context" in stdout

    assert_no_logging_messages_or_tracebacks(caplog, result)

def test_tap_new_with_non_existant_suite(caplog, empty_data_context):
    """
    We call the "tap new" command on a data context that has a datasource
    configured and no suites.

    The command should:
    - exit with a clear error message
    """
    empty_data_context.add_datasource(
        "my_datasource",
        module_name="great_expectations.datasource",
        class_name="PandasDatasource",
    )
    root_dir = empty_data_context.root_directory
    runner = CliRunner(mix_stderr=False)
    result = runner.invoke(
        cli, f"tap new not_a_suite tap.py -d {root_dir}", catch_exceptions=False,
    )
    stdout = result.stdout

    assert result.exit_code == 1
    assert "This is a BETA feature which may change" in stdout
    assert "Could not find a suite named `not_a_suite`" in stdout

    assert_no_logging_messages_or_tracebacks(caplog, result)

def test_tap_new_on_context_with_2_datasources_with_no_datasource_option_prompts_user(
    caplog, empty_data_context
):
    """
    We call the "tap new" command on a data context that has 2 datasources
    configured.

    The command should:
    - prompt the user to choose a datasource
    """
    empty_data_context.add_datasource(
        "1_datasource",
        module_name="great_expectations.datasource",
        class_name="PandasDatasource",
    )
    empty_data_context.add_datasource(
        "2_my_datasource",
        module_name="great_expectations.datasource",
        class_name="PandasDatasource",
    )
    root_dir = empty_data_context.root_directory
    runner = CliRunner(mix_stderr=False)
    result = runner.invoke(
        cli,
        f"tap new not_a_suite tap.py -d {root_dir}",
        input="1\n",
        catch_exceptions=False,
    )
    stdout = result.stdout

    assert "This is a BETA feature which may change" in stdout
    assert "Select a datasource" in stdout
    assert result.exit_code == 1

    assert_no_logging_messages_or_tracebacks(caplog, result)

def test_tap_new_on_context_builds_runnable_tap_file(
    caplog, empty_data_context, filesystem_csv
):
    """
    We call the "tap new" command on a data context that has 2 datasources
    configured.

    The command should:
    - prompt the user to choose a datasource
    - create the tap file

    This test then runs the tap file to verify it is runnable.
    """
    context = empty_data_context
    root_dir = context.root_directory
    csv = os.path.join(filesystem_csv, "f1.csv")

    context.add_datasource(
        "1_datasource",
        module_name="great_expectations.datasource",
        class_name="PandasDatasource",
    )
    context.create_expectation_suite("sweet_suite")
    suite = context.get_expectation_suite("sweet_suite")
    context.save_expectation_suite(suite)
    runner = CliRunner(mix_stderr=False)
    result = runner.invoke(
        cli,
        f"tap new sweet_suite tap.py -d {root_dir}",
        input=f"{csv}\n",
        catch_exceptions=False,
    )
    stdout = result.stdout

    assert "This is a BETA feature which may change" in stdout
    assert "Enter the path (relative or absolute) of a data file" in stdout
    assert "A new tap has been generated" in stdout
    assert result.exit_code == 0

    assert_no_logging_messages_or_tracebacks(caplog, result)

    tap_file = os.path.abspath(os.path.join(root_dir, "..", "tap.py"))
    # In travis on osx, python may not execute from the build dir
    cmdstring = f"python {tap_file}"
    if os.environ.get("TRAVIS_OS_NAME") == "osx":
        print("HERE!!!")
        build_dir = os.environ.get("TRAVIS_BUILD_DIR")
        cmdstring = f"PYTHONPATH={build_dir} " + cmdstring
    status, output = subprocess.getstatusoutput(cmdstring)
    assert status == 0
    assert output == "Validation Succeeded!"

def test_tap_new_on_context_builds_runnable_tap_file_that_fails_validation(
    caplog, empty_data_context, filesystem_csv
):
    """
    We call the "tap new" command on a data context that has 1 datasource
    configured with a suite that will fail.

    The command should:
    - prompt the user to choose a datasource
    - create the tap file

    This test then runs the tap file to verify it is runnable and fails
    correctly.
    """
    context = empty_data_context
    root_dir = context.root_directory
    csv = os.path.join(filesystem_csv, "f1.csv")

    context.add_datasource(
        "1_datasource",
        module_name="great_expectations.datasource",
        class_name="PandasDatasource",
    )
    context.create_expectation_suite("sweet_suite")
    suite = context.get_expectation_suite("sweet_suite")
    context.save_expectation_suite(suite)
    batch_kwargs = {"datasource": "1_datasource", "path": csv}
    batch = context.get_batch(batch_kwargs, suite)
    # Make an expectation that will fail and save it.
    batch.expect_table_column_count_to_equal(9999)
    batch.save_expectation_suite(discard_failed_expectations=False)

    runner = CliRunner(mix_stderr=False)
    result = runner.invoke(
        cli,
        f"tap new sweet_suite tap.py -d {root_dir}",
        input=f"{csv}\n",
        catch_exceptions=False,
    )
    stdout = result.stdout

    assert "This is a BETA feature which may change" in stdout
    assert "Enter the path (relative or absolute) of a data file" in stdout
    assert "A new tap has been generated" in stdout
    assert result.exit_code == 0

    assert_no_logging_messages_or_tracebacks(caplog, result)
    # In travis on osx, python may not execute from the build dir
    tap_file = os.path.abspath(os.path.join(root_dir, "..", "tap.py"))
    cmdstring = f"python {tap_file}"
    if os.environ.get("TRAVIS_OS_NAME") == "osx":
        build_dir = os.environ.get("TRAVIS_BUILD_DIR")
        cmdstring = f"PYTHONPATH={build_dir} " + cmdstring

    status, output = subprocess.getstatusoutput(f"python {tap_file}")
    assert status == 1
    assert output == "Validation Failed!"

def test_tap_new_on_context_with_1_datasources_with_no_datasource_option_prompts_user_and_generates_runnable_tap_file(
    caplog, empty_data_context, filesystem_csv
):
    """
    We call the "tap new" command on a data context that has 1 datasources
    configured.

    The command should:
    - NOT prompt the user to choose a datasource
    - create the tap file

    This test then runs the tap file to verify it is runnable.
    """
    context = empty_data_context
    root_dir = context.root_directory
    csv = os.path.join(filesystem_csv, "f1.csv")

    context.add_datasource(
        "1_datasource",
        module_name="great_expectations.datasource",
        class_name="PandasDatasource",
    )
    context.create_expectation_suite("sweet_suite")
    suite = context.get_expectation_suite("sweet_suite")
    context.save_expectation_suite(suite)
    runner = CliRunner(mix_stderr=False)
    result = runner.invoke(
        cli,
        f"tap new sweet_suite tap.py -d {root_dir}",
        input=f"{csv}\n",
        catch_exceptions=False,
    )
    stdout = result.stdout

    assert "This is a BETA feature which may change" in stdout
    assert "Select a datasource" not in stdout
    assert "A new tap has been generated" in stdout
    assert result.exit_code == 0

    assert_no_logging_messages_or_tracebacks(caplog, result)

    tap_file = os.path.abspath(os.path.join(root_dir, "..", "tap.py"))
    status, output = subprocess.getstatusoutput(f"python {tap_file}")
    assert status == 0
    assert output == "Validation Succeeded!"
<|endoftext|>


<commit_before>#!/usr/bin/env python
# DialsIntegrater.py
#   Copyright (C) 2014 Diamond Light Source, Richard Gildea & Graeme Winter
#
#   This code is distributed under the BSD license, a copy of which is
#   included in the root directory of this package.
#
# An implementation of the Integrater interface using Dials. This depends on the
# Dials wrappers to actually implement the functionality.
#

import os
import sys
import math
import copy
import shutil

if not os.environ.has_key('XIA2_ROOT'):
  raise RuntimeError, 'XIA2_ROOT not defined'

if not os.environ['XIA2_ROOT'] in sys.path:
  sys.path.append(os.environ['XIA2_ROOT'])

# wrappers for programs that this needs

from Wrappers.Dials.Refine import Refine as _Refine
from Wrappers.Dials.Integrate import Integrate as _Integrate
from Wrappers.Dials.ExportMtz import ExportMtz as _ExportMtz
from Wrappers.Dials.ShowIsigRmsd import ShowIsigRmsd as _ShowIsigRmsd

# interfaces that this must implement to be an integrater

from Schema.Interfaces.Integrater import Integrater
from Schema.Interfaces.FrameProcessor import FrameProcessor
from Schema.Exceptions.BadLatticeError import BadLatticeError

# indexing functionality if not already provided - even if it is
# we still need to reindex with XDS.

from Modules.Indexer.DialsIndexer import DialsIndexer
from Wrappers.CCP4.Reindex import Reindex

# odds and sods that are needed

from lib.bits import auto_logfiler
from lib.SymmetryLib import lattice_to_spacegroup
from Handlers.Streams import Chatter, Debug, Journal
from Handlers.Flags import Flags
from Handlers.Files import FileHandler
from Handlers.Phil import PhilIndex

from Experts.SymmetryExpert import lattice_to_spacegroup_number

class DialsIntegrater(FrameProcessor,
                      Integrater):
  '''A class to implement the Integrater interface using *only* XDS
  programs.'''

  def __init__(self):

    # set up the inherited objects

    FrameProcessor.__init__(self)
    Integrater.__init__(self)

    # check that the programs exist - this will raise an exception if
    # they do not...

    integrate = _Integrate()

    # admin junk
    self._working_directory = os.getcwd()

    # place to store working data
    self._data_files = { }

    # internal parameters to pass around
    self._integrate_parameters = { }
    self._intgr_integrated_filename = None

    return

  # overload these methods as we don't want the resolution range
  # feeding back... aha - but we may want to assign them
  # from outside!

  def set_integrater_resolution(self, dmin, dmax, user = False):
    if user:
      Integrater.set_integrater_resolution(self, dmin, dmax, user)
    return

  def set_integrater_high_resolution(self, dmin, user = False):
    if user:
      Integrater.set_integrater_high_resolution(self, dmin, user)
    return

  def set_integrater_low_resolution(self, dmax, user = False):
    self._intgr_reso_low = dmax
    return

  # admin functions

  def set_working_directory(self, working_directory):
    self._working_directory = working_directory
    return

  def get_working_directory(self):
    return self._working_directory

  def get_integrated_filename(self):
    return self._intgr_integrated_filename

  # factory functions

  def Refine(self):
    refine = _Refine()
    params = PhilIndex.params.dials.refine
    refine.set_phil_file(params.phil_file)
    refine.set_working_directory(self.get_working_directory())
    refine.setup_from_image(self.get_image_name(self._intgr_wedge[0]))
    refine.set_experiments_filename(self._intgr_experiments_filename)
    refine.set_indexed_filename(
      self._intgr_indexer.get_indexed_filename())
    refine.set_scan_varying(params.scan_varying)
    refine.set_use_all_reflections(params.scan_varying)
    auto_logfiler(refine, 'REFINE')

    return refine

  def Integrate(self):
    params = PhilIndex.params.dials.integrate
    integrate = _Integrate()
    integrate.set_phil_file(params.phil_file)
    integrate.set_intensity_algorithm(params.intensity_algorithm)
    integrate.set_background_outlier_algorithm(
      params.background_outlier_algorithm)
    integrate.set_working_directory(self.get_working_directory())

    integrate.setup_from_image(self.get_image_name(
        self._intgr_wedge[0]))
    integrate.set_experiments_filename(self._intgr_experiments_filename)
    integrate.set_reflections_filename(
      self._intgr_indexer.get_indexed_filename())

    if self.get_distance():
      integrate.set_distance(self.get_distance())

    if self.get_wavelength():
      integrate.set_wavelength(self.get_wavelength())

    auto_logfiler(integrate, 'INTEGRATE')

    return integrate

  def ExportMtz(self):
    export = _ExportMtz()
    export.set_working_directory(self.get_working_directory())

    export.set_experiments_filename(self._intgr_experiments_filename)

    auto_logfiler(export, 'EXPORTMTZ')

    return export

  def ShowIsigRmsd(self):
    show = _ShowIsigRmsd()
    show.set_working_directory(self.get_working_directory())
    auto_logfiler(show, 'SHOWISIGRMSD')
    return show

  # now some real functions, which do useful things

  def _integrater_reset_callback(self):
    '''Delete all results on a reset.'''
    Debug.write('Deleting all stored results.')
    self._data_files = { }
    self._integrate_parameters = { }
    return

  def _integrate_prepare(self):
    '''Prepare for integration - in XDS terms this may mean rerunning
    IDXREF to get the XPARM etc. DEFPIX is considered part of the full
    integration as it is resolution dependent.'''

    from Handlers.Citations import Citations
    Citations.cite('dials')

    # decide what images we are going to process, if not already
    # specified
    if not self._intgr_wedge:
      images = self.get_matching_images()
      self.set_integrater_wedge(min(images),
                                max(images))

    Debug.write('DIALS INTEGRATE PREPARE:')
    Debug.write('Wavelength: %.6f' % self.get_wavelength())
    Debug.write('Distance: %.2f' % self.get_distance())

    if not self._intgr_indexer:
      self.set_integrater_indexer(DialsIndexer())
      self.get_integrater_indexer().set_indexer_sweep(
          self.get_integrater_sweep())

      self._intgr_indexer.set_working_directory(
          self.get_working_directory())

      self._intgr_indexer.setup_from_image(self.get_image_name(
          self._intgr_wedge[0]))

      if self.get_frame_wedge():
        wedge = self.get_frame_wedge()
        Debug.write('Propogating wedge limit: %d %d' % wedge)
        self._intgr_indexer.set_frame_wedge(wedge[0], wedge[1],
                                            apply_offset = False)

      # this needs to be set up from the contents of the
      # Integrater frame processer - wavelength &c.

      if self.get_beam_centre():
        self._intgr_indexer.set_beam_centre(self.get_beam_centre())

      if self.get_distance():
        self._intgr_indexer.set_distance(self.get_distance())

      if self.get_wavelength():
        self._intgr_indexer.set_wavelength(
            self.get_wavelength())

    # get the unit cell from this indexer to initiate processing
    # if it is new... and also copy out all of the information for
    # the Dials indexer if not...

    experiments = self._intgr_indexer.get_indexer_experiment_list()
    assert len(experiments) == 1 # currently only handle one lattice/sweep
    experiment = experiments[0]
    crystal_model = experiment.crystal
    lattice = self._intgr_indexer.get_indexer_lattice()

    # check if the lattice was user assigned...
    user_assigned = self._intgr_indexer.get_indexer_user_input_lattice()

    # XXX check that the indexer is an Dials indexer - if not then
    # create one...

    # set a low resolution limit (which isn't really used...)
    # this should perhaps be done more intelligently from an
    # analysis of the spot list or something...?

    if not self.get_integrater_low_resolution():

      dmax = self._intgr_indexer.get_indexer_low_resolution()
      self.set_integrater_low_resolution(dmax)

      Debug.write('Low resolution set to: %s' % \
                  self.get_integrater_low_resolution())

    # copy the data across
    from dxtbx.serialize import load, dump
    self._intgr_experiments_filename = os.path.join(
      self.get_working_directory(), "experiments.json")
    dump.experiment_list(experiments, self._intgr_experiments_filename)
    self._intgr_indexed_filename = os.path.join(
      self.get_working_directory(), os.path.basename(
        self._intgr_indexer.get_indexed_filename()))

    refiner = self.Refine()
    refiner.run()
    self._intgr_experiments_filename \
      = refiner.get_refined_experiments_filename()
    experiments = load.experiment_list(self._intgr_experiments_filename)
    experiment = experiments[0]

    #shutil.copyfile(self._intgr_indexer.get_experiments_filename(),
                #self._intgr_experiments_filename)
    if self._intgr_indexer.get_indexed_filename() != self._intgr_indexed_filename:
      shutil.copyfile(self._intgr_indexer.get_indexed_filename(),
                      self._intgr_indexed_filename)

    Debug.write('Files available at the end of DIALS integrate prepare:')
    for f in self._data_files.keys():
      Debug.write('%s' % f)

    self.set_detector(experiment.detector)
    self.set_beam_obj(experiment.beam)
    self.set_goniometer(experiment.goniometer)

    return

  def _integrate(self):
    '''Actually do the integration - in XDS terms this will mean running
    DEFPIX and INTEGRATE to measure all the reflections.'''

    images_str = '%d to %d' % self._intgr_wedge
    cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
               self._intgr_indexer.get_indexer_cell()

    if len(self._fp_directory) <= 50:
      dirname = self._fp_directory
    else:
      dirname = '...%s' % self._fp_directory[-46:]

    Journal.block(
        'integrating', self._intgr_sweep_name, 'DIALS',
        {'images':images_str,
         'cell':cell_str,
         'lattice':self.get_integrater_indexer().get_indexer_lattice(),
         'template':self._fp_template,
         'directory':dirname,
         'resolution':'%.2f' % self._intgr_reso_high})

    integrate = self.Integrate()

    if self._integrate_parameters:
      integrate.set_updates(self._integrate_parameters)

    # decide what images we are going to process, if not already
    # specified

    if not self._intgr_wedge:
      images = self.get_matching_images()
      self.set_integrater_wedge(min(images),
                                max(images))

    first_image_in_wedge = self.get_image_name(self._intgr_wedge[0])

    integrate.set_experiments_filename(self._intgr_experiments_filename)
    integrate.set_reflections_filename(self._intgr_indexed_filename)

    integrate.run()

    # FIXME (i) record the log file, (ii) get more information out from the
    # integration log on the quality of the data and (iii) the mosaic spread
    # range observed and R.M.S. deviations.

    self._intgr_integrated_pickle \
      = os.path.join(self.get_working_directory(), 'integrated.pickle')

    show = self.ShowIsigRmsd()
    show.set_reflections_filename(self._intgr_integrated_pickle)
    show.run()
    data = show.data()

    spot_status = ''

    for frame in range(self._intgr_wedge[0], self._intgr_wedge[1] + 1):
      n, isig, rmsd = data.get(frame, (0, 0.0, 0.0))

      if isig < 1.0:
        status = '.'
      elif rmsd > 2.5:
        status = '!'
      elif rmsd > 1.0:
        status = '%'
      else:
        status = 'o'

      spot_statuss += status

    if len(spot_status) > 60:
      Chatter.write('Integration status per image (60/record):')
    else:
      Chatter.write('Integration status per image:')

    for chunk in [spot_status[i:i + 60] \
                  for i in range(0, len(spot_status), 60)]:
      Chatter.write(chunk)

    Chatter.write(
      '"o" => good        "%" => ok        "!" => bad rmsd')
    Chatter.write(
      '"O" => overloaded  "#" => many bad  "." => weak')
    Chatter.write(
      '"@" => abandoned')

    mosaic = integrate.get_mosaic()
    Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
                  (mosaic, mosaic, mosaic))

    return self._intgr_integrated_pickle

  def _integrate_finish(self):
    '''Finish off the integration by running dials.export_mtz.'''

    exporter = self.ExportMtz()
    exporter.set_reflections_filename(self._intgr_integrated_pickle)
    mtz_filename = os.path.join(self.get_working_directory(), 'integrated.mtz')
    exporter.set_mtz_filename(mtz_filename)
    exporter.run()
    self._intgr_integrated_filename = mtz_filename

    if self._intgr_reindex_operator is None and \
      self._intgr_spacegroup_number == lattice_to_spacegroup(
        self.get_integrater_indexer().get_indexer_lattice()):
      return mtz_filename

    if self._intgr_reindex_operator is None and \
      self._intgr_spacegroup_number == 0:
      return mtz_filename

    Debug.write('Reindexing to spacegroup %d (%s)' % \
                (self._intgr_spacegroup_number,
                 self._intgr_reindex_operator))

    hklin = mtz_filename
    reindex = Reindex()
    reindex.set_working_directory(self.get_working_directory())
    auto_logfiler(reindex)

    reindex.set_operator(self._intgr_reindex_operator)

    if self._intgr_spacegroup_number:
      reindex.set_spacegroup(self._intgr_spacegroup_number)
    hklout = '%s_reindex.mtz' % hklin[:-4]
    reindex.set_hklin(hklin)
    reindex.set_hklout(hklout)
    reindex.reindex()
    self._intgr_integrated_filename = hklout
    return hklout

    return mtz_filename

if __name__ == '__main__':

  # run a demo test

  if not os.environ.has_key('XIA2_ROOT'):
    raise RuntimeError, 'XIA2_ROOT not defined'

  di = DialsIntegrater()
  di.setup_from_image(sys.argv[1])
  from Schema.XCrystal import XCrystal
  from Schema.XWavelength import XWavelength
  from Schema.XSweep import XSweep
  cryst = XCrystal("CRYST1", None)
  wav = XWavelength("WAVE1", cryst, di.get_wavelength())
  directory, image = os.path.split(sys.argv[1])
  sweep = XSweep('SWEEP1', wav, directory=directory, image=image)
  di.set_integrater_sweep(sweep)
  di.integrate()
<commit_msg>Too much s<commit_after>#!/usr/bin/env python
# DialsIntegrater.py
#   Copyright (C) 2014 Diamond Light Source, Richard Gildea & Graeme Winter
#
#   This code is distributed under the BSD license, a copy of which is
#   included in the root directory of this package.
#
# An implementation of the Integrater interface using Dials. This depends on the
# Dials wrappers to actually implement the functionality.
#

import os
import sys
import math
import copy
import shutil

if not os.environ.has_key('XIA2_ROOT'):
  raise RuntimeError, 'XIA2_ROOT not defined'

if not os.environ['XIA2_ROOT'] in sys.path:
  sys.path.append(os.environ['XIA2_ROOT'])

# wrappers for programs that this needs

from Wrappers.Dials.Refine import Refine as _Refine
from Wrappers.Dials.Integrate import Integrate as _Integrate
from Wrappers.Dials.ExportMtz import ExportMtz as _ExportMtz
from Wrappers.Dials.ShowIsigRmsd import ShowIsigRmsd as _ShowIsigRmsd

# interfaces that this must implement to be an integrater

from Schema.Interfaces.Integrater import Integrater
from Schema.Interfaces.FrameProcessor import FrameProcessor
from Schema.Exceptions.BadLatticeError import BadLatticeError

# indexing functionality if not already provided - even if it is
# we still need to reindex with XDS.

from Modules.Indexer.DialsIndexer import DialsIndexer
from Wrappers.CCP4.Reindex import Reindex

# odds and sods that are needed

from lib.bits import auto_logfiler
from lib.SymmetryLib import lattice_to_spacegroup
from Handlers.Streams import Chatter, Debug, Journal
from Handlers.Flags import Flags
from Handlers.Files import FileHandler
from Handlers.Phil import PhilIndex

from Experts.SymmetryExpert import lattice_to_spacegroup_number

class DialsIntegrater(FrameProcessor,
                      Integrater):
  '''A class to implement the Integrater interface using *only* XDS
  programs.'''

  def __init__(self):

    # set up the inherited objects

    FrameProcessor.__init__(self)
    Integrater.__init__(self)

    # check that the programs exist - this will raise an exception if
    # they do not...

    integrate = _Integrate()

    # admin junk
    self._working_directory = os.getcwd()

    # place to store working data
    self._data_files = { }

    # internal parameters to pass around
    self._integrate_parameters = { }
    self._intgr_integrated_filename = None

    return

  # overload these methods as we don't want the resolution range
  # feeding back... aha - but we may want to assign them
  # from outside!

  def set_integrater_resolution(self, dmin, dmax, user = False):
    if user:
      Integrater.set_integrater_resolution(self, dmin, dmax, user)
    return

  def set_integrater_high_resolution(self, dmin, user = False):
    if user:
      Integrater.set_integrater_high_resolution(self, dmin, user)
    return

  def set_integrater_low_resolution(self, dmax, user = False):
    self._intgr_reso_low = dmax
    return

  # admin functions

  def set_working_directory(self, working_directory):
    self._working_directory = working_directory
    return

  def get_working_directory(self):
    return self._working_directory

  def get_integrated_filename(self):
    return self._intgr_integrated_filename

  # factory functions

  def Refine(self):
    refine = _Refine()
    params = PhilIndex.params.dials.refine
    refine.set_phil_file(params.phil_file)
    refine.set_working_directory(self.get_working_directory())
    refine.setup_from_image(self.get_image_name(self._intgr_wedge[0]))
    refine.set_experiments_filename(self._intgr_experiments_filename)
    refine.set_indexed_filename(
      self._intgr_indexer.get_indexed_filename())
    refine.set_scan_varying(params.scan_varying)
    refine.set_use_all_reflections(params.scan_varying)
    auto_logfiler(refine, 'REFINE')

    return refine

  def Integrate(self):
    params = PhilIndex.params.dials.integrate
    integrate = _Integrate()
    integrate.set_phil_file(params.phil_file)
    integrate.set_intensity_algorithm(params.intensity_algorithm)
    integrate.set_background_outlier_algorithm(
      params.background_outlier_algorithm)
    integrate.set_working_directory(self.get_working_directory())

    integrate.setup_from_image(self.get_image_name(
        self._intgr_wedge[0]))
    integrate.set_experiments_filename(self._intgr_experiments_filename)
    integrate.set_reflections_filename(
      self._intgr_indexer.get_indexed_filename())

    if self.get_distance():
      integrate.set_distance(self.get_distance())

    if self.get_wavelength():
      integrate.set_wavelength(self.get_wavelength())

    auto_logfiler(integrate, 'INTEGRATE')

    return integrate

  def ExportMtz(self):
    export = _ExportMtz()
    export.set_working_directory(self.get_working_directory())

    export.set_experiments_filename(self._intgr_experiments_filename)

    auto_logfiler(export, 'EXPORTMTZ')

    return export

  def ShowIsigRmsd(self):
    show = _ShowIsigRmsd()
    show.set_working_directory(self.get_working_directory())
    auto_logfiler(show, 'SHOWISIGRMSD')
    return show

  # now some real functions, which do useful things

  def _integrater_reset_callback(self):
    '''Delete all results on a reset.'''
    Debug.write('Deleting all stored results.')
    self._data_files = { }
    self._integrate_parameters = { }
    return

  def _integrate_prepare(self):
    '''Prepare for integration - in XDS terms this may mean rerunning
    IDXREF to get the XPARM etc. DEFPIX is considered part of the full
    integration as it is resolution dependent.'''

    from Handlers.Citations import Citations
    Citations.cite('dials')

    # decide what images we are going to process, if not already
    # specified
    if not self._intgr_wedge:
      images = self.get_matching_images()
      self.set_integrater_wedge(min(images),
                                max(images))

    Debug.write('DIALS INTEGRATE PREPARE:')
    Debug.write('Wavelength: %.6f' % self.get_wavelength())
    Debug.write('Distance: %.2f' % self.get_distance())

    if not self._intgr_indexer:
      self.set_integrater_indexer(DialsIndexer())
      self.get_integrater_indexer().set_indexer_sweep(
          self.get_integrater_sweep())

      self._intgr_indexer.set_working_directory(
          self.get_working_directory())

      self._intgr_indexer.setup_from_image(self.get_image_name(
          self._intgr_wedge[0]))

      if self.get_frame_wedge():
        wedge = self.get_frame_wedge()
        Debug.write('Propogating wedge limit: %d %d' % wedge)
        self._intgr_indexer.set_frame_wedge(wedge[0], wedge[1],
                                            apply_offset = False)

      # this needs to be set up from the contents of the
      # Integrater frame processer - wavelength &c.

      if self.get_beam_centre():
        self._intgr_indexer.set_beam_centre(self.get_beam_centre())

      if self.get_distance():
        self._intgr_indexer.set_distance(self.get_distance())

      if self.get_wavelength():
        self._intgr_indexer.set_wavelength(
            self.get_wavelength())

    # get the unit cell from this indexer to initiate processing
    # if it is new... and also copy out all of the information for
    # the Dials indexer if not...

    experiments = self._intgr_indexer.get_indexer_experiment_list()
    assert len(experiments) == 1 # currently only handle one lattice/sweep
    experiment = experiments[0]
    crystal_model = experiment.crystal
    lattice = self._intgr_indexer.get_indexer_lattice()

    # check if the lattice was user assigned...
    user_assigned = self._intgr_indexer.get_indexer_user_input_lattice()

    # XXX check that the indexer is an Dials indexer - if not then
    # create one...

    # set a low resolution limit (which isn't really used...)
    # this should perhaps be done more intelligently from an
    # analysis of the spot list or something...?

    if not self.get_integrater_low_resolution():

      dmax = self._intgr_indexer.get_indexer_low_resolution()
      self.set_integrater_low_resolution(dmax)

      Debug.write('Low resolution set to: %s' % \
                  self.get_integrater_low_resolution())

    # copy the data across
    from dxtbx.serialize import load, dump
    self._intgr_experiments_filename = os.path.join(
      self.get_working_directory(), "experiments.json")
    dump.experiment_list(experiments, self._intgr_experiments_filename)
    self._intgr_indexed_filename = os.path.join(
      self.get_working_directory(), os.path.basename(
        self._intgr_indexer.get_indexed_filename()))

    refiner = self.Refine()
    refiner.run()
    self._intgr_experiments_filename \
      = refiner.get_refined_experiments_filename()
    experiments = load.experiment_list(self._intgr_experiments_filename)
    experiment = experiments[0]

    #shutil.copyfile(self._intgr_indexer.get_experiments_filename(),
                #self._intgr_experiments_filename)
    if self._intgr_indexer.get_indexed_filename() != self._intgr_indexed_filename:
      shutil.copyfile(self._intgr_indexer.get_indexed_filename(),
                      self._intgr_indexed_filename)

    Debug.write('Files available at the end of DIALS integrate prepare:')
    for f in self._data_files.keys():
      Debug.write('%s' % f)

    self.set_detector(experiment.detector)
    self.set_beam_obj(experiment.beam)
    self.set_goniometer(experiment.goniometer)

    return

  def _integrate(self):
    '''Actually do the integration - in XDS terms this will mean running
    DEFPIX and INTEGRATE to measure all the reflections.'''

    images_str = '%d to %d' % self._intgr_wedge
    cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
               self._intgr_indexer.get_indexer_cell()

    if len(self._fp_directory) <= 50:
      dirname = self._fp_directory
    else:
      dirname = '...%s' % self._fp_directory[-46:]

    Journal.block(
        'integrating', self._intgr_sweep_name, 'DIALS',
        {'images':images_str,
         'cell':cell_str,
         'lattice':self.get_integrater_indexer().get_indexer_lattice(),
         'template':self._fp_template,
         'directory':dirname,
         'resolution':'%.2f' % self._intgr_reso_high})

    integrate = self.Integrate()

    if self._integrate_parameters:
      integrate.set_updates(self._integrate_parameters)

    # decide what images we are going to process, if not already
    # specified

    if not self._intgr_wedge:
      images = self.get_matching_images()
      self.set_integrater_wedge(min(images),
                                max(images))

    first_image_in_wedge = self.get_image_name(self._intgr_wedge[0])

    integrate.set_experiments_filename(self._intgr_experiments_filename)
    integrate.set_reflections_filename(self._intgr_indexed_filename)

    integrate.run()

    # FIXME (i) record the log file, (ii) get more information out from the
    # integration log on the quality of the data and (iii) the mosaic spread
    # range observed and R.M.S. deviations.

    self._intgr_integrated_pickle \
      = os.path.join(self.get_working_directory(), 'integrated.pickle')

    show = self.ShowIsigRmsd()
    show.set_reflections_filename(self._intgr_integrated_pickle)
    show.run()
    data = show.data()

    spot_status = ''

    for frame in range(self._intgr_wedge[0], self._intgr_wedge[1] + 1):
      n, isig, rmsd = data.get(frame, (0, 0.0, 0.0))

      if isig < 1.0:
        status = '.'
      elif rmsd > 2.5:
        status = '!'
      elif rmsd > 1.0:
        status = '%'
      else:
        status = 'o'

      spot_status += status

    if len(spot_status) > 60:
      Chatter.write('Integration status per image (60/record):')
    else:
      Chatter.write('Integration status per image:')

    for chunk in [spot_status[i:i + 60] \
                  for i in range(0, len(spot_status), 60)]:
      Chatter.write(chunk)

    Chatter.write(
      '"o" => good        "%" => ok        "!" => bad rmsd')
    Chatter.write(
      '"O" => overloaded  "#" => many bad  "." => weak')
    Chatter.write(
      '"@" => abandoned')

    mosaic = integrate.get_mosaic()
    Chatter.write('Mosaic spread: %.3f < %.3f < %.3f' % \
                  (mosaic, mosaic, mosaic))

    return self._intgr_integrated_pickle

  def _integrate_finish(self):
    '''Finish off the integration by running dials.export_mtz.'''

    exporter = self.ExportMtz()
    exporter.set_reflections_filename(self._intgr_integrated_pickle)
    mtz_filename = os.path.join(self.get_working_directory(), 'integrated.mtz')
    exporter.set_mtz_filename(mtz_filename)
    exporter.run()
    self._intgr_integrated_filename = mtz_filename

    if self._intgr_reindex_operator is None and \
      self._intgr_spacegroup_number == lattice_to_spacegroup(
        self.get_integrater_indexer().get_indexer_lattice()):
      return mtz_filename

    if self._intgr_reindex_operator is None and \
      self._intgr_spacegroup_number == 0:
      return mtz_filename

    Debug.write('Reindexing to spacegroup %d (%s)' % \
                (self._intgr_spacegroup_number,
                 self._intgr_reindex_operator))

    hklin = mtz_filename
    reindex = Reindex()
    reindex.set_working_directory(self.get_working_directory())
    auto_logfiler(reindex)

    reindex.set_operator(self._intgr_reindex_operator)

    if self._intgr_spacegroup_number:
      reindex.set_spacegroup(self._intgr_spacegroup_number)
    hklout = '%s_reindex.mtz' % hklin[:-4]
    reindex.set_hklin(hklin)
    reindex.set_hklout(hklout)
    reindex.reindex()
    self._intgr_integrated_filename = hklout
    return hklout

    return mtz_filename

if __name__ == '__main__':

  # run a demo test

  if not os.environ.has_key('XIA2_ROOT'):
    raise RuntimeError, 'XIA2_ROOT not defined'

  di = DialsIntegrater()
  di.setup_from_image(sys.argv[1])
  from Schema.XCrystal import XCrystal
  from Schema.XWavelength import XWavelength
  from Schema.XSweep import XSweep
  cryst = XCrystal("CRYST1", None)
  wav = XWavelength("WAVE1", cryst, di.get_wavelength())
  directory, image = os.path.split(sys.argv[1])
  sweep = XSweep('SWEEP1', wav, directory=directory, image=image)
  di.set_integrater_sweep(sweep)
  di.integrate()
<|endoftext|>


<commit_before>import collections
import logging
import io
import warnings

import dispatch

logger = logging.getLogger("remote.stream")

class Stream(io.RawIOBase):
    """Base class for stream wrappers: streams which wrap other streams, adding functionality.

    """
    onReadFinished = dispatch.Signal(["requestedBytes", "dataRead"])
    onWriteFinished = dispatch.Signal(["bytesWritten", "data"])

    def __init__(self, targetStream, *args, **kwargs):
        self.targetStream = targetStream

        super(Stream, self).__init__(*args, **kwargs)

    def _emit(self, signal, eventName, **kwargs):
        logger.debug("Emitting %r signal.", eventName)
        signal.send_robust(self, **kwargs)

    @classmethod
    def factory(cls, **kwargs):
        def fact(*args, **kwargsOverrides):
            kwa = kwargs.copy()
            kwa.update(kwargsOverrides)
            return cls(*args, **kwa)

        return fact

    def readable(self):
        return self.targetStream.readable()

    def read(self, requestedBytes=-1):
        """Read incoming data from the target stream.

        """
        dataRead, metadata = self.targetStream.read(requestedBytes)

        self._emit(self.onReadFinished, "read finished", requestedBytes=requestedBytes, dataRead=dataRead)

        return dataRead, metadata

    def write(self, data):
        """Write outgoing data to the target stream.

        """
        bytesWritten = self.targetStream.write(data)

        self._emit(self.onWriteFinished, "write finished", bytesWritten=bytesWritten, data=data)

        return bytesWritten

class OutgoingQueuedStream(Stream):
    """Stream wrapper which queues all outgoing messages instead of writing them instantly.

    """
    onOutgoingQueueEmpty = dispatch.Signal()
    onOutgoingMessageQueued = dispatch.Signal(["message", "queueLength"])

    def __init__(self, *args, **kwargs):
        self._outgoingMessages = collections.deque()

        super(OutgoingQueuedStream, self).__init__(*args, **kwargs)

    def write(self, message):
        """Write message to the outgoing queue.

        """
        self._outgoingMessages.append(message)

        self._emit(self.onOutgoingMessageQueued, "outgoing message queued",
                message=message, queueLength=len(self._outgoingMessages))

        return len(message)

    def handleWrite(self):
        """Write the next outgoing message to the socket.

        This should be called by a ready-polling class like SelectCommunicator.

        """
        if len(self._outgoingMessages) == 0:
            # If we got here, we probably weren't properly removed from the list of streams with outgoing messages;
            # signal whatever's managing outgoing writes that we're done.
            self._emit(self.onOutgoingQueueEmpty, "outgoing queue empty")
            return

        outgoingMessage = self._outgoingMessages.popleft()

        bytesWritten = self.targetStream.write(outgoingMessage)

        if bytesWritten < len(outgoingMessage):
            logger.debug("Wrote first %d bytes of %r; re-queuing rest.",
                    bytesWritten, outgoingMessage)
            self._outgoingMessages.appendleft(outgoingMessage[bytesWritten:])
        else:
            logger.debug("Wrote all %d bytes of %r",
                    bytesWritten, outgoingMessage)

        if len(self._outgoingMessages) == 0:
            self._emit(self.onOutgoingQueueEmpty, "outgoing queue empty")

class IncomingQueuedStream(Stream):
    """Stream wrapper which can read incoming messages into a queue, instead of waiting for a read() call to read from
    the target stream.

    """
    def __init__(self, *args, **kwargs):
        self._incomingMessages = collections.deque()

        self.onIncomingQueueEmpty = []
        self.onIncomingMessageQueued = []

        super(IncomingQueuedStream, self).__init__(*args, **kwargs)

    def read(self, requestedBytes=-1):
        """Read message from the incoming queue.

        """
        if len(self._incomingMessages) == 0:
            self._emit(self.onIncomingQueueEmpty, "incoming queue empty")
            return None

        try:
            #TODO: Implement requestedBytes support
            if requestedBytes != -1:
                warnings.warn("requestedBytes support not currently implemented.", FutureWarning)
            incomingMessage = self._incomingMessages.popleft()

        except IndexError:
            incomingMessage = None

        if len(self._incomingMessages) == 0:
            self._emit(self.onIncomingQueueEmpty, "incoming queue empty")

        return incomingMessage

    def handleRead(self):
        """Read an incoming message from the socket, and queue it for reading.

        This should be called by a ready-polling class like SelectCommunicator.

        """
        incomingMessage = self.targetStream.read()
        logger.debug("Read %d bytes: %r", len(incomingMessage),
                incomingMessage)

        self._incomingMessages.append(incomingMessage)

        self._emit(self.onIncomingMessageQueued, "incoming message queued")

class IOQueuedStream(IncomingQueuedStream, OutgoingQueuedStream):
    """Stream wrapper which queues both incoming and outgoing messages.

    This allows messages to be read and written without blocking, provided handleRead() and handleWrite() are called at
    appropriate times.

    """
<commit_msg>Missed a couple of handler lists when converting to Signals earlier. Converted now.<commit_after>import collections
import logging
import io
import warnings

import dispatch

logger = logging.getLogger("remote.stream")

class Stream(io.RawIOBase):
    """Base class for stream wrappers: streams which wrap other streams, adding functionality.

    """
    onReadFinished = dispatch.Signal(["requestedBytes", "dataRead"])
    onWriteFinished = dispatch.Signal(["bytesWritten", "data"])

    def __init__(self, targetStream, *args, **kwargs):
        self.targetStream = targetStream

        super(Stream, self).__init__(*args, **kwargs)

    def _emit(self, signal, eventName, **kwargs):
        logger.debug("Emitting %r signal.", eventName)
        signal.send_robust(self, **kwargs)

    @classmethod
    def factory(cls, **kwargs):
        def fact(*args, **kwargsOverrides):
            kwa = kwargs.copy()
            kwa.update(kwargsOverrides)
            return cls(*args, **kwa)

        return fact

    def readable(self):
        return self.targetStream.readable()

    def read(self, requestedBytes=-1):
        """Read incoming data from the target stream.

        """
        dataRead, metadata = self.targetStream.read(requestedBytes)

        self._emit(self.onReadFinished, "read finished", requestedBytes=requestedBytes, dataRead=dataRead)

        return dataRead, metadata

    def write(self, data):
        """Write outgoing data to the target stream.

        """
        bytesWritten = self.targetStream.write(data)

        self._emit(self.onWriteFinished, "write finished", bytesWritten=bytesWritten, data=data)

        return bytesWritten

class OutgoingQueuedStream(Stream):
    """Stream wrapper which queues all outgoing messages instead of writing them instantly.

    """
    onOutgoingQueueEmpty = dispatch.Signal()
    onOutgoingMessageQueued = dispatch.Signal(["message", "queueLength"])

    def __init__(self, *args, **kwargs):
        self._outgoingMessages = collections.deque()

        super(OutgoingQueuedStream, self).__init__(*args, **kwargs)

    def write(self, message):
        """Write message to the outgoing queue.

        """
        self._outgoingMessages.append(message)

        self._emit(self.onOutgoingMessageQueued, "outgoing message queued",
                message=message, queueLength=len(self._outgoingMessages))

        return len(message)

    def handleWrite(self):
        """Write the next outgoing message to the socket.

        This should be called by a ready-polling class like SelectCommunicator.

        """
        if len(self._outgoingMessages) == 0:
            # If we got here, we probably weren't properly removed from the list of streams with outgoing messages;
            # signal whatever's managing outgoing writes that we're done.
            self._emit(self.onOutgoingQueueEmpty, "outgoing queue empty")
            return

        outgoingMessage = self._outgoingMessages.popleft()

        bytesWritten = self.targetStream.write(outgoingMessage)

        if bytesWritten < len(outgoingMessage):
            logger.debug("Wrote first %d bytes of %r; re-queuing rest.",
                    bytesWritten, outgoingMessage)
            self._outgoingMessages.appendleft(outgoingMessage[bytesWritten:])
        else:
            logger.debug("Wrote all %d bytes of %r",
                    bytesWritten, outgoingMessage)

        if len(self._outgoingMessages) == 0:
            self._emit(self.onOutgoingQueueEmpty, "outgoing queue empty")

class IncomingQueuedStream(Stream):
    """Stream wrapper which can read incoming messages into a queue, instead of waiting for a read() call to read from
    the target stream.

    """
    onIncomingQueueEmpty = dispatch.Signal()
    onIncomingMessageQueued = dispatch.Signal(["message", "queueLength"])

    def __init__(self, *args, **kwargs):
        self._incomingMessages = collections.deque()

        super(IncomingQueuedStream, self).__init__(*args, **kwargs)

    def read(self, requestedBytes=-1):
        """Read message from the incoming queue.

        """
        if len(self._incomingMessages) == 0:
            self._emit(self.onIncomingQueueEmpty, "incoming queue empty")
            return None

        try:
            #TODO: Implement requestedBytes support
            if requestedBytes != -1:
                warnings.warn("requestedBytes support not currently implemented.", FutureWarning)
            incomingMessage = self._incomingMessages.popleft()

        except IndexError:
            incomingMessage = None

        if len(self._incomingMessages) == 0:
            self._emit(self.onIncomingQueueEmpty, "incoming queue empty")

        return incomingMessage

    def handleRead(self):
        """Read an incoming message from the socket, and queue it for reading.

        This should be called by a ready-polling class like SelectCommunicator.

        """
        message = self.targetStream.read()
        logger.debug("Read %d bytes: %r", len(message), message)

        self._incomingMessages.append(message)

        self._emit(self.onIncomingMessageQueued, "incoming message queued",
                message=message, queueLength=len(self._outgoingMessages))

class IOQueuedStream(IncomingQueuedStream, OutgoingQueuedStream):
    """Stream wrapper which queues both incoming and outgoing messages.

    This allows messages to be read and written without blocking, provided handleRead() and handleWrite() are called at
    appropriate times.

    """
<|endoftext|>


<commit_before>from django import forms

from core import models

class EditPublication(forms.ModelForm):
    class Meta:
        model = models.Publication
        fields = ('title', 'description', 'state_publication', 'pet')

        widgets = {
            'title': forms.TextInput(),
            'description': forms.TextInput(),
            'pet': forms.HiddenInput(),
        }

class NewPet(forms.Form):

    CATEGORIES = [
        ('dog', 'Perro'),
        ('cat', 'Gato'),
        ('bird', 'Pájaro'),
        ('reptile', 'Reptil'),
        ('other', 'Otro'),
    ]

    name = forms.CharField(label='Nombre')
    type_animal = forms.ChoiceField(label='Tipo', choices=CATEGORIES)
    breed = forms.CharField(label='Raza')
    description = forms.CharField(label='Descripción')
    picture = forms.ImageField(label='Foto', required=False)
    color = forms.MultipleChoiceField(
        choices=models.Color.objects.values_list('id', 'name'), widget=forms.CheckboxSelectMultiple)
    location = forms.CharField(label='Ubicación')

    def execute(self, usuario):
        cleaned_data = self.cleaned_data
        pet = models.Pet.objects.create(
                name=cleaned_data['name'],
                type_animal=cleaned_data['type_animal'],
                breed=cleaned_data['breed'],
                description=cleaned_data['description'],
                location=cleaned_data['location'],
                picture=cleaned_data['picture'],
            )
        pet.color = cleaned_data['color']

        member = usuario
        publication = models.Publication.objects.create(
            title='', description='', state_publication='', pet=pet, member=member)
        return publication

class NewEmail(forms.Form):

    subject = forms.CharField(label='Asunto')
    message = forms.CharField(label='Mensaje', widget=forms.TextInput)
    email = forms.CharField(label='Tu email')
    mobile = forms.CharField(label="Tu telefono")
<commit_msg>formularios desripciones en grande<commit_after>from django import forms

from core import models

class EditPublication(forms.ModelForm):
    class Meta:
        model = models.Publication
        fields = ('title', 'description', 'state_publication', 'pet')

        widgets = {
            'title': forms.TextInput(),
            'description': forms.Textarea(attrs={'rows': 4, 'cols': 40}),
            'pet': forms.HiddenInput(),
        }

class NewPet(forms.Form):

    CATEGORIES = [
        ('dog', 'Perro'),
        ('cat', 'Gato'),
        ('bird', 'Pájaro'),
        ('reptile', 'Reptil'),
        ('other', 'Otro'),
    ]

    name = forms.CharField(label='Nombre')
    type_animal = forms.ChoiceField(label='Tipo', choices=CATEGORIES)
    breed = forms.CharField(label='Raza')
    description = forms.CharField(label='Descripción', widget=forms.Textarea(attrs={'rows': 4, 'cols': 40}))
    picture = forms.ImageField(label='Foto', required=False)
    color = forms.MultipleChoiceField(
        choices=models.Color.objects.values_list('id', 'name'), widget=forms.CheckboxSelectMultiple)
    location = forms.CharField(label='Ubicación')

    def execute(self, usuario):
        cleaned_data = self.cleaned_data
        pet = models.Pet.objects.create(
                name=cleaned_data['name'],
                type_animal=cleaned_data['type_animal'],
                breed=cleaned_data['breed'],
                description=cleaned_data['description'],
                location=cleaned_data['location'],
                picture=cleaned_data['picture'],
            )
        pet.color = cleaned_data['color']

        member = usuario
        publication = models.Publication.objects.create(
            title='', description='', state_publication='', pet=pet, member=member)
        return publication

class NewEmail(forms.Form):

    subject = forms.CharField(label='Asunto')
    message = forms.CharField(label='Mensaje', widget=forms.Textarea(attrs={'rows': 4, 'cols': 40}))
    email = forms.CharField(label='Tu email')
    mobile = forms.CharField(label="Tu telefono")
<|endoftext|>


<commit_before>#
# gdb commands for debugging/inspecting beng-proxy.
#
# Author: Max Kellermann <mk@cm4all.com>
#

import gdb

class DumpHashmapSlot(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_hashmap_slot", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        arg_list = gdb.string_to_argv(arg)
        if len(arg_list) != 2:
            print "usage: bp_dump_hashmap ptr i"
            return

        h = gdb.parse_and_eval(arg_list[0])
        if h.type.code != gdb.lookup_type('struct hashmap').pointer().code:
            print "%s is not a hashmap*" % arg_list[0]
            return

        i = int(arg_list[1])
        slot = h['slots'][i]

        if not slot['pair']['key']:
            print "empty"
            return

        while slot:
            print slot['pair']['key'].string()
            slot = slot['next']

class DumpHashmap(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_hashmap", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        h = gdb.parse_and_eval(arg)
        if h.type.code != gdb.lookup_type('struct hashmap').pointer().code:
            print "%s is not a hashmap*" % arg_list[0]
            return

        n_slots = 0
        n_total = 0
        biggest_slot = 0
        i_biggest_slot = -1

        capacity = h['capacity']
        for i in range(capacity):
            slot = h['slots'][i]
            if slot['pair']['key']:
                s = slot['next']
                n = 1
                while s:
                    n += 1
                    s = s['next']
                if n > biggest_slot:
                    biggest_slot = n
                    i_biggest_slot = i
                if n > 1000:
                    print "big", i, n
                n_total += n
                n_slots += 1
                #print n, slot['pair']['key']
                if n_slots % 256 == 0:
                    print n_slots, n_total, biggest_slot, i_biggest_slot
        print n_slots, n_total, biggest_slot, i_biggest_slot

class DumpHashmap2(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_hashmap2", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        h = gdb.parse_and_eval(arg)
        if h.type.code != gdb.lookup_type('struct hashmap').pointer().code:
            print "%s is not a hashmap*" % arg_list[0]
            return

        string_type = gdb.lookup_type('char').pointer()

        capacity = h['capacity']
        for i in range(capacity):
            slot = h['slots'][i]
            if not slot['pair']['key']:
                continue

            while slot:
                pair = slot['pair']
                print i, pair['key'].string()
                slot = slot['next']

class DumpStrmap(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_strmap", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        h = gdb.parse_and_eval(arg)
        if h.type.code != gdb.lookup_type('struct strmap').pointer().code:
            print "%s is not a strmap*" % arg_list[0]
            return

        h = h['hashmap']

        string_type = gdb.lookup_type('char').pointer()

        capacity = h['capacity']
        for i in range(capacity):
            slot = h['slots'][i]
            if not slot['pair']['key']:
                continue

            while slot:
                pair = slot['pair']
                print pair['key'].string(), '=', pair['value'].cast(string_type).string()
                slot = slot['next']

def pool_size(pool):
    return int(pool['size'])

def pool_sizes(pool):
    netto_size = pool_size(pool)
    if pool['type'] == 1:
        # linear pool
        brutto_size = 0
        area = pool['current_area']['linear']
        while area:
            brutto_size += int(area['size'])
            area = area['prev']
    else:
        brutto_size = netto_size
    return brutto_size, netto_size

def for_each_list_head(head):
    item = head['next']
    head_address = head.address
    while item != head_address:
        yield item
        item = item['next']

def for_each_list_head_reverse(head):
    item = head['prev']
    head_address = head.address
    while item != head_address:
        yield item
        item = item['prev']

def for_each_list_item_reverse(head, cast):
    item = head['prev']
    head_address = head.address
    while item != head_address:
        yield item.cast(cast)
        item = item['prev']

def pool_recursive_sizes(pool):
    pool_pointer = gdb.lookup_type('struct pool').pointer()

    brutto_size, netto_size = pool_sizes(pool)
    for child in for_each_list_head(pool['children']):
        child_brutto_size, child_netto_size = pool_recursive_sizes(child.cast(pool_pointer))
        brutto_size += child_brutto_size
        netto_size += child_netto_size

    return brutto_size, netto_size

class DumpPoolStats(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_pool_stats", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def _dump_refs(self, head):
        pass

    def invoke(self, arg, from_tty):
        pool = gdb.parse_and_eval(arg)
        if pool.type.code != gdb.lookup_type('struct pool').pointer().code:
            print "%s is not a strmap*" % arg_list[0]
            return

        print "pool '%s' type=%d" % (pool['name'].string(), pool['type'])
        #print "refs:"
        #self._dump_refs(pool['refs'])
        #print "unrefs:"
        #self._dump_refs(pool['unrefs'])
        print "size", pool_sizes(pool)
        print "recursive_size", pool_recursive_sizes(pool)

class DumpPoolAllocations(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_pool_allocations", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        pool = gdb.parse_and_eval(arg)
        if pool.type.code != gdb.lookup_type('struct pool').pointer().code:
            print "%s is not a strmap*" % arg_list[0]
            return

        allocation_pointer = gdb.lookup_type('struct allocation_info').pointer()
        for a in for_each_list_item_reverse(pool['allocations'], allocation_pointer):
            print '%8u %s:%u' % (a['size'], a['file'].string().replace('../', ''), a['line'])

class DumpPools(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_pools", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        h = gdb.parse_and_eval(arg)
        if h.type.code != gdb.lookup_type('struct pool').pointer().code:
            print "%s is not a strmap*" % arg_list[0]
            return

        h = h['hashmap']

        string_type = gdb.lookup_type('char').pointer()

        capacity = h['capacity']
        for i in range(capacity):
            slot = h['slots'][i]
            if not slot['pair']['key']:
                continue

            while slot:
                pair = slot['pair']
                print pair['key'].string(), '=', pair['value'].cast(string_type).string()
                slot = slot['next']

DumpHashmapSlot()
DumpHashmap()
DumpHashmap2()
DumpStrmap()
DumpPoolStats()
DumpPoolAllocations()
DumpPools()
<commit_msg>gdb: add command "bp_dump_pool_refs"<commit_after>#
# gdb commands for debugging/inspecting beng-proxy.
#
# Author: Max Kellermann <mk@cm4all.com>
#

import gdb

class DumpHashmapSlot(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_hashmap_slot", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        arg_list = gdb.string_to_argv(arg)
        if len(arg_list) != 2:
            print "usage: bp_dump_hashmap ptr i"
            return

        h = gdb.parse_and_eval(arg_list[0])
        if h.type.code != gdb.lookup_type('struct hashmap').pointer().code:
            print "%s is not a hashmap*" % arg_list[0]
            return

        i = int(arg_list[1])
        slot = h['slots'][i]

        if not slot['pair']['key']:
            print "empty"
            return

        while slot:
            print slot['pair']['key'].string()
            slot = slot['next']

class DumpHashmap(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_hashmap", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        h = gdb.parse_and_eval(arg)
        if h.type.code != gdb.lookup_type('struct hashmap').pointer().code:
            print "%s is not a hashmap*" % arg_list[0]
            return

        n_slots = 0
        n_total = 0
        biggest_slot = 0
        i_biggest_slot = -1

        capacity = h['capacity']
        for i in range(capacity):
            slot = h['slots'][i]
            if slot['pair']['key']:
                s = slot['next']
                n = 1
                while s:
                    n += 1
                    s = s['next']
                if n > biggest_slot:
                    biggest_slot = n
                    i_biggest_slot = i
                if n > 1000:
                    print "big", i, n
                n_total += n
                n_slots += 1
                #print n, slot['pair']['key']
                if n_slots % 256 == 0:
                    print n_slots, n_total, biggest_slot, i_biggest_slot
        print n_slots, n_total, biggest_slot, i_biggest_slot

class DumpHashmap2(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_hashmap2", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        h = gdb.parse_and_eval(arg)
        if h.type.code != gdb.lookup_type('struct hashmap').pointer().code:
            print "%s is not a hashmap*" % arg_list[0]
            return

        string_type = gdb.lookup_type('char').pointer()

        capacity = h['capacity']
        for i in range(capacity):
            slot = h['slots'][i]
            if not slot['pair']['key']:
                continue

            while slot:
                pair = slot['pair']
                print i, pair['key'].string()
                slot = slot['next']

class DumpStrmap(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_strmap", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        h = gdb.parse_and_eval(arg)
        if h.type.code != gdb.lookup_type('struct strmap').pointer().code:
            print "%s is not a strmap*" % arg_list[0]
            return

        h = h['hashmap']

        string_type = gdb.lookup_type('char').pointer()

        capacity = h['capacity']
        for i in range(capacity):
            slot = h['slots'][i]
            if not slot['pair']['key']:
                continue

            while slot:
                pair = slot['pair']
                print pair['key'].string(), '=', pair['value'].cast(string_type).string()
                slot = slot['next']

def pool_size(pool):
    return int(pool['size'])

def pool_sizes(pool):
    netto_size = pool_size(pool)
    if pool['type'] == 1:
        # linear pool
        brutto_size = 0
        area = pool['current_area']['linear']
        while area:
            brutto_size += int(area['size'])
            area = area['prev']
    else:
        brutto_size = netto_size
    return brutto_size, netto_size

def for_each_list_head(head):
    item = head['next']
    head_address = head.address
    while item != head_address:
        yield item
        item = item['next']

def for_each_list_head_reverse(head):
    item = head['prev']
    head_address = head.address
    while item != head_address:
        yield item
        item = item['prev']

def for_each_list_item_reverse(head, cast):
    item = head['prev']
    head_address = head.address
    while item != head_address:
        yield item.cast(cast)
        item = item['prev']

def pool_recursive_sizes(pool):
    pool_pointer = gdb.lookup_type('struct pool').pointer()

    brutto_size, netto_size = pool_sizes(pool)
    for child in for_each_list_head(pool['children']):
        child_brutto_size, child_netto_size = pool_recursive_sizes(child.cast(pool_pointer))
        brutto_size += child_brutto_size
        netto_size += child_netto_size

    return brutto_size, netto_size

class DumpPoolStats(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_pool_stats", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        pool = gdb.parse_and_eval(arg)
        if pool.type.code != gdb.lookup_type('struct pool').pointer().code:
            print "%s is not a strmap*" % arg_list[0]
            return

        print "pool '%s' type=%d" % (pool['name'].string(), pool['type'])
        print "size", pool_sizes(pool)
        print "recursive_size", pool_recursive_sizes(pool)

class DumpPoolRefs(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_pool_refs", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def _dump_refs(self, pool, label, head):
        print "pool '%s' %s:" % (pool['name'].string(), label)

        ref_pointer = gdb.lookup_type('struct pool_ref').pointer()
        for r in for_each_list_item_reverse(head, ref_pointer):
            print '%4u %s:%u' % (r['count'], r['file'].string().replace('../', ''), r['line'])

    def invoke(self, arg, from_tty):
        pool = gdb.parse_and_eval(arg)
        if pool.type.code != gdb.lookup_type('struct pool').pointer().code:
            print "%s is not a pool*" % arg_list[0]
            return

        for i in ('refs', 'unrefs'):
            self._dump_refs(pool, i, pool[i])

class DumpPoolAllocations(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_pool_allocations", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        pool = gdb.parse_and_eval(arg)
        if pool.type.code != gdb.lookup_type('struct pool').pointer().code:
            print "%s is not a pool*" % arg_list[0]
            return

        allocation_pointer = gdb.lookup_type('struct allocation_info').pointer()
        for a in for_each_list_item_reverse(pool['allocations'], allocation_pointer):
            print '%8u %s:%u' % (a['size'], a['file'].string().replace('../', ''), a['line'])

class DumpPools(gdb.Command):
    def __init__(self):
        gdb.Command.__init__(self, "bp_dump_pools", gdb.COMMAND_DATA, gdb.COMPLETE_SYMBOL, True)

    def invoke(self, arg, from_tty):
        h = gdb.parse_and_eval(arg)
        if h.type.code != gdb.lookup_type('struct pool').pointer().code:
            print "%s is not a strmap*" % arg_list[0]
            return

        h = h['hashmap']

        string_type = gdb.lookup_type('char').pointer()

        capacity = h['capacity']
        for i in range(capacity):
            slot = h['slots'][i]
            if not slot['pair']['key']:
                continue

            while slot:
                pair = slot['pair']
                print pair['key'].string(), '=', pair['value'].cast(string_type).string()
                slot = slot['next']

DumpHashmapSlot()
DumpHashmap()
DumpHashmap2()
DumpStrmap()
DumpPoolStats()
DumpPoolRefs()
DumpPoolAllocations()
DumpPools()
<|endoftext|>