<commit_before>from .enrichment_service import EnrichmentService, prepare_variables, AGGREGATION_DEFAULT, AGGREGATION_NONE

class Enrichment(EnrichmentService):
    """This is the main class to enrich your own data with data from the
    `Data Observatory <https://carto.com/platform/location-data-streams/>`__

    To be able to use the `Enrichment` functions you need A CARTO account with Data Observatory v2 enabled. Contact
    us at support@carto.com for more information about this.

    Please, see the :obj:`Catalog` discovery and subscription guides, to understand how to explore the Data Observatory
    repository and subscribe to premium datasets to be used in your enrichment workflows.

    Args:
        credentials (:py:class:`Credentials <cartoframes.auth.Credentials>`, optional):
            credentials of user account. If not provided,
            a default credentials (if set with :py:meth:`set_default_credentials
            <cartoframes.auth.set_default_credentials>`) will attempted to be
            used.
    """

    def __init__(self, credentials=None):
        super(Enrichment, self).__init__(credentials)

    def enrich_points(self, dataframe, variables, geom_column='geometry', filters={}):
        """Enrich your points `DataFrame` with columns (:obj:`Variable`) from one or more :obj:`Dataset`
        in the Data Observatory, intersecting the points in the source `DataFrame` with the geographies in the
        Data Observatory.

        Extra columns as `area` and `population` will be provided in the resulting `DataFrame` for normalization
        purposes.

        Args:
            dataframe (pandas `DataFrame`, geopandas `GeoDataFrame`
                or :py:class:`CartoDataFrame <cartoframes.CartoDataFrame>`): a `DataFrame` instance to be enriched.
            variables (:py:class:`Variable <cartoframes.data.observatory.Variable>`, list, str):
                variable ID, slug or :obj:`Variable` instance or list of variable IDs, slugs
                or :obj:`Variable` instances taken from the Data Observatory :obj:`Catalog`.
            geom_column (str): string indicating the 4326 geometry column name in the
                source `DataFrame`. Defaults to `geometry`.
            filters (list, optional): list of :obj:`VariableFilter` to filter rows from
                the enrichment data. Example: `[VariableFilter(variable1, "= 'a string'")]`

        Returns:
            A :py:class:`CartoDataFrame <cartoframes.CartoDataFrame>` enriched with the variables passed as argument.

        *Note that if the points of the `dataframe` you provide are contained in more than one geometry
        in the enrichment dataset, the number of rows of the returned `CartoDataFrame` could be different
        than the `dataframe` argument number of rows.*

        Examples:

            Enrich a points `DataFrame` with Catalog classes:

            .. code::

                import pandas
                from cartoframes.auth import set_default_credentials
                from cartoframes.data.observatory import Enrichment, Catalog

                set_default_credentials()

                df = pandas.read_csv('...')

                catalog = Catalog()
                variables = catalog.country('usa').category('demographics').datasets[0].variables

                enrichment = Enrichment()
                cdf_enrich = enrichment.enrich_points(df, variables)

            Enrich a points dataframe with several Variables using their ids:

            .. code::

                import pandas
                from cartoframes.auth import set_default_credentials
                from cartoframes.data.observatory import Enrichment, Catalog

                set_default_credentials()

                df = pandas.read_csv('...')

                catalog = Catalog()
                all_variables = catalog.country('usa').category('demographics').datasets[0].variables
                variable1 = all_variables[0]
                variable2 = all_variables[1]
                variables = [
                    variable1.id,
                    variable2.id
                ]

                enrichment = Enrichment()
                cdf_enrich = enrichment.enrich_points(df, variables)

            Enrich a points dataframe with filters:

            .. code::

                import pandas
                from cartoframes.auth import set_default_credentials
                from cartoframes.data.observatory import Enrichment, Catalog, VariableFilter

                set_default_credentials()

                df = pandas.read_csv('...')

                catalog = Catalog()
                variable = catalog.country('usa').category('demographics').datasets[0].variables[0]
                filter = VariableFilter(variable, "= '2019-09-01'")

                enrichment = Enrichment()
                cdf_enrich = enrichment.enrich_points(df, variables=[variable], filters=[filter])
        """
        variables = prepare_variables(variables)
        cartodataframe = self._prepare_data(dataframe, geom_column)

        temp_table_name = self._get_temp_table_name()
        self._upload_data(temp_table_name, cartodataframe)

        queries = self._get_points_enrichment_sql(temp_table_name, variables, filters)
        return self._execute_enrichment(queries, cartodataframe)

    AGGREGATION_DEFAULT = AGGREGATION_DEFAULT
    """Use default aggregation method for polygons enrichment. More info in :py:attr:`Enrichment.enrich_polygons`"""

    AGGREGATION_NONE = AGGREGATION_NONE
    """Do not aggregate data in polygons enrichment. More info in :py:attr:`Enrichment.enrich_polygons`"""

    def enrich_polygons(self, dataframe, variables, geom_column='geometry', filters=[],
                        aggregation=AGGREGATION_DEFAULT):
        """Enrich your polygons `DataFrame` with columns (:obj:`Variable`) from one or more :obj:`Dataset` in
        the Data Observatory by intersecting the polygons in the source `DataFrame` with geographies in the
        Data Observatory.

        When a polygon intersects with multiple geographies, the proportional part of the intersection will be used
        to interpolate the quantity of the polygon value intersected, aggregating them. Most of :obj:`Variable`
        instances have a :py:attr:`Variable.agg_method` property what is used by default as aggregation function, but
        you can overwrite it using the `aggregation` parameter (not even doing the aggregation). If a variable does not
        have the `agg_method` property set and you do not overwrite it either (with the `aggregation` parameter), the
        variable column will be skipped from the enrichment.

        Args:
            dataframe (pandas `DataFrame`, geopandas `GeoDataFrame`
                or :py:class:`CartoDataFrame <cartoframes.CartoDataFrame>`): a `DataFrame` instance to be enriched.
            variables (:py:class:`Variable <cartoframes.data.observatory.Variable>`, list, str):
                variable ID, slug or :obj:`Variable` instance or list of variable IDs, slugs
                or :obj:`Variable` instances taken from the Data Observatory :obj:`Catalog`.
            geom_column (str): string indicating the 4326 geometry column name in the source `DataFrame`.
                Defaults to `geometry`.
            filters (list, optional): list of :obj:`VariableFilter` to filter rows from
                the enrichment data. Example: `[VariableFilter(variable1, "= 'a string'")]`
            aggregation (str, list, optional): sets the data aggregation. The polygons in the source `DataFrame` can
                intersect with one or more polygons from the Data Observatory. With this method you can select how to
                aggregate the resulting data.

                An aggregation method can be one of these values: 'MIN', 'MAX', 'SUM', 'AVG', 'COUNT',
                'ARRAY_AGG', 'ARRAY_CONCAT_AGG', 'STRING_AGG' but check this
                `documentation <https://cloud.google.com/bigquery/docs/reference/standard-sql/aggregate_functions>`__
                for a complete list of aggregate functions.

                The options are:
                    - :py:attr:`Enrichment.AGGREGATION_DEFAULT` (default): Every :obj:`Variable` has a default
                    aggregation method in the :py:attr:`Variable.agg_method` property and it will be used to aggregate
                    the data (a variable could not have `agg_method` defined and in this case, the variables will be
                    skipped).
                    - :py:attr:`Enrichment.AGGREGATION_NONE`: use this option to do the aggregation locally by yourself.
                    you will receive an array with all the data from each polygon instersected.
                    - str: if you want to overwrite every default aggregation method, you can pass a string with the
                    aggregation method to use.
                    - dictionary: if you want to overwrite some default aggregation methods from your selected
                    variables, use a dict as
                    :py:attr:`Variable.id`: aggregation method pairs,
                    for example: `{variable1.id: 'SUM', variable3.id: 'AVG'}`.

        Returns:
            A :py:class:`CartoDataFrame <cartoframes.CartoDataFrame>` enriched with the variables passed as argument.

        *Note that if the geometry of the `dataframe` you provide intersects with more than one geometry
        in the enrichment dataset, the number of rows of the returned `CartoDataFrame` could be different
        than the `dataframe` argument number of rows.*

        Examples:

            Enrich a polygons dataframe with one Variable:

            .. code::

                import pandas
                from cartoframes.auth import set_default_credentials
                from cartoframes.data.observatory import Enrichment, Catalog

                set_default_credentials()

                df = pandas.read_csv('...')

                catalog = Catalog()
                variable = catalog.country('usa').category('demographics').datasets[0].variables[0]
                variables = [variable]

                enrichment = Enrichment()
                cdf_enrich = enrichment.enrich_polygons(df, variables)

            Enrich a polygons dataframe with all Variables from a Catalog Dataset:

            .. code::

                import pandas
                from cartoframes.auth import set_default_credentials
                from cartoframes.data.observatory import Enrichment, Catalog

                set_default_credentials()

                df = pandas.read_csv('...')

                catalog = Catalog()
                variables = catalog.country('usa').category('demographics').datasets[0].variables

                enrichment = Enrichment()
                cdf_enrich = enrichment.enrich_polygons(df, variables)

            Enrich a polygons dataframe with several Variables using their ids:

            .. code::

                import pandas
                from cartoframes.auth import set_default_credentials
                from cartoframes.data.observatory import Enrichment, Catalog

                set_default_credentials()

                df = pandas.read_csv('...')

                catalog = Catalog()
                all_variables = catalog.country('usa').category('demographics').datasets[0].variables
                variable1 = all_variables[0]
                variable2 = all_variables[1]
                variables = [
                    variable1.id,
                    variable2.id
                ]

                enrichment = Enrichment()
                cdf_enrich = enrichment.enrich_polygons(df, variables)

            Enrich a polygons dataframe with filters:

            .. code::

                import pandas
                from cartoframes.data.observatory import Enrichment, Catalog, VariableFilter
                from cartoframes.auth import set_default_credentials, Credentials

                set_default_credentials()

                df = pandas.read_csv('...')

                catalog = Catalog()
                variable = catalog.country('usa').category('demographics').datasets[0].variables[0]
                filter = VariableFilter(variable, "= '2019-09-01'")

                enrichment = Enrichment()
                cdf_enrich = enrichment.enrich_polygons(df, variables=[variable], filters=[filter])

            Enrich a polygons dataframe overwriting every variables aggregation methods to use `SUM` function:

            .. code::

                import pandas
                from cartoframes.data.observatory import Enrichment, Catalog
                from cartoframes.auth import set_default_credentials, Credentials

                set_default_credentials()

                df = pandas.read_csv('...')

                catalog = Catalog()
                all_variables = catalog.country('usa').category('demographics').datasets[0].variables
                variable1 = all_variables[0] // variable1.agg_method is 'AVG' but you want 'SUM'
                variable2 = all_variables[1] // variable2.agg_method is 'AVG' and it is what you want
                variable3 = all_variables[2] // variable3.agg_method is 'SUM' but you want 'AVG'

                variables = [variable1, variable2, variable3]

                enrichment = Enrichment()
                cdf_enrich = enrichment.enrich_polygons(df, variables, aggregation='SUM')

            Enrich a polygons dataframe overwriting some of the variables aggregation methods:

            .. code::

                import pandas
                from cartoframes.data.observatory import Enrichment, Catalog
                from cartoframes.auth import set_default_credentials, Credentials

                set_default_credentials()

                df = pandas.read_csv('...')

                catalog = Catalog()
                all_variables = catalog.country('usa').category('demographics').datasets[0].variables
                variable1 = all_variables[0] // variable1.agg_method is 'AVG' but you want 'SUM'
                variable2 = all_variables[1] // variable2.agg_method is 'AVG' and it is what you want
                variable3 = all_variables[2] // variable3.agg_method is 'SUM' but you want 'AVG'

                variables = [variable1, variable2, variable3]

                aggregation = {
                    variable1.id: 'SUM',
                    variable3.id: 'AVG'
                }

                enrichment = Enrichment()
                cdf_enrich = enrichment.enrich_polygons(df, variables, aggregation=aggregation)
        """

        variables = prepare_variables(variables, aggregation)
        cartodataframe = self._prepare_data(dataframe, geom_column)

        temp_table_name = self._get_temp_table_name()
        self._upload_data(temp_table_name, cartodataframe)

        queries = self._get_polygon_enrichment_sql(temp_table_name, variables, filters, aggregation)
        return self._execute_enrichment(queries, cartodataframe)
<commit_msg>docs detail<commit_after>from .enrichment_service import EnrichmentService, prepare_variables, AGGREGATION_DEFAULT, AGGREGATION_NONE

class Enrichment(EnrichmentService):
    """This is the main class to enrich your own data with data from the
    `Data Observatory <https://carto.com/platform/location-data-streams/>`__

    To be able to use the `Enrichment` functions you need A CARTO account with Data Observatory v2 enabled. Contact
    us at support@carto.com for more information about this.

    Please, see the :obj:`Catalog` discovery and subscription guides, to understand how to explore the Data Observatory
    repository and subscribe to premium datasets to be used in your enrichment workflows.

    Args:
        credentials (:py:class:`Credentials <cartoframes.auth.Credentials>`, optional):
            credentials of user account. If not provided,
            a default credentials (if set with :py:meth:`set_default_credentials
            <cartoframes.auth.set_default_credentials>`) will attempted to be
            used.
    """

    def __init__(self, credentials=None):
        super(Enrichment, self).__init__(credentials)

    def enrich_points(self, dataframe, variables, geom_column='geometry', filters={}):
        """Enrich your points `DataFrame` with columns (:obj:`Variable`) from one or more :obj:`Dataset`
        in the Data Observatory, intersecting the points in the source `DataFrame` with the geographies in the
        Data Observatory.

        Extra columns as `area` and `population` will be provided in the resulting `DataFrame` for normalization
        purposes.

        Args:
            dataframe (pandas `DataFrame`, geopandas `GeoDataFrame`
                or :py:class:`CartoDataFrame <cartoframes.CartoDataFrame>`): a `DataFrame` instance to be enriched.
            variables (:py:class:`Variable <cartoframes.data.observatory.Variable>`, list, str):
                variable ID, slug or :obj:`Variable` instance or list of variable IDs, slugs
                or :obj:`Variable` instances taken from the Data Observatory :obj:`Catalog`.
            geom_column (str): string indicating the 4326 geometry column name in the
                source `DataFrame`. Defaults to `geometry`.
            filters (list, optional): list of :obj:`VariableFilter` to filter rows from
                the enrichment data. Example: `[VariableFilter(variable1, "= 'a string'")]`

        Returns:
            A :py:class:`CartoDataFrame <cartoframes.CartoDataFrame>` enriched with the variables passed as argument.

        *Note that if the points of the `dataframe` you provide are contained in more than one geometry
        in the enrichment dataset, the number of rows of the returned `CartoDataFrame` could be different
        than the `dataframe` argument number of rows.*

        Examples:

            Enrich a points `DataFrame` with Catalog classes:

            .. code::

                import pandas
                from cartoframes.auth import set_default_credentials
                from cartoframes.data.observatory import Enrichment, Catalog

                set_default_credentials()

                df = pandas.read_csv('...')

                catalog = Catalog()
                variables = catalog.country('usa').category('demographics').datasets[0].variables

                enrichment = Enrichment()
                cdf_enrich = enrichment.enrich_points(df, variables)

            Enrich a points dataframe with several Variables using their ids:

            .. code::

                import pandas
                from cartoframes.auth import set_default_credentials
                from cartoframes.data.observatory import Enrichment, Catalog

                set_default_credentials()

                df = pandas.read_csv('...')

                catalog = Catalog()
                all_variables = catalog.country('usa').category('demographics').datasets[0].variables
                variable1 = all_variables[0]
                variable2 = all_variables[1]
                variables = [
                    variable1.id,
                    variable2.id
                ]

                enrichment = Enrichment()
                cdf_enrich = enrichment.enrich_points(df, variables)

            Enrich a points dataframe with filters:

            .. code::

                import pandas
                from cartoframes.auth import set_default_credentials
                from cartoframes.data.observatory import Enrichment, Catalog, VariableFilter

                set_default_credentials()

                df = pandas.read_csv('...')

                catalog = Catalog()
                variable = catalog.country('usa').category('demographics').datasets[0].variables[0]
                filter = VariableFilter(variable, "= '2019-09-01'")

                enrichment = Enrichment()
                cdf_enrich = enrichment.enrich_points(df, variables=[variable], filters=[filter])
        """
        variables = prepare_variables(variables)
        cartodataframe = self._prepare_data(dataframe, geom_column)

        temp_table_name = self._get_temp_table_name()
        self._upload_data(temp_table_name, cartodataframe)

        queries = self._get_points_enrichment_sql(temp_table_name, variables, filters)
        return self._execute_enrichment(queries, cartodataframe)

    AGGREGATION_DEFAULT = AGGREGATION_DEFAULT
    """Use default aggregation method for polygons enrichment. More info in :py:attr:`Enrichment.enrich_polygons`"""

    AGGREGATION_NONE = AGGREGATION_NONE
    """Do not aggregate data in polygons enrichment. More info in :py:attr:`Enrichment.enrich_polygons`"""

    def enrich_polygons(self, dataframe, variables, geom_column='geometry', filters=[],
                        aggregation=AGGREGATION_DEFAULT):
        """Enrich your polygons `DataFrame` with columns (:obj:`Variable`) from one or more :obj:`Dataset` in
        the Data Observatory by intersecting the polygons in the source `DataFrame` with geographies in the
        Data Observatory.

        When a polygon intersects with multiple geographies, the proportional part of the intersection will be used
        to interpolate the quantity of the polygon value intersected, aggregating them. Most of :obj:`Variable`
        instances have a :py:attr:`Variable.agg_method` property what is used by default as aggregation function, but
        you can overwrite it using the `aggregation` parameter (not even doing the aggregation). If a variable does not
        have the `agg_method` property set and you do not overwrite it either (with the `aggregation` parameter), the
        variable column will be skipped from the enrichment.

        Args:
            dataframe (pandas `DataFrame`, geopandas `GeoDataFrame`
                or :py:class:`CartoDataFrame <cartoframes.CartoDataFrame>`): a `DataFrame` instance to be enriched.
            variables (:py:class:`Variable <cartoframes.data.observatory.Variable>`, list, str):
                variable ID, slug or :obj:`Variable` instance or list of variable IDs, slugs
                or :obj:`Variable` instances taken from the Data Observatory :obj:`Catalog`.
            geom_column (str): string indicating the 4326 geometry column name in the source `DataFrame`.
                Defaults to `geometry`.
            filters (list, optional): list of :obj:`VariableFilter` to filter rows from
                the enrichment data. Example: `[VariableFilter(variable1, "= 'a string'")]`
            aggregation (str, list, optional): sets the data aggregation. The polygons in the source `DataFrame` can
                intersect with one or more polygons from the Data Observatory. With this method you can select how to
                aggregate the resulting data.

                An aggregation method can be one of these values: 'MIN', 'MAX', 'SUM', 'AVG', 'COUNT',
                'ARRAY_AGG', 'ARRAY_CONCAT_AGG', 'STRING_AGG' but check this
                `documentation <https://cloud.google.com/bigquery/docs/reference/standard-sql/aggregate_functions>`__
                for a complete list of aggregate functions.

                The options are:
                    - :py:attr:`Enrichment.AGGREGATION_DEFAULT` (default): Every :obj:`Variable` has a default
                    aggregation method in the :py:attr:`Variable.agg_method` property and it will be used to aggregate
                    the data (a variable could not have `agg_method` defined and in this case, the variables will be
                    skipped).
                    - :py:attr:`Enrichment.AGGREGATION_NONE`: use this option to do the aggregation locally by yourself.
                    You will receive a row of data from each polygon instersected.
                    - str: if you want to overwrite every default aggregation method, you can pass a string with the
                    aggregation method to use.
                    - dictionary: if you want to overwrite some default aggregation methods from your selected
                    variables, use a dict as :py:attr:`Variable.id`: aggregation method pairs, for example:
                    `{variable1.id: 'SUM', variable3.id: 'AVG'}`.

        Returns:
            A :py:class:`CartoDataFrame <cartoframes.CartoDataFrame>` enriched with the variables passed as argument.

        *Note that if the geometry of the `dataframe` you provide intersects with more than one geometry
        in the enrichment dataset, the number of rows of the returned `CartoDataFrame` could be different
        than the `dataframe` argument number of rows.*

        Examples:

            Enrich a polygons dataframe with one Variable:

            .. code::

                import pandas
                from cartoframes.auth import set_default_credentials
                from cartoframes.data.observatory import Enrichment, Catalog

                set_default_credentials()

                df = pandas.read_csv('...')

                catalog = Catalog()
                variable = catalog.country('usa').category('demographics').datasets[0].variables[0]
                variables = [variable]

                enrichment = Enrichment()
                cdf_enrich = enrichment.enrich_polygons(df, variables)

            Enrich a polygons dataframe with all Variables from a Catalog Dataset:

            .. code::

                import pandas
                from cartoframes.auth import set_default_credentials
                from cartoframes.data.observatory import Enrichment, Catalog

                set_default_credentials()

                df = pandas.read_csv('...')

                catalog = Catalog()
                variables = catalog.country('usa').category('demographics').datasets[0].variables

                enrichment = Enrichment()
                cdf_enrich = enrichment.enrich_polygons(df, variables)

            Enrich a polygons dataframe with several Variables using their ids:

            .. code::

                import pandas
                from cartoframes.auth import set_default_credentials
                from cartoframes.data.observatory import Enrichment, Catalog

                set_default_credentials()

                df = pandas.read_csv('...')

                catalog = Catalog()
                all_variables = catalog.country('usa').category('demographics').datasets[0].variables
                variable1 = all_variables[0]
                variable2 = all_variables[1]
                variables = [
                    variable1.id,
                    variable2.id
                ]

                enrichment = Enrichment()
                cdf_enrich = enrichment.enrich_polygons(df, variables)

            Enrich a polygons dataframe with filters:

            .. code::

                import pandas
                from cartoframes.data.observatory import Enrichment, Catalog, VariableFilter
                from cartoframes.auth import set_default_credentials, Credentials

                set_default_credentials()

                df = pandas.read_csv('...')

                catalog = Catalog()
                variable = catalog.country('usa').category('demographics').datasets[0].variables[0]
                filter = VariableFilter(variable, "= '2019-09-01'")

                enrichment = Enrichment()
                cdf_enrich = enrichment.enrich_polygons(df, variables=[variable], filters=[filter])

            Enrich a polygons dataframe overwriting every variables aggregation methods to use `SUM` function:

            .. code::

                import pandas
                from cartoframes.data.observatory import Enrichment, Catalog
                from cartoframes.auth import set_default_credentials, Credentials

                set_default_credentials()

                df = pandas.read_csv('...')

                catalog = Catalog()
                all_variables = catalog.country('usa').category('demographics').datasets[0].variables
                variable1 = all_variables[0] // variable1.agg_method is 'AVG' but you want 'SUM'
                variable2 = all_variables[1] // variable2.agg_method is 'AVG' and it is what you want
                variable3 = all_variables[2] // variable3.agg_method is 'SUM' but you want 'AVG'

                variables = [variable1, variable2, variable3]

                enrichment = Enrichment()
                cdf_enrich = enrichment.enrich_polygons(df, variables, aggregation='SUM')

            Enrich a polygons dataframe overwriting some of the variables aggregation methods:

            .. code::

                import pandas
                from cartoframes.data.observatory import Enrichment, Catalog
                from cartoframes.auth import set_default_credentials, Credentials

                set_default_credentials()

                df = pandas.read_csv('...')

                catalog = Catalog()
                all_variables = catalog.country('usa').category('demographics').datasets[0].variables
                variable1 = all_variables[0] // variable1.agg_method is 'AVG' but you want 'SUM'
                variable2 = all_variables[1] // variable2.agg_method is 'AVG' and it is what you want
                variable3 = all_variables[2] // variable3.agg_method is 'SUM' but you want 'AVG'

                variables = [variable1, variable2, variable3]

                aggregation = {
                    variable1.id: 'SUM',
                    variable3.id: 'AVG'
                }

                enrichment = Enrichment()
                cdf_enrich = enrichment.enrich_polygons(df, variables, aggregation=aggregation)
        """

        variables = prepare_variables(variables, aggregation)
        cartodataframe = self._prepare_data(dataframe, geom_column)

        temp_table_name = self._get_temp_table_name()
        self._upload_data(temp_table_name, cartodataframe)

        queries = self._get_polygon_enrichment_sql(temp_table_name, variables, filters, aggregation)
        return self._execute_enrichment(queries, cartodataframe)
<|endoftext|>


<commit_before># This file is part of Archivematica.
#
# Copyright 2010-2013 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

from lxml import etree
import os
import sys

from django.core.urlresolvers import reverse
from django.contrib import messages
from django.shortcuts import redirect
from django.shortcuts import render

from main import models
from components import helpers
from components.helpers import hidden_features

path = "/usr/lib/archivematica/archivematicaCommon"
if path not in sys.path:
    sys.path.append(path)
import storageService as storage_service

class PreconfiguredChoices:
    xml     = None
    choices = None

    def __init__(self):
        self.xml = etree.Element('processingMCP')
        self.choices = etree.Element('preconfiguredChoices')

    def add_choice(self, applies_to_text, go_to_chain_text, delay_duration=None):
        choice = etree.Element('preconfiguredChoice')

        applies_to = etree.Element('appliesTo')
        applies_to.text = applies_to_text
        choice.append(applies_to)

        go_to_chain = etree.Element('goToChain')
        go_to_chain.text = go_to_chain_text
        choice.append(go_to_chain)

        if delay_duration != None:
            delay = etree.Element('delay', unitCtime='yes')
            delay.text = delay_duration
            choice.append(delay)

        self.choices.append(choice)

    def write_to_file(self, file_path):
        self.xml.append(self.choices)
        file = open(file_path, 'w')
        file.write(etree.tostring(self.xml, pretty_print=True))

def index(request):
    file_path = os.path.join(
        helpers.get_server_config_value('sharedDirectory'),
        'sharedMicroServiceTasksConfigs/processingMCPConfigs/defaultProcessingMCP.xml'
    )

    # Lists of dicts declare what options to display, and where to look for
    # the options
    # Name: name in HTML
    # Label: text to display, <label> in HTML, required.  Fallback value for
    #   applies_to, lookup_description
    boolean_select_fields = [
        {
            "name":         "quarantine_transfer",
            "choice_uuid":  "755b4177-c587-41a7-8c52-015277568302", # Workflow decision - send transfer to quarantine
            "label":        "Send transfer to quarantine",
            "yes_option":   "97ea7702-e4d5-48bc-b4b5-d15d897806ab", # Quarantine
            "no_option":    "d4404ab1-dc7f-4e9e-b1f8-aa861e766b8e" # Skip quarantine
        },
        {
            "name":         "normalize_transfer",
            "choice_uuid":  "de909a42-c5b5-46e1-9985-c031b50e9d30",
            "label":        "Approve normalization",
            "yes_option":   "1e0df175-d56d-450d-8bee-7df1dc7ae815", # Approve
            "action":       "Approve"
        },
        {
            "name":         "store_aip",
            "choice_uuid":  "2d32235c-02d4-4686-88a6-96f4d6c7b1c3",
            "label":        "Store AIP",
            "yes_option":   "9efab23c-31dc-4cbd-a39d-bb1665460cbe", # Store AIP
            "action":       "Store AIP"
        }
    ]

    # 'label': text to display, <label> in HTML.
    # 'link_uuid': If there are conflicts on lookup_description, specify this
    #   to use that MicroServiceChainLink
    chain_choice_fields = [
        {
            "name":  "create_sip",
            "label": "Create SIP(s)",
            "choice_uuid": "bb194013-597c-4e4a-8493-b36d190f8717"
        },
        {
            "name":  "extract_packages",
            "label": "Extract packages",
            "choice_uuid": "dec97e3c-5598-4b99-b26e-f87a435a6b7f",
            "uuid": "01d80b27-4ad1-4bd1-8f8d-f819f18bf685"
        },
        {
            "name":  "normalize",
            "label": "Normalize",
            "choice_uuid": "cb8e5706-e73f-472f-ad9b-d1236af8095f",
        }
    ]

    populate_select_fields_with_chain_choice_options(chain_choice_fields)

    # 'label': text to display, <label> in HTML.  Also, the
    # 'lookup_description': TasksConfig description to search against to find
    #   the MicroServiceChainLink for the
    #   MicroServiceChainChoice.choiceavailableatlink  If not specified, use label
    # 'link_uuid': If there are conflicts on lookup_description, specify this
    #   to use that MicroServiceChainLink
    # 'applies_to': Description of the TasksConfig that the choice applies to.
    #   Put in defaultProcessingMCP.xml
    replace_dict_fields = [
        {
            "name": "select_format_id_tool_transfer",
            "label": "Select file format identification command (Transfer)",
            "choice_uuid": 'f09847c2-ee51-429a-9478-a860477f6b8d'
        },
        {
            "name": "select_format_id_tool_ingest",
            "label": "Select file format identification command (Ingest)",
            "choice_uuid": '7a024896-c4f7-4808-a240-44c87c762bc5'
        },
        {
            "name":  "delete_packages",
            "label": "Delete packages after extraction",
            "choice_uuid": "f19926dd-8fb5-4c79-8ade-c83f61f55b40",
            "uuid": "85b1e45d-8f98-4cae-8336-72f40e12cbef"
        },
        {
            "name":  "compression_algo",
            "label": "Select compression algorithm",
            "choice_uuid": "01d64f58-8295-4b7b-9cab-8f1b153a504f"
        },
        {
            "name":  "compression_level",
            "label": "Select compression level",
            "choice_uuid": "01c651cb-c174-4ba4-b985-1d87a44d6754"
        }
    ]

    """ Return a dict of AIP Storage Locations and their descriptions."""
    storage_directory_options = [{'value': '', 'label': '--Actions--'}]
    try:
        storage_directories = storage_service.get_location(purpose="AS")
        if storage_directories == None:
            raise Exception("Storage server improperly configured.")
    except Exception:
        messages.warning(request, 'Error retrieving AIP storage locations: is the storage server running? Please contact an administrator.')
    else:
        for storage_dir in storage_directories:
            storage_directory_options.append({
                'value': storage_dir['resource_uri'],
                'label': storage_dir['description']
            })
    other_fields = [
        {
            "name":        "store_aip_location",
            "label":       "Store AIP location",
            "choice_uuid": "b320ce81-9982-408a-9502-097d0daa48fa",
            "options":     storage_directory_options
        }
    ]

    populate_select_fields_with_replace_dict_options(replace_dict_fields)

    select_fields = chain_choice_fields + replace_dict_fields + other_fields

    if request.method == 'POST':
        # render XML using request data
        xmlChoices = PreconfiguredChoices()

        # use toggle field submissions to add to XML
        for field in boolean_select_fields:
            enabled = request.POST.get(field['name'])
            if enabled == 'yes':
                if 'yes_option' in field:
                    # can be set to either yes or no
                    toggle = request.POST.get(field['name'] + '_toggle', '')
                    if toggle == 'yes':
                        go_to_chain_text = field['yes_option']
                    elif 'no_option' in field:
                        go_to_chain_text = field['no_option']

                    if 'no_option' in field:
                        xmlChoices.add_choice(
                            field['choice_uuid'],
                            go_to_chain_text
                        )
                    else:
                        if toggle == 'yes':
                            xmlChoices.add_choice(
                                field['choice_uuid'],
                                go_to_chain_text
                            )

        # set quarantine duration if applicable
        quarantine_expiry_enabled = request.POST.get('quarantine_expiry_enabled', '')
        quarantine_expiry         = request.POST.get('quarantine_expiry', '')
        if quarantine_expiry_enabled == 'yes' and quarantine_expiry != '':
            xmlChoices.add_choice(
                '19adb668-b19a-4fcb-8938-f49d7485eaf3', # Remove from quarantine
                '333643b7-122a-4019-8bef-996443f3ecc5', # Unquarantine
                str(float(quarantine_expiry) * (24 * 60 * 60))
            )

        # use select field submissions to add to XML
        for field in select_fields:
            enabled = request.POST.get(field['name'] + '_enabled')
            if enabled == 'yes':
                field_value = request.POST.get(field['name'], '')
                if field_value != '':
                    xmlChoices.add_choice(
                        field['choice_uuid'],
                        uuid_from_description(field_value)
                    )

        xmlChoices.write_to_file(file_path)

        messages.info(request, 'Saved!')

        return redirect('components.administration.views.processing')
    else:
        debug = request.GET.get('debug', '')
        quarantine_expiry = ''

        file = open(file_path, 'r')
        xml = file.read()

        # parse XML to work out locals()
        root = etree.fromstring(xml)
        choices = root.findall('.//preconfiguredChoice')

        for item in boolean_select_fields:
            item['checked']     = ''
            item['yes_checked'] = ''
            item['no_checked']  = ''

        for choice in choices:
            applies_to = choice.find('appliesTo').text
            go_to_chain = choice.find('goToChain').text

            # use toggle field submissions to add to XML
            for field in boolean_select_fields:
                if applies_to == field['choice_uuid']:
                    set_field_property_by_name(boolean_select_fields, field['name'], 'checked', 'checked')

                    if 'yes_option' in field:
                        if go_to_chain == field['yes_option']:
                            set_field_property_by_name(boolean_select_fields, field['name'], 'yes_checked', 'selected')
                        else:
                            set_field_property_by_name(boolean_select_fields, field['name'], 'no_checked', 'selected')

            # a quarantine expiry was found
            if applies_to == 'Remove from quarantine':
                quarantine_expiry_enabled_checked = 'checked'
                quarantine_expiry = float(choice.find('delay').text) / (24 * 60 * 60)

            # check select fields for defaults
            for field in select_fields:
                if applies_to == field['choice_uuid'] and go_to_chain:
                    try:
                        chain = models.MicroServiceChain.objects.get(pk=go_to_chain)
                        choice = chain.description
                    except models.MicroServiceChain.DoesNotExist:
                        try:
                            choice = models.MicroServiceChoiceReplacementDic.objects.get(pk=go_to_chain).description
                        except models.MicroServiceChoiceReplacementDic.DoesNotExist:
                            continue

                    field['selected'] = choice
                    field['checked'] = 'checked'

    hide_features = hidden_features()
    return render(request, 'administration/processing.html', locals())

def lookup_chain_link(field):
    return models.MicroServiceChainLink.objects.get(pk=field['choice_uuid'])

def remove_option_by_value(options, value):
    for option in options:
        if option['value'] == value:
            options.remove(option)

def uuid_from_description(description):
    try:
        return models.MicroServiceChain.objects.get(description=description).pk
    except models.MicroServiceChain.DoesNotExist:
        return models.MicroServiceChoiceReplacementDic.objects.filter(description=description)[0].pk

def populate_select_field_options_with_chain_choices(field):
    link = lookup_chain_link(field)

    choices = models.MicroServiceChainChoice.objects.filter(choiceavailableatlink=link.pk)

    field['options'] = [{'value': '', 'label': '--Actions--'}]
    options = []
    for choice in choices:
        chain = models.MicroServiceChain.objects.get(pk=choice.chainavailable)
        option = {'value': chain.description, 'label': chain.description}
        options.append(option)

    if field['label'] == 'Create SIP(s)':
        remove_option_by_value(options, 'Reject transfer')
        remove_option_by_value(options, 'Create SIP(s) manually')

    if field['label'] == 'Normalize':
        remove_option_by_value(options, 'Reject SIP')

    options.sort()
    field['options'] += options

def populate_select_field_options_with_replace_dict_values(field):
    link = lookup_chain_link(field)

    replace_dicts = models.MicroServiceChoiceReplacementDic.objects.filter(
        choiceavailableatlink=link.pk
    )

    field['options'] = [{'value': '', 'label': '--Actions--'}]
    options = []
    for dict in replace_dicts:
        option = {'value': dict.description, 'label': dict.description}
        options.append(option)

    options.sort()
    field['options'] += options

def populate_select_fields_with_chain_choice_options(fields):
    for field in fields:
        populate_select_field_options_with_chain_choices(field)

def populate_select_fields_with_replace_dict_options(fields):
    for field in fields:
        populate_select_field_options_with_replace_dict_values(field)

def set_field_property_by_name(fields, name, property, value):
    for field in fields:
        if field['name'] == name:
            field[property] = value
<commit_msg>views_processing: Don't look up UUIDs for storage service<commit_after># This file is part of Archivematica.
#
# Copyright 2010-2013 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

from lxml import etree
import os
import sys

from django.core.urlresolvers import reverse
from django.contrib import messages
from django.shortcuts import redirect
from django.shortcuts import render

from main import models
from components import helpers
from components.helpers import hidden_features

path = "/usr/lib/archivematica/archivematicaCommon"
if path not in sys.path:
    sys.path.append(path)
import storageService as storage_service

class PreconfiguredChoices:
    xml     = None
    choices = None

    def __init__(self):
        self.xml = etree.Element('processingMCP')
        self.choices = etree.Element('preconfiguredChoices')

    def add_choice(self, applies_to_text, go_to_chain_text, delay_duration=None):
        choice = etree.Element('preconfiguredChoice')

        applies_to = etree.Element('appliesTo')
        applies_to.text = applies_to_text
        choice.append(applies_to)

        go_to_chain = etree.Element('goToChain')
        go_to_chain.text = go_to_chain_text
        choice.append(go_to_chain)

        if delay_duration != None:
            delay = etree.Element('delay', unitCtime='yes')
            delay.text = delay_duration
            choice.append(delay)

        self.choices.append(choice)

    def write_to_file(self, file_path):
        self.xml.append(self.choices)
        file = open(file_path, 'w')
        file.write(etree.tostring(self.xml, pretty_print=True))

def index(request):
    file_path = os.path.join(
        helpers.get_server_config_value('sharedDirectory'),
        'sharedMicroServiceTasksConfigs/processingMCPConfigs/defaultProcessingMCP.xml'
    )

    # Lists of dicts declare what options to display, and where to look for
    # the options
    # Name: name in HTML
    # Label: text to display, <label> in HTML, required.  Fallback value for
    #   applies_to, lookup_description
    boolean_select_fields = [
        {
            "name":         "quarantine_transfer",
            "choice_uuid":  "755b4177-c587-41a7-8c52-015277568302", # Workflow decision - send transfer to quarantine
            "label":        "Send transfer to quarantine",
            "yes_option":   "97ea7702-e4d5-48bc-b4b5-d15d897806ab", # Quarantine
            "no_option":    "d4404ab1-dc7f-4e9e-b1f8-aa861e766b8e" # Skip quarantine
        },
        {
            "name":         "normalize_transfer",
            "choice_uuid":  "de909a42-c5b5-46e1-9985-c031b50e9d30",
            "label":        "Approve normalization",
            "yes_option":   "1e0df175-d56d-450d-8bee-7df1dc7ae815", # Approve
            "action":       "Approve"
        },
        {
            "name":         "store_aip",
            "choice_uuid":  "2d32235c-02d4-4686-88a6-96f4d6c7b1c3",
            "label":        "Store AIP",
            "yes_option":   "9efab23c-31dc-4cbd-a39d-bb1665460cbe", # Store AIP
            "action":       "Store AIP"
        }
    ]

    # 'label': text to display, <label> in HTML.
    # 'link_uuid': If there are conflicts on lookup_description, specify this
    #   to use that MicroServiceChainLink
    chain_choice_fields = [
        {
            "name":  "create_sip",
            "label": "Create SIP(s)",
            "choice_uuid": "bb194013-597c-4e4a-8493-b36d190f8717"
        },
        {
            "name":  "extract_packages",
            "label": "Extract packages",
            "choice_uuid": "dec97e3c-5598-4b99-b26e-f87a435a6b7f",
            "uuid": "01d80b27-4ad1-4bd1-8f8d-f819f18bf685"
        },
        {
            "name":  "normalize",
            "label": "Normalize",
            "choice_uuid": "cb8e5706-e73f-472f-ad9b-d1236af8095f",
        }
    ]

    populate_select_fields_with_chain_choice_options(chain_choice_fields)

    # 'label': text to display, <label> in HTML.  Also, the
    # 'lookup_description': TasksConfig description to search against to find
    #   the MicroServiceChainLink for the
    #   MicroServiceChainChoice.choiceavailableatlink  If not specified, use label
    # 'link_uuid': If there are conflicts on lookup_description, specify this
    #   to use that MicroServiceChainLink
    # 'applies_to': Description of the TasksConfig that the choice applies to.
    #   Put in defaultProcessingMCP.xml
    replace_dict_fields = [
        {
            "name": "select_format_id_tool_transfer",
            "label": "Select file format identification command (Transfer)",
            "choice_uuid": 'f09847c2-ee51-429a-9478-a860477f6b8d'
        },
        {
            "name": "select_format_id_tool_ingest",
            "label": "Select file format identification command (Ingest)",
            "choice_uuid": '7a024896-c4f7-4808-a240-44c87c762bc5'
        },
        {
            "name":  "delete_packages",
            "label": "Delete packages after extraction",
            "choice_uuid": "f19926dd-8fb5-4c79-8ade-c83f61f55b40",
            "uuid": "85b1e45d-8f98-4cae-8336-72f40e12cbef"
        },
        {
            "name":  "compression_algo",
            "label": "Select compression algorithm",
            "choice_uuid": "01d64f58-8295-4b7b-9cab-8f1b153a504f"
        },
        {
            "name":  "compression_level",
            "label": "Select compression level",
            "choice_uuid": "01c651cb-c174-4ba4-b985-1d87a44d6754"
        }
    ]

    """ Return a dict of AIP Storage Locations and their descriptions."""
    storage_directory_options = [{'value': '', 'label': '--Actions--'}]
    try:
        storage_directories = storage_service.get_location(purpose="AS")
        if storage_directories == None:
            raise Exception("Storage server improperly configured.")
    except Exception:
        messages.warning(request, 'Error retrieving AIP storage locations: is the storage server running? Please contact an administrator.')
    else:
        for storage_dir in storage_directories:
            storage_directory_options.append({
                'value': storage_dir['resource_uri'],
                'label': storage_dir['description']
            })

    storage_service_options = [
        {
            "name":          "store_aip_location",
            "label":         "Store AIP location",
            "choice_uuid":   "b320ce81-9982-408a-9502-097d0daa48fa",
            "options":       storage_directory_options,
            # Unlike other options, the correct value here is a literal string,
            # not a pointer to a chain or dict in the database.
            "do_not_lookup": True
        }
    ]

    populate_select_fields_with_replace_dict_options(replace_dict_fields)

    select_fields = chain_choice_fields + replace_dict_fields + storage_service_options

    if request.method == 'POST':
        # render XML using request data
        xmlChoices = PreconfiguredChoices()

        # use toggle field submissions to add to XML
        for field in boolean_select_fields:
            enabled = request.POST.get(field['name'])
            if enabled == 'yes':
                if 'yes_option' in field:
                    # can be set to either yes or no
                    toggle = request.POST.get(field['name'] + '_toggle', '')
                    if toggle == 'yes':
                        go_to_chain_text = field['yes_option']
                    elif 'no_option' in field:
                        go_to_chain_text = field['no_option']

                    if 'no_option' in field:
                        xmlChoices.add_choice(
                            field['choice_uuid'],
                            go_to_chain_text
                        )
                    else:
                        if toggle == 'yes':
                            xmlChoices.add_choice(
                                field['choice_uuid'],
                                go_to_chain_text
                            )

        # set quarantine duration if applicable
        quarantine_expiry_enabled = request.POST.get('quarantine_expiry_enabled', '')
        quarantine_expiry         = request.POST.get('quarantine_expiry', '')
        if quarantine_expiry_enabled == 'yes' and quarantine_expiry != '':
            xmlChoices.add_choice(
                '19adb668-b19a-4fcb-8938-f49d7485eaf3', # Remove from quarantine
                '333643b7-122a-4019-8bef-996443f3ecc5', # Unquarantine
                str(float(quarantine_expiry) * (24 * 60 * 60))
            )

        # use select field submissions to add to XML
        for field in select_fields:
            enabled = request.POST.get(field['name'] + '_enabled')
            if enabled == 'yes':
                field_value = request.POST.get(field['name'], '')
                if field_value != '':
                    if field.get('do_not_lookup', False):
                        target = field_value
                    else:
                        target = uuid_from_description(field_value)

                    xmlChoices.add_choice(
                        field['choice_uuid'],
                        target
                    )

        xmlChoices.write_to_file(file_path)

        messages.info(request, 'Saved!')

        return redirect('components.administration.views.processing')
    else:
        debug = request.GET.get('debug', '')
        quarantine_expiry = ''

        file = open(file_path, 'r')
        xml = file.read()

        # parse XML to work out locals()
        root = etree.fromstring(xml)
        choices = root.findall('.//preconfiguredChoice')

        for item in boolean_select_fields:
            item['checked']     = ''
            item['yes_checked'] = ''
            item['no_checked']  = ''

        for choice in choices:
            applies_to = choice.find('appliesTo').text
            go_to_chain = choice.find('goToChain').text

            # use toggle field submissions to add to XML
            for field in boolean_select_fields:
                if applies_to == field['choice_uuid']:
                    set_field_property_by_name(boolean_select_fields, field['name'], 'checked', 'checked')

                    if 'yes_option' in field:
                        if go_to_chain == field['yes_option']:
                            set_field_property_by_name(boolean_select_fields, field['name'], 'yes_checked', 'selected')
                        else:
                            set_field_property_by_name(boolean_select_fields, field['name'], 'no_checked', 'selected')

            # a quarantine expiry was found
            if applies_to == 'Remove from quarantine':
                quarantine_expiry_enabled_checked = 'checked'
                quarantine_expiry = float(choice.find('delay').text) / (24 * 60 * 60)

            # check select fields for defaults
            for field in select_fields:
                if applies_to == field['choice_uuid'] and go_to_chain:
                    try:
                        chain = models.MicroServiceChain.objects.get(pk=go_to_chain)
                        choice = chain.description
                    except models.MicroServiceChain.DoesNotExist:
                        try:
                            choice = models.MicroServiceChoiceReplacementDic.objects.get(pk=go_to_chain).description
                        except models.MicroServiceChoiceReplacementDic.DoesNotExist:
                            # fallback for storage service options, which are
                            # strings that don't map to chains or dicts in
                            # the database
                            choice = go_to_chain

                    field['selected'] = choice
                    field['checked'] = 'checked'

    hide_features = hidden_features()
    return render(request, 'administration/processing.html', locals())

def lookup_chain_link(field):
    return models.MicroServiceChainLink.objects.get(pk=field['choice_uuid'])

def remove_option_by_value(options, value):
    for option in options:
        if option['value'] == value:
            options.remove(option)

def uuid_from_description(description):
    try:
        return models.MicroServiceChain.objects.get(description=description).pk
    except models.MicroServiceChain.DoesNotExist:
        return models.MicroServiceChoiceReplacementDic.objects.filter(description=description)[0].pk

def populate_select_field_options_with_chain_choices(field):
    link = lookup_chain_link(field)

    choices = models.MicroServiceChainChoice.objects.filter(choiceavailableatlink=link.pk)

    field['options'] = [{'value': '', 'label': '--Actions--'}]
    options = []
    for choice in choices:
        chain = models.MicroServiceChain.objects.get(pk=choice.chainavailable)
        option = {'value': chain.description, 'label': chain.description}
        options.append(option)

    if field['label'] == 'Create SIP(s)':
        remove_option_by_value(options, 'Reject transfer')
        remove_option_by_value(options, 'Create SIP(s) manually')

    if field['label'] == 'Normalize':
        remove_option_by_value(options, 'Reject SIP')

    options.sort()
    field['options'] += options

def populate_select_field_options_with_replace_dict_values(field):
    link = lookup_chain_link(field)

    replace_dicts = models.MicroServiceChoiceReplacementDic.objects.filter(
        choiceavailableatlink=link.pk
    )

    field['options'] = [{'value': '', 'label': '--Actions--'}]
    options = []
    for dict in replace_dicts:
        option = {'value': dict.description, 'label': dict.description}
        options.append(option)

    options.sort()
    field['options'] += options

def populate_select_fields_with_chain_choice_options(fields):
    for field in fields:
        populate_select_field_options_with_chain_choices(field)

def populate_select_fields_with_replace_dict_options(fields):
    for field in fields:
        populate_select_field_options_with_replace_dict_values(field)

def set_field_property_by_name(fields, name, property, value):
    for field in fields:
        if field['name'] == name:
            field[property] = value
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

__description__ = 'Malcom - Malware communications analyzer'
__author__ = '@tomchop_'
__version__ = '1.1 alpha'
__license__ = "GPL"

import os, sys, argparse, threading
import netifaces as ni
from time import sleep

from flask import Flask

from Malcom.analytics.analytics import Analytics
from Malcom.feeds.feed import FeedEngine
from Malcom.web.webserver import MalcomWeb
from Malcom.networking.tlsproxy.tlsproxy import MalcomTLSProxy
from Malcom.networking import netsniffer
import Malcom # this is the configuraiton

# this should be stored and loaded from a configuration file
Malcom.config['DEBUG'] = True
Malcom.config['VERSION'] = "1.1 alpha"
Malcom.config['LISTEN_INTERFACE'] = "0.0.0.0"
Malcom.config['LISTEN_PORT'] = 8080
Malcom.config['MAX_WORKERS'] = 4
Malcom.config['PUBLIC'] = False
Malcom.config['NO_FEED'] = False
Malcom.config['TLS_PROXY_PORT'] = False
Malcom.config['BASE_PATH'] = os.getcwd() + '/Malcom'
Malcom.config['SNIFFER_DIR'] = Malcom.config['BASE_PATH'] + '/sniffer'
Malcom.config['FEEDS_DIR'] = Malcom.config['BASE_PATH'] + '/feeds'

Malcom.config['IFACES'] = {}
for i in [i for i in ni.interfaces() if i.find('eth') != -1]:
	Malcom.config['IFACES'][i] = ni.ifaddresses(i).get(2,[{'addr':'Not defined'}])[0]['addr']

if __name__ == "__main__":

	# options
	parser = argparse.ArgumentParser(description="Malcom - malware communications analyzer")
	parser.add_argument("-a", "--analytics", help="Run analytics", action="store_true", default=False)
	parser.add_argument("-f", "--feeds", help="Run feeds (use -ff to force run on all feeds)", action="count")
	parser.add_argument("-i", "--interface", help="Listen interface", default=Malcom.config['LISTEN_INTERFACE'])
	parser.add_argument("-p", "--port", help="Listen port", type=int, default=Malcom.config['LISTEN_PORT'])
	parser.add_argument("--public", help="Run a public instance (Feeds and network sniffing disabled)", action="store_true", default=Malcom.config['PUBLIC'])
	parser.add_argument("--max-workers", help="Number of worker processes to use (default 4)", type=int, default=Malcom.config['MAX_WORKERS'])
	parser.add_argument("--tls-proxy-port", help="Port number on which to start the TLS proxy on. No proxy started if not specified.", type=int, default=Malcom.config['TLS_PROXY_PORT'])

	#parser.add_argument("--no-feeds", help="Disable automatic feeding", action="store_true", default=app.config['NO_FEED'])
	args = parser.parse_args()

	os.system('clear')
	Malcom.config['LISTEN_INTERFACE'] = args.interface
	Malcom.config['LISTEN_PORT'] = args.port
	Malcom.config['MAX_WORKERS'] = args.max_workers
	Malcom.config['PUBLIC'] = args.public

	sys.stderr.write("===== Malcom %s - Malware Communications Analyzer =====\n\n" % Malcom.config['VERSION'])

	sys.stderr.write("Detected interfaces:\n")
	for iface in Malcom.config['IFACES']:
		sys.stderr.write("%s:\t%s\n" % (iface, Malcom.config['IFACES'][iface]))

	Malcom.analytics_engine = Analytics()

	if args.tls_proxy_port:
		Malcom.config['TLS_PROXY_PORT'] = args.tls_proxy_port
		sys.stderr.write("Starting TLS proxy on port %s\n" % args.tls_proxy_port)
		Malcom.tls_proxy = MalcomTLSProxy(args.tls_proxy_port)
		Malcom.tls_proxy.start()
	else:
		Malcom.tls_proxy = None

	sys.stderr.write("Importing feeds...\n")
	Malcom.feed_engine = FeedEngine(Malcom.analytics_engine)
	Malcom.feed_engine.load_feeds()

	sys.stderr.write("Importing packet captures...\n")

	for s in Malcom.analytics_engine.data.get_sniffer_sessions():
		Malcom.sniffer_sessions[s['name']] = netsniffer.Sniffer(Malcom.analytics_engine,
																s['name'],
																None,
																None,
																filter_restore=s['filter'],
																intercept_tls=s['intercept_tls'] if args.tls_proxy_port else False)

		Malcom.sniffer_sessions[s['name']].pcap = True

	# call malcom to run feeds - this will not start the web interface
	if args.feeds >= 1:
		if args.feeds == 1:
			try:
				Malcom.feed_engine.start()
				sys.stderr.write("Starting feed scheduler...\n")
				while True:
					raw_input()
			except KeyboardInterrupt:
				sys.stderr.write("\nStopping all feeds...\n")
				Malcom.feed_engine.stop_all_feeds()
				exit(0)

		elif args.feeds == 2:
			Malcom.feed_engine.run_all_feeds()
		exit(0)

	elif args.analytics: # run analytics
		while True:
			Malcom.analytics_engine.process()
			sleep(10) # sleep 10 seconds
		pass

	else: # run webserver
		web = MalcomWeb(Malcom.config['PUBLIC'], Malcom.config['LISTEN_PORT'], Malcom.config['LISTEN_INTERFACE'])
		try:
			Malcom.tls_proxy.stop()
		except Exception, e:
			pass

		exit(0)
<commit_msg>Bugfix<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

__description__ = 'Malcom - Malware communications analyzer'
__author__ = '@tomchop_'
__version__ = '1.1 alpha'
__license__ = "GPL"

import os, sys, argparse, threading
import netifaces as ni
from time import sleep

from flask import Flask

from Malcom.analytics.analytics import Analytics
from Malcom.feeds.feed import FeedEngine
from Malcom.web.webserver import MalcomWeb
from Malcom.networking.tlsproxy.tlsproxy import MalcomTLSProxy
from Malcom.networking import netsniffer
import Malcom # this is the configuraiton

# this should be stored and loaded from a configuration file
Malcom.config['DEBUG'] = True
Malcom.config['VERSION'] = "1.1 alpha"
Malcom.config['LISTEN_INTERFACE'] = "0.0.0.0"
Malcom.config['LISTEN_PORT'] = 8080
Malcom.config['MAX_WORKERS'] = 4
Malcom.config['PUBLIC'] = False
Malcom.config['NO_FEED'] = False
Malcom.config['TLS_PROXY_PORT'] = False
Malcom.config['BASE_PATH'] = os.getcwd() + '/Malcom'
Malcom.config['SNIFFER_DIR'] = Malcom.config['BASE_PATH'] + '/sniffer'
Malcom.config['FEEDS_DIR'] = Malcom.config['BASE_PATH'] + '/feeds'

Malcom.config['IFACES'] = {}
for i in [i for i in ni.interfaces() if i.find('eth') != -1]:
	Malcom.config['IFACES'][i] = ni.ifaddresses(i).get(2,[{'addr':'Not defined'}])[0]['addr']

if __name__ == "__main__":

	# options
	parser = argparse.ArgumentParser(description="Malcom - malware communications analyzer")
	parser.add_argument("-a", "--analytics", help="Run analytics", action="store_true", default=False)
	parser.add_argument("-f", "--feeds", help="Run feeds (use -ff to force run on all feeds)", action="count")
	parser.add_argument("-i", "--interface", help="Listen interface", default=Malcom.config['LISTEN_INTERFACE'])
	parser.add_argument("-p", "--port", help="Listen port", type=int, default=Malcom.config['LISTEN_PORT'])
	parser.add_argument("--public", help="Run a public instance (Feeds and network sniffing disabled)", action="store_true", default=Malcom.config['PUBLIC'])
	parser.add_argument("--max-workers", help="Number of worker processes to use (default 4)", type=int, default=Malcom.config['MAX_WORKERS'])
	parser.add_argument("--tls-proxy-port", help="Port number on which to start the TLS proxy on. No proxy started if not specified.", type=int, default=Malcom.config['TLS_PROXY_PORT'])

	#parser.add_argument("--no-feeds", help="Disable automatic feeding", action="store_true", default=app.config['NO_FEED'])
	args = parser.parse_args()

	os.system('clear')
	Malcom.config['LISTEN_INTERFACE'] = args.interface
	Malcom.config['LISTEN_PORT'] = args.port
	Malcom.config['MAX_WORKERS'] = args.max_workers
	Malcom.config['PUBLIC'] = args.public

	sys.stderr.write("===== Malcom %s - Malware Communications Analyzer =====\n\n" % Malcom.config['VERSION'])

	sys.stderr.write("Detected interfaces:\n")
	for iface in Malcom.config['IFACES']:
		sys.stderr.write("%s:\t%s\n" % (iface, Malcom.config['IFACES'][iface]))

	Malcom.analytics_engine = Analytics()

	if args.tls_proxy_port:
		Malcom.config['TLS_PROXY_PORT'] = args.tls_proxy_port
		sys.stderr.write("Starting TLS proxy on port %s\n" % args.tls_proxy_port)
		Malcom.tls_proxy = MalcomTLSProxy(args.tls_proxy_port)
		Malcom.tls_proxy.start()
	else:
		Malcom.tls_proxy = None

	sys.stderr.write("Importing feeds...\n")
	Malcom.feed_engine = FeedEngine(Malcom.analytics_engine)
	Malcom.feed_engine.load_feeds()

	sys.stderr.write("Importing packet captures...\n")

	for s in Malcom.analytics_engine.data.get_sniffer_sessions():
		Malcom.sniffer_sessions[s['name']] = netsniffer.Sniffer(Malcom.analytics_engine,
																s['name'],
																None,
																None,
																filter_restore=s['filter'],
																intercept_tls=s['intercept_tls'] if Malcom.tls_proxy else False)

		Malcom.sniffer_sessions[s['name']].pcap = True

	# call malcom to run feeds - this will not start the web interface
	if args.feeds >= 1:
		if args.feeds == 1:
			try:
				Malcom.feed_engine.start()
				sys.stderr.write("Starting feed scheduler...\n")
				while True:
					raw_input()
			except KeyboardInterrupt:
				sys.stderr.write("\nStopping all feeds...\n")
				Malcom.feed_engine.stop_all_feeds()
				exit(0)

		elif args.feeds == 2:
			Malcom.feed_engine.run_all_feeds()
		exit(0)

	elif args.analytics: # run analytics
		while True:
			Malcom.analytics_engine.process()
			sleep(10) # sleep 10 seconds
		pass

	else: # run webserver
		web = MalcomWeb(Malcom.config['PUBLIC'], Malcom.config['LISTEN_PORT'], Malcom.config['LISTEN_INTERFACE'])
		try:
			Malcom.tls_proxy.stop()
		except Exception, e:
			pass

		exit(0)
<|endoftext|>


<commit_before># This file is part of Archivematica.
#
# Copyright 2010-2013 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

from django.core.urlresolvers import reverse
from django.shortcuts import render
from django.http import HttpResponse, HttpResponseRedirect
from main import models
import sys, os, ConfigParser
from lxml import etree

class PreconfiguredChoices:
    xml     = None
    choices = None

    def __init__(self):
        self.xml = etree.Element('processingMCP')
        self.choices = etree.Element('preconfiguredChoices')

    def add_choice(self, applies_to_text, go_to_chain_text, delay_duration=None):
        choice = etree.Element('preconfiguredChoice')

        applies_to = etree.Element('appliesTo')
        applies_to.text = applies_to_text
        choice.append(applies_to)

        go_to_chain = etree.Element('goToChain')
        go_to_chain.text = go_to_chain_text
        choice.append(go_to_chain)

        if delay_duration != None:
            delay = etree.Element('delay', unitCtime='yes')
            delay.text = delay_duration
            choice.append(delay)

        self.choices.append(choice)

    def write_to_file(self, file_path):
        self.xml.append(self.choices)
        file = open(file_path, 'w')
        file.write(etree.tostring(self.xml, pretty_print=True))

def administration_processing(request):
    clientConfigFilePath = '/etc/archivematica/MCPClient/clientConfig.conf'
    config               = ConfigParser.SafeConfigParser()

    config.read(clientConfigFilePath)

    shared_directory     = config.get('MCPClient', "sharedDirectoryMounted")
    file_path            = os.path.join(shared_directory, 'sharedMicroServiceTasksConfigs/processingMCPConfigs/defaultProcessingMCP.xml')

    boolean_select_fields = [
        {
            "name":       "backup_transfer",
            "label":      "Create transfer backup",
            "yes_option": "Backup transfer",
            "no_option":  "Do not backup transfer",
            "applies_to": "Workflow decision - create transfer backup"
        },
        {
            "name":       "quarantine_transfer",
            "label":      "Send transfer to quarantine",
            "yes_option": "Quarantine",
            "no_option":  "Skip quarantine",
            "applies_to": "Workflow decision - send transfer to quarantine"
        },
        {
            "name":       "normalize_transfer",
            "label":      "Approve normalization",
            "applies_to": "Approve normalization",
            "yes_option": "Approve normalization",
            "action":     "Approve"
        },
        {
            "name":       "store_aip",
            "label":      "Store AIP",
            "yes_option": "Store AIP",
            "applies_to": "Store AIP",
            "action":     "Store AIP"
        }
    ]

    chain_choice_fields = [
        {
            "name":  "create_sip",
            "label": "Create SIP(s)"
        },
        {
            "name":  "select_format_id_tool",
            "label": "Select format identification tool"
        },
        {
            "name":  "normalize",
            "label": "Normalize"
        }
    ]

    populate_select_fields_with_chain_choice_options(chain_choice_fields)

    replace_dict_fields = [
        {
            "name":  "compression_algo",
            "label": "Select compression algorithm"
        },
        {
            "name":  "compression_level",
            "label": "Select compression level"
        },
        {
            "name":  "store_aip_location",
            "label": "Store AIP location"
        }
    ]

    populate_select_fields_with_replace_dict_options(replace_dict_fields)

    select_fields = chain_choice_fields + replace_dict_fields

    if request.method == 'POST':
        # render XML using request data
        xmlChoices = PreconfiguredChoices()

        # use toggle field submissions to add to XML
        for field in boolean_select_fields:
            enabled = request.POST.get(field['name'])
            if enabled == 'yes':
                if 'yes_option' in field:
                    # can be set to either yes or no
                    toggle = request.POST.get(field['name'] + '_toggle', '')
                    if toggle == 'yes':
                        go_to_chain_text = field['yes_option']
                    elif 'no_option' in field:
                        go_to_chain_text = field['no_option']

                    if 'no_option' in field:
                        xmlChoices.add_choice(
                            field['applies_to'],
                            go_to_chain_text
                        )
                    else:
                        if toggle == 'yes':
                            xmlChoices.add_choice(
                                field['applies_to'],
                                go_to_chain_text
                            )
                else:
                    xmlChoices.add_choice(
                        field['label'],
                        field['action']
                    )

        # set quarantine duration if applicable
        quarantine_expiry_enabled = request.POST.get('quarantine_expiry_enabled', '')
        quarantine_expiry         = request.POST.get('quarantine_expiry', '')
        if quarantine_expiry_enabled == 'yes' and quarantine_expiry != '':
            xmlChoices.add_choice(
                'Remove from quarantine',
                'Unquarantine',
                quarantine_expiry
            )

        # use select field submissions to add to XML
        for field in select_fields:
            enabled = request.POST.get(field['name'] + '_enabled')
            if enabled == 'yes':
                field_value = request.POST.get(field['name'], '')
                if field_value != '':
                    xmlChoices.add_choice(
                        field['label'],
                        field_value
                    )

        xmlChoices.write_to_file(file_path)

        return HttpResponseRedirect(reverse('components.administration.views.administration_processing'))
    else:
        debug = request.GET.get('debug', '')
        quarantine_expiry = ''

        file = open(file_path, 'r')
        xml = file.read()

        # parse XML to work out locals()
        root = etree.fromstring(xml)
        choices = root.find('preconfiguredChoices')

        for item in boolean_select_fields:
            item['checked']     = ''
            item['yes_checked'] = ''
            item['no_checked']  = ''

        for choice in choices:
            applies_to = choice.find('appliesTo').text
            go_to_chain = choice.find('goToChain').text

            # use toggle field submissions to add to XML
            for field in boolean_select_fields:
                if applies_to == field['applies_to']:
                    set_field_property_by_name(boolean_select_fields, field['name'], 'checked', 'checked')

                    if 'yes_option' in field:
                        if go_to_chain == field['yes_option']:
                            set_field_property_by_name(boolean_select_fields, field['name'], 'yes_checked', 'selected')
                        else:
                            set_field_property_by_name(boolean_select_fields, field['name'], 'no_checked', 'selected')

            # a quarantine expiry was found
            if applies_to == 'Remove from quarantine':
                quarantine_expiry_enabled_checked = 'checked'
                quarantine_expiry = choice.find('delay').text

            # check select fields for defaults
            for field in select_fields:
                if applies_to == field['label']:
                    field['selected'] = go_to_chain
                    field['checked'] = 'checked'

    return render(request, 'administration/processing.html', locals())

def lookup_chain_link_by_description(field):
    try:
        lookup_description = field['lookup_description']
    except:
        lookup_description = field['label']

    task = models.TaskConfig.objects.filter(description=lookup_description)[0]
    link = models.MicroServiceChainLink.objects.get(currenttask=task.pk)

    return link

def populate_select_field_options_with_chain_choices(field):
    link = lookup_chain_link_by_description(field)

    choices = models.MicroServiceChainChoice.objects.filter(choiceavailableatlink=link.pk)

    field['options'] = [{'value': '', 'label': '--Actions--'}]
    options = []
    for choice in choices:
        chain = models.MicroServiceChain.objects.get(pk=choice.chainavailable)
        option = {'value': chain.description, 'label': chain.description}
        options.append(option)

    options.sort()
    field['options'] += options

def populate_select_field_options_with_replace_dict_values(field):
    link = lookup_chain_link_by_description(field)

    replace_dicts = models.MicroServiceChoiceReplacementDic.objects.filter(
        choiceavailableatlink=link.pk
    )

    field['options'] = [{'value': '', 'label': '--Actions--'}]
    options = []
    for dict in replace_dicts:
        option = {'value': dict.description, 'label': dict.description}
        options.append(option)

    options.sort()
    field['options'] += options

def populate_select_fields_with_chain_choice_options(fields):
    for field in fields:
        populate_select_field_options_with_chain_choices(field)

def populate_select_fields_with_replace_dict_options(fields):
    for field in fields:
        populate_select_field_options_with_replace_dict_values(field)

def set_field_property_by_name(fields, name, property, value):
    for field in fields:
        if field['name'] == name:
            field[property] = value
<commit_msg>Removed option from processing config admin.<commit_after># This file is part of Archivematica.
#
# Copyright 2010-2013 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

from django.core.urlresolvers import reverse
from django.shortcuts import render
from django.http import HttpResponse, HttpResponseRedirect
from main import models
import sys, os, ConfigParser
from lxml import etree

class PreconfiguredChoices:
    xml     = None
    choices = None

    def __init__(self):
        self.xml = etree.Element('processingMCP')
        self.choices = etree.Element('preconfiguredChoices')

    def add_choice(self, applies_to_text, go_to_chain_text, delay_duration=None):
        choice = etree.Element('preconfiguredChoice')

        applies_to = etree.Element('appliesTo')
        applies_to.text = applies_to_text
        choice.append(applies_to)

        go_to_chain = etree.Element('goToChain')
        go_to_chain.text = go_to_chain_text
        choice.append(go_to_chain)

        if delay_duration != None:
            delay = etree.Element('delay', unitCtime='yes')
            delay.text = delay_duration
            choice.append(delay)

        self.choices.append(choice)

    def write_to_file(self, file_path):
        self.xml.append(self.choices)
        file = open(file_path, 'w')
        file.write(etree.tostring(self.xml, pretty_print=True))

def administration_processing(request):
    clientConfigFilePath = '/etc/archivematica/MCPClient/clientConfig.conf'
    config               = ConfigParser.SafeConfigParser()

    config.read(clientConfigFilePath)

    shared_directory     = config.get('MCPClient', "sharedDirectoryMounted")
    file_path            = os.path.join(shared_directory, 'sharedMicroServiceTasksConfigs/processingMCPConfigs/defaultProcessingMCP.xml')

    boolean_select_fields = [
        {
            "name":       "backup_transfer",
            "label":      "Create transfer backup",
            "yes_option": "Backup transfer",
            "no_option":  "Do not backup transfer",
            "applies_to": "Workflow decision - create transfer backup"
        },
        {
            "name":       "quarantine_transfer",
            "label":      "Send transfer to quarantine",
            "yes_option": "Quarantine",
            "no_option":  "Skip quarantine",
            "applies_to": "Workflow decision - send transfer to quarantine"
        },
        {
            "name":       "normalize_transfer",
            "label":      "Approve normalization",
            "applies_to": "Approve normalization",
            "yes_option": "Approve normalization",
            "action":     "Approve"
        },
        {
            "name":       "store_aip",
            "label":      "Store AIP",
            "yes_option": "Store AIP",
            "applies_to": "Store AIP",
            "action":     "Store AIP"
        }
    ]

    chain_choice_fields = [
        {
            "name":  "create_sip",
            "label": "Create SIP(s)"
        },
        {
            "name":  "select_format_id_tool",
            "label": "Select format identification tool"
        },
        {
            "name":  "normalize",
            "label": "Normalize"
        }
    ]

    populate_select_fields_with_chain_choice_options(chain_choice_fields)

    replace_dict_fields = [
        {
            "name":  "compression_algo",
            "label": "Select compression algorithm"
        },
        {
            "name":  "compression_level",
            "label": "Select compression level"
        },
        {
            "name":  "store_aip_location",
            "label": "Store AIP location"
        }
    ]

    populate_select_fields_with_replace_dict_options(replace_dict_fields)

    select_fields = chain_choice_fields + replace_dict_fields

    if request.method == 'POST':
        # render XML using request data
        xmlChoices = PreconfiguredChoices()

        # use toggle field submissions to add to XML
        for field in boolean_select_fields:
            enabled = request.POST.get(field['name'])
            if enabled == 'yes':
                if 'yes_option' in field:
                    # can be set to either yes or no
                    toggle = request.POST.get(field['name'] + '_toggle', '')
                    if toggle == 'yes':
                        go_to_chain_text = field['yes_option']
                    elif 'no_option' in field:
                        go_to_chain_text = field['no_option']

                    if 'no_option' in field:
                        xmlChoices.add_choice(
                            field['applies_to'],
                            go_to_chain_text
                        )
                    else:
                        if toggle == 'yes':
                            xmlChoices.add_choice(
                                field['applies_to'],
                                go_to_chain_text
                            )
                else:
                    xmlChoices.add_choice(
                        field['label'],
                        field['action']
                    )

        # set quarantine duration if applicable
        quarantine_expiry_enabled = request.POST.get('quarantine_expiry_enabled', '')
        quarantine_expiry         = request.POST.get('quarantine_expiry', '')
        if quarantine_expiry_enabled == 'yes' and quarantine_expiry != '':
            xmlChoices.add_choice(
                'Remove from quarantine',
                'Unquarantine',
                quarantine_expiry
            )

        # use select field submissions to add to XML
        for field in select_fields:
            enabled = request.POST.get(field['name'] + '_enabled')
            if enabled == 'yes':
                field_value = request.POST.get(field['name'], '')
                if field_value != '':
                    xmlChoices.add_choice(
                        field['label'],
                        field_value
                    )

        xmlChoices.write_to_file(file_path)

        return HttpResponseRedirect(reverse('components.administration.views.administration_processing'))
    else:
        debug = request.GET.get('debug', '')
        quarantine_expiry = ''

        file = open(file_path, 'r')
        xml = file.read()

        # parse XML to work out locals()
        root = etree.fromstring(xml)
        choices = root.find('preconfiguredChoices')

        for item in boolean_select_fields:
            item['checked']     = ''
            item['yes_checked'] = ''
            item['no_checked']  = ''

        for choice in choices:
            applies_to = choice.find('appliesTo').text
            go_to_chain = choice.find('goToChain').text

            # use toggle field submissions to add to XML
            for field in boolean_select_fields:
                if applies_to == field['applies_to']:
                    set_field_property_by_name(boolean_select_fields, field['name'], 'checked', 'checked')

                    if 'yes_option' in field:
                        if go_to_chain == field['yes_option']:
                            set_field_property_by_name(boolean_select_fields, field['name'], 'yes_checked', 'selected')
                        else:
                            set_field_property_by_name(boolean_select_fields, field['name'], 'no_checked', 'selected')

            # a quarantine expiry was found
            if applies_to == 'Remove from quarantine':
                quarantine_expiry_enabled_checked = 'checked'
                quarantine_expiry = choice.find('delay').text

            # check select fields for defaults
            for field in select_fields:
                if applies_to == field['label']:
                    field['selected'] = go_to_chain
                    field['checked'] = 'checked'

    return render(request, 'administration/processing.html', locals())

def lookup_chain_link_by_description(field):
    try:
        lookup_description = field['lookup_description']
    except:
        lookup_description = field['label']

    task = models.TaskConfig.objects.filter(description=lookup_description)[0]
    link = models.MicroServiceChainLink.objects.get(currenttask=task.pk)

    return link

def remove_option_by_value(options, value):
    for option in options:
        if option['value'] == value:
            options.remove(option)

def populate_select_field_options_with_chain_choices(field):
    link = lookup_chain_link_by_description(field)

    choices = models.MicroServiceChainChoice.objects.filter(choiceavailableatlink=link.pk)

    field['options'] = [{'value': '', 'label': '--Actions--'}]
    options = []
    for choice in choices:
        chain = models.MicroServiceChain.objects.get(pk=choice.chainavailable)
        option = {'value': chain.description, 'label': chain.description}
        options.append(option)

    if field['label'] == 'Create SIP(s)':
        remove_option_by_value(options, 'Reject transfer')

    #if field['label'] == 'Normalize':
    #    remove_option_by_value(options, 'Reject transfer')

    options.sort()
    field['options'] += options

def populate_select_field_options_with_replace_dict_values(field):
    link = lookup_chain_link_by_description(field)

    replace_dicts = models.MicroServiceChoiceReplacementDic.objects.filter(
        choiceavailableatlink=link.pk
    )

    field['options'] = [{'value': '', 'label': '--Actions--'}]
    options = []
    for dict in replace_dicts:
        option = {'value': dict.description, 'label': dict.description}
        options.append(option)

    options.sort()
    field['options'] += options

def populate_select_fields_with_chain_choice_options(fields):
    for field in fields:
        populate_select_field_options_with_chain_choices(field)

def populate_select_fields_with_replace_dict_options(fields):
    for field in fields:
        populate_select_field_options_with_replace_dict_values(field)

def set_field_property_by_name(fields, name, property, value):
    for field in fields:
        if field['name'] == name:
            field[property] = value
<|endoftext|>