<commit_before>try:
    # Only exists in Python 2.4+
    from threading import local
except ImportError:
    # Import copy of _thread_local.py from Python 2.4
    from django.utils._threading_local import local

from django.db.backends import util
from django.utils import datetime_safe

class BaseDatabaseWrapper(local):
    """
    Represents a database connection.
    """
    ops = None
    def __init__(self, **kwargs):
        self.connection = None
        self.queries = []
        self.options = kwargs

    def _commit(self):
        if self.connection is not None:
            return self.connection.commit()

    def _rollback(self):
        if self.connection is not None:
            return self.connection.rollback()

    def close(self):
        if self.connection is not None:
            self.connection.close()
            self.connection = None

    def cursor(self):
        from django.conf import settings
        cursor = self._cursor(settings)
        if settings.DEBUG:
            return self.make_debug_cursor(cursor)
        return cursor

    def make_debug_cursor(self, cursor):
        return util.CursorDebugWrapper(cursor, self)

class BaseDatabaseFeatures(object):
    allows_group_by_ordinal = True
    inline_fk_references = True
    # True if django.db.backend.utils.typecast_timestamp is used on values
    # returned from dates() calls.
    needs_datetime_string_cast = True
    supports_constraints = True
    supports_tablespaces = False
    uses_case_insensitive_names = False
    uses_custom_query_class = False
    empty_fetchmany_value = []
    update_can_self_select = True
    interprets_empty_strings_as_nulls = False
    can_use_chunked_reads = True

class BaseDatabaseOperations(object):
    """
    This class encapsulates all backend-specific differences, such as the way
    a backend performs ordering or calculates the ID of a recently-inserted
    row.
    """
    def autoinc_sql(self, table, column):
        """
        Returns any SQL needed to support auto-incrementing primary keys, or
        None if no SQL is necessary.

        This SQL is executed when a table is created.
        """
        return None

    def date_extract_sql(self, lookup_type, field_name):
        """
        Given a lookup_type of 'year', 'month' or 'day', returns the SQL that
        extracts a value from the given date field field_name.
        """
        raise NotImplementedError()

    def date_trunc_sql(self, lookup_type, field_name):
        """
        Given a lookup_type of 'year', 'month' or 'day', returns the SQL that
        truncates the given date field field_name to a DATE object with only
        the given specificity.
        """
        raise NotImplementedError()

    def datetime_cast_sql(self):
        """
        Returns the SQL necessary to cast a datetime value so that it will be
        retrieved as a Python datetime object instead of a string.

        This SQL should include a '%s' in place of the field's name.
        """
        return "%s"

    def deferrable_sql(self):
        """
        Returns the SQL necessary to make a constraint "initially deferred"
        during a CREATE TABLE statement.
        """
        return ''

    def drop_foreignkey_sql(self):
        """
        Returns the SQL command that drops a foreign key.
        """
        return "DROP CONSTRAINT"

    def drop_sequence_sql(self, table):
        """
        Returns any SQL necessary to drop the sequence for the given table.
        Returns None if no SQL is necessary.
        """
        return None

    def field_cast_sql(self, db_type):
        """
        Given a column type (e.g. 'BLOB', 'VARCHAR'), returns the SQL necessary
        to cast it before using it in a WHERE statement. Note that the
        resulting string should contain a '%s' placeholder for the column being
        searched against.
        """
        return '%s'

    def fulltext_search_sql(self, field_name):
        """
        Returns the SQL WHERE clause to use in order to perform a full-text
        search of the given field_name. Note that the resulting string should
        contain a '%s' placeholder for the value being searched against.
        """
        raise NotImplementedError('Full-text search is not implemented for this database backend')

    def last_executed_query(self, cursor, sql, params):
        """
        Returns a string of the query last executed by the given cursor, with
        placeholders replaced with actual values.

        `sql` is the raw query containing placeholders, and `params` is the
        sequence of parameters. These are used by default, but this method
        exists for database backends to provide a better implementation
        according to their own quoting schemes.
        """
        from django.utils.encoding import smart_unicode, force_unicode

        # Convert params to contain Unicode values.
        to_unicode = lambda s: force_unicode(s, strings_only=True)
        if isinstance(params, (list, tuple)):
            u_params = tuple([to_unicode(val) for val in params])
        else:
            u_params = dict([(to_unicode(k), to_unicode(v)) for k, v in params.items()])

        return smart_unicode(sql) % u_params

    def last_insert_id(self, cursor, table_name, pk_name):
        """
        Given a cursor object that has just performed an INSERT statement into
        a table that has an auto-incrementing ID, returns the newly created ID.

        This method also receives the table name and the name of the primary-key
        column.
        """
        return cursor.lastrowid

    def lookup_cast(self, lookup_type):
        """
        Returns the string to use in a query when performing lookups
        ("contains", "like", etc). The resulting string should contain a '%s'
        placeholder for the column being searched against.
        """
        return "%s"

    def max_name_length(self):
        """
        Returns the maximum length of table and column names, or None if there
        is no limit.
        """
        return None

    def no_limit_value(self):
        """
        Returns the value to use for the LIMIT when we are wanting "LIMIT
        infinity". Returns None if the limit clause can be omitted in this case.
        """
        # FIXME: API may need to change once Oracle backend is repaired.
        raise NotImplementedError()

    def pk_default_value(self):
        """
        Returns the value to use during an INSERT statement to specify that
        the field should use its default value.
        """
        return 'DEFAULT'

    def query_class(self, DefaultQueryClass):
        """
        Given the default Query class, returns a custom Query class
        to use for this backend. Returns None if a custom Query isn't used.
        See also BaseDatabaseFeatures.uses_custom_query_class, which regulates
        whether this method is called at all.
        """
        return None

    def quote_name(self, name):
        """
        Returns a quoted version of the given table, index or column name. Does
        not quote the given name if it's already been quoted.
        """
        raise NotImplementedError()

    def random_function_sql(self):
        """
        Returns a SQL expression that returns a random value.
        """
        return 'RANDOM()'

    def regex_lookup(self, lookup_type):
        """
        Returns the string to use in a query when performing regular expression
        lookups (using "regex" or "iregex"). The resulting string should
        contain a '%s' placeholder for the column being searched against.

        If the feature is not supported (or part of it is not supported), a
        NotImplementedError exception can be raised.
        """
        raise NotImplementedError

    def sql_flush(self, style, tables, sequences):
        """
        Returns a list of SQL statements required to remove all data from
        the given database tables (without actually removing the tables
        themselves).

        The `style` argument is a Style object as returned by either
        color_style() or no_style() in django.core.management.color.
        """
        raise NotImplementedError()

    def sequence_reset_sql(self, style, model_list):
        """
        Returns a list of the SQL statements required to reset sequences for
        the given models.

        The `style` argument is a Style object as returned by either
        color_style() or no_style() in django.core.management.color.
        """
        return [] # No sequence reset required by default.

    def start_transaction_sql(self):
        """
        Returns the SQL statement required to start a transaction.
        """
        return "BEGIN;"

    def tablespace_sql(self, tablespace, inline=False):
        """
        Returns the tablespace SQL, or None if the backend doesn't use
        tablespaces.
        """
        return None

    def prep_for_like_query(self, x):
        """Prepares a value for use in a LIKE query."""
        from django.utils.encoding import smart_unicode
        return smart_unicode(x).replace("\\", "\\\\").replace("%", "\%").replace("_", "\_")

    def value_to_db_date(self, value):
        """
        Transform a date value to an object compatible with what is expected
        by the backend driver for date columns.
        """
        if value is None:
            return None
        return datetime_safe.new_date(value).strftime('%Y-%m-%d')

    def value_to_db_datetime(self, value):
        """
        Transform a datetime value to an object compatible with what is expected
        by the backend driver for date columns.
        """
        if value is None:
            return None
        return unicode(value)

    def value_to_db_time(self, value):
        """
        Transform a datetime value to an object compatible with what is expected
        by the backend driver for date columns.
        """
        if value is None:
            return None
        return unicode(value)

    def value_to_db_decimal(self, value, max_digits, decimal_places):
        """
        Transform a decimal.Decimal value to an object compatible with what is
        expected by the backend driver for decimal (numeric) columns.
        """
        if value is None:
            return None
        return util.format_number(value, max_digits, decimal_places)

    def year_lookup_bounds(self, value):
        """
        Returns a two-elements list with the lower and upper bound to be used
        with a BETWEEN operator to query a field value using a year lookup

        `value` is an int, containing the looked-up year.
        """
        first = '%s-01-01 00:00:00'
        second = '%s-12-31 23:59:59.999999'
        return [first % value, second % value]

    def year_lookup_bounds_for_date_field(self, value):
        """
        Returns a two-elements list with the lower and upper bound to be used
        with a BETWEEN operator to query a DateField value using a year lookup

        `value` is an int, containing the looked-up year.

        By default, it just calls `self.year_lookup_bounds`. Some backends need
        this hook because on their DB date fields can't be compared to values
        which include a time part.
        """
        return self.year_lookup_bounds(value)

<commit_msg>Fixed #8028 -- A couple of small docstring fixes from [8131]. Thanks, Leo Soto.<commit_after>try:
    # Only exists in Python 2.4+
    from threading import local
except ImportError:
    # Import copy of _thread_local.py from Python 2.4
    from django.utils._threading_local import local

from django.db.backends import util
from django.utils import datetime_safe

class BaseDatabaseWrapper(local):
    """
    Represents a database connection.
    """
    ops = None
    def __init__(self, **kwargs):
        self.connection = None
        self.queries = []
        self.options = kwargs

    def _commit(self):
        if self.connection is not None:
            return self.connection.commit()

    def _rollback(self):
        if self.connection is not None:
            return self.connection.rollback()

    def close(self):
        if self.connection is not None:
            self.connection.close()
            self.connection = None

    def cursor(self):
        from django.conf import settings
        cursor = self._cursor(settings)
        if settings.DEBUG:
            return self.make_debug_cursor(cursor)
        return cursor

    def make_debug_cursor(self, cursor):
        return util.CursorDebugWrapper(cursor, self)

class BaseDatabaseFeatures(object):
    allows_group_by_ordinal = True
    inline_fk_references = True
    # True if django.db.backend.utils.typecast_timestamp is used on values
    # returned from dates() calls.
    needs_datetime_string_cast = True
    supports_constraints = True
    supports_tablespaces = False
    uses_case_insensitive_names = False
    uses_custom_query_class = False
    empty_fetchmany_value = []
    update_can_self_select = True
    interprets_empty_strings_as_nulls = False
    can_use_chunked_reads = True

class BaseDatabaseOperations(object):
    """
    This class encapsulates all backend-specific differences, such as the way
    a backend performs ordering or calculates the ID of a recently-inserted
    row.
    """
    def autoinc_sql(self, table, column):
        """
        Returns any SQL needed to support auto-incrementing primary keys, or
        None if no SQL is necessary.

        This SQL is executed when a table is created.
        """
        return None

    def date_extract_sql(self, lookup_type, field_name):
        """
        Given a lookup_type of 'year', 'month' or 'day', returns the SQL that
        extracts a value from the given date field field_name.
        """
        raise NotImplementedError()

    def date_trunc_sql(self, lookup_type, field_name):
        """
        Given a lookup_type of 'year', 'month' or 'day', returns the SQL that
        truncates the given date field field_name to a DATE object with only
        the given specificity.
        """
        raise NotImplementedError()

    def datetime_cast_sql(self):
        """
        Returns the SQL necessary to cast a datetime value so that it will be
        retrieved as a Python datetime object instead of a string.

        This SQL should include a '%s' in place of the field's name.
        """
        return "%s"

    def deferrable_sql(self):
        """
        Returns the SQL necessary to make a constraint "initially deferred"
        during a CREATE TABLE statement.
        """
        return ''

    def drop_foreignkey_sql(self):
        """
        Returns the SQL command that drops a foreign key.
        """
        return "DROP CONSTRAINT"

    def drop_sequence_sql(self, table):
        """
        Returns any SQL necessary to drop the sequence for the given table.
        Returns None if no SQL is necessary.
        """
        return None

    def field_cast_sql(self, db_type):
        """
        Given a column type (e.g. 'BLOB', 'VARCHAR'), returns the SQL necessary
        to cast it before using it in a WHERE statement. Note that the
        resulting string should contain a '%s' placeholder for the column being
        searched against.
        """
        return '%s'

    def fulltext_search_sql(self, field_name):
        """
        Returns the SQL WHERE clause to use in order to perform a full-text
        search of the given field_name. Note that the resulting string should
        contain a '%s' placeholder for the value being searched against.
        """
        raise NotImplementedError('Full-text search is not implemented for this database backend')

    def last_executed_query(self, cursor, sql, params):
        """
        Returns a string of the query last executed by the given cursor, with
        placeholders replaced with actual values.

        `sql` is the raw query containing placeholders, and `params` is the
        sequence of parameters. These are used by default, but this method
        exists for database backends to provide a better implementation
        according to their own quoting schemes.
        """
        from django.utils.encoding import smart_unicode, force_unicode

        # Convert params to contain Unicode values.
        to_unicode = lambda s: force_unicode(s, strings_only=True)
        if isinstance(params, (list, tuple)):
            u_params = tuple([to_unicode(val) for val in params])
        else:
            u_params = dict([(to_unicode(k), to_unicode(v)) for k, v in params.items()])

        return smart_unicode(sql) % u_params

    def last_insert_id(self, cursor, table_name, pk_name):
        """
        Given a cursor object that has just performed an INSERT statement into
        a table that has an auto-incrementing ID, returns the newly created ID.

        This method also receives the table name and the name of the primary-key
        column.
        """
        return cursor.lastrowid

    def lookup_cast(self, lookup_type):
        """
        Returns the string to use in a query when performing lookups
        ("contains", "like", etc). The resulting string should contain a '%s'
        placeholder for the column being searched against.
        """
        return "%s"

    def max_name_length(self):
        """
        Returns the maximum length of table and column names, or None if there
        is no limit.
        """
        return None

    def no_limit_value(self):
        """
        Returns the value to use for the LIMIT when we are wanting "LIMIT
        infinity". Returns None if the limit clause can be omitted in this case.
        """
        # FIXME: API may need to change once Oracle backend is repaired.
        raise NotImplementedError()

    def pk_default_value(self):
        """
        Returns the value to use during an INSERT statement to specify that
        the field should use its default value.
        """
        return 'DEFAULT'

    def query_class(self, DefaultQueryClass):
        """
        Given the default Query class, returns a custom Query class
        to use for this backend. Returns None if a custom Query isn't used.
        See also BaseDatabaseFeatures.uses_custom_query_class, which regulates
        whether this method is called at all.
        """
        return None

    def quote_name(self, name):
        """
        Returns a quoted version of the given table, index or column name. Does
        not quote the given name if it's already been quoted.
        """
        raise NotImplementedError()

    def random_function_sql(self):
        """
        Returns a SQL expression that returns a random value.
        """
        return 'RANDOM()'

    def regex_lookup(self, lookup_type):
        """
        Returns the string to use in a query when performing regular expression
        lookups (using "regex" or "iregex"). The resulting string should
        contain a '%s' placeholder for the column being searched against.

        If the feature is not supported (or part of it is not supported), a
        NotImplementedError exception can be raised.
        """
        raise NotImplementedError

    def sql_flush(self, style, tables, sequences):
        """
        Returns a list of SQL statements required to remove all data from
        the given database tables (without actually removing the tables
        themselves).

        The `style` argument is a Style object as returned by either
        color_style() or no_style() in django.core.management.color.
        """
        raise NotImplementedError()

    def sequence_reset_sql(self, style, model_list):
        """
        Returns a list of the SQL statements required to reset sequences for
        the given models.

        The `style` argument is a Style object as returned by either
        color_style() or no_style() in django.core.management.color.
        """
        return [] # No sequence reset required by default.

    def start_transaction_sql(self):
        """
        Returns the SQL statement required to start a transaction.
        """
        return "BEGIN;"

    def tablespace_sql(self, tablespace, inline=False):
        """
        Returns the tablespace SQL, or None if the backend doesn't use
        tablespaces.
        """
        return None

    def prep_for_like_query(self, x):
        """Prepares a value for use in a LIKE query."""
        from django.utils.encoding import smart_unicode
        return smart_unicode(x).replace("\\", "\\\\").replace("%", "\%").replace("_", "\_")

    def value_to_db_date(self, value):
        """
        Transform a date value to an object compatible with what is expected
        by the backend driver for date columns.
        """
        if value is None:
            return None
        return datetime_safe.new_date(value).strftime('%Y-%m-%d')

    def value_to_db_datetime(self, value):
        """
        Transform a datetime value to an object compatible with what is expected
        by the backend driver for datetime columns.
        """
        if value is None:
            return None
        return unicode(value)

    def value_to_db_time(self, value):
        """
        Transform a datetime value to an object compatible with what is expected
        by the backend driver for time columns.
        """
        if value is None:
            return None
        return unicode(value)

    def value_to_db_decimal(self, value, max_digits, decimal_places):
        """
        Transform a decimal.Decimal value to an object compatible with what is
        expected by the backend driver for decimal (numeric) columns.
        """
        if value is None:
            return None
        return util.format_number(value, max_digits, decimal_places)

    def year_lookup_bounds(self, value):
        """
        Returns a two-elements list with the lower and upper bound to be used
        with a BETWEEN operator to query a field value using a year lookup

        `value` is an int, containing the looked-up year.
        """
        first = '%s-01-01 00:00:00'
        second = '%s-12-31 23:59:59.999999'
        return [first % value, second % value]

    def year_lookup_bounds_for_date_field(self, value):
        """
        Returns a two-elements list with the lower and upper bound to be used
        with a BETWEEN operator to query a DateField value using a year lookup

        `value` is an int, containing the looked-up year.

        By default, it just calls `self.year_lookup_bounds`. Some backends need
        this hook because on their DB date fields can't be compared to values
        which include a time part.
        """
        return self.year_lookup_bounds(value)

<|endoftext|>


<commit_before>"""
Classes to represent the definitions of aggregate functions.
"""
from django.core.exceptions import FieldError
from django.db.models.expressions import Case, Func, Star, When
from django.db.models.fields import DecimalField, FloatField, IntegerField
from django.db.models.query_utils import Q

__all__ = [
    'Aggregate', 'Avg', 'Count', 'Max', 'Min', 'StdDev', 'Sum', 'Variance',
]

class Aggregate(Func):
    contains_aggregate = True
    name = None
    filter_template = '%s FILTER (WHERE %%(filter)s)'

    def __init__(self, *args, filter=None, **kwargs):
        self.filter = filter
        super().__init__(*args, **kwargs)

    def get_source_fields(self):
        # Don't return the filter expression since it's not a source field.
        return [e._output_field_or_none for e in super().get_source_expressions()]

    def get_source_expressions(self):
        source_expressions = super().get_source_expressions()
        if self.filter:
            source_expressions += [self.filter]
        return source_expressions

    def set_source_expressions(self, exprs):
        if self.filter:
            self.filter = exprs.pop()
        return super().set_source_expressions(exprs)

    def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
        # Aggregates are not allowed in UPDATE queries, so ignore for_save
        c = super().resolve_expression(query, allow_joins, reuse, summarize)
        if c.filter:
            c.filter = c.filter.resolve_expression(query, allow_joins, reuse, summarize)
        if not summarize:
            # Call Aggregate.get_source_expressions() to avoid
            # returning self.filter and including that in this loop.
            expressions = super(Aggregate, c).get_source_expressions()
            for index, expr in enumerate(expressions):
                if expr.contains_aggregate:
                    before_resolved = self.get_source_expressions()[index]
                    name = before_resolved.name if hasattr(before_resolved, 'name') else repr(before_resolved)
                    raise FieldError("Cannot compute %s('%s'): '%s' is an aggregate" % (c.name, name, name))
        return c

    @property
    def default_alias(self):
        expressions = self.get_source_expressions()
        if len(expressions) == 1 and hasattr(expressions[0], 'name'):
            return '%s__%s' % (expressions[0].name, self.name.lower())
        raise TypeError("Complex expressions require an alias")

    def get_group_by_cols(self):
        return []

    def as_sql(self, compiler, connection, **extra_context):
        if self.filter:
            if connection.features.supports_aggregate_filter_clause:
                filter_sql, filter_params = self.filter.as_sql(compiler, connection)
                template = self.filter_template % extra_context.get('template', self.template)
                sql, params = super().as_sql(compiler, connection, template=template, filter=filter_sql)
                return sql, params + filter_params
            else:
                copy = self.copy()
                copy.filter = None
                condition = When(Q())
                source_expressions = copy.get_source_expressions()
                condition.set_source_expressions([self.filter, source_expressions[0]])
                copy.set_source_expressions([Case(condition)] + source_expressions[1:])
                return super(Aggregate, copy).as_sql(compiler, connection, **extra_context)
        return super().as_sql(compiler, connection, **extra_context)

    def _get_repr_options(self):
        options = super()._get_repr_options()
        if self.filter:
            options.update({'filter': self.filter})
        return options

class Avg(Aggregate):
    function = 'AVG'
    name = 'Avg'

    def _resolve_output_field(self):
        source_field = self.get_source_fields()[0]
        if isinstance(source_field, (IntegerField, DecimalField)):
            return FloatField()
        return super()._resolve_output_field()

    def as_oracle(self, compiler, connection):
        if self.output_field.get_internal_type() == 'DurationField':
            expression = self.get_source_expressions()[0]
            from django.db.backends.oracle.functions import IntervalToSeconds, SecondsToInterval
            return compiler.compile(
                SecondsToInterval(Avg(IntervalToSeconds(expression), filter=self.filter))
            )
        return super().as_sql(compiler, connection)

class Count(Aggregate):
    function = 'COUNT'
    name = 'Count'
    template = '%(function)s(%(distinct)s%(expressions)s)'

    def __init__(self, expression, distinct=False, filter=None, **extra):
        if expression == '*':
            expression = Star()
        if isinstance(expression, Star) and filter is not None:
            raise ValueError('Star cannot be used with filter. Please specify a field.')
        super().__init__(
            expression, distinct='DISTINCT ' if distinct else '',
            output_field=IntegerField(), filter=filter, **extra
        )

    def _get_repr_options(self):
        options = super()._get_repr_options()
        return dict(options, distinct=self.extra['distinct'] != '')

    def convert_value(self, value, expression, connection):
        return 0 if value is None else value

class Max(Aggregate):
    function = 'MAX'
    name = 'Max'

class Min(Aggregate):
    function = 'MIN'
    name = 'Min'

class StdDev(Aggregate):
    name = 'StdDev'

    def __init__(self, expression, sample=False, **extra):
        self.function = 'STDDEV_SAMP' if sample else 'STDDEV_POP'
        super().__init__(expression, output_field=FloatField(), **extra)

    def _get_repr_options(self):
        options = super()._get_repr_options()
        return dict(options, sample=self.function == 'STDDEV_SAMP')

    def convert_value(self, value, expression, connection):
        if value is None:
            return value
        return float(value)

class Sum(Aggregate):
    function = 'SUM'
    name = 'Sum'

    def as_oracle(self, compiler, connection):
        if self.output_field.get_internal_type() == 'DurationField':
            expression = self.get_source_expressions()[0]
            from django.db.backends.oracle.functions import IntervalToSeconds, SecondsToInterval
            return compiler.compile(
                SecondsToInterval(Sum(IntervalToSeconds(expression)))
            )
        return super().as_sql(compiler, connection)

class Variance(Aggregate):
    name = 'Variance'

    def __init__(self, expression, sample=False, **extra):
        self.function = 'VAR_SAMP' if sample else 'VAR_POP'
        super().__init__(expression, output_field=FloatField(), **extra)

    def _get_repr_options(self):
        options = super()._get_repr_options()
        return dict(options, sample=self.function == 'VAR_SAMP')

    def convert_value(self, value, expression, connection):
        if value is None:
            return value
        return float(value)
<commit_msg>Removed unneeded StdDev.convert_value() and Variance.convert_value().<commit_after>"""
Classes to represent the definitions of aggregate functions.
"""
from django.core.exceptions import FieldError
from django.db.models.expressions import Case, Func, Star, When
from django.db.models.fields import DecimalField, FloatField, IntegerField
from django.db.models.query_utils import Q

__all__ = [
    'Aggregate', 'Avg', 'Count', 'Max', 'Min', 'StdDev', 'Sum', 'Variance',
]

class Aggregate(Func):
    contains_aggregate = True
    name = None
    filter_template = '%s FILTER (WHERE %%(filter)s)'

    def __init__(self, *args, filter=None, **kwargs):
        self.filter = filter
        super().__init__(*args, **kwargs)

    def get_source_fields(self):
        # Don't return the filter expression since it's not a source field.
        return [e._output_field_or_none for e in super().get_source_expressions()]

    def get_source_expressions(self):
        source_expressions = super().get_source_expressions()
        if self.filter:
            source_expressions += [self.filter]
        return source_expressions

    def set_source_expressions(self, exprs):
        if self.filter:
            self.filter = exprs.pop()
        return super().set_source_expressions(exprs)

    def resolve_expression(self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False):
        # Aggregates are not allowed in UPDATE queries, so ignore for_save
        c = super().resolve_expression(query, allow_joins, reuse, summarize)
        if c.filter:
            c.filter = c.filter.resolve_expression(query, allow_joins, reuse, summarize)
        if not summarize:
            # Call Aggregate.get_source_expressions() to avoid
            # returning self.filter and including that in this loop.
            expressions = super(Aggregate, c).get_source_expressions()
            for index, expr in enumerate(expressions):
                if expr.contains_aggregate:
                    before_resolved = self.get_source_expressions()[index]
                    name = before_resolved.name if hasattr(before_resolved, 'name') else repr(before_resolved)
                    raise FieldError("Cannot compute %s('%s'): '%s' is an aggregate" % (c.name, name, name))
        return c

    @property
    def default_alias(self):
        expressions = self.get_source_expressions()
        if len(expressions) == 1 and hasattr(expressions[0], 'name'):
            return '%s__%s' % (expressions[0].name, self.name.lower())
        raise TypeError("Complex expressions require an alias")

    def get_group_by_cols(self):
        return []

    def as_sql(self, compiler, connection, **extra_context):
        if self.filter:
            if connection.features.supports_aggregate_filter_clause:
                filter_sql, filter_params = self.filter.as_sql(compiler, connection)
                template = self.filter_template % extra_context.get('template', self.template)
                sql, params = super().as_sql(compiler, connection, template=template, filter=filter_sql)
                return sql, params + filter_params
            else:
                copy = self.copy()
                copy.filter = None
                condition = When(Q())
                source_expressions = copy.get_source_expressions()
                condition.set_source_expressions([self.filter, source_expressions[0]])
                copy.set_source_expressions([Case(condition)] + source_expressions[1:])
                return super(Aggregate, copy).as_sql(compiler, connection, **extra_context)
        return super().as_sql(compiler, connection, **extra_context)

    def _get_repr_options(self):
        options = super()._get_repr_options()
        if self.filter:
            options.update({'filter': self.filter})
        return options

class Avg(Aggregate):
    function = 'AVG'
    name = 'Avg'

    def _resolve_output_field(self):
        source_field = self.get_source_fields()[0]
        if isinstance(source_field, (IntegerField, DecimalField)):
            return FloatField()
        return super()._resolve_output_field()

    def as_oracle(self, compiler, connection):
        if self.output_field.get_internal_type() == 'DurationField':
            expression = self.get_source_expressions()[0]
            from django.db.backends.oracle.functions import IntervalToSeconds, SecondsToInterval
            return compiler.compile(
                SecondsToInterval(Avg(IntervalToSeconds(expression), filter=self.filter))
            )
        return super().as_sql(compiler, connection)

class Count(Aggregate):
    function = 'COUNT'
    name = 'Count'
    template = '%(function)s(%(distinct)s%(expressions)s)'

    def __init__(self, expression, distinct=False, filter=None, **extra):
        if expression == '*':
            expression = Star()
        if isinstance(expression, Star) and filter is not None:
            raise ValueError('Star cannot be used with filter. Please specify a field.')
        super().__init__(
            expression, distinct='DISTINCT ' if distinct else '',
            output_field=IntegerField(), filter=filter, **extra
        )

    def _get_repr_options(self):
        options = super()._get_repr_options()
        return dict(options, distinct=self.extra['distinct'] != '')

    def convert_value(self, value, expression, connection):
        return 0 if value is None else value

class Max(Aggregate):
    function = 'MAX'
    name = 'Max'

class Min(Aggregate):
    function = 'MIN'
    name = 'Min'

class StdDev(Aggregate):
    name = 'StdDev'

    def __init__(self, expression, sample=False, **extra):
        self.function = 'STDDEV_SAMP' if sample else 'STDDEV_POP'
        super().__init__(expression, output_field=FloatField(), **extra)

    def _get_repr_options(self):
        options = super()._get_repr_options()
        return dict(options, sample=self.function == 'STDDEV_SAMP')

class Sum(Aggregate):
    function = 'SUM'
    name = 'Sum'

    def as_oracle(self, compiler, connection):
        if self.output_field.get_internal_type() == 'DurationField':
            expression = self.get_source_expressions()[0]
            from django.db.backends.oracle.functions import IntervalToSeconds, SecondsToInterval
            return compiler.compile(
                SecondsToInterval(Sum(IntervalToSeconds(expression)))
            )
        return super().as_sql(compiler, connection)

class Variance(Aggregate):
    name = 'Variance'

    def __init__(self, expression, sample=False, **extra):
        self.function = 'VAR_SAMP' if sample else 'VAR_POP'
        super().__init__(expression, output_field=FloatField(), **extra)

    def _get_repr_options(self):
        options = super()._get_repr_options()
        return dict(options, sample=self.function == 'VAR_SAMP')
<|endoftext|>


<commit_before>class MergeDict(object):
    """
    A simple class for creating new "virtual" dictionaries that actualy look
    up values in more than one dictionary, passed in the constructor.
    """
    def __init__(self, *dicts):
        self.dicts = dicts

    def __getitem__(self, key):
        for dict in self.dicts:
            try:
                return dict[key]
            except KeyError:
                pass
        raise KeyError

    def __copy__(self):
        return self.__class__(*self.dicts)

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def getlist(self, key):
        for dict in self.dicts:
            try:
                return dict.getlist(key)
            except KeyError:
                pass
        raise KeyError

    def items(self):
        item_list = []
        for dict in self.dicts:
            item_list.extend(dict.items())
        return item_list

    def has_key(self, key):
        for dict in self.dicts:
            if key in dict:
                return True
        return False

    __contains__ = has_key

    def copy(self):
        """Returns a copy of this object."""
        return self.__copy__()

class SortedDict(dict):
    """
    A dictionary that keeps its keys in the order in which they're inserted.
    """
    def __init__(self, data=None):
        if data is None:
            data = {}
        dict.__init__(self, data)
        if isinstance(data, dict):
            self.keyOrder = data.keys()
        else:
            self.keyOrder = [key for key, value in data]

    def __deepcopy__(self,memo):
        from copy import deepcopy
        obj = self.__class__()
        for k, v in self.items():
            obj[k] = deepcopy(v, memo)
        return obj

    def __setitem__(self, key, value):
        dict.__setitem__(self, key, value)
        if key not in self.keyOrder:
            self.keyOrder.append(key)

    def __delitem__(self, key):
        dict.__delitem__(self, key)
        self.keyOrder.remove(key)

    def __iter__(self):
        for k in self.keyOrder:
            yield k

    def pop(self, k, *args):
        result = dict.pop(self, k, *args)
        try:
            self.keyOrder.remove(k)
        except ValueError:
            # Key wasn't in the dictionary in the first place. No problem.
            pass
        return result

    def popitem(self):
        result = dict.popitem(self)
        self.keyOrder.remove(result[0])
        return result

    def items(self):
        return zip(self.keyOrder, self.values())

    def iteritems(self):
        for key in self.keyOrder:
            yield key, dict.__getitem__(self, key)

    def keys(self):
        return self.keyOrder[:]

    def iterkeys(self):
        return iter(self.keyOrder)

    def values(self):
        return [dict.__getitem__(self, k) for k in self.keyOrder]

    def itervalues(self):
        for key in self.keyOrder:
            yield dict.__getitem__(self, key)

    def update(self, dict):
        for k, v in dict.items():
            self.__setitem__(k, v)

    def setdefault(self, key, default):
        if key not in self.keyOrder:
            self.keyOrder.append(key)
        return dict.setdefault(self, key, default)

    def value_for_index(self, index):
        """Returns the value of the item at the given zero-based index."""
        return self[self.keyOrder[index]]

    def insert(self, index, key, value):
        """Inserts the key, value pair before the item with the given index."""
        if key in self.keyOrder:
            n = self.keyOrder.index(key)
            del self.keyOrder[n]
            if n < index:
                index -= 1
        self.keyOrder.insert(index, key)
        dict.__setitem__(self, key, value)

    def copy(self):
        """Returns a copy of this object."""
        # This way of initializing the copy means it works for subclasses, too.
        obj = self.__class__(self)
        obj.keyOrder = self.keyOrder
        return obj

    def __repr__(self):
        """
        Replaces the normal dict.__repr__ with a version that returns the keys
        in their sorted order.
        """
        return '{%s}' % ', '.join(['%r: %r' % (k, v) for k, v in self.items()])

class MultiValueDictKeyError(KeyError):
    pass

class MultiValueDict(dict):
    """
    A subclass of dictionary customized to handle multiple values for the
    same key.

    >>> d = MultiValueDict({'name': ['Adrian', 'Simon'], 'position': ['Developer']})
    >>> d['name']
    'Simon'
    >>> d.getlist('name')
    ['Adrian', 'Simon']
    >>> d.get('lastname', 'nonexistent')
    'nonexistent'
    >>> d.setlist('lastname', ['Holovaty', 'Willison'])

    This class exists to solve the irritating problem raised by cgi.parse_qs,
    which returns a list for every key, even though most Web forms submit
    single name-value pairs.
    """
    def __init__(self, key_to_list_mapping=()):
        dict.__init__(self, key_to_list_mapping)

    def __repr__(self):
        return "<%s: %s>" % (self.__class__.__name__, dict.__repr__(self))

    def __getitem__(self, key):
        """
        Returns the last data value for this key, or [] if it's an empty list;
        raises KeyError if not found.
        """
        try:
            list_ = dict.__getitem__(self, key)
        except KeyError:
            raise MultiValueDictKeyError, "Key %r not found in %r" % (key, self)
        try:
            return list_[-1]
        except IndexError:
            return []

    def __setitem__(self, key, value):
        dict.__setitem__(self, key, [value])

    def __copy__(self):
        return self.__class__(dict.items(self))

    def __deepcopy__(self, memo=None):
        import copy
        if memo is None:
            memo = {}
        result = self.__class__()
        memo[id(self)] = result
        for key, value in dict.items(self):
            dict.__setitem__(result, copy.deepcopy(key, memo),
                             copy.deepcopy(value, memo))
        return result

    def get(self, key, default=None):
        """Returns the default value if the requested data doesn't exist."""
        try:
            val = self[key]
        except KeyError:
            return default
        if val == []:
            return default
        return val

    def getlist(self, key):
        """Returns an empty list if the requested data doesn't exist."""
        try:
            return dict.__getitem__(self, key)
        except KeyError:
            return []

    def setlist(self, key, list_):
        dict.__setitem__(self, key, list_)

    def setdefault(self, key, default=None):
        if key not in self:
            self[key] = default
        return self[key]

    def setlistdefault(self, key, default_list=()):
        if key not in self:
            self.setlist(key, default_list)
        return self.getlist(key)

    def appendlist(self, key, value):
        """Appends an item to the internal list associated with key."""
        self.setlistdefault(key, [])
        dict.__setitem__(self, key, self.getlist(key) + [value])

    def items(self):
        """
        Returns a list of (key, value) pairs, where value is the last item in
        the list associated with the key.
        """
        return [(key, self[key]) for key in self.keys()]

    def lists(self):
        """Returns a list of (key, list) pairs."""
        return dict.items(self)

    def values(self):
        """Returns a list of the last value on every key list."""
        return [self[key] for key in self.keys()]

    def copy(self):
        """Returns a copy of this object."""
        return self.__deepcopy__()

    def update(self, *args, **kwargs):
        """
        update() extends rather than replaces existing key lists.
        Also accepts keyword args.
        """
        if len(args) > 1:
            raise TypeError, "update expected at most 1 arguments, got %d" % len(args)
        if args:
            other_dict = args[0]
            if isinstance(other_dict, MultiValueDict):
                for key, value_list in other_dict.lists():
                    self.setlistdefault(key, []).extend(value_list)
            else:
                try:
                    for key, value in other_dict.items():
                        self.setlistdefault(key, []).append(value)
                except TypeError:
                    raise ValueError, "MultiValueDict.update() takes either a MultiValueDict or dictionary"
        for key, value in kwargs.iteritems():
            self.setlistdefault(key, []).append(value)

class DotExpandedDict(dict):
    """
    A special dictionary constructor that takes a dictionary in which the keys
    may contain dots to specify inner dictionaries. It's confusing, but this
    example should make sense.

    >>> d = DotExpandedDict({'person.1.firstname': ['Simon'], \
            'person.1.lastname': ['Willison'], \
            'person.2.firstname': ['Adrian'], \
            'person.2.lastname': ['Holovaty']})
    >>> d
    {'person': {'1': {'lastname': ['Willison'], 'firstname': ['Simon']}, '2': {'lastname': ['Holovaty'], 'firstname': ['Adrian']}}}
    >>> d['person']
    {'1': {'lastname': ['Willison'], 'firstname': ['Simon']}, '2': {'lastname': ['Holovaty'], 'firstname': ['Adrian']}}
    >>> d['person']['1']
    {'lastname': ['Willison'], 'firstname': ['Simon']}

    # Gotcha: Results are unpredictable if the dots are "uneven":
    >>> DotExpandedDict({'c.1': 2, 'c.2': 3, 'c': 1})
    {'c': 1}
    """
    def __init__(self, key_to_list_mapping):
        for k, v in key_to_list_mapping.items():
            current = self
            bits = k.split('.')
            for bit in bits[:-1]:
                current = current.setdefault(bit, {})
            # Now assign value to current position
            try:
                current[bits[-1]] = v
            except TypeError: # Special-case if current isn't a dict.
                current = {bits[-1] : v}

class FileDict(dict):
    """
    A dictionary used to hold uploaded file contents. The only special feature
    here is that repr() of this object won't dump the entire contents of the
    file to the output. A handy safeguard for a large file upload.
    """
    def __repr__(self):
        if 'content' in self:
            d = dict(self, content='<omitted>')
            return dict.__repr__(d)
        return dict.__repr__(self)
<commit_msg>Simplified `SortedDict.__deepcopy__` now that the its constructor can take a sequence of tuples.<commit_after>class MergeDict(object):
    """
    A simple class for creating new "virtual" dictionaries that actualy look
    up values in more than one dictionary, passed in the constructor.
    """
    def __init__(self, *dicts):
        self.dicts = dicts

    def __getitem__(self, key):
        for dict in self.dicts:
            try:
                return dict[key]
            except KeyError:
                pass
        raise KeyError

    def __copy__(self):
        return self.__class__(*self.dicts)

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def getlist(self, key):
        for dict in self.dicts:
            try:
                return dict.getlist(key)
            except KeyError:
                pass
        raise KeyError

    def items(self):
        item_list = []
        for dict in self.dicts:
            item_list.extend(dict.items())
        return item_list

    def has_key(self, key):
        for dict in self.dicts:
            if key in dict:
                return True
        return False

    __contains__ = has_key

    def copy(self):
        """Returns a copy of this object."""
        return self.__copy__()

class SortedDict(dict):
    """
    A dictionary that keeps its keys in the order in which they're inserted.
    """
    def __init__(self, data=None):
        if data is None:
            data = {}
        dict.__init__(self, data)
        if isinstance(data, dict):
            self.keyOrder = data.keys()
        else:
            self.keyOrder = [key for key, value in data]

    def __deepcopy__(self, memo):
        from copy import deepcopy
        return self.__class__([(key, deepcopy(value, memo))
                               for key, value in self.iteritems()])

    def __setitem__(self, key, value):
        dict.__setitem__(self, key, value)
        if key not in self.keyOrder:
            self.keyOrder.append(key)

    def __delitem__(self, key):
        dict.__delitem__(self, key)
        self.keyOrder.remove(key)

    def __iter__(self):
        for k in self.keyOrder:
            yield k

    def pop(self, k, *args):
        result = dict.pop(self, k, *args)
        try:
            self.keyOrder.remove(k)
        except ValueError:
            # Key wasn't in the dictionary in the first place. No problem.
            pass
        return result

    def popitem(self):
        result = dict.popitem(self)
        self.keyOrder.remove(result[0])
        return result

    def items(self):
        return zip(self.keyOrder, self.values())

    def iteritems(self):
        for key in self.keyOrder:
            yield key, dict.__getitem__(self, key)

    def keys(self):
        return self.keyOrder[:]

    def iterkeys(self):
        return iter(self.keyOrder)

    def values(self):
        return [dict.__getitem__(self, k) for k in self.keyOrder]

    def itervalues(self):
        for key in self.keyOrder:
            yield dict.__getitem__(self, key)

    def update(self, dict):
        for k, v in dict.items():
            self.__setitem__(k, v)

    def setdefault(self, key, default):
        if key not in self.keyOrder:
            self.keyOrder.append(key)
        return dict.setdefault(self, key, default)

    def value_for_index(self, index):
        """Returns the value of the item at the given zero-based index."""
        return self[self.keyOrder[index]]

    def insert(self, index, key, value):
        """Inserts the key, value pair before the item with the given index."""
        if key in self.keyOrder:
            n = self.keyOrder.index(key)
            del self.keyOrder[n]
            if n < index:
                index -= 1
        self.keyOrder.insert(index, key)
        dict.__setitem__(self, key, value)

    def copy(self):
        """Returns a copy of this object."""
        # This way of initializing the copy means it works for subclasses, too.
        obj = self.__class__(self)
        obj.keyOrder = self.keyOrder
        return obj

    def __repr__(self):
        """
        Replaces the normal dict.__repr__ with a version that returns the keys
        in their sorted order.
        """
        return '{%s}' % ', '.join(['%r: %r' % (k, v) for k, v in self.items()])

class MultiValueDictKeyError(KeyError):
    pass

class MultiValueDict(dict):
    """
    A subclass of dictionary customized to handle multiple values for the
    same key.

    >>> d = MultiValueDict({'name': ['Adrian', 'Simon'], 'position': ['Developer']})
    >>> d['name']
    'Simon'
    >>> d.getlist('name')
    ['Adrian', 'Simon']
    >>> d.get('lastname', 'nonexistent')
    'nonexistent'
    >>> d.setlist('lastname', ['Holovaty', 'Willison'])

    This class exists to solve the irritating problem raised by cgi.parse_qs,
    which returns a list for every key, even though most Web forms submit
    single name-value pairs.
    """
    def __init__(self, key_to_list_mapping=()):
        dict.__init__(self, key_to_list_mapping)

    def __repr__(self):
        return "<%s: %s>" % (self.__class__.__name__, dict.__repr__(self))

    def __getitem__(self, key):
        """
        Returns the last data value for this key, or [] if it's an empty list;
        raises KeyError if not found.
        """
        try:
            list_ = dict.__getitem__(self, key)
        except KeyError:
            raise MultiValueDictKeyError, "Key %r not found in %r" % (key, self)
        try:
            return list_[-1]
        except IndexError:
            return []

    def __setitem__(self, key, value):
        dict.__setitem__(self, key, [value])

    def __copy__(self):
        return self.__class__(dict.items(self))

    def __deepcopy__(self, memo=None):
        import copy
        if memo is None:
            memo = {}
        result = self.__class__()
        memo[id(self)] = result
        for key, value in dict.items(self):
            dict.__setitem__(result, copy.deepcopy(key, memo),
                             copy.deepcopy(value, memo))
        return result

    def get(self, key, default=None):
        """Returns the default value if the requested data doesn't exist."""
        try:
            val = self[key]
        except KeyError:
            return default
        if val == []:
            return default
        return val

    def getlist(self, key):
        """Returns an empty list if the requested data doesn't exist."""
        try:
            return dict.__getitem__(self, key)
        except KeyError:
            return []

    def setlist(self, key, list_):
        dict.__setitem__(self, key, list_)

    def setdefault(self, key, default=None):
        if key not in self:
            self[key] = default
        return self[key]

    def setlistdefault(self, key, default_list=()):
        if key not in self:
            self.setlist(key, default_list)
        return self.getlist(key)

    def appendlist(self, key, value):
        """Appends an item to the internal list associated with key."""
        self.setlistdefault(key, [])
        dict.__setitem__(self, key, self.getlist(key) + [value])

    def items(self):
        """
        Returns a list of (key, value) pairs, where value is the last item in
        the list associated with the key.
        """
        return [(key, self[key]) for key in self.keys()]

    def lists(self):
        """Returns a list of (key, list) pairs."""
        return dict.items(self)

    def values(self):
        """Returns a list of the last value on every key list."""
        return [self[key] for key in self.keys()]

    def copy(self):
        """Returns a copy of this object."""
        return self.__deepcopy__()

    def update(self, *args, **kwargs):
        """
        update() extends rather than replaces existing key lists.
        Also accepts keyword args.
        """
        if len(args) > 1:
            raise TypeError, "update expected at most 1 arguments, got %d" % len(args)
        if args:
            other_dict = args[0]
            if isinstance(other_dict, MultiValueDict):
                for key, value_list in other_dict.lists():
                    self.setlistdefault(key, []).extend(value_list)
            else:
                try:
                    for key, value in other_dict.items():
                        self.setlistdefault(key, []).append(value)
                except TypeError:
                    raise ValueError, "MultiValueDict.update() takes either a MultiValueDict or dictionary"
        for key, value in kwargs.iteritems():
            self.setlistdefault(key, []).append(value)

class DotExpandedDict(dict):
    """
    A special dictionary constructor that takes a dictionary in which the keys
    may contain dots to specify inner dictionaries. It's confusing, but this
    example should make sense.

    >>> d = DotExpandedDict({'person.1.firstname': ['Simon'], \
            'person.1.lastname': ['Willison'], \
            'person.2.firstname': ['Adrian'], \
            'person.2.lastname': ['Holovaty']})
    >>> d
    {'person': {'1': {'lastname': ['Willison'], 'firstname': ['Simon']}, '2': {'lastname': ['Holovaty'], 'firstname': ['Adrian']}}}
    >>> d['person']
    {'1': {'lastname': ['Willison'], 'firstname': ['Simon']}, '2': {'lastname': ['Holovaty'], 'firstname': ['Adrian']}}
    >>> d['person']['1']
    {'lastname': ['Willison'], 'firstname': ['Simon']}

    # Gotcha: Results are unpredictable if the dots are "uneven":
    >>> DotExpandedDict({'c.1': 2, 'c.2': 3, 'c': 1})
    {'c': 1}
    """
    def __init__(self, key_to_list_mapping):
        for k, v in key_to_list_mapping.items():
            current = self
            bits = k.split('.')
            for bit in bits[:-1]:
                current = current.setdefault(bit, {})
            # Now assign value to current position
            try:
                current[bits[-1]] = v
            except TypeError: # Special-case if current isn't a dict.
                current = {bits[-1] : v}

class FileDict(dict):
    """
    A dictionary used to hold uploaded file contents. The only special feature
    here is that repr() of this object won't dump the entire contents of the
    file to the output. A handy safeguard for a large file upload.
    """
    def __repr__(self):
        if 'content' in self:
            d = dict(self, content='<omitted>')
            return dict.__repr__(d)
        return dict.__repr__(self)
<|endoftext|>


<commit_before>from django.db import models
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _

from core.utils import ChoiceEnum

class ExecutionStatus(ChoiceEnum):
    INIT = "Initiating"
    QUEUEING = "Waiting in the queue"
    RUNNING = "Running"
    SUCCESSFUL = "Successful"
    FAILED = "Failed"

class AbstractPipelineModel(models.Model):
    analysis_name = models.CharField(max_length=255)
    date_created = models.DateTimeField(
        verbose_name=_('date created'),
        default=timezone.now,
        editable=False,
    )
    date_finished = models.DateTimeField(
        verbose_name=_('date finished'),
        blank=True,
        default=None,
        editable=False,
    )
    execution_status = models.CharField(
        choices=ExecutionStatus.choices(),
        max_length=63,
        default=ExecutionStatus.INIT.name,
        editable=False,
    )

    class Meta:
        abstract = True
<commit_msg>Add owner to PipelineModel<commit_after>from django.conf import settings
from django.db import models
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _

from core.utils import ChoiceEnum

class ExecutionStatus(ChoiceEnum):
    INIT = "Initiating"
    QUEUEING = "Waiting in the queue"
    RUNNING = "Running"
    SUCCESSFUL = "Successful"
    FAILED = "Failed"

class AbstractPipelineModel(models.Model):
    owner = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        related_name='analyses'
    )
    analysis_name = models.CharField(max_length=255)
    date_created = models.DateTimeField(
        verbose_name=_('date created'),
        default=timezone.now,
        editable=False,
    )
    date_finished = models.DateTimeField(
        verbose_name=_('date finished'),
        blank=True,
        default=None,
        editable=False,
    )
    execution_status = models.CharField(
        choices=ExecutionStatus.choices(),
        max_length=63,
        default=ExecutionStatus.INIT.name,
        editable=False,
    )

    class Meta:
        abstract = True
<|endoftext|>


<commit_before>from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.mail.backends.base import BaseEmailBackend
from django.core.mail.message import sanitize_address
from django.utils import simplejson as json

import requests

class DjrillBackendHTTPError(Exception):
   """An exception that will turn into an HTTP error response."""
   def __init__(self, status_code, log_message=None, *args):
      self.status_code = status_code
      self.log_message = log_message
      self.args = args

      def __str__(self):
         message = "DjrillBackendHTTP %d: %s" % (
            self.status_code, httplib.responses[self.status_code])
         if self.log_message:
            return message + " (" + (self.log_message % self.args) + ")"
         else:
            return message

class DjrillBackend(BaseEmailBackend):
    """
    Mandrill API Email Backend
    """

    def __init__(self, fail_silently=False, **kwargs):
        """
        Set the API key, API url and set the action url.
        """
        super(DjrillBackend, self).__init__(**kwargs)
        self.api_key = getattr(settings, "MANDRILL_API_KEY", None)
        self.api_url = getattr(settings, "MANDRILL_API_URL", None)
        self.connection = None

        if not self.api_key:
            raise ImproperlyConfigured("You have not set your mandrill api key "
                "in the settings.py file.")
        if not self.api_url:
            raise ImproperlyConfigured("You have not added the Mandrill api "
                "url to your settings.py")

        self.api_action = self.api_url + "/messages/send.json"
        self.api_verify = self.api_url + "/users/ping.json"

    def open(self, sender):
        """
        """
        self.connection = None

        valid_sender = requests.post(
            self.api_verify, data={"key": self.api_key})

        if valid_sender.status_code == 200:
                self.connection = True
                return True
        else:
            if not self.fail_silently:
                raise

    def send_messages(self, email_messages):
        if not email_messages:
            return

        num_sent = 0
        for message in email_messages:
            self.open(message.from_email)
            if not self.connection:
                return

            sent = self._send(message)

            if sent:
                num_sent += 1

        return num_sent

    def _send(self, message):
        if not message.recipients():
            return False

        self.sender = sanitize_address(message.from_email, message.encoding)
        recipients_list = [sanitize_address(addr, message.encoding)
            for addr in message.recipients()]
        from email.utils import parseaddr
        self.recipients = [{"email": e, "name": n} for n,e in [
            parseaddr(r) for r in recipients_list]]

        self.msg_dict = self._build_standard_message_dict(message)

        if getattr(message, "alternative_subtype", None):
            if message.alternative_subtype == "mandrill":
                self._build_advanced_message_dict(message)
                if message.alternatives:
                    self._add_alternatives(message)

        djrill_it = requests.post(self.api_action, data=json.dumps({
            "key": self.api_key,
            "message": self.msg_dict
        }))

        if djrill_it.status_code != 200:
            if not self.fail_silently:
                raise DjrillBackendHTTPError(status_code=djrill_it.status_code, log_message="Failed to send a message to %s, from %s" % (self.recipients, self.sender))
            return False
        return True

    def _build_standard_message_dict(self, message):
        """
        Build standard message dict.

        Builds the standard dict that Django's send_mail and send_mass_mail
        use by default. Standard text email messages sent through Django will
        still work through Mandrill.
        """
        return {
            "text": message.body,
            "subject": message.subject,
            "from_email": self.sender,
            "to": self.recipients
        }

    def _build_advanced_message_dict(self, message):
        """
        Builds advanced message dict and attaches any accepted extra headers.
        """
        self.msg_dict.update({
            "from_name": message.from_name,
            "tags": message.tags,
            "track_opens": message.track_opens,
        })

        if message.extra_headers:
            accepted_headers = {}

            for k in message.extra_headers.keys():
                if k.startswith("X-") or k == "Reply-To":
                    accepted_headers.update(
                        {"%s" % k: message.extra_headers[k]})
            self.msg_dict.update({"headers": accepted_headers})

    def _add_alternatives(self, message):
        """
        There can be only one! ... alternative attachment.

        Since mandrill does not accept image attachments or anything other
        than HTML, the assumption is the only thing you are attaching is
        the HTML output for your email.
        """
        if len(message.alternatives) > 1:
            raise ImproperlyConfigured(
                "Mandrill only accepts plain text and html emails. Please "
                "check the alternatives you have attached to your message.")

        self.msg_dict.update({
            "html": message.alternatives[0][0],
            "track_clicks": message.track_clicks
        })
<commit_msg>Backend: fix exception messaging<commit_after>from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.mail.backends.base import BaseEmailBackend
from django.core.mail.message import sanitize_address
from django.utils import simplejson as json

import requests

class DjrillBackendHTTPError(Exception):
    """An exception that will turn into an HTTP error response."""
    def __init__(self, status_code, log_message=None):
        super(DjrillBackendHTTPError, self).__init__()
        self.status_code = status_code
        self.log_message = log_message

    def __str__(self):
        message = "DjrillBackendHTTP %d" % self.status_code
        if self.log_message:
            return message + " " + self.log_message
        else:
            return message

class DjrillBackend(BaseEmailBackend):
    """
    Mandrill API Email Backend
    """

    def __init__(self, fail_silently=False, **kwargs):
        """
        Set the API key, API url and set the action url.
        """
        super(DjrillBackend, self).__init__(**kwargs)
        self.api_key = getattr(settings, "MANDRILL_API_KEY", None)
        self.api_url = getattr(settings, "MANDRILL_API_URL", None)
        self.connection = None

        if not self.api_key:
            raise ImproperlyConfigured("You have not set your mandrill api key "
                "in the settings.py file.")
        if not self.api_url:
            raise ImproperlyConfigured("You have not added the Mandrill api "
                "url to your settings.py")

        self.api_action = self.api_url + "/messages/send.json"
        self.api_verify = self.api_url + "/users/ping.json"

    def open(self, sender):
        """
        """
        self.connection = None

        valid_sender = requests.post(
            self.api_verify, data={"key": self.api_key})

        if valid_sender.status_code == 200:
                self.connection = True
                return True
        else:
            if not self.fail_silently:
                raise

    def send_messages(self, email_messages):
        if not email_messages:
            return

        num_sent = 0
        for message in email_messages:
            self.open(message.from_email)
            if not self.connection:
                return

            sent = self._send(message)

            if sent:
                num_sent += 1

        return num_sent

    def _send(self, message):
        if not message.recipients():
            return False

        self.sender = sanitize_address(message.from_email, message.encoding)
        recipients_list = [sanitize_address(addr, message.encoding)
            for addr in message.recipients()]
        from email.utils import parseaddr
        self.recipients = [{"email": e, "name": n} for n,e in [
            parseaddr(r) for r in recipients_list]]

        self.msg_dict = self._build_standard_message_dict(message)

        if getattr(message, "alternative_subtype", None):
            if message.alternative_subtype == "mandrill":
                self._build_advanced_message_dict(message)
                if message.alternatives:
                    self._add_alternatives(message)

        djrill_it = requests.post(self.api_action, data=json.dumps({
            "key": self.api_key,
            "message": self.msg_dict
        }))

        if djrill_it.status_code != 200:
            if not self.fail_silently:
                raise DjrillBackendHTTPError(status_code=djrill_it.status_code, log_message="Failed to send a message to %s, from %s" % (self.recipients, self.sender))
            return False
        return True

    def _build_standard_message_dict(self, message):
        """
        Build standard message dict.

        Builds the standard dict that Django's send_mail and send_mass_mail
        use by default. Standard text email messages sent through Django will
        still work through Mandrill.
        """
        return {
            "text": message.body,
            "subject": message.subject,
            "from_email": self.sender,
            "to": self.recipients
        }

    def _build_advanced_message_dict(self, message):
        """
        Builds advanced message dict and attaches any accepted extra headers.
        """
        self.msg_dict.update({
            "from_name": message.from_name,
            "tags": message.tags,
            "track_opens": message.track_opens,
        })

        if message.extra_headers:
            accepted_headers = {}

            for k in message.extra_headers.keys():
                if k.startswith("X-") or k == "Reply-To":
                    accepted_headers.update(
                        {"%s" % k: message.extra_headers[k]})
            self.msg_dict.update({"headers": accepted_headers})

    def _add_alternatives(self, message):
        """
        There can be only one! ... alternative attachment.

        Since mandrill does not accept image attachments or anything other
        than HTML, the assumption is the only thing you are attaching is
        the HTML output for your email.
        """
        if len(message.alternatives) > 1:
            raise ImproperlyConfigured(
                "Mandrill only accepts plain text and html emails. Please "
                "check the alternatives you have attached to your message.")

        self.msg_dict.update({
            "html": message.alternatives[0][0],
            "track_clicks": message.track_clicks
        })
<|endoftext|>


<commit_before>from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.mail.backends.base import BaseEmailBackend
from django.core.mail.message import sanitize_address
from django.utils import simplejson as json

import requests

class DjrillBackendHTTPError(Exception):
   """An exception that will turn into an HTTP error response."""
   def __init__(self, status_code, log_message=None, *args):
      self.status_code = status_code
      self.log_message = log_message
      self.args = args

      def __str__(self):
         message = "DjrillBackendHTTP %d: %s" % (
            self.status_code, httplib.responses[self.status_code])
         if self.log_message:
            return message + " (" + (self.log_message % self.args) + ")"
         else:
            return message

class DjrillBackend(BaseEmailBackend):
    """
    Mandrill API Email Backend
    """

    def __init__(self, fail_silently=False, **kwargs):
        """
        Set the API key, API url and set the action url.
        """
        super(DjrillBackend, self).__init__(**kwargs)
        self.api_key = getattr(settings, "MANDRILL_API_KEY", None)
        self.api_url = getattr(settings, "MANDRILL_API_URL", None)
        self.connection = None

        if not self.api_key:
            raise ImproperlyConfigured("You have not set your mandrill api key "
                "in the settings.py file.")
        if not self.api_url:
            raise ImproperlyConfigured("You have not added the Mandrill api "
                "url to your settings.py")

        self.api_action = self.api_url + "/messages/send.json"
        self.api_verify = self.api_url + "/users/verify-sender.json"

    def open(self, sender):
        """
        """
        self.connection = True
        # self.connection = None
        #
        # valid_sender = requests.post(
        #     self.api_verify, data={"key": self.api_key, "email": sender})
        #
        # if valid_sender.status_code == 200:
        #     data = json.loads(valid_sender.content)
        #     if data["is_enabled"]:
        #         self.connection = True
        #         return True
        # else:
        #     if not self.fail_silently:
        #         raise

    def send_messages(self, email_messages):
        if not email_messages:
            return

        num_sent = 0
        for message in email_messages:
            self.open(message.from_email)
            if not self.connection:
                return

            sent = self._send(message)

            if sent:
                num_sent += 1

        return num_sent

    def _send(self, message):
        if not message.recipients():
            return False

        self.sender = sanitize_address(message.from_email, message.encoding)
        recipients_list = [sanitize_address(addr, message.encoding)
            for addr in message.recipients()]
        from email.utils import parseaddr
        self.recipients = [{"email": e, "name": n} for n,e in [
            parseaddr(r) for r in recipients_list]]

        self.msg_dict = self._build_standard_message_dict(message)

        if getattr(message, "alternative_subtype", None):
            if message.alternative_subtype == "mandrill":
                self._build_advanced_message_dict(message)
                if message.alternatives:
                    self._add_alternatives(message)

        djrill_it = requests.post(self.api_action, data=json.dumps({
            "key": self.api_key,
            "message": self.msg_dict
        }))

        if djrill_it.status_code != 200:
            if not self.fail_silently:
                raise DjrillBackendHTTPError(status_code=djrill_it.status_code, log_message="Failed to send a message to %s, from %s" % (self.recipients, self.sender))
            return False
        return True

    def _build_standard_message_dict(self, message):
        """
        Build standard message dict.

        Builds the standard dict that Django's send_mail and send_mass_mail
        use by default. Standard text email messages sent through Django will
        still work through Mandrill.
        """
        return {
            "text": message.body,
            "subject": message.subject,
            "from_email": self.sender,
            "to": self.recipients
        }

    def _build_advanced_message_dict(self, message):
        """
        Builds advanced message dict and attaches any accepted extra headers.
        """
        self.msg_dict.update({
            "from_name": message.from_name,
            "tags": message.tags,
            "track_opens": message.track_opens,
        })

        if message.extra_headers:
            accepted_headers = {}

            for k in message.extra_headers.keys():
                if k.startswith("X-") or k == "Reply-To":
                    accepted_headers.update(
                        {"%s" % k: message.extra_headers[k]})
            self.msg_dict.update({"headers": accepted_headers})

    def _add_alternatives(self, message):
        """
        There can be only one! ... alternative attachment.

        Since mandrill does not accept image attachments or anything other
        than HTML, the assumption is the only thing you are attaching is
        the HTML output for your email.
        """
        if len(message.alternatives) > 1:
            raise ImproperlyConfigured(
                "Mandrill only accepts plain text and html emails. Please "
                "check the alternatives you have attached to your message.")

        self.msg_dict.update({
            "html": message.alternatives[0][0],
            "track_clicks": message.track_clicks
        })
<commit_msg>Backend: fix exception messaging<commit_after>from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.mail.backends.base import BaseEmailBackend
from django.core.mail.message import sanitize_address
from django.utils import simplejson as json

import requests

class DjrillBackendHTTPError(Exception):
    """An exception that will turn into an HTTP error response."""
    def __init__(self, status_code, log_message=None):
        super(DjrillBackendHTTPError, self).__init__()
        self.status_code = status_code
        self.log_message = log_message

    def __str__(self):
        message = "DjrillBackendHTTP %d" % self.status_code
        if self.log_message:
            return message + " " + self.log_message
        else:
            return message

class DjrillBackend(BaseEmailBackend):
    """
    Mandrill API Email Backend
    """

    def __init__(self, fail_silently=False, **kwargs):
        """
        Set the API key, API url and set the action url.
        """
        super(DjrillBackend, self).__init__(**kwargs)
        self.api_key = getattr(settings, "MANDRILL_API_KEY", None)
        self.api_url = getattr(settings, "MANDRILL_API_URL", None)
        self.connection = None

        if not self.api_key:
            raise ImproperlyConfigured("You have not set your mandrill api key "
                "in the settings.py file.")
        if not self.api_url:
            raise ImproperlyConfigured("You have not added the Mandrill api "
                "url to your settings.py")

        self.api_action = self.api_url + "/messages/send.json"
        self.api_verify = self.api_url + "/users/verify-sender.json"

    def open(self, sender):
        """
        """
        self.connection = True
        # self.connection = None
        #
        # valid_sender = requests.post(
        #     self.api_verify, data={"key": self.api_key, "email": sender})
        #
        # if valid_sender.status_code == 200:
        #     data = json.loads(valid_sender.content)
        #     if data["is_enabled"]:
        #         self.connection = True
        #         return True
        # else:
        #     if not self.fail_silently:
        #         raise

    def send_messages(self, email_messages):
        if not email_messages:
            return

        num_sent = 0
        for message in email_messages:
            self.open(message.from_email)
            if not self.connection:
                return

            sent = self._send(message)

            if sent:
                num_sent += 1

        return num_sent

    def _send(self, message):
        if not message.recipients():
            return False

        self.sender = sanitize_address(message.from_email, message.encoding)
        recipients_list = [sanitize_address(addr, message.encoding)
            for addr in message.recipients()]
        from email.utils import parseaddr
        self.recipients = [{"email": e, "name": n} for n,e in [
            parseaddr(r) for r in recipients_list]]

        self.msg_dict = self._build_standard_message_dict(message)

        if getattr(message, "alternative_subtype", None):
            if message.alternative_subtype == "mandrill":
                self._build_advanced_message_dict(message)
                if message.alternatives:
                    self._add_alternatives(message)

        djrill_it = requests.post(self.api_action, data=json.dumps({
            "key": self.api_key,
            "message": self.msg_dict
        }))

        if djrill_it.status_code != 200:
            if not self.fail_silently:
                raise DjrillBackendHTTPError(status_code=djrill_it.status_code, log_message="Failed to send a message to %s, from %s" % (self.recipients, self.sender))
            return False
        return True

    def _build_standard_message_dict(self, message):
        """
        Build standard message dict.

        Builds the standard dict that Django's send_mail and send_mass_mail
        use by default. Standard text email messages sent through Django will
        still work through Mandrill.
        """
        return {
            "text": message.body,
            "subject": message.subject,
            "from_email": self.sender,
            "to": self.recipients
        }

    def _build_advanced_message_dict(self, message):
        """
        Builds advanced message dict and attaches any accepted extra headers.
        """
        self.msg_dict.update({
            "from_name": message.from_name,
            "tags": message.tags,
            "track_opens": message.track_opens,
        })

        if message.extra_headers:
            accepted_headers = {}

            for k in message.extra_headers.keys():
                if k.startswith("X-") or k == "Reply-To":
                    accepted_headers.update(
                        {"%s" % k: message.extra_headers[k]})
            self.msg_dict.update({"headers": accepted_headers})

    def _add_alternatives(self, message):
        """
        There can be only one! ... alternative attachment.

        Since mandrill does not accept image attachments or anything other
        than HTML, the assumption is the only thing you are attaching is
        the HTML output for your email.
        """
        if len(message.alternatives) > 1:
            raise ImproperlyConfigured(
                "Mandrill only accepts plain text and html emails. Please "
                "check the alternatives you have attached to your message.")

        self.msg_dict.update({
            "html": message.alternatives[0][0],
            "track_clicks": message.track_clicks
        })
<|endoftext|>


<commit_before>#!/usr/bin/env python2

import sys
import logging
import time
import subprocess
import shlex

ACCESSION_ANALYSIS_APPLET = '/applets/accession_analysis'

logger = logging.getLogger(__name__)

EPILOG = '''Notes:

Examples:

    %(prog)s
'''

class InputError(Exception):
    pass

def get_args():
    import argparse
    parser = argparse.ArgumentParser(
        description=__doc__, epilog=EPILOG,
        formatter_class=argparse.RawDescriptionHelpFormatter)

    def t_or_f(arg):
        ua = str(arg).upper()
        if ua == 'TRUE'[:len(ua)]:
            return True
        elif ua == 'FALSE'[:len(ua)]:
            return False
        else:
            assert not (True or False), "Cannot parse %s to boolean" % (arg)

    timestring = time.strftime("%y%m%d%H%M%S")

    parser.add_argument('analysis_ids', help='List of analysis IDs to accession', nargs='*', default=None)
    parser.add_argument('--infile', help='Local file containing analysis IDs', type=argparse.FileType('r'), default=sys.stdin)
    parser.add_argument('--outfile', help='DNAnexus file to save output summary table', default="accession_%s.csv" % (timestring))
    parser.add_argument('--destination', help='DNAnexus folder for output summary table', default='/accession_log/')
    parser.add_argument('--name', help='DNAnexus name for the accessioning run', default="accession_%s" % (timestring))
    parser.add_argument('--watch', help="Watch the run log", default=False, action='store_true')
    # Any applet arguments with default None will not be passed to the DNAnexus
    # applet, so that its defaults will be used
    # Otherwise the string will be cast to boolean
    parser.add_argument('--project', help='DNAnexus project in which to run', default=None)
    parser.add_argument('--pipeline', help='Over-ride automatic pipeline determination', default=None)
    parser.add_argument('--key', help="The keypair identifier from the keyfile.", default=None)
    parser.add_argument('--keyfile', help="The keyfile.", default=None)
    parser.add_argument('--debug', help="Print debug messages", type=t_or_f, default=None)
    parser.add_argument('--dryrun', help="Set up runs but don't change anything.", type=t_or_f, default=None)
    parser.add_argument('--force_patch', help="Force patching metadata for existing files", type=t_or_f, default=None)
    parser.add_argument('--fqcheck', help="Check that analysis is based on latest fastqs on ENCODEd", type=t_or_f, default=None)
    parser.add_argument('--accession_raw', help="Accession unfiltered bams", type=t_or_f, default=None)
    parser.add_argument('--signal_only', help="Accession unfiltered bams", type=t_or_f, default=None)
    args = parser.parse_args()

    return args

def main():

    args = get_args()
    if args.debug:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

    if args.analysis_ids:
        ids = args.analysis_ids
    elif args.infile:
        ids = args.infile
    else:
        # never reached because inile defaults to stdin
        raise InputError("Must supply analysis id's in arguments or --infile")

    tokens = [
        'dx run applets/accession_analysis',
        '-i "outfn=%s"' % (args.outfile),
        '--destination "%s"' % (args.destination),
        '--name "%s"' % (args.name),
        '--yes'
    ]
    if args.watch:
        tokens.append('--watch')
    if args.project is not None:
        tokens.append('-i "project=%s"' % (args.project))
    if args.pipeline is not None:
        tokens.append('-i "pipeline=%s"' % (args.pipeline))
    if args.key is not None:
        tokens.append('-i "key=%s"' % (args.key))
    if args.keyfile is not None:
        tokens.append('-i "keyfile=%s"' % (args.keyfile))
    if args.debug is not None:
        tokens.append('-i "debug=%s"' % (args.debug))
    if args.dryrun is not None:
        tokens.append('-i "dryrun=%s"' % (args.dryrun))
    if args.force_patch is not None:
        tokens.append('-i "force_patch=%s"' % (args.force_patch))
    if args.fqcheck is not None:
        tokens.append('-i "fqcheck=%s"' % (args.fqcheck))
    if args.accession_raw is not None:
        tokens.append('-i "accession_raw=%s"' % (args.accession_raw))
    if args.signal_only is not None:
        tokens.append('-i "signal_only=%s"' % (args.signal_only))

    for (i, analysis_id) in enumerate(ids):
        if analysis_id.startswith('#'):
            continue
        analysis_id = analysis_id.rstrip()
        logger.debug('%s' % (analysis_id))
        tokens.append('-i "analysis_ids=%s"' % (analysis_id))

    command_string = ' '.join(tokens)
    logger.debug(command_string)
    subprocess.check_call(shlex.split(command_string))

if __name__ == '__main__':
    main()
<commit_msg>Update args<commit_after>#!/usr/bin/env python2

import sys
import logging
import time
import subprocess
import shlex

ACCESSION_ANALYSIS_APPLET = '/applets/accession_analysis'

logger = logging.getLogger(__name__)

EPILOG = '''Notes:

Examples:

    %(prog)s
'''

class InputError(Exception):
    pass

def get_args():
    import argparse
    parser = argparse.ArgumentParser(
        description=__doc__, epilog=EPILOG,
        formatter_class=argparse.RawDescriptionHelpFormatter)

    def t_or_f(arg):
        ua = str(arg).upper()
        if ua == 'TRUE'[:len(ua)]:
            return True
        elif ua == 'FALSE'[:len(ua)]:
            return False
        else:
            assert not (True or False), "Cannot parse %s to boolean" % (arg)

    timestring = time.strftime("%y%m%d%H%M%S")

    parser.add_argument('analysis_ids', help='List of analysis IDs to accession', nargs='*', default=None)
    parser.add_argument('--infile', help='Local file containing analysis IDs', type=argparse.FileType('r'), default=sys.stdin)
    parser.add_argument('--outfile', help='DNAnexus file to save output summary table', default="accession_%s.csv" % (timestring))
    parser.add_argument('--destination', help='DNAnexus folder for output summary table', default='/accession_log/')
    parser.add_argument('--name', help='DNAnexus name for the accessioning run', default="accession_%s" % (timestring))
    parser.add_argument('--watch', help="Watch the run log", default=False, action='store_true')
    # Any applet arguments with default None will not be passed to the DNAnexus
    # applet, so that its defaults will be used
    # Otherwise the string will be cast to boolean
    parser.add_argument('--project', help='DNAnexus project in which to run', default=None)
    parser.add_argument('--pipeline', help='Over-ride automatic pipeline determination', default=None)
    parser.add_argument('--key', help="The keypair identifier from the keyfile.", default=None)
    parser.add_argument('--keyfile', help="The keyfile.", default=None)
    parser.add_argument('--debug', help="Print debug messages", type=t_or_f, default=None)
    parser.add_argument('--dryrun', help="Set up runs but don't change anything.", type=t_or_f, default=None)
    parser.add_argument('--force_patch', help="Force patching metadata for existing files", type=t_or_f, default=None)
    parser.add_argument('--fqcheck', help="Check that analysis is based on latest fastqs on ENCODEd", type=t_or_f, default=None)
    parser.add_argument('--accession_raw', help="Accession unfiltered bams", type=t_or_f, default=None)
    parser.add_argument('--signal_only', help="Accession through signal generation only", type=t_or_f, default=None)
    args = parser.parse_args()

    return args

def main():

    args = get_args()
    if args.debug:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

    if args.analysis_ids:
        ids = args.analysis_ids
    elif args.infile:
        ids = args.infile
    else:
        # never reached because inile defaults to stdin
        raise InputError("Must supply analysis id's in arguments or --infile")

    tokens = [
        'dx run applets/accession_analysis',
        '-i "outfn=%s"' % (args.outfile),
        '--destination "%s"' % (args.destination),
        '--name "%s"' % (args.name),
        '--yes'
    ]
    if args.watch:
        tokens.append('--watch')
    if args.project is not None:
        tokens.append('-i "project=%s"' % (args.project))
    if args.pipeline is not None:
        tokens.append('-i "pipeline=%s"' % (args.pipeline))
    if args.key is not None:
        tokens.append('-i "key=%s"' % (args.key))
    if args.keyfile is not None:
        tokens.append('-i "keyfile=%s"' % (args.keyfile))
    if args.debug is not None:
        tokens.append('-i "debug=%s"' % (args.debug))
    if args.dryrun is not None:
        tokens.append('-i "dryrun=%s"' % (args.dryrun))
    if args.force_patch is not None:
        tokens.append('-i "force_patch=%s"' % (args.force_patch))
    if args.fqcheck is not None:
        tokens.append('-i "fqcheck=%s"' % (args.fqcheck))
    if args.accession_raw is not None:
        tokens.append('-i "accession_raw=%s"' % (args.accession_raw))
    if args.signal_only is not None:
        tokens.append('-i "signal_only=%s"' % (args.signal_only))

    for (i, analysis_id) in enumerate(ids):
        if analysis_id.startswith('#'):
            continue
        analysis_id = analysis_id.rstrip()
        logger.debug('%s' % (analysis_id))
        tokens.append('-i "analysis_ids=%s"' % (analysis_id))

    command_string = ' '.join(tokens)
    logger.debug(command_string)
    subprocess.check_call(shlex.split(command_string))

if __name__ == '__main__':
    main()
<|endoftext|>