<commit_before>from datetime import datetime
from django.core.management import sql
from django.core.management.color import no_style
from django.db import connection, transaction, settings, models
from django.db.backends.util import truncate_name
from django.db.models.loading import cache
from django.utils.datastructures import SortedDict
from django.utils.functional import curry
from django_evolution import signature, is_multi_db
from django_evolution.tests import models as evo_test
from django_evolution.utils import write_sql, execute_sql
import copy

if is_multi_db():
    from django.db import connections
    from django.db.utils import DEFAULT_DB_ALIAS

DEFAULT_TEST_ATTRIBUTE_VALUES = {
    models.CharField: 'TestCharField',
    models.IntegerField: '123',
    models.AutoField: None,
    models.DateTimeField: datetime.now(),
    models.PositiveIntegerField: '42'
}

def wrap_sql_func(func, evo_test, style, db_name=None):
    if is_multi_db():
        return func(evo_test, style, connections[db_name or DEFAULT_DB_ALIAS])
    else:
        return func(evo_test, style)

# Wrap the sql.* functions to work with the multi-db support
sql_create = curry(wrap_sql_func, sql.sql_create)
sql_indexes = curry(wrap_sql_func, sql.sql_indexes)
sql_delete = curry(wrap_sql_func, sql.sql_delete)

def _register_models(app_label='tests', db_name='default', *models):
    app_cache = SortedDict()

    my_connection = connection

    if is_multi_db():
        my_connection = connections[db_name or DEFAULT_DB_ALIAS]

    max_name_length = my_connection.ops.max_name_length()

    for name, model in reversed(models):
        if model._meta.module_name in cache.app_models['django_evolution']:
            del cache.app_models['django_evolution'][model._meta.module_name]

            orig_db_table = model._meta.db_table
            orig_object_name = model._meta.object_name
            orig_module_name = model._meta.module_name

            generated_db_table = truncate_name(
                '%s_%s' % (model._meta.app_label, model._meta.module_name),
                max_name_length)

            if orig_db_table.startswith(generated_db_table):
                model._meta.db_table = '%s_%s' % (app_label, name.lower())

            model._meta.db_table = truncate_name(model._meta.db_table,
                                                 max_name_length)
            model._meta.app_label = app_label
            model._meta.object_name = name
            model._meta.module_name = name.lower()

            add_app_test_model(model, app_label=app_label)

            for field in model._meta.local_many_to_many:
                if not field.rel.through:
                    continue

                through = field.rel.through

                generated_db_table = truncate_name(
                    '%s_%s' % (orig_db_table, field.name),
                    max_name_length)

                if through._meta.db_table == generated_db_table:
                    through._meta.app_label = app_label

                    # Transform the 'through' table information only
                    # if we've transformed the parent db_table.
                    if model._meta.db_table != orig_db_table:
                        through._meta.db_table = \
                            '%s_%s' % (model._meta.db_table, field.name)

                        through._meta.object_name = \
                            through._meta.object_name.replace(
                                orig_object_name,
                                model._meta.object_name)

                        through._meta.module_name = \
                            through._meta.module_name.replace(
                                orig_module_name,
                                model._meta.module_name)

                through._meta.db_table = \
                    truncate_name(through._meta.db_table, max_name_length)

                for field in through._meta.local_fields:
                    if field.rel and field.rel.to:
                        column = field.column

                        if (column.startswith(orig_module_name) or
                            column.startswith('to_%s' % orig_module_name) or
                            column.startswith('from_%s' % orig_module_name)):

                            field.column = column.replace(
                                orig_module_name,
                                model._meta.module_name)

                if (through._meta.module_name in
                    cache.app_models['django_evolution']):
                    del cache.app_models['django_evolution'][
                        through._meta.module_name]

                app_cache[through._meta.module_name] = through
                add_app_test_model(through, app_label=app_label)

        app_cache[model._meta.module_name] = model

    return app_cache

def register_models(*models):
    return _register_models('tests', 'default', *models)

def register_models_multi(app_label, db_name, *models):
    return _register_models(app_label, db_name, *models)

def _test_proj_sig(app_label, *models, **kwargs):
    "Generate a dummy project signature based around a single model"
    version = kwargs.get('version', 1)
    proj_sig = {
        app_label: SortedDict(),
        '__version__': version,
    }

    # Compute the project siguature
    for full_name, model in models:
        parts = full_name.split('.')

        if len(parts) == 1:
            name = parts[0]
            app = app_label
        else:
            app, name = parts

        proj_sig.setdefault(app, SortedDict())[name] = \
            signature.create_model_sig(model)

    return proj_sig

def test_proj_sig(*models, **kwargs):
    return _test_proj_sig('tests', *models, **kwargs)

def test_proj_sig_multi(app_label, *models, **kwargs):
    return _test_proj_sig(app_label, *models, **kwargs)

def execute_transaction(sql, output=False, database='default'):
    "A transaction wrapper for executing a list of SQL statements"
    my_connection = connection
    using_args = {}

    if is_multi_db():
        if not database:
            database = DEFAULT_DB_ALIAS

        my_connection = connections[database]
        using_args['using'] = database

    try:
        # Begin Transaction
        transaction.enter_transaction_management(**using_args)
        transaction.managed(True, **using_args)

        cursor = my_connection.cursor()

        # Perform the SQL
        if output:
            write_sql(sql, database)

        execute_sql(cursor, sql)

        transaction.commit(**using_args)
        transaction.leave_transaction_management(**using_args)
    except Exception:
        transaction.rollback(**using_args)
        raise

def execute_test_sql(start, end, sql, debug=False, app_label='tests',
                     database='default'):
    """
    Execute a test SQL sequence. This method also creates and destroys the
    database tables required by the models registered against the test
    application.

    start and end are the start- and end-point states of the application cache.

    sql is the list of sql statements to execute.

    cleanup is a list of extra sql statements required to clean up. This is
    primarily for any extra m2m tables that were added during a test that won't
    be cleaned up by Django's sql_delete() implementation.

    debug is a helper flag. It displays the ALL the SQL that would be executed,
    (including setup and teardown SQL), and executes the Django-derived
    setup/teardown SQL.
    """
    # Set up the initial state of the app cache
    set_app_test_models(copy.deepcopy(start), app_label=app_label)

    # Install the initial tables and indicies
    style = no_style()
    execute_transaction(sql_create(evo_test, style, database),
                        output=debug, database=database)
    execute_transaction(sql_indexes(evo_test, style, database),
                        output=debug, database=database)
    create_test_data(models.get_models(evo_test), database)

    # Set the app cache to the end state
    set_app_test_models(copy.deepcopy(end), app_label=app_label)

    try:
        # Execute the test sql
        if debug:
            write_sql(sql, database)
        else:
            execute_transaction(sql, output=True, database=database)
    finally:
        # Cleanup the apps.
        if debug:
            print sql_delete(evo_test, style, database)
        else:
            execute_transaction(sql_delete(evo_test, style, database),
                                output=debug, database=database)

def create_test_data(app_models, database):
    deferred_models = []
    deferred_fields = {}

    using_args = {}

    if is_multi_db():
        using_args['using'] = database

    for model in app_models:
        params = {}
        deferred = False
        for field in model._meta.fields:
            if not deferred:
                if type(field) in (models.ForeignKey, models.ManyToManyField):
                    related_model = field.rel.to

                    related_q = related_model.objects.all()

                    if is_multi_db():
                        related_q = related_q.using(database)

                    if related_q.count():
                        related_instance = related_q[0]
                    else:
                        if field.null == False:
                            # Field cannot be null yet the related object
                            # hasn't been created yet Defer the creation of
                            # this model
                            deferred = True
                            deferred_models.append(model)
                        else:
                            # Field cannot be set yet but null is acceptable
                            # for the moment
                            deferred_fields[type(model)] = \
                                deferred_fields.get(type(model),
                                                    []).append(field)
                            related_instance = None

                    if not deferred:
                        if type(field) == models.ForeignKey:
                            params[field.name] = related_instance
                        else:
                            params[field.name] = [related_instance]
                else:
                    params[field.name] = \
                        DEFAULT_TEST_ATTRIBUTE_VALUES[type(field)]

        if not deferred:
            model(**params).save(**using_args)

    # Create all deferred models.
    if deferred_models:
        create_test_data(deferred_models, database)

    # All models should be created (Not all deferred fields have been populated
    # yet) Populate deferred fields that we know about.  Here lies untested
    # code!
    if deferred_fields:
        for model, field_list in deferred_fields.items():
            for field in field_list:
                related_model = field.rel.to
                related_instance = related_model.objects.using(database)[0]

                if type(field) == models.ForeignKey:
                    setattr(model, field.name, related_instance)
                else:
                    getattr(model, field.name).add(related_instance,
                                                   **using_args)

            model.save(**using_args)

def test_sql_mapping(test_field_name, db_name='default'):
    if is_multi_db():
        engine = settings.DATABASES[db_name]['ENGINE'].split('.')[-1]
    else:
        engine = settings.DATABASE_ENGINE

    sql_for_engine = __import__('django_evolution.tests.db.%s' % (engine),
                                {}, {}, [''])

    return getattr(sql_for_engine, test_field_name)

def deregister_models(app_label='tests'):
    "Clear the test section of the app cache"
    del cache.app_models[app_label]
    clear_models_cache()

def clear_models_cache():
    """Clears the Django models cache.

    This cache is used in Django >= 1.2 to quickly return results from
    cache.get_models(). It needs to be cleared when modifying the model
    registry.
    """
    if hasattr(cache, '_get_models_cache'):
        # On Django 1.2, we need to clear this cache when unregistering models.
        cache._get_models_cache.clear()

def set_app_test_models(models, app_label):
    """Sets the list of models in the Django test models registry."""
    cache.app_models[app_label] = models
    clear_models_cache()

def add_app_test_model(model, app_label):
    """Adds a model to the Django test models registry."""
    key = model._meta.object_name.lower()
    cache.app_models.setdefault(app_label, SortedDict())[key] = model
    clear_models_cache()
<commit_msg>Fix compatibility with Django SVN in the unit tests.<commit_after>from datetime import datetime
from django.core.management import sql
from django.core.management.color import no_style
from django.db import connection, transaction, settings, models
from django.db.backends.util import truncate_name
from django.db.models.loading import cache
from django.utils.datastructures import SortedDict
from django.utils.functional import curry
from django_evolution import signature, is_multi_db
from django_evolution.tests import models as evo_test
from django_evolution.utils import write_sql, execute_sql
import copy

if is_multi_db():
    from django.db import connections
    from django.db.utils import DEFAULT_DB_ALIAS

DEFAULT_TEST_ATTRIBUTE_VALUES = {
    models.CharField: 'TestCharField',
    models.IntegerField: '123',
    models.AutoField: None,
    models.DateTimeField: datetime.now(),
    models.PositiveIntegerField: '42'
}

def wrap_sql_func(func, evo_test, style, db_name=None):
    if is_multi_db():
        return func(evo_test, style, connections[db_name or DEFAULT_DB_ALIAS])
    else:
        return func(evo_test, style)

# Wrap the sql.* functions to work with the multi-db support
sql_create = curry(wrap_sql_func, sql.sql_create)
sql_indexes = curry(wrap_sql_func, sql.sql_indexes)
sql_delete = curry(wrap_sql_func, sql.sql_delete)

def _register_models(app_label='tests', db_name='default', *models):
    app_cache = SortedDict()

    my_connection = connection

    if is_multi_db():
        my_connection = connections[db_name or DEFAULT_DB_ALIAS]

    max_name_length = my_connection.ops.max_name_length()

    for name, model in reversed(models):
        if model._meta.module_name in cache.app_models['django_evolution']:
            del cache.app_models['django_evolution'][model._meta.module_name]

            orig_db_table = model._meta.db_table
            orig_object_name = model._meta.object_name
            orig_module_name = model._meta.module_name

            generated_db_table = truncate_name(
                '%s_%s' % (model._meta.app_label, model._meta.module_name),
                max_name_length)

            if orig_db_table.startswith(generated_db_table):
                model._meta.db_table = '%s_%s' % (app_label, name.lower())

            model._meta.db_table = truncate_name(model._meta.db_table,
                                                 max_name_length)
            model._meta.app_label = app_label
            model._meta.object_name = name
            model._meta.module_name = name.lower()

            add_app_test_model(model, app_label=app_label)

            for field in model._meta.local_many_to_many:
                if not field.rel.through:
                    continue

                through = field.rel.through

                generated_db_table = truncate_name(
                    '%s_%s' % (orig_db_table, field.name),
                    max_name_length)

                if through._meta.db_table == generated_db_table:
                    through._meta.app_label = app_label

                    # Transform the 'through' table information only
                    # if we've transformed the parent db_table.
                    if model._meta.db_table != orig_db_table:
                        through._meta.db_table = \
                            '%s_%s' % (model._meta.db_table, field.name)

                        through._meta.object_name = \
                            through._meta.object_name.replace(
                                orig_object_name,
                                model._meta.object_name)

                        through._meta.module_name = \
                            through._meta.module_name.replace(
                                orig_module_name,
                                model._meta.module_name)

                through._meta.db_table = \
                    truncate_name(through._meta.db_table, max_name_length)

                for field in through._meta.local_fields:
                    if field.rel and field.rel.to:
                        column = field.column

                        if (column.startswith(orig_module_name) or
                            column.startswith('to_%s' % orig_module_name) or
                            column.startswith('from_%s' % orig_module_name)):

                            field.column = column.replace(
                                orig_module_name,
                                model._meta.module_name)

                if (through._meta.module_name in
                    cache.app_models['django_evolution']):
                    del cache.app_models['django_evolution'][
                        through._meta.module_name]

                app_cache[through._meta.module_name] = through
                add_app_test_model(through, app_label=app_label)

        app_cache[model._meta.module_name] = model

    if evo_test not in cache.app_store:
        cache.app_store[evo_test] = len(cache.app_store)

        if hasattr(cache, 'app_labels'):
            cache.app_labels[app_label] = evo_test

    return app_cache

def register_models(*models):
    return _register_models('tests', 'default', *models)

def register_models_multi(app_label, db_name, *models):
    return _register_models(app_label, db_name, *models)

def _test_proj_sig(app_label, *models, **kwargs):
    "Generate a dummy project signature based around a single model"
    version = kwargs.get('version', 1)
    proj_sig = {
        app_label: SortedDict(),
        '__version__': version,
    }

    # Compute the project siguature
    for full_name, model in models:
        parts = full_name.split('.')

        if len(parts) == 1:
            name = parts[0]
            app = app_label
        else:
            app, name = parts

        proj_sig.setdefault(app, SortedDict())[name] = \
            signature.create_model_sig(model)

    return proj_sig

def test_proj_sig(*models, **kwargs):
    return _test_proj_sig('tests', *models, **kwargs)

def test_proj_sig_multi(app_label, *models, **kwargs):
    return _test_proj_sig(app_label, *models, **kwargs)

def execute_transaction(sql, output=False, database='default'):
    "A transaction wrapper for executing a list of SQL statements"
    my_connection = connection
    using_args = {}

    if is_multi_db():
        if not database:
            database = DEFAULT_DB_ALIAS

        my_connection = connections[database]
        using_args['using'] = database

    try:
        # Begin Transaction
        transaction.enter_transaction_management(**using_args)
        transaction.managed(True, **using_args)

        cursor = my_connection.cursor()

        # Perform the SQL
        if output:
            write_sql(sql, database)

        execute_sql(cursor, sql)

        transaction.commit(**using_args)
        transaction.leave_transaction_management(**using_args)
    except Exception:
        transaction.rollback(**using_args)
        raise

def execute_test_sql(start, end, sql, debug=False, app_label='tests',
                     database='default'):
    """
    Execute a test SQL sequence. This method also creates and destroys the
    database tables required by the models registered against the test
    application.

    start and end are the start- and end-point states of the application cache.

    sql is the list of sql statements to execute.

    cleanup is a list of extra sql statements required to clean up. This is
    primarily for any extra m2m tables that were added during a test that won't
    be cleaned up by Django's sql_delete() implementation.

    debug is a helper flag. It displays the ALL the SQL that would be executed,
    (including setup and teardown SQL), and executes the Django-derived
    setup/teardown SQL.
    """
    # Set up the initial state of the app cache
    set_app_test_models(copy.deepcopy(start), app_label=app_label)

    # Install the initial tables and indicies
    style = no_style()
    execute_transaction(sql_create(evo_test, style, database),
                        output=debug, database=database)
    execute_transaction(sql_indexes(evo_test, style, database),
                        output=debug, database=database)
    create_test_data(models.get_models(evo_test), database)

    # Set the app cache to the end state
    set_app_test_models(copy.deepcopy(end), app_label=app_label)

    try:
        # Execute the test sql
        if debug:
            write_sql(sql, database)
        else:
            execute_transaction(sql, output=True, database=database)
    finally:
        # Cleanup the apps.
        if debug:
            print sql_delete(evo_test, style, database)
        else:
            execute_transaction(sql_delete(evo_test, style, database),
                                output=debug, database=database)

def create_test_data(app_models, database):
    deferred_models = []
    deferred_fields = {}

    using_args = {}

    if is_multi_db():
        using_args['using'] = database

    for model in app_models:
        params = {}
        deferred = False
        for field in model._meta.fields:
            if not deferred:
                if type(field) in (models.ForeignKey, models.ManyToManyField):
                    related_model = field.rel.to

                    related_q = related_model.objects.all()

                    if is_multi_db():
                        related_q = related_q.using(database)

                    if related_q.count():
                        related_instance = related_q[0]
                    else:
                        if field.null == False:
                            # Field cannot be null yet the related object
                            # hasn't been created yet Defer the creation of
                            # this model
                            deferred = True
                            deferred_models.append(model)
                        else:
                            # Field cannot be set yet but null is acceptable
                            # for the moment
                            deferred_fields[type(model)] = \
                                deferred_fields.get(type(model),
                                                    []).append(field)
                            related_instance = None

                    if not deferred:
                        if type(field) == models.ForeignKey:
                            params[field.name] = related_instance
                        else:
                            params[field.name] = [related_instance]
                else:
                    params[field.name] = \
                        DEFAULT_TEST_ATTRIBUTE_VALUES[type(field)]

        if not deferred:
            model(**params).save(**using_args)

    # Create all deferred models.
    if deferred_models:
        create_test_data(deferred_models, database)

    # All models should be created (Not all deferred fields have been populated
    # yet) Populate deferred fields that we know about.  Here lies untested
    # code!
    if deferred_fields:
        for model, field_list in deferred_fields.items():
            for field in field_list:
                related_model = field.rel.to
                related_instance = related_model.objects.using(database)[0]

                if type(field) == models.ForeignKey:
                    setattr(model, field.name, related_instance)
                else:
                    getattr(model, field.name).add(related_instance,
                                                   **using_args)

            model.save(**using_args)

def test_sql_mapping(test_field_name, db_name='default'):
    if is_multi_db():
        engine = settings.DATABASES[db_name]['ENGINE'].split('.')[-1]
    else:
        engine = settings.DATABASE_ENGINE

    sql_for_engine = __import__('django_evolution.tests.db.%s' % (engine),
                                {}, {}, [''])

    return getattr(sql_for_engine, test_field_name)

def deregister_models(app_label='tests'):
    "Clear the test section of the app cache"
    del cache.app_models[app_label]
    clear_models_cache()

def clear_models_cache():
    """Clears the Django models cache.

    This cache is used in Django >= 1.2 to quickly return results from
    cache.get_models(). It needs to be cleared when modifying the model
    registry.
    """
    if hasattr(cache, '_get_models_cache'):
        # On Django 1.2, we need to clear this cache when unregistering models.
        cache._get_models_cache.clear()

def set_app_test_models(models, app_label):
    """Sets the list of models in the Django test models registry."""
    cache.app_models[app_label] = models
    clear_models_cache()

def add_app_test_model(model, app_label):
    """Adds a model to the Django test models registry."""
    key = model._meta.object_name.lower()
    cache.app_models.setdefault(app_label, SortedDict())[key] = model
    clear_models_cache()
<|endoftext|>


<commit_before>from django.conf import settings
from django.utils.deprecation import MiddlewareMixin
from prometheus_client import Counter, Histogram

from django_prometheus.conf import NAMESPACE
from django_prometheus.utils import PowersOf, Time, TimeSince

DEFAULT_LATENCY_BUCKETS = (
    0.01,
    0.025,
    0.05,
    0.075,
    0.1,
    0.25,
    0.5,
    0.75,
    1.0,
    2.5,
    5.0,
    7.5,
    10.0,
    25.0,
    50.0,
    75.0,
    float("inf"),
)

class Metrics:
    _instance = None

    @classmethod
    def get_instance(cls):
        if not cls._instance:
            cls._instance = cls()
        return cls._instance

    def register_metric(self, metric_cls, name, documentation, labelnames=(), **kwargs):
        return metric_cls(name, documentation, labelnames=labelnames, **kwargs)

    def __init__(self, *args, **kwargs):
        self.register()

    def register(self):
        self.requests_total = self.register_metric(
            Counter,
            "django_http_requests_before_middlewares_total",
            "Total count of requests before middlewares run.",
            namespace=NAMESPACE,
        )
        self.responses_total = self.register_metric(
            Counter,
            "django_http_responses_before_middlewares_total",
            "Total count of responses before middlewares run.",
            namespace=NAMESPACE,
        )
        self.requests_latency_before = self.register_metric(
            Histogram,
            "django_http_requests_latency_including_middlewares_seconds",
            (
                "Histogram of requests processing time (including middleware "
                "processing time)."
            ),
            namespace=NAMESPACE,
        )
        self.requests_unknown_latency_before = self.register_metric(
            Counter,
            "django_http_requests_unknown_latency_including_middlewares_total",
            (
                "Count of requests for which the latency was unknown (when computing "
                "django_http_requests_latency_including_middlewares_seconds)."
            ),
            namespace=NAMESPACE,
        )
        self.requests_latency_by_view_method = self.register_metric(
            Histogram,
            "django_http_requests_latency_seconds_by_view_method",
            "Histogram of request processing time labelled by view.",
            ["view", "method"],
            buckets=getattr(
                settings, "PROMETHEUS_LATENCY_BUCKETS", DEFAULT_LATENCY_BUCKETS
            ),
            namespace=NAMESPACE,
        )
        self.requests_unknown_latency = self.register_metric(
            Counter,
            "django_http_requests_unknown_latency_total",
            "Count of requests for which the latency was unknown.",
            namespace=NAMESPACE,
        )
        # Set in process_request
        self.requests_ajax = self.register_metric(
            Counter,
            "django_http_ajax_requests_total",
            "Count of AJAX requests.",
            namespace=NAMESPACE,
        )
        self.requests_by_method = self.register_metric(
            Counter,
            "django_http_requests_total_by_method",
            "Count of requests by method.",
            ["method"],
            namespace=NAMESPACE,
        )
        self.requests_by_transport = self.register_metric(
            Counter,
            "django_http_requests_total_by_transport",
            "Count of requests by transport.",
            ["transport"],
            namespace=NAMESPACE,
        )
        # Set in process_view
        self.requests_by_view_transport_method = self.register_metric(
            Counter,
            "django_http_requests_total_by_view_transport_method",
            "Count of requests by view, transport, method.",
            ["view", "transport", "method"],
            namespace=NAMESPACE,
        )
        self.requests_body_bytes = self.register_metric(
            Histogram,
            "django_http_requests_body_total_bytes",
            "Histogram of requests by body size.",
            buckets=PowersOf(2, 30),
            namespace=NAMESPACE,
        )
        # Set in process_template_response
        self.responses_by_templatename = self.register_metric(
            Counter,
            "django_http_responses_total_by_templatename",
            "Count of responses by template name.",
            ["templatename"],
            namespace=NAMESPACE,
        )
        # Set in process_response
        self.responses_by_status = self.register_metric(
            Counter,
            "django_http_responses_total_by_status",
            "Count of responses by status.",
            ["status"],
            namespace=NAMESPACE,
        )
        self.responses_by_status_view_method = self.register_metric(
            Counter,
            "django_http_responses_total_by_status_view_method",
            "Count of responses by status, view, method.",
            ["status", "view", "method"],
            namespace=NAMESPACE,
        )
        self.responses_body_bytes = self.register_metric(
            Histogram,
            "django_http_responses_body_total_bytes",
            "Histogram of responses by body size.",
            buckets=PowersOf(2, 30),
            namespace=NAMESPACE,
        )
        self.responses_by_charset = self.register_metric(
            Counter,
            "django_http_responses_total_by_charset",
            "Count of responses by charset.",
            ["charset"],
            namespace=NAMESPACE,
        )
        self.responses_streaming = self.register_metric(
            Counter,
            "django_http_responses_streaming_total",
            "Count of streaming responses.",
            namespace=NAMESPACE,
        )
        # Set in process_exception
        self.exceptions_by_type = self.register_metric(
            Counter,
            "django_http_exceptions_total_by_type",
            "Count of exceptions by object type.",
            ["type"],
            namespace=NAMESPACE,
        )
        self.exceptions_by_view = self.register_metric(
            Counter,
            "django_http_exceptions_total_by_view",
            "Count of exceptions by view.",
            ["view"],
            namespace=NAMESPACE,
        )

class PrometheusBeforeMiddleware(MiddlewareMixin):
    """Monitoring middleware that should run before other middlewares."""

    metrics_cls = Metrics

    def __init__(self, get_response=None):
        super().__init__(get_response)
        self.metrics = self.metrics_cls.get_instance()

    def process_request(self, request):
        self.metrics.requests_total.inc()
        request.prometheus_before_middleware_event = Time()

    def process_response(self, request, response):
        self.metrics.responses_total.inc()
        if hasattr(request, "prometheus_before_middleware_event"):
            self.metrics.requests_latency_before.observe(
                TimeSince(request.prometheus_before_middleware_event)
            )
        else:
            self.metrics.requests_unknown_latency_before.inc()
        return response

class PrometheusAfterMiddleware(MiddlewareMixin):
    """Monitoring middleware that should run after other middlewares."""

    metrics_cls = Metrics

    def __init__(self, get_response=None):
        super().__init__(get_response)
        self.metrics = self.metrics_cls.get_instance()

    def _transport(self, request):
        return "https" if request.is_secure() else "http"

    def _method(self, request):
        m = request.method
        if m not in (
            "GET",
            "HEAD",
            "POST",
            "PUT",
            "DELETE",
            "TRACE",
            "OPTIONS",
            "CONNECT",
            "PATCH",
        ):
            return "<invalid method>"
        return m

    def label_metric(self, metric, request, response=None, **labels):
        return metric.labels(**labels) if labels else metric

    def process_request(self, request):
        transport = self._transport(request)
        method = self._method(request)
        self.label_metric(self.metrics.requests_by_method, request, method=method).inc()
        self.label_metric(
            self.metrics.requests_by_transport, request, transport=transport
        ).inc()

        # Mimic the behaviour of the deprecated "Request.is_ajax()" method.
        if request.META.get("HTTP_X_REQUESTED_WITH") == "XMLHttpRequest":
            self.label_metric(self.metrics.requests_ajax, request).inc()

        content_length = int(request.META.get("CONTENT_LENGTH") or 0)
        self.label_metric(self.metrics.requests_body_bytes, request).observe(
            content_length
        )
        request.prometheus_after_middleware_event = Time()

    def _get_view_name(self, request):
        view_name = "<unnamed view>"
        if hasattr(request, "resolver_match"):
            if request.resolver_match is not None:
                if request.resolver_match.view_name is not None:
                    view_name = request.resolver_match.view_name
        return view_name

    def process_view(self, request, view_func, *view_args, **view_kwargs):
        transport = self._transport(request)
        method = self._method(request)
        if hasattr(request, "resolver_match"):
            name = request.resolver_match.view_name or "<unnamed view>"
            self.label_metric(
                self.metrics.requests_by_view_transport_method,
                request,
                view=name,
                transport=transport,
                method=method,
            ).inc()

    def process_template_response(self, request, response):
        if hasattr(response, "template_name"):
            self.label_metric(
                self.metrics.responses_by_templatename,
                request,
                response=response,
                templatename=str(response.template_name),
            ).inc()
        return response

    def process_response(self, request, response):
        method = self._method(request)
        name = self._get_view_name(request)
        status = str(response.status_code)
        self.label_metric(
            self.metrics.responses_by_status, request, response, status=status
        ).inc()
        self.label_metric(
            self.metrics.responses_by_status_view_method,
            request,
            response,
            status=status,
            view=name,
            method=method,
        ).inc()
        if hasattr(response, "charset"):
            self.label_metric(
                self.metrics.responses_by_charset,
                request,
                response,
                charset=str(response.charset),
            ).inc()
        if hasattr(response, "streaming") and response.streaming:
            self.label_metric(self.metrics.responses_streaming, request, response).inc()
        if hasattr(response, "content"):
            self.label_metric(
                self.metrics.responses_body_bytes, request, response
            ).observe(len(response.content))
        if hasattr(request, "prometheus_after_middleware_event"):
            self.label_metric(
                self.metrics.requests_latency_by_view_method,
                request,
                response,
                view=self._get_view_name(request),
                method=request.method,
            ).observe(TimeSince(request.prometheus_after_middleware_event))
        else:
            self.label_metric(
                self.metrics.requests_unknown_latency, request, response
            ).inc()
        return response

    def process_exception(self, request, exception):
        self.label_metric(
            self.metrics.exceptions_by_type, request, type=type(exception).__name__
        ).inc()
        if hasattr(request, "resolver_match"):
            name = request.resolver_match.view_name or "<unnamed view>"
            self.label_metric(self.metrics.exceptions_by_view, request, view=name).inc()
        if hasattr(request, "prometheus_after_middleware_event"):
            self.label_metric(
                self.metrics.requests_latency_by_view_method,
                request,
                view=self._get_view_name(request),
                method=request.method,
            ).observe(TimeSince(request.prometheus_after_middleware_event))
        else:
            self.label_metric(self.metrics.requests_unknown_latency, request).inc()
<commit_msg>make middleware init signature compatible with django 4 (#322)<commit_after>from django.conf import settings
from django.utils.deprecation import MiddlewareMixin
from prometheus_client import Counter, Histogram

from django_prometheus.conf import NAMESPACE
from django_prometheus.utils import PowersOf, Time, TimeSince

DEFAULT_LATENCY_BUCKETS = (
    0.01,
    0.025,
    0.05,
    0.075,
    0.1,
    0.25,
    0.5,
    0.75,
    1.0,
    2.5,
    5.0,
    7.5,
    10.0,
    25.0,
    50.0,
    75.0,
    float("inf"),
)

class Metrics:
    _instance = None

    @classmethod
    def get_instance(cls):
        if not cls._instance:
            cls._instance = cls()
        return cls._instance

    def register_metric(self, metric_cls, name, documentation, labelnames=(), **kwargs):
        return metric_cls(name, documentation, labelnames=labelnames, **kwargs)

    def __init__(self, *args, **kwargs):
        self.register()

    def register(self):
        self.requests_total = self.register_metric(
            Counter,
            "django_http_requests_before_middlewares_total",
            "Total count of requests before middlewares run.",
            namespace=NAMESPACE,
        )
        self.responses_total = self.register_metric(
            Counter,
            "django_http_responses_before_middlewares_total",
            "Total count of responses before middlewares run.",
            namespace=NAMESPACE,
        )
        self.requests_latency_before = self.register_metric(
            Histogram,
            "django_http_requests_latency_including_middlewares_seconds",
            (
                "Histogram of requests processing time (including middleware "
                "processing time)."
            ),
            namespace=NAMESPACE,
        )
        self.requests_unknown_latency_before = self.register_metric(
            Counter,
            "django_http_requests_unknown_latency_including_middlewares_total",
            (
                "Count of requests for which the latency was unknown (when computing "
                "django_http_requests_latency_including_middlewares_seconds)."
            ),
            namespace=NAMESPACE,
        )
        self.requests_latency_by_view_method = self.register_metric(
            Histogram,
            "django_http_requests_latency_seconds_by_view_method",
            "Histogram of request processing time labelled by view.",
            ["view", "method"],
            buckets=getattr(
                settings, "PROMETHEUS_LATENCY_BUCKETS", DEFAULT_LATENCY_BUCKETS
            ),
            namespace=NAMESPACE,
        )
        self.requests_unknown_latency = self.register_metric(
            Counter,
            "django_http_requests_unknown_latency_total",
            "Count of requests for which the latency was unknown.",
            namespace=NAMESPACE,
        )
        # Set in process_request
        self.requests_ajax = self.register_metric(
            Counter,
            "django_http_ajax_requests_total",
            "Count of AJAX requests.",
            namespace=NAMESPACE,
        )
        self.requests_by_method = self.register_metric(
            Counter,
            "django_http_requests_total_by_method",
            "Count of requests by method.",
            ["method"],
            namespace=NAMESPACE,
        )
        self.requests_by_transport = self.register_metric(
            Counter,
            "django_http_requests_total_by_transport",
            "Count of requests by transport.",
            ["transport"],
            namespace=NAMESPACE,
        )
        # Set in process_view
        self.requests_by_view_transport_method = self.register_metric(
            Counter,
            "django_http_requests_total_by_view_transport_method",
            "Count of requests by view, transport, method.",
            ["view", "transport", "method"],
            namespace=NAMESPACE,
        )
        self.requests_body_bytes = self.register_metric(
            Histogram,
            "django_http_requests_body_total_bytes",
            "Histogram of requests by body size.",
            buckets=PowersOf(2, 30),
            namespace=NAMESPACE,
        )
        # Set in process_template_response
        self.responses_by_templatename = self.register_metric(
            Counter,
            "django_http_responses_total_by_templatename",
            "Count of responses by template name.",
            ["templatename"],
            namespace=NAMESPACE,
        )
        # Set in process_response
        self.responses_by_status = self.register_metric(
            Counter,
            "django_http_responses_total_by_status",
            "Count of responses by status.",
            ["status"],
            namespace=NAMESPACE,
        )
        self.responses_by_status_view_method = self.register_metric(
            Counter,
            "django_http_responses_total_by_status_view_method",
            "Count of responses by status, view, method.",
            ["status", "view", "method"],
            namespace=NAMESPACE,
        )
        self.responses_body_bytes = self.register_metric(
            Histogram,
            "django_http_responses_body_total_bytes",
            "Histogram of responses by body size.",
            buckets=PowersOf(2, 30),
            namespace=NAMESPACE,
        )
        self.responses_by_charset = self.register_metric(
            Counter,
            "django_http_responses_total_by_charset",
            "Count of responses by charset.",
            ["charset"],
            namespace=NAMESPACE,
        )
        self.responses_streaming = self.register_metric(
            Counter,
            "django_http_responses_streaming_total",
            "Count of streaming responses.",
            namespace=NAMESPACE,
        )
        # Set in process_exception
        self.exceptions_by_type = self.register_metric(
            Counter,
            "django_http_exceptions_total_by_type",
            "Count of exceptions by object type.",
            ["type"],
            namespace=NAMESPACE,
        )
        self.exceptions_by_view = self.register_metric(
            Counter,
            "django_http_exceptions_total_by_view",
            "Count of exceptions by view.",
            ["view"],
            namespace=NAMESPACE,
        )

class PrometheusBeforeMiddleware(MiddlewareMixin):
    """Monitoring middleware that should run before other middlewares."""

    metrics_cls = Metrics

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.metrics = self.metrics_cls.get_instance()

    def process_request(self, request):
        self.metrics.requests_total.inc()
        request.prometheus_before_middleware_event = Time()

    def process_response(self, request, response):
        self.metrics.responses_total.inc()
        if hasattr(request, "prometheus_before_middleware_event"):
            self.metrics.requests_latency_before.observe(
                TimeSince(request.prometheus_before_middleware_event)
            )
        else:
            self.metrics.requests_unknown_latency_before.inc()
        return response

class PrometheusAfterMiddleware(MiddlewareMixin):
    """Monitoring middleware that should run after other middlewares."""

    metrics_cls = Metrics

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.metrics = self.metrics_cls.get_instance()

    def _transport(self, request):
        return "https" if request.is_secure() else "http"

    def _method(self, request):
        m = request.method
        if m not in (
            "GET",
            "HEAD",
            "POST",
            "PUT",
            "DELETE",
            "TRACE",
            "OPTIONS",
            "CONNECT",
            "PATCH",
        ):
            return "<invalid method>"
        return m

    def label_metric(self, metric, request, response=None, **labels):
        return metric.labels(**labels) if labels else metric

    def process_request(self, request):
        transport = self._transport(request)
        method = self._method(request)
        self.label_metric(self.metrics.requests_by_method, request, method=method).inc()
        self.label_metric(
            self.metrics.requests_by_transport, request, transport=transport
        ).inc()

        # Mimic the behaviour of the deprecated "Request.is_ajax()" method.
        if request.META.get("HTTP_X_REQUESTED_WITH") == "XMLHttpRequest":
            self.label_metric(self.metrics.requests_ajax, request).inc()

        content_length = int(request.META.get("CONTENT_LENGTH") or 0)
        self.label_metric(self.metrics.requests_body_bytes, request).observe(
            content_length
        )
        request.prometheus_after_middleware_event = Time()

    def _get_view_name(self, request):
        view_name = "<unnamed view>"
        if hasattr(request, "resolver_match"):
            if request.resolver_match is not None:
                if request.resolver_match.view_name is not None:
                    view_name = request.resolver_match.view_name
        return view_name

    def process_view(self, request, view_func, *view_args, **view_kwargs):
        transport = self._transport(request)
        method = self._method(request)
        if hasattr(request, "resolver_match"):
            name = request.resolver_match.view_name or "<unnamed view>"
            self.label_metric(
                self.metrics.requests_by_view_transport_method,
                request,
                view=name,
                transport=transport,
                method=method,
            ).inc()

    def process_template_response(self, request, response):
        if hasattr(response, "template_name"):
            self.label_metric(
                self.metrics.responses_by_templatename,
                request,
                response=response,
                templatename=str(response.template_name),
            ).inc()
        return response

    def process_response(self, request, response):
        method = self._method(request)
        name = self._get_view_name(request)
        status = str(response.status_code)
        self.label_metric(
            self.metrics.responses_by_status, request, response, status=status
        ).inc()
        self.label_metric(
            self.metrics.responses_by_status_view_method,
            request,
            response,
            status=status,
            view=name,
            method=method,
        ).inc()
        if hasattr(response, "charset"):
            self.label_metric(
                self.metrics.responses_by_charset,
                request,
                response,
                charset=str(response.charset),
            ).inc()
        if hasattr(response, "streaming") and response.streaming:
            self.label_metric(self.metrics.responses_streaming, request, response).inc()
        if hasattr(response, "content"):
            self.label_metric(
                self.metrics.responses_body_bytes, request, response
            ).observe(len(response.content))
        if hasattr(request, "prometheus_after_middleware_event"):
            self.label_metric(
                self.metrics.requests_latency_by_view_method,
                request,
                response,
                view=self._get_view_name(request),
                method=request.method,
            ).observe(TimeSince(request.prometheus_after_middleware_event))
        else:
            self.label_metric(
                self.metrics.requests_unknown_latency, request, response
            ).inc()
        return response

    def process_exception(self, request, exception):
        self.label_metric(
            self.metrics.exceptions_by_type, request, type=type(exception).__name__
        ).inc()
        if hasattr(request, "resolver_match"):
            name = request.resolver_match.view_name or "<unnamed view>"
            self.label_metric(self.metrics.exceptions_by_view, request, view=name).inc()
        if hasattr(request, "prometheus_after_middleware_event"):
            self.label_metric(
                self.metrics.requests_latency_by_view_method,
                request,
                view=self._get_view_name(request),
                method=request.method,
            ).observe(TimeSince(request.prometheus_after_middleware_event))
        else:
            self.label_metric(self.metrics.requests_unknown_latency, request).inc()
<|endoftext|>


<commit_before>VERSION = (0, 2, 4)

def get_version():
    return ".".join(map(str, VERSION))

__version__ = get_version()
<commit_msg>Bump version<commit_after>VERSION = (0, 2, 5)

def get_version():
    return ".".join(map(str, VERSION))

__version__ = get_version()
<|endoftext|>


<commit_before>import hashlib
import json
import os
import pickle

from datetime import date, datetime

import shutil
from django.conf import settings
from django.utils import timezone

BLOCKSIZE = 65536

def create_dated(filename):
    """
    Based on the filename will create a full path filename including the date and time in '%Y%m%d_%H%M' format.
    The path to the filename will be set in the TEST_OUTPUT_PATH settings variable.

    :param filename: base filename. my_excel_data.xlsx for example
    :return: string, full path to file with date and time in the TEST_OUTPUT_PATH folder
    """
    if getattr(settings, 'TEST_OUTPUT_PATH', None) is None:
        msg = 'You need a the variable TEST_OUTPUT_PATH in settings. It should point to a folder' \
              'for temporary data to be written and reviewed.'
        raise ValueError(msg)
    if not os.path.exists(settings.TEST_OUTPUT_PATH):
        os.makedirs(settings.TEST_OUTPUT_PATH)
    return add_date(os.path.join(settings.TEST_OUTPUT_PATH, filename))

def hash_file(filename, algorithm='sha1', block_size=BLOCKSIZE):
    try:
        hasher = getattr(hashlib, algorithm)()
    except AttributeError:
        raise ValueError('{} is not a valid hashing algorithm'.format(algorithm))

    with open(filename, 'rb') as afile:
        buf = afile.read(block_size)
        while len(buf) > 0:
            hasher.update(buf)
            buf = afile.read(block_size)
    return hasher.hexdigest()

def parametrized(dec):
    """
    Need to study this code.
    Got it from http://stackoverflow.com/questions/5929107/python-decorators-with-parameters
    :param dec:
    :return:
    """

    def layer(*args, **kwargs):
        def repl(f):
            return dec(f, *args, **kwargs)
        return repl
    return layer

@parametrized
def temporary_file(func, extension, delete_on_exit=True):
    """
    This method decorator creates a filename with date using the provided extension and delete the file after the method
    has been executed.

    The settings.TEST_OUTPUT_PATH must be configured in your settings file.

    .. code-block:: python

        @temporary_file('json')
        def test_temporary_file_decorator(self):
            filename = self.test_temporary_file_decorator.filename
            ... write to the file ...

    :param func: function to decorate
    :param extension: extension of the filename without the dot
    :param delete_on_exit: If True the filename will be deleted.
    :return: the function
    """
    filename = create_dated('{}.{}'.format(func.__name__, extension))

    def function_t_return(*args):
        results = func(*args)
        if os.path.exists(filename) and delete_on_exit:
            os.remove(filename)
        return results

    function_t_return.filename = filename
    return function_t_return

def json_serial(obj):
    """JSON serializer for objects not serializable by default json code
    taken from: https://stackoverflow.com/questions/11875770/how-to-overcome-datetime-datetime-not-json-serializable
    """

    if isinstance(obj, (datetime, date)):
        serial = obj.isoformat()
        return serial
    raise TypeError("Type %s not serializable" % type(obj))

def serialize_data(data, output_file=None, format='json', **kwargs):
    """
    Quick function to serialize a data to file. The data keys will be saved in an alphabetical order
    for consistency purposes.
    If no output_file is supplied the function will created a dated file in the settings.TEST_OUTPUT_PATH folder.
    if the output_file is a folder the dated file will be created on the supplied folder with the serialized date.
    if the output_file is a file the data will be serialized to thar file

    :param data: Dictionary or list to serialize
    :param format: Format to serialize to. Currently json is the only one supported
    :param output_file: File to output the data to
    :param kwargs:
    """
    assert format in ['json', 'pickle'], 'Unsupported format {}'.format(format)
    if output_file is None:
        filename = create_dated('{}.{}'.format('serialize_data_q', format))
    elif os.path.isdir(output_file):
        filename = os.path.join(output_file, '{}.{}'.format('serialize_data_f', format))
    else:
        filename = output_file
    if format == 'json':
        with open(filename, 'w', encoding=kwargs.get('encoding', 'utf-8'), newline='\n') as fp:
            json.dump(data, fp, indent=kwargs.get('indent', 4),
                      default=json_serial, sort_keys=True)
    elif format == 'pickle':
        with open(filename, 'wb') as output:
            pickle.dump(data, output, pickle.HIGHEST_PROTOCOL)

    return filename

def add_date(filename, **kwargs):
    """
    Adds to a filename the current date and time in '%Y%m%d_%H%M' format.
    For a filename /my/path/myexcel.xlsx the function would return /my/path/myexcel_20170101_1305.xlsx.
    If the file already exists the function will add seconds to the date to attempt to get a unique name.

    The function will detect if another file exists with the same name if it exist it will append seconds to the
    filename. For example if file /my/path/myexcel_20170101_1305.xlsx alread exist thte function will return
    /my/path/myexcel_20170101_130521.xlsx.

    :param filename: string with fullpath to file or just the filename
    :param kwargs: dictionary. date_position: suffix or preffix, extension: string to replace extension
    :return: string with full path string including the date and time
    """
    current_datetime = timezone.localtime(timezone.now()).strftime('%Y%m%d_%H%M%S')
    new_filename_data = dict()
    suffix_template = '{path}{separator}{filename_with_out_extension}_{datetime}.{extension}'
    prefix_template = '{path}{separator}{datetime}_{filename_with_out_extension}.{extension}'
    if '/' in filename and '\\' in filename:
        raise ValueError('Filename %s contains both / and \\ separators' % filename)
    if '\\' in filename:
        path_parts = filename.split('\\')
        file = path_parts[-1]
        path = '\\'.join(path_parts[:-1])
        separator = '\\'
    elif '/' in filename:
        path_parts = filename.split('/')
        file = path_parts[-1]
        path = '/'.join(path_parts[:-1])
        separator = '/'
    else:
        file = filename
        path = ''
        separator = ''

    new_filename_data['path'] = path
    parts = file.split('.')
    if kwargs.get('extension', None) is not None:
        new_filename_data['extension'] = kwargs['extension']
    else:
        if len(parts) > 1:
            new_filename_data['extension'] = parts[-1]
        else:
            new_filename_data['extension'] = ''

    new_filename_data['separator'] = separator
    if new_filename_data['extension'] == '':
        new_filename_data['filename_with_out_extension'] = parts[0]
    else:
        new_filename_data['filename_with_out_extension'] = '.'.join(parts[:-1])
    new_filename_data['datetime'] = current_datetime[:-2]  # Seconds are stripped

    date_position = kwargs.get('date_position', 'suffix')
    if date_position == 'suffix':
        new_filename = suffix_template.format(**new_filename_data)
        if os.path.exists(new_filename):
            new_filename_data['datetime'] = current_datetime
            new_filename = suffix_template.format(**new_filename_data)
        if new_filename_data['extension'] == '':
            new_filename = new_filename[:-1]
    else:
        new_filename = prefix_template.format(**new_filename_data)
        if os.path.exists(new_filename):
            new_filename_data['datetime'] = current_datetime
            new_filename = prefix_template.format(**new_filename_data)
        if new_filename_data['extension'] == '':
            new_filename = new_filename[:-1]

    return new_filename

class TemporaryFolder:
    def __init__(self, base_name, delete_on_exit=True):
        self.new_path = create_dated(base_name)
        self.delete_on_exit = delete_on_exit

    def __enter__(self):
        os.mkdir(self.new_path)
        self.saved_path = os.getcwd()
        os.chdir(self.new_path)

        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        os.chdir(self.saved_path)
        if self.delete_on_exit:
            shutil.rmtree(self.new_path)

    def write(self, filename, content):
        with open(filename, 'w', encoding='utf-8') as file:
            if isinstance(content, str):
                file.writelines(content)
            elif isinstance(content, list):
                for line in content:
                    file.write(line)
                    file.write('\n')
            else:
                file.writelines(str(content))
        return os.path.join(self.new_path, filename)
<commit_msg>Added base_filename keyword argument to serialze_data<commit_after>import hashlib
import json
import os
import pickle

from datetime import date, datetime

import shutil
from django.conf import settings
from django.utils import timezone

BLOCKSIZE = 65536

def create_dated(filename):
    """
    Based on the filename will create a full path filename including the date and time in '%Y%m%d_%H%M' format.
    The path to the filename will be set in the TEST_OUTPUT_PATH settings variable.

    :param filename: base filename. my_excel_data.xlsx for example
    :return: string, full path to file with date and time in the TEST_OUTPUT_PATH folder
    """
    if getattr(settings, 'TEST_OUTPUT_PATH', None) is None:
        msg = 'You need a the variable TEST_OUTPUT_PATH in settings. It should point to a folder' \
              'for temporary data to be written and reviewed.'
        raise ValueError(msg)
    if not os.path.exists(settings.TEST_OUTPUT_PATH):
        os.makedirs(settings.TEST_OUTPUT_PATH)
    return add_date(os.path.join(settings.TEST_OUTPUT_PATH, filename))

def hash_file(filename, algorithm='sha1', block_size=BLOCKSIZE):
    try:
        hasher = getattr(hashlib, algorithm)()
    except AttributeError:
        raise ValueError('{} is not a valid hashing algorithm'.format(algorithm))

    with open(filename, 'rb') as afile:
        buf = afile.read(block_size)
        while len(buf) > 0:
            hasher.update(buf)
            buf = afile.read(block_size)
    return hasher.hexdigest()

def parametrized(dec):
    """
    Need to study this code.
    Got it from http://stackoverflow.com/questions/5929107/python-decorators-with-parameters
    :param dec:
    :return:
    """

    def layer(*args, **kwargs):
        def repl(f):
            return dec(f, *args, **kwargs)
        return repl
    return layer

@parametrized
def temporary_file(func, extension, delete_on_exit=True):
    """
    This method decorator creates a filename with date using the provided extension and delete the file after the method
    has been executed.

    The settings.TEST_OUTPUT_PATH must be configured in your settings file.

    .. code-block:: python

        @temporary_file('json')
        def test_temporary_file_decorator(self):
            filename = self.test_temporary_file_decorator.filename
            ... write to the file ...

    :param func: function to decorate
    :param extension: extension of the filename without the dot
    :param delete_on_exit: If True the filename will be deleted.
    :return: the function
    """
    filename = create_dated('{}.{}'.format(func.__name__, extension))

    def function_t_return(*args):
        results = func(*args)
        if os.path.exists(filename) and delete_on_exit:
            os.remove(filename)
        return results

    function_t_return.filename = filename
    return function_t_return

def json_serial(obj):
    """JSON serializer for objects not serializable by default json code
    taken from: https://stackoverflow.com/questions/11875770/how-to-overcome-datetime-datetime-not-json-serializable
    """

    if isinstance(obj, (datetime, date)):
        serial = obj.isoformat()
        return serial
    raise TypeError("Type %s not serializable" % type(obj))

def serialize_data(data, output_file=None, format='json', **kwargs):
    """
    Quick function to serialize a data to file. The data keys will be saved in an alphabetical order
    for consistency purposes.
    If no output_file is supplied the function will created a dated file in the settings.TEST_OUTPUT_PATH folder.
    if the output_file is a folder the dated file will be created on the supplied folder with the serialized date.
    if the output_file is a file the data will be serialized to thar file

    :param data: Dictionary or list to serialize
    :param format: Format to serialize to. Currently json is the only one supported
    :param output_file: File to output the data to
    :param kwargs:
    """
    assert format in ['json', 'pickle'], 'Unsupported format {}'.format(format)
    base_filename = kwargs.get('base_filename', 'serialized_data')

    if output_file is None:
        filename = create_dated('{}.{}'.format(base_filename, format))
    elif os.path.isdir(output_file):
        filename = os.path.join(output_file, '{}.{}'.format(base_filename, format))
    else:
        filename = output_file
    if format == 'json':
        with open(filename, 'w', encoding=kwargs.get('encoding', 'utf-8'), newline='\n') as fp:
            json.dump(data, fp, indent=kwargs.get('indent', 4),
                      default=json_serial, sort_keys=True)
    elif format == 'pickle':
        with open(filename, 'wb') as output:
            pickle.dump(data, output, pickle.HIGHEST_PROTOCOL)

    return filename

def add_date(filename, **kwargs):
    """
    Adds to a filename the current date and time in '%Y%m%d_%H%M' format.
    For a filename /my/path/myexcel.xlsx the function would return /my/path/myexcel_20170101_1305.xlsx.
    If the file already exists the function will add seconds to the date to attempt to get a unique name.

    The function will detect if another file exists with the same name if it exist it will append seconds to the
    filename. For example if file /my/path/myexcel_20170101_1305.xlsx alread exist thte function will return
    /my/path/myexcel_20170101_130521.xlsx.

    :param filename: string with fullpath to file or just the filename
    :param kwargs: dictionary. date_position: suffix or preffix, extension: string to replace extension
    :return: string with full path string including the date and time
    """
    current_datetime = timezone.localtime(timezone.now()).strftime('%Y%m%d_%H%M%S')
    new_filename_data = dict()
    suffix_template = '{path}{separator}{filename_with_out_extension}_{datetime}.{extension}'
    prefix_template = '{path}{separator}{datetime}_{filename_with_out_extension}.{extension}'
    if '/' in filename and '\\' in filename:
        raise ValueError('Filename %s contains both / and \\ separators' % filename)
    if '\\' in filename:
        path_parts = filename.split('\\')
        file = path_parts[-1]
        path = '\\'.join(path_parts[:-1])
        separator = '\\'
    elif '/' in filename:
        path_parts = filename.split('/')
        file = path_parts[-1]
        path = '/'.join(path_parts[:-1])
        separator = '/'
    else:
        file = filename
        path = ''
        separator = ''

    new_filename_data['path'] = path
    parts = file.split('.')
    if kwargs.get('extension', None) is not None:
        new_filename_data['extension'] = kwargs['extension']
    else:
        if len(parts) > 1:
            new_filename_data['extension'] = parts[-1]
        else:
            new_filename_data['extension'] = ''

    new_filename_data['separator'] = separator
    if new_filename_data['extension'] == '':
        new_filename_data['filename_with_out_extension'] = parts[0]
    else:
        new_filename_data['filename_with_out_extension'] = '.'.join(parts[:-1])
    new_filename_data['datetime'] = current_datetime[:-2]  # Seconds are stripped

    date_position = kwargs.get('date_position', 'suffix')
    if date_position == 'suffix':
        new_filename = suffix_template.format(**new_filename_data)
        if os.path.exists(new_filename):
            new_filename_data['datetime'] = current_datetime
            new_filename = suffix_template.format(**new_filename_data)
        if new_filename_data['extension'] == '':
            new_filename = new_filename[:-1]
    else:
        new_filename = prefix_template.format(**new_filename_data)
        if os.path.exists(new_filename):
            new_filename_data['datetime'] = current_datetime
            new_filename = prefix_template.format(**new_filename_data)
        if new_filename_data['extension'] == '':
            new_filename = new_filename[:-1]

    return new_filename

class TemporaryFolder:
    def __init__(self, base_name, delete_on_exit=True):
        self.new_path = create_dated(base_name)
        self.delete_on_exit = delete_on_exit

    def __enter__(self):
        os.mkdir(self.new_path)
        self.saved_path = os.getcwd()
        os.chdir(self.new_path)

        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        os.chdir(self.saved_path)
        if self.delete_on_exit:
            shutil.rmtree(self.new_path)

    def write(self, filename, content):
        with open(filename, 'w', encoding='utf-8') as file:
            if isinstance(content, str):
                file.writelines(content)
            elif isinstance(content, list):
                for line in content:
                    file.write(line)
                    file.write('\n')
            else:
                file.writelines(str(content))
        return os.path.join(self.new_path, filename)
<|endoftext|>


<commit_before>__version__ = '0.0.1'
<commit_msg>version bump<commit_after>__version__ = '0.0.2'
<|endoftext|>


<commit_before>from django.template import RequestContext
from django.conf import settings
from django.shortcuts import render_to_response, HttpResponse

def jasmine_spec(request):
    context = RequestContext(request)
    context['DEBUG'] = settings.DEBUG
    context['livereload_address'] = "http://%s:35729/livereload.js" % request.META['REMOTE_ADDR']
    return render_to_response("jasmine_spec.html", context_instance=context)

def sandbox(request):
    context = RequestContext(request)
    context['DEBUG'] = settings.DEBUG
    context['livereload_address'] = "http://%s:35729/livereload.js" % request.META['REMOTE_ADDR']
    context['run_jasmine'] = False
    context['include_coffeefile'] = "kobo/stylesheets/pages/form_builder.coffee"
    return render_to_response("sandbox.html", context_instance=context)
<commit_msg>livereload displayed in sandbox only if specified<commit_after>from django.template import RequestContext
from django.conf import settings
from django.shortcuts import render_to_response, HttpResponse

def jasmine_spec(request):
    context = RequestContext(request)
    context['DEBUG'] = settings.DEBUG
    if settings.LIVE_RELOAD:
        context['livereload_address'] = "http://%s:35729/livereload.js" % request.META['REMOTE_ADDR']
    return render_to_response("jasmine_spec.html", context_instance=context)

def sandbox(request):
    context = RequestContext(request)
    context['DEBUG'] = settings.DEBUG
    if settings.LIVE_RELOAD:
        context['livereload_address'] = "http://%s:35729/livereload.js" % request.META['REMOTE_ADDR']
    context['run_jasmine'] = False
    context['include_coffeefile'] = "kobo/stylesheets/pages/form_builder.coffee"
    return render_to_response("sandbox.html", context_instance=context)
<|endoftext|>


<commit_before>"""
Django settings for do2things project.

Generated by 'django-admin startproject' using Django 1.8.3.

For more information on this file, see
https://docs.djangoproject.com/en/1.8/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.8/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.environ.get('SECRET_KEY')

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False

ALLOWED_HOSTS = ['127.0.0.1', 'do2things.herokuapp.com']

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'decohere',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.middleware.security.SecurityMiddleware',
)

ROOT_URLCONF = 'do2things.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'do2things.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.8/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'NAME': 'do2things',
        'USER': '',
        'PASSWORD': '',
        'HOST': 'localhost',
        'PORT': '',
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.8/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.8/howto/static-files/

STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles')

STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage'
<commit_msg>allowed hosts update<commit_after>"""
Django settings for do2things project.

Generated by 'django-admin startproject' using Django 1.8.3.

For more information on this file, see
https://docs.djangoproject.com/en/1.8/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.8/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.environ.get('SECRET_KEY')

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False

ALLOWED_HOSTS = ['127.0.0.1', 'do2things.herokuapp.com', '*']

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'decohere',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.middleware.security.SecurityMiddleware',
)

ROOT_URLCONF = 'do2things.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'do2things.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.8/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'NAME': 'do2things',
        'USER': '',
        'PASSWORD': '',
        'HOST': 'localhost',
        'PORT': '',
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.8/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.8/howto/static-files/

STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles')

STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage'
<|endoftext|>


<commit_before>from django.contrib.auth.models import User
from django.db import models

from common import fields

class UserData(models.Model):
    user = fields.AutoOneToOneField(User, related_name='pgp')

class HuId(models.Model):
    user_data = models.ForeignKey(UserData, related_name='huids')

    value = models.CharField(primary_key=True, max_length=64)

class DataFile(BaseDataFile):
    """
    Storage for an PGP data file.
    """
    user_data = models.ForeignKey(UserData)
    task = models.ForeignKey(DataRetrievalTask, related_name='datafile_pgp')

    def __unicode__(self):
        return '%s:%s:%s' % (self.user_data.user, 'pgp', self.file)
<commit_msg>🐛 add missing import<commit_after>from django.contrib.auth.models import User
from django.db import models

from common import fields
from data_import.models import BaseDataFile, DataRetrievalTask

class UserData(models.Model):
    user = fields.AutoOneToOneField(User, related_name='pgp')

class HuId(models.Model):
    user_data = models.ForeignKey(UserData, related_name='huids')

    value = models.CharField(primary_key=True, max_length=64)

class DataFile(BaseDataFile):
    """
    Storage for an PGP data file.
    """
    user_data = models.ForeignKey(UserData)
    task = models.ForeignKey(DataRetrievalTask, related_name='datafile_pgp')

    def __unicode__(self):
        return '%s:%s:%s' % (self.user_data.user, 'pgp', self.file)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('wagtailcore', '0001_squashed_0016_change_page_url_path_to_text_field'),
    ]

    operations = [
        migrations.AlterField(
            model_name='grouppagepermission',
            name='permission_type',
            field=models.CharField(choices=[('add', 'Add/edit pages you own'), ('edit', 'Edit any page'), ('publish', 'Publish any page'), ('lock', 'Lock/unlock any page')], max_length=20, verbose_name='Permission type'),
            preserve_default=True,
        ),
    ]
<commit_msg>Make migrations depend on non-squashed versions - fixes #1817<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('wagtailcore', '0016_change_page_url_path_to_text_field'),
    ]

    operations = [
        migrations.AlterField(
            model_name='grouppagepermission',
            name='permission_type',
            field=models.CharField(choices=[('add', 'Add/edit pages you own'), ('edit', 'Edit any page'), ('publish', 'Publish any page'), ('lock', 'Lock/unlock any page')], max_length=20, verbose_name='Permission type'),
            preserve_default=True,
        ),
    ]
<|endoftext|>