<commit_before>"""Builder module

A builder connect a :class:`umongo.document.Template` with a
:class:`umongo.instance.BaseInstance` by generating an
:class:`umongo.document.Implementation`.
"""
import re
import inspect
from copy import copy

import marshmallow as ma

from .template import Template, Implementation
from .data_proxy import data_proxy_factory
from .document import DocumentTemplate, DocumentOpts, DocumentImplementation
from .embedded_document import (
    EmbeddedDocumentTemplate, EmbeddedDocumentOpts, EmbeddedDocumentImplementation)
from .exceptions import DocumentDefinitionError, NotRegisteredDocumentError
from .schema import Schema
from .indexes import parse_index
from . import fields

def camel_to_snake(name):
    tmp_str = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
    return re.sub('([a-z0-9])([A-Z])', r'\1_\2', tmp_str).lower()

def _is_child(bases):
    """Find if the given inheritance leeds to a child document (i.e.
    a document that shares the same collection with a parent)
    """
    return any(b for b in bases if issubclass(b, DocumentImplementation) and not b.opts.abstract)

def _is_child_embedded_document(bases):
    """Same thing than _is_child, but for EmbeddedDocument...
    """
    return any(b for b in bases
               if issubclass(b, EmbeddedDocumentImplementation) and
               b is not EmbeddedDocumentImplementation)

def _on_need_add_id_field(bases, fields_dict):
    """
    If the given fields make no reference to `_id`, add an `id` field
    (type ObjectId, dump_only=True, attribute=`_id`) to handle it
    """

    def find_id_field(fields_dict):
        for name, field in fields_dict.items():
            # Skip fake fields present in schema (e.g. `post_load` decorated function)
            if not isinstance(field, ma.fields.Field):
                continue
            if (name == '_id' and not field.attribute) or field.attribute == '_id':
                return name
        return None

    # Search among parents for the id field
    for base in bases:
        schema = base()
        name = find_id_field(schema.fields)
        if name is not None:
            return name

    # Search among our own fields
    name = find_id_field(fields_dict)
    if name is not None:
        return name

    # No id field found, add a default one
    fields_dict['id'] = fields.ObjectIdField(attribute='_id', dump_only=True)
    return 'id'

def _add_child_field(name, fields_dict):
    fields_dict['cls'] = fields.StringField(attribute='_cls', default=name, dump_only=True)

def _collect_schema_attrs(nmspc):
    """
    Split dict between schema fields and non-fields elements and retrieve
    marshmallow tags if any.
    """
    schema_fields = {}
    schema_non_fields = {}
    doc_nmspc = {}
    for key, item in nmspc.items():
        if hasattr(item, '__marshmallow_hook__'):
            # Decorated special functions (e.g. `post_load`)
            schema_non_fields[key] = item
        elif isinstance(item, ma.fields.Field):
            # Given the fields provided by the template are going to be
            # customized in the implementation, we copy them to avoid
            # overwriting if two implementations are created
            schema_fields[key] = copy(item)
        else:
            doc_nmspc[key] = item
    return doc_nmspc, schema_fields, schema_non_fields

def _collect_indexes(meta, schema_nmspc, bases):
    """
    Retrieve all indexes (custom defined in meta class, by inheritances
    and unique attribut in fields)
    """
    indexes = []
    is_child = _is_child(bases)

    # First collect parent indexes (including inherited field's unique indexes)
    for base in bases:
        if issubclass(base, DocumentImplementation):
            indexes += base.opts.indexes

    # Then get our own custom indexes
    if is_child:
        custom_indexes = [
            parse_index(x, base_compound_field='_cls')
            for x in getattr(meta, 'indexes', ())
        ]
    else:
        custom_indexes = [parse_index(x) for x in getattr(meta, 'indexes', ())]
    indexes += custom_indexes

    if is_child:
        indexes.append(parse_index('_cls'))

    # Finally parse our own fields (i.e. not inherited) for unique indexes
    def parse_field(mongo_path, path, field):
        if field.unique:
            index = {'unique': True, 'key': [mongo_path]}
            if not field.required or field.allow_none:
                index['sparse'] = True
            if is_child:
                index['key'].append('_cls')
            indexes.append(parse_index(index))

    for name, field in schema_nmspc.items():
        parse_field(name or field.attribute, name, field)
        if hasattr(field, 'map_to_field'):
            field.map_to_field(name or field.attribute, name, parse_field)

    return indexes

def _build_document_opts(instance, template, name, nmspc, bases):
    kwargs = {}
    meta = nmspc.get('Meta')
    collection_name = getattr(meta, 'collection_name', None)
    kwargs['instance'] = instance
    kwargs['template'] = template
    kwargs['abstract'] = getattr(meta, 'abstract', False)
    kwargs['allow_inheritance'] = getattr(meta, 'allow_inheritance', None)
    kwargs['is_child'] = _is_child(bases)
    kwargs['strict'] = getattr(meta, 'strict', True)

    # Handle option inheritance and integrity checks
    for base in bases:
        if not issubclass(base, DocumentImplementation):
            continue
        popts = base.opts
        if not popts.allow_inheritance:
            raise DocumentDefinitionError("Document %r doesn't allow inheritance" % base)
        if kwargs['abstract'] and not popts.abstract:
            raise DocumentDefinitionError(
                "Abstract document should have all it parents abstract")
        if popts.collection_name:
            if collection_name:
                raise DocumentDefinitionError(
                    "Cannot redefine collection_name in a child, use abstract instead")
            collection_name = popts.collection_name

    if collection_name:
        if kwargs['abstract']:
            raise DocumentDefinitionError(
                'Abstract document cannot define collection_name')
    elif not kwargs['abstract']:
        # Determine the collection name from the class name
        collection_name = camel_to_snake(name)

    return DocumentOpts(collection_name=collection_name, **kwargs)

def _build_embedded_document_opts(instance, template, name, nmspc, bases):
    kwargs = {}
    meta = nmspc.get('Meta')
    kwargs['instance'] = instance
    kwargs['template'] = template
    kwargs['abstract'] = getattr(meta, 'abstract', False)
    kwargs['allow_inheritance'] = getattr(meta, 'allow_inheritance', True)
    kwargs['is_child'] = _is_child_embedded_document(bases)
    kwargs['strict'] = getattr(meta, 'strict', True)

    # Handle option inheritance and integrity checks
    for base in bases:
        if not issubclass(base, EmbeddedDocumentImplementation):
            continue
        popts = base.opts
        if not popts.allow_inheritance:
            raise DocumentDefinitionError("EmbeddedDocument %r doesn't allow inheritance" % base)
        if kwargs['abstract'] and not popts.abstract:
            raise DocumentDefinitionError(
                "Abstract embedded document should have all it parents abstract")

    return EmbeddedDocumentOpts(**kwargs)

class BaseBuilder:
    """
    A builder connect a :class:`umongo.document.Template` with a
    :class:`umongo.instance.BaseInstance` by generating an
    :class:`umongo.document.Implementation`.

    .. note:: This class should not be used directly, it should be inherited by
              concrete implementations such as :class:`umongo.frameworks.pymongo.PyMongoBuilder`
    """

    BASE_DOCUMENT_CLS = None

    def __init__(self, instance):
        assert self.BASE_DOCUMENT_CLS
        self.instance = instance
        self._templates_lookup = {
            DocumentTemplate: self.BASE_DOCUMENT_CLS,
            EmbeddedDocumentTemplate: EmbeddedDocumentImplementation
        }

    def _convert_bases(self, bases):
        "Replace template parents by their implementation inside this instance"
        converted_bases = []
        for base in bases:
            assert not issubclass(base, Implementation), \
                'Document cannot inherit of implementations'
            if issubclass(base, Template):
                if base not in self._templates_lookup:
                    raise NotRegisteredDocumentError('Unknown document `%r`' % base)
                converted_bases.append(self._templates_lookup[base])
            else:
                converted_bases.append(base)
        return tuple(converted_bases)

    def _patch_field(self, field):
        # Recursively set the `instance` attribute to all fields
        field.instance = self.instance
        if isinstance(field, fields.ListField):
            self._patch_field(field.inner)
        elif isinstance(field, fields.DictField):
            if field.key_field:
                self._patch_field(field.key_field)
            if field.value_field:
                self._patch_field(field.value_field)
        elif isinstance(field, fields.EmbeddedField):
            for embedded_field in field.schema.fields.values():
                self._patch_field(embedded_field)

    def _build_schema(self, template, schema_bases, schema_fields, schema_non_fields):
        # Recursively set the `instance` attribute to all fields
        for field in schema_fields.values():
            self._patch_field(field)

        # Finally build the schema class
        schema_nmspc = {}
        schema_nmspc.update(schema_fields)
        schema_nmspc.update(schema_non_fields)
        schema_nmspc['MA_BASE_SCHEMA_CLS'] = template.MA_BASE_SCHEMA_CLS
        return type('%sSchema' % template.__name__, schema_bases, schema_nmspc)

    def build_document_from_template(self, template):
        """
        Generate a :class:`umongo.document.DocumentImplementation` for this
        instance from the given :class:`umongo.document.DocumentTemplate`.
        """
        assert issubclass(template, DocumentTemplate)
        name = template.__name__
        bases = self._convert_bases(template.__bases__)
        opts = _build_document_opts(self.instance, template, name, template.__dict__, bases)
        nmspc, schema_fields, schema_non_fields = _collect_schema_attrs(template.__dict__)
        nmspc['opts'] = opts

        # Create schema by retrieving inherited schema classes
        schema_bases = tuple([base.Schema for base in bases
                              if hasattr(base, 'Schema')])
        if not schema_bases:
            schema_bases = (Schema, )
        nmspc['pk_field'] = _on_need_add_id_field(schema_bases, schema_fields)
        # If Document is a child, _cls field must be added to the schema
        if opts.is_child:
            _add_child_field(name, schema_fields)
        schema_cls = self._build_schema(template, schema_bases, schema_fields, schema_non_fields)
        nmspc['Schema'] = schema_cls
        schema = schema_cls()
        nmspc['schema'] = schema
        nmspc['DataProxy'] = data_proxy_factory(name, schema, strict=opts.strict)

        # _build_document_opts cannot determine the indexes given we need to
        # visit the document's fields which weren't defined at this time
        opts.indexes = _collect_indexes(nmspc.get('Meta'), schema.fields, bases)

        implementation = type(name, bases, nmspc)
        self._templates_lookup[template] = implementation
        # Notify the parent & grand parents of the newborn !
        for base in bases:
            for parent in inspect.getmro(base):
                if (not issubclass(parent, DocumentImplementation) or
                        parent is DocumentImplementation):
                    continue
                parent.opts.offspring.add(implementation)
        return implementation

    def build_embedded_document_from_template(self, template):
        """
        Generate a :class:`umongo.document.EmbeddedDocumentImplementation` for this
        instance from the given :class:`umongo.document.EmbeddedDocumentTemplate`.
        """
        assert issubclass(template, EmbeddedDocumentTemplate)
        name = template.__name__
        bases = self._convert_bases(template.__bases__)
        opts = _build_embedded_document_opts(
            self.instance, template, name, template.__dict__, bases)

        nmspc, schema_fields, schema_non_fields = _collect_schema_attrs(template.__dict__)
        nmspc['opts'] = opts

        # If EmbeddedDocument is a child, _cls field must be added to the schema
        if opts.is_child:
            _add_child_field(name, schema_fields)

        # Create schema by retrieving inherited schema classes
        schema_bases = tuple([base.Schema for base in bases
                              if hasattr(base, 'Schema')])
        if not schema_bases:
            schema_bases = (Schema, )
        schema_cls = self._build_schema(template, schema_bases, schema_fields, schema_non_fields)
        nmspc['Schema'] = schema_cls
        schema = schema_cls()
        nmspc['schema'] = schema
        nmspc['DataProxy'] = data_proxy_factory(name, schema, strict=opts.strict)

        implementation = type(name, bases, nmspc)
        self._templates_lookup[template] = implementation
        # Notify the parent & grand parents of the newborn !
        for base in bases:
            for parent in inspect.getmro(base):
                if (not issubclass(parent, EmbeddedDocumentImplementation) or
                        parent is EmbeddedDocumentImplementation):
                    continue
                parent.opts.offspring.add(implementation)
        return implementation
<commit_msg>_collect_schema_args: take template as argument<commit_after>"""Builder module

A builder connect a :class:`umongo.document.Template` with a
:class:`umongo.instance.BaseInstance` by generating an
:class:`umongo.document.Implementation`.
"""
import re
import inspect
from copy import copy

import marshmallow as ma

from .template import Template, Implementation
from .data_proxy import data_proxy_factory
from .document import DocumentTemplate, DocumentOpts, DocumentImplementation
from .embedded_document import (
    EmbeddedDocumentTemplate, EmbeddedDocumentOpts, EmbeddedDocumentImplementation)
from .exceptions import DocumentDefinitionError, NotRegisteredDocumentError
from .schema import Schema
from .indexes import parse_index
from . import fields

def camel_to_snake(name):
    tmp_str = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
    return re.sub('([a-z0-9])([A-Z])', r'\1_\2', tmp_str).lower()

def _is_child(bases):
    """Find if the given inheritance leeds to a child document (i.e.
    a document that shares the same collection with a parent)
    """
    return any(b for b in bases if issubclass(b, DocumentImplementation) and not b.opts.abstract)

def _is_child_embedded_document(bases):
    """Same thing than _is_child, but for EmbeddedDocument...
    """
    return any(b for b in bases
               if issubclass(b, EmbeddedDocumentImplementation) and
               b is not EmbeddedDocumentImplementation)

def _on_need_add_id_field(bases, fields_dict):
    """
    If the given fields make no reference to `_id`, add an `id` field
    (type ObjectId, dump_only=True, attribute=`_id`) to handle it
    """

    def find_id_field(fields_dict):
        for name, field in fields_dict.items():
            # Skip fake fields present in schema (e.g. `post_load` decorated function)
            if not isinstance(field, ma.fields.Field):
                continue
            if (name == '_id' and not field.attribute) or field.attribute == '_id':
                return name
        return None

    # Search among parents for the id field
    for base in bases:
        schema = base()
        name = find_id_field(schema.fields)
        if name is not None:
            return name

    # Search among our own fields
    name = find_id_field(fields_dict)
    if name is not None:
        return name

    # No id field found, add a default one
    fields_dict['id'] = fields.ObjectIdField(attribute='_id', dump_only=True)
    return 'id'

def _add_child_field(name, fields_dict):
    fields_dict['cls'] = fields.StringField(attribute='_cls', default=name, dump_only=True)

def _collect_schema_attrs(template):
    """
    Split dict between schema fields and non-fields elements and retrieve
    marshmallow tags if any.
    """
    schema_fields = {}
    schema_non_fields = {}
    nmspc = {}
    for key, item in template.__dict__.items():
        if hasattr(item, '__marshmallow_hook__'):
            # Decorated special functions (e.g. `post_load`)
            schema_non_fields[key] = item
        elif isinstance(item, ma.fields.Field):
            # Given the fields provided by the template are going to be
            # customized in the implementation, we copy them to avoid
            # overwriting if two implementations are created
            schema_fields[key] = copy(item)
        else:
            nmspc[key] = item
    return nmspc, schema_fields, schema_non_fields

def _collect_indexes(meta, schema_nmspc, bases):
    """
    Retrieve all indexes (custom defined in meta class, by inheritances
    and unique attribut in fields)
    """
    indexes = []
    is_child = _is_child(bases)

    # First collect parent indexes (including inherited field's unique indexes)
    for base in bases:
        if issubclass(base, DocumentImplementation):
            indexes += base.opts.indexes

    # Then get our own custom indexes
    if is_child:
        custom_indexes = [
            parse_index(x, base_compound_field='_cls')
            for x in getattr(meta, 'indexes', ())
        ]
    else:
        custom_indexes = [parse_index(x) for x in getattr(meta, 'indexes', ())]
    indexes += custom_indexes

    if is_child:
        indexes.append(parse_index('_cls'))

    # Finally parse our own fields (i.e. not inherited) for unique indexes
    def parse_field(mongo_path, path, field):
        if field.unique:
            index = {'unique': True, 'key': [mongo_path]}
            if not field.required or field.allow_none:
                index['sparse'] = True
            if is_child:
                index['key'].append('_cls')
            indexes.append(parse_index(index))

    for name, field in schema_nmspc.items():
        parse_field(name or field.attribute, name, field)
        if hasattr(field, 'map_to_field'):
            field.map_to_field(name or field.attribute, name, parse_field)

    return indexes

def _build_document_opts(instance, template, name, nmspc, bases):
    kwargs = {}
    meta = nmspc.get('Meta')
    collection_name = getattr(meta, 'collection_name', None)
    kwargs['instance'] = instance
    kwargs['template'] = template
    kwargs['abstract'] = getattr(meta, 'abstract', False)
    kwargs['allow_inheritance'] = getattr(meta, 'allow_inheritance', None)
    kwargs['is_child'] = _is_child(bases)
    kwargs['strict'] = getattr(meta, 'strict', True)

    # Handle option inheritance and integrity checks
    for base in bases:
        if not issubclass(base, DocumentImplementation):
            continue
        popts = base.opts
        if not popts.allow_inheritance:
            raise DocumentDefinitionError("Document %r doesn't allow inheritance" % base)
        if kwargs['abstract'] and not popts.abstract:
            raise DocumentDefinitionError(
                "Abstract document should have all it parents abstract")
        if popts.collection_name:
            if collection_name:
                raise DocumentDefinitionError(
                    "Cannot redefine collection_name in a child, use abstract instead")
            collection_name = popts.collection_name

    if collection_name:
        if kwargs['abstract']:
            raise DocumentDefinitionError(
                'Abstract document cannot define collection_name')
    elif not kwargs['abstract']:
        # Determine the collection name from the class name
        collection_name = camel_to_snake(name)

    return DocumentOpts(collection_name=collection_name, **kwargs)

def _build_embedded_document_opts(instance, template, name, nmspc, bases):
    kwargs = {}
    meta = nmspc.get('Meta')
    kwargs['instance'] = instance
    kwargs['template'] = template
    kwargs['abstract'] = getattr(meta, 'abstract', False)
    kwargs['allow_inheritance'] = getattr(meta, 'allow_inheritance', True)
    kwargs['is_child'] = _is_child_embedded_document(bases)
    kwargs['strict'] = getattr(meta, 'strict', True)

    # Handle option inheritance and integrity checks
    for base in bases:
        if not issubclass(base, EmbeddedDocumentImplementation):
            continue
        popts = base.opts
        if not popts.allow_inheritance:
            raise DocumentDefinitionError("EmbeddedDocument %r doesn't allow inheritance" % base)
        if kwargs['abstract'] and not popts.abstract:
            raise DocumentDefinitionError(
                "Abstract embedded document should have all it parents abstract")

    return EmbeddedDocumentOpts(**kwargs)

class BaseBuilder:
    """
    A builder connect a :class:`umongo.document.Template` with a
    :class:`umongo.instance.BaseInstance` by generating an
    :class:`umongo.document.Implementation`.

    .. note:: This class should not be used directly, it should be inherited by
              concrete implementations such as :class:`umongo.frameworks.pymongo.PyMongoBuilder`
    """

    BASE_DOCUMENT_CLS = None

    def __init__(self, instance):
        assert self.BASE_DOCUMENT_CLS
        self.instance = instance
        self._templates_lookup = {
            DocumentTemplate: self.BASE_DOCUMENT_CLS,
            EmbeddedDocumentTemplate: EmbeddedDocumentImplementation
        }

    def _convert_bases(self, bases):
        "Replace template parents by their implementation inside this instance"
        converted_bases = []
        for base in bases:
            assert not issubclass(base, Implementation), \
                'Document cannot inherit of implementations'
            if issubclass(base, Template):
                if base not in self._templates_lookup:
                    raise NotRegisteredDocumentError('Unknown document `%r`' % base)
                converted_bases.append(self._templates_lookup[base])
            else:
                converted_bases.append(base)
        return tuple(converted_bases)

    def _patch_field(self, field):
        # Recursively set the `instance` attribute to all fields
        field.instance = self.instance
        if isinstance(field, fields.ListField):
            self._patch_field(field.inner)
        elif isinstance(field, fields.DictField):
            if field.key_field:
                self._patch_field(field.key_field)
            if field.value_field:
                self._patch_field(field.value_field)
        elif isinstance(field, fields.EmbeddedField):
            for embedded_field in field.schema.fields.values():
                self._patch_field(embedded_field)

    def _build_schema(self, template, schema_bases, schema_fields, schema_non_fields):
        # Recursively set the `instance` attribute to all fields
        for field in schema_fields.values():
            self._patch_field(field)

        # Finally build the schema class
        schema_nmspc = {}
        schema_nmspc.update(schema_fields)
        schema_nmspc.update(schema_non_fields)
        schema_nmspc['MA_BASE_SCHEMA_CLS'] = template.MA_BASE_SCHEMA_CLS
        return type('%sSchema' % template.__name__, schema_bases, schema_nmspc)

    def build_document_from_template(self, template):
        """
        Generate a :class:`umongo.document.DocumentImplementation` for this
        instance from the given :class:`umongo.document.DocumentTemplate`.
        """
        assert issubclass(template, DocumentTemplate)
        name = template.__name__
        bases = self._convert_bases(template.__bases__)
        opts = _build_document_opts(self.instance, template, name, template.__dict__, bases)
        nmspc, schema_fields, schema_non_fields = _collect_schema_attrs(template)
        nmspc['opts'] = opts

        # Create schema by retrieving inherited schema classes
        schema_bases = tuple([base.Schema for base in bases
                              if hasattr(base, 'Schema')])
        if not schema_bases:
            schema_bases = (Schema, )
        nmspc['pk_field'] = _on_need_add_id_field(schema_bases, schema_fields)
        # If Document is a child, _cls field must be added to the schema
        if opts.is_child:
            _add_child_field(name, schema_fields)
        schema_cls = self._build_schema(template, schema_bases, schema_fields, schema_non_fields)
        nmspc['Schema'] = schema_cls
        schema = schema_cls()
        nmspc['schema'] = schema
        nmspc['DataProxy'] = data_proxy_factory(name, schema, strict=opts.strict)

        # _build_document_opts cannot determine the indexes given we need to
        # visit the document's fields which weren't defined at this time
        opts.indexes = _collect_indexes(nmspc.get('Meta'), schema.fields, bases)

        implementation = type(name, bases, nmspc)
        self._templates_lookup[template] = implementation
        # Notify the parent & grand parents of the newborn !
        for base in bases:
            for parent in inspect.getmro(base):
                if (not issubclass(parent, DocumentImplementation) or
                        parent is DocumentImplementation):
                    continue
                parent.opts.offspring.add(implementation)
        return implementation

    def build_embedded_document_from_template(self, template):
        """
        Generate a :class:`umongo.document.EmbeddedDocumentImplementation` for this
        instance from the given :class:`umongo.document.EmbeddedDocumentTemplate`.
        """
        assert issubclass(template, EmbeddedDocumentTemplate)
        name = template.__name__
        bases = self._convert_bases(template.__bases__)
        opts = _build_embedded_document_opts(
            self.instance, template, name, template.__dict__, bases)

        nmspc, schema_fields, schema_non_fields = _collect_schema_attrs(template)
        nmspc['opts'] = opts

        # If EmbeddedDocument is a child, _cls field must be added to the schema
        if opts.is_child:
            _add_child_field(name, schema_fields)

        # Create schema by retrieving inherited schema classes
        schema_bases = tuple([base.Schema for base in bases
                              if hasattr(base, 'Schema')])
        if not schema_bases:
            schema_bases = (Schema, )
        schema_cls = self._build_schema(template, schema_bases, schema_fields, schema_non_fields)
        nmspc['Schema'] = schema_cls
        schema = schema_cls()
        nmspc['schema'] = schema
        nmspc['DataProxy'] = data_proxy_factory(name, schema, strict=opts.strict)

        implementation = type(name, bases, nmspc)
        self._templates_lookup[template] = implementation
        # Notify the parent & grand parents of the newborn !
        for base in bases:
            for parent in inspect.getmro(base):
                if (not issubclass(parent, EmbeddedDocumentImplementation) or
                        parent is EmbeddedDocumentImplementation):
                    continue
                parent.opts.offspring.add(implementation)
        return implementation
<|endoftext|>


<commit_before>import random
from django.urls import reverse
from django.db.models import Count

from django.test import TestCase
from silk.config import SilkyConfig
from silk import models
from silk.middleware import silky_reverse

from .test_lib.mock_suite import MockSuite

class TestEndPoints(TestCase):
    """
    Hit all the endpoints to check everything actually renders/no error 500s etc.
    Each test will ensure that an object with something to display is chosen to be rendered e.g.
    a request/profile that has queries
    """

    @classmethod
    def setUpClass(cls):
        super(TestEndPoints, cls).setUpClass()
        # We're not testing auth here.
        SilkyConfig().SILKY_AUTHORISATION = False
        SilkyConfig().SILKY_AUTHENTICATION = False
        mock_suite = MockSuite()
        for _ in range(0, 100):
            mock_suite.mock_request()

    def test_summary(self):
        response = self.client.get(silky_reverse('summary'))
        self.assertTrue(response.status_code == 200)

    def test_requests(self):
        response = self.client.get(silky_reverse('requests'))
        self.assertTrue(response.status_code == 200)

    def test_request_detail(self):
        request_id = random.choice(models.Request.objects.all()).pk
        response = self.client.get(silky_reverse('request_detail', kwargs={
            'request_id': request_id
        }))
        self.assertTrue(response.status_code == 200)

    def test_request_sql(self):
        request_id = random.choice(models.SQLQuery.objects.values('request_id').filter(request_id__isnull=False))
        response = self.client.get(silky_reverse('request_sql', kwargs=request_id))
        self.assertTrue(response.status_code == 200)

    def test_request_sql_detail(self):
        fields = {
            'sql_id': 'silk_sqlquery.id',
            'request_id': 'request_id'
        }
        kwargs = random.choice(models.SQLQuery.objects.extra(select=fields).values(*fields.keys()).filter(request_id__isnull=False))
        response = self.client.get(silky_reverse('request_sql_detail', kwargs=kwargs))
        self.assertTrue(response.status_code == 200)

    def test_raw(self):
        request_id = random.choice(models.Request.objects.filter(body__isnull=False)).pk
        url = reverse('silk:raw', kwargs={
            'request_id': request_id
        })+'?typ=request&subtyp=processed'
        response = self.client.get(url)
        code = response.status_code
        self.assertTrue(code == 200)

    def test_request_profiling(self):
        request_id = random.choice(models.Profile.objects.values('request_id').filter(request_id__isnull=False))
        response = self.client.get(silky_reverse('request_profiling', kwargs=request_id))
        self.assertTrue(response.status_code == 200)

    def test_request_profile_detail(self):
        fields = {
            'profile_id': 'silk_profile.id',
            'request_id': 'request_id'
        }
        kwargs = random.choice(models.Profile.objects.extra(select=fields).values(*fields.keys()).filter(request_id__isnull=False))
        response = self.client.get(silky_reverse('request_profile_detail', kwargs=kwargs))
        self.assertTrue(response.status_code == 200)

    def test_request_and_profile_sql(self):
        fields = {
            'profile_id': 'profile_id',
            # 'sql_id': 'sqlquery_id',
            'request_id': 'request_id'
        }
        kwargs = random.choice(models.Profile.objects.extra(select=fields).annotate(num=Count('queries')).values(*fields.keys()).filter(request_id__isnull=False, num__gt=0))
        response = self.client.get(silky_reverse('request_and_profile_sql', kwargs=kwargs))
        self.assertTrue(response.status_code == 200)

    def test_request_and_profile_sql_detail(self):
        fields = {
            'profile_id': 'profile_id',
            'sql_id': 'sqlquery_id',
            'request_id': 'request_id'
        }
        kwargs = random.choice(models.Profile.objects.extra(select=fields).annotate(num=Count('queries')).values(*fields.keys()).filter(request_id__isnull=False, num__gt=0))
        response = self.client.get(silky_reverse('request_and_profile_sql_detail', kwargs=kwargs))
        self.assertTrue(response.status_code == 200)

    def test_profile_detail(self):
        profile_id = random.choice(models.Profile.objects.all()).pk
        response = self.client.get(silky_reverse('profile_detail', kwargs={
            'profile_id': profile_id
        }))
        self.assertTrue(response.status_code == 200)

    def test_profile_sql(self):
        profile_id = random.choice(models.Profile.objects.annotate(num=Count('queries')).values_list('id').filter(num__gt=0))[0]
        response = self.client.get(silky_reverse('profile_sql', kwargs={'profile_id': profile_id}))
        self.assertTrue(response.status_code == 200)

    def test_profile_sql_detail(self):
        profile_id = random.choice(models.Profile.objects.annotate(num=Count('queries')).values_list('id').filter(num__gt=0))[0]
        sql_id = random.choice(models.SQLQuery.objects.filter(profiles=profile_id)).pk
        response = self.client.get(silky_reverse('profile_sql_detail', kwargs={'profile_id': profile_id,
                                                                              'sql_id': sql_id}))
        self.assertTrue(response.status_code == 200)

    def test_profiling(self):
        response = self.client.get(silky_reverse('profiling'))
        self.assertTrue(response.status_code == 200)
<commit_msg>Update tests<commit_after>import random
from django.urls import reverse
from django.db.models import Count, F

from django.test import TestCase
from silk.config import SilkyConfig
from silk import models
from silk.middleware import silky_reverse

from .test_lib.mock_suite import MockSuite

class TestEndPoints(TestCase):
    """
    Hit all the endpoints to check everything actually renders/no error 500s etc.
    Each test will ensure that an object with something to display is chosen to be rendered e.g.
    a request/profile that has queries
    """

    @classmethod
    def setUpClass(cls):
        super(TestEndPoints, cls).setUpClass()
        # We're not testing auth here.
        SilkyConfig().SILKY_AUTHORISATION = False
        SilkyConfig().SILKY_AUTHENTICATION = False
        mock_suite = MockSuite()
        for _ in range(0, 100):
            mock_suite.mock_request()

    def test_summary(self):
        response = self.client.get(silky_reverse('summary'))
        self.assertTrue(response.status_code == 200)

    def test_requests(self):
        response = self.client.get(silky_reverse('requests'))
        self.assertTrue(response.status_code == 200)

    def test_request_detail(self):
        request_query_data = random.choice(models.Request.objects.values('id'))
        request_id = request_query_data['id']
        response = self.client.get(silky_reverse('request_detail', kwargs={
            'request_id': request_id
        }))
        self.assertTrue(response.status_code == 200)

    def test_request_sql(self):
        request_query_data = random.choice(
            models.SQLQuery.objects
            .values('request_id')
            .filter(request_id__isnull=False)
        )
        request_id = request_query_data['request_id']
        response = self.client.get(silky_reverse('request_sql', kwargs={'request_id': request_id}))
        self.assertTrue(response.status_code == 200)

    def test_request_sql_detail(self):
        kwargs = random.choice(
            models.SQLQuery.objects
            .annotate(sql_id=F('id'))
            .values('sql_id', 'request_id')
            .filter(request_id__isnull=False)
        )
        response = self.client.get(silky_reverse('request_sql_detail', kwargs=kwargs))
        self.assertTrue(response.status_code == 200)

    def test_raw(self):
        request_query_data = random.choice(
            models.Request.objects
            .values('id')
            .filter(body__isnull=False)
        )
        request_id = request_query_data['id']
        url = reverse('silk:raw', kwargs={
            'request_id': request_id
        })+'?typ=request&subtyp=processed'
        response = self.client.get(url)
        code = response.status_code
        self.assertTrue(code == 200)

    def test_request_profiling(self):
        request_id = random.choice(
            models.Profile.objects
            .values('request_id')
            .filter(request_id__isnull=False)
        )
        response = self.client.get(silky_reverse('request_profiling', kwargs=request_id))
        self.assertTrue(response.status_code == 200)

    def test_request_profile_detail(self):
        kwargs = random.choice(
            models.Profile.objects
            .annotate(profile_id=F('id'))
            .values('profile_id', 'request_id')
            .filter(request_id__isnull=False)
        )
        response = self.client.get(silky_reverse('request_profile_detail', kwargs=kwargs))
        self.assertTrue(response.status_code == 200)

    def test_request_and_profile_sql(self):
        kwargs = random.choice(
            models.Profile.objects
            .annotate(num=Count('queries'), profile_id=F('id'))
            .values('profile_id', 'request_id')
            .filter(request_id__isnull=False, num__gt=0)
        )
        response = self.client.get(silky_reverse('request_and_profile_sql', kwargs=kwargs))
        self.assertTrue(response.status_code == 200)

    def test_request_and_profile_sql_detail(self):
        random_profile = random.choice(
            models.Profile.objects
            .annotate(num=Count('queries'), profile_id=F('id'))
            .values('profile_id', 'request_id')
            .filter(request_id__isnull=False, num__gt=0)
        )
        random_sql_query = random.choice(
            models.SQLQuery.objects
            .annotate(sql_id=F('id'))
            .values('sql_id')
            .filter(profiles__id=random_profile['profile_id'])
        )
        kwargs = {}
        kwargs.update(random_profile)
        kwargs.update(random_sql_query)
        response = self.client.get(silky_reverse('request_and_profile_sql_detail', kwargs=kwargs))
        self.assertTrue(response.status_code == 200)

    def test_profile_detail(self):
        profile_query_data = random.choice(models.Profile.objects.values('id'))
        profile_id = profile_query_data['id']
        response = self.client.get(silky_reverse('profile_detail', kwargs={
            'profile_id': profile_id
        }))
        self.assertTrue(response.status_code == 200)

    def test_profile_sql(self):
        profile_query_data = random.choice(
            models.Profile.objects
            .annotate(num=Count('queries'))
            .values('id')
            .filter(num__gt=0)
        )
        profile_id = profile_query_data['id']
        response = self.client.get(silky_reverse('profile_sql', kwargs={'profile_id': profile_id}))
        self.assertTrue(response.status_code == 200)

    def test_profile_sql_detail(self):
        profile_query_data = random.choice(
            models.Profile.objects
            .annotate(num=Count('queries'))
            .values('id')
            .filter(num__gt=0)
        )
        profile_id = profile_query_data['id']
        sql_id = random.choice(models.SQLQuery.objects.filter(profiles=profile_id)).pk
        response = self.client.get(silky_reverse('profile_sql_detail', kwargs={'profile_id': profile_id,
                                                                              'sql_id': sql_id}))
        self.assertTrue(response.status_code == 200)

    def test_profiling(self):
        response = self.client.get(silky_reverse('profiling'))
        self.assertTrue(response.status_code == 200)
<|endoftext|>


<commit_before>from django.contrib import admin

from djcelery.models import TaskMeta

from models import Service, Layer, Check, SpatialReferenceSystem

class ServiceAdmin(admin.ModelAdmin):
    model = Service
    list_display = ('type', 'title', 'active', )
    list_display_links = ('title', )

class SpatialReferenceSystemAdmin(admin.ModelAdmin):
    model = SpatialReferenceSystem
    list_display = ('code', )

class LayerAdmin(admin.ModelAdmin):
    model = Layer
    list_display = ('name', 'title', 'service', )
    search_fields = ['name', 'title', ]

class CheckAdmin(admin.ModelAdmin):
    model = Check
    list_display = ('resource', 'checked_datetime', 'success', 'response_time', )
    list_display_links = ('resource', )
    search_fields = ['resource__title', ]

admin.site.register(Service, ServiceAdmin)
admin.site.register(Check, CheckAdmin)
admin.site.register(SpatialReferenceSystem, SpatialReferenceSystemAdmin)
admin.site.register(Layer, LayerAdmin)

# we like to see celery results using the admin
class TaskMetaAdmin(admin.ModelAdmin):
    list_display = ('task_id', 'date_done', 'status', )

admin.site.register(TaskMeta, TaskMetaAdmin)
<commit_msg>Improved admin for services<commit_after>from django.contrib import admin

from djcelery.models import TaskMeta

from models import Service, Layer, Check, SpatialReferenceSystem

class ServiceAdmin(admin.ModelAdmin):
    model = Service
    list_display = ('id', 'type', 'title', 'active', )
    list_display_links = ('id', )
    search_fields = ['title', 'url', ]
    list_filter = ('type', )

class SpatialReferenceSystemAdmin(admin.ModelAdmin):
    model = SpatialReferenceSystem
    list_display = ('code', )

class LayerAdmin(admin.ModelAdmin):
    model = Layer
    list_display = ('name', 'title', 'service', )
    search_fields = ['name', 'title', ]

class CheckAdmin(admin.ModelAdmin):
    model = Check
    list_display = ('resource', 'checked_datetime', 'success', 'response_time', )
    list_display_links = ('resource', )
    search_fields = ['resource__title', ]

admin.site.register(Service, ServiceAdmin)
admin.site.register(Check, CheckAdmin)
admin.site.register(SpatialReferenceSystem, SpatialReferenceSystemAdmin)
admin.site.register(Layer, LayerAdmin)

# we like to see celery results using the admin
class TaskMetaAdmin(admin.ModelAdmin):
    list_display = ('task_id', 'date_done', 'status', )

admin.site.register(TaskMeta, TaskMetaAdmin)
<|endoftext|>


<commit_before>"""This runs Step 1 of The Cannon:
    uses the training set to solve for the best-fit model."""

# Currently, this is a bit sketchy...there were sections of the original code that I didn't understand. have e-mailed MKN, will update.

from dataset import Dataset
import numpy as np
import math
import pylab
import matplotlib.pyplot as plt
from matplotlib import rc

def do_one_regression_at_fixed_scatter(spectra, x, scatter):
    """
    Params
    ------
    spectra: ndarray, [nstars, 3]
        "3" corresponds to wavelength, flux, flux_err

    x=coefficients of the model: ndarray, [nstars, nlabels]

    scatter: ndarray, [nstars]

    Returns
    ------
    coeff: ndarray
        coefficients of the fit

    xTCinvx: ndarray
        inverse covariance matrix for fit coefficients

    chi: float
        chi-squared at best fit

    logdet_Cinv: float
        inverse of the log determinant of the cov matrix
    """
    Cinv = 1. / (spectra[:, 2] ** 2 + scatter ** 2)
    xTCinvx = np.dot(x.T, Cinv[:, None] * x)
    fluxes = spectra[:, 1]
    xTCinvf = np.dot(x.T, Cinv * fluxes)
    try:
        coeff = np.linalg.solve(xTCinvx, xTCinvf) # this is the model!
    except np.linalg.linalg.LinAlgError:
        print "np.linalg.linalg.LinAlgError, do_one_regression_at_fixed_scatter"
        print MTCinvM, MTCinvx, spectra[:,0], spectra[:,1], spectra[:,2]
        print fluxes
    assert np.all(np.isfinite(coeff))
    chi = np.sqrt(Cinv) * (fluxes - np.dot(x, coeff))
    logdet_Cinv = np.sum(np.log(Cinv))
    return (coeff, xTCinvx, chi, logdet_Cinv)

def do_one_regression(spectra, x):
    """
    Optimizes to find the scatter associated with the best-fit model.

    This scatter is the deviation between the observed spectrum and the model.
    It is wavelength-independent, so we perform this at a single wavelength.

    Input
    -----
    spectra: ndarray, [nstars, 3]
    x = coefficients of the model: ndarray, [nstars, nlabels]

    Output
    -----
    output of do_one_regression_at_fixed_scatter
    """
    # cover the full order of the scatter term, scatter << max term
    ln_scatter_vals = np.arange(np.log(0.0001), 0., 0.5)
    # will minimize over the range of scatter possibilities
    chis_eval = np.zeros_like(ln_scatter_vals)
    for jj, ln_scatter_val in enumerate(ln_scatter_vals):
        coeff, xTCinvx, chi, logdet_Cinv = do_one_regression_at_fixed_scatter(
                spectra, x, scatter = np.exp(ln_scatter_val))
        chis_eval[jj] = np.sum(chi * chi) - logdet_Cinv
    # What do the below two cases *mean*?
    if np.any(np.isnan(chis_eval)):
        best_scatter = np.exp(ln_scatter_vals[-1])
        return do_one_regression_at_fixed_scatter(
                spectra, x, scatter = best_scatter) + (best_scatter, )
    lowest = np.argmin(chis_eval) # the best-fit scatter value?
    # If it was unsuccessful at finding it...
    if lowest == 0 or lowest == len(ln_scatter_vals) + 1:
        best_scatter = np.exp(ln_scatter_vals[lowest])
        return do_one_regression_at_fixed_scatter(spectra, x,
                scatter = best_scatter) + (best_scatter, )
    ln_scatter_vals_short = ln_scatter_vals[np.array(
        [lowest-1, lowest, lowest+1])]
    chis_eval_short = chis_eval[np.array([lowest-1, lowest, lowest+1])]
    z = np.polyfit(ln_scatter_vals_short, chis_eval_short, 2)
    f = np.poly1d(z)
    fit_pder = np.polyder(z)
    fit_pder2 = pylab.polyder(fit_pder)
    best_scatter = np.exp(np.roots(fit_pder)[0])
    return do_one_regression_at_fixed_scatter(
            spectra, x, scatter = best_scatter) + (best_scatter, )

def train_model(training_set):
    """
    This determines the coefficients of the model using the training data

    Input: the training_set, a Dataset object (see dataset.py)
    Returns: the model, which consists of...
    -------
    coefficients: ndarray, (npixels, nstars, 15)
    the covariance matrix
    scatter values
    chi squareds
    the pivot values
    """
    label_names = training_set.label_names
    label_values = training_set.label_values #(nstars, nlabels)
    nlabels = len(label_names)
    spectra = training_set.spectra #(nstars, npixels, 3)
    nstars = spectra.shape[0]
    npixels = spectra.shape[1]

    # Establish label vector
    pivots = np.mean(label_values, axis=0)
    ones = np.ones((nstars, 1))
    linear_offsets = label_values - pivots
    quadratic_offsets = np.array([np.outer(m, m)[np.triu_indices(nlabels)]
        for m in (label_values - pivots)])
    x = np.hstack((ones, linear_offsets, quadratic_offsets))
    x_full = np.array([x,]*npixels)

    # Perform REGRESSIONS
    spectra = spectra.swapaxes(0,1) # for consistency with x_full
    blob = map(do_one_regression, spectra, x_full) # one regression per pixel
    coeffs = np.array([b[0] for b in blob])
    covs = np.array([np.linalg.inv(b[1]) for b in blob])
    chis = np.array([b[2] for b in blob])
    chisqs = np.array([np.dot(b[2],b[2]) - b[3] for b in blob])
    scatters = np.array([b[4] for b in blob])
    model = coeffs, covs, scatters, chis, chisqs, pivots
    return model

def model_diagnostics(training_set, model):
    """Run a set of diagnostics on the model.

    Plot the 0th order coefficients as the baseline spectrum.
    Overplot the continuum pixels.

    Plot each label's leading coefficient as a function of wavelength.
    Color-code by label.
    """
    # Baseline spectrum with continuum
    coeffs_all, covs, scatters, chis, chisqs, pivots = model
    pixels = training_set.spectra[0,:,0]
    baseline_spec = coeffs_all[:,0]
    plt.plot(pixels, baseline_spec)
    contpix = list(np.loadtxt("pixtest4_lambda.txt",
        usecols = (0,), unpack =1))
    y = [1]*len(contpix)
    plt.scatter(contpix, y)
    plt.title("Baseline Spectrum with Continuum Pixels")
    plt.xlabel(r"Wavelength $\lambda (\AA)$")
    plt.ylabel(r"$\theta_0$")
    filename = "baseline_spec_with_cont_pix.png"
    print "Diagnostic plot: baseline spectrum from the model with continuum pixels overlaid. Saved as %s" %filename
    plt.savefig(filename)
    plt.close()

    # Leading coefficients for each label
    nlabels = len(pivots)
    fig, axarr = plt.subplots(nlabels, sharex=True)
    plt.xlabel(r"Wavelength $\lambda (\AA)$")
    for i in range(nlabels):
        ax = axarr[i]
        ax.set_ylabel(r"$\theta_%s$" %i)
        ax.set_title("%s" %training_set.label_names[i])
        ax.plot(pixels, coeffs_all[:,i+1])
    print "Diagnostic plot: leading coefficients as a function of wavelength."
    filename = "leading_coeffs.png"
    print "Saved as %s" %filename
    fig.savefig(filename)
    plt.close(fig)

<commit_msg>hist of chi squareds<commit_after>"""This runs Step 1 of The Cannon:
    uses the training set to solve for the best-fit model."""

# Currently, this is a bit sketchy...there were sections of the original code that I didn't understand. have e-mailed MKN, will update.

from dataset import Dataset
import numpy as np
import math
import pylab
import matplotlib.pyplot as plt
from matplotlib import rc

def do_one_regression_at_fixed_scatter(spectra, x, scatter):
    """
    Params
    ------
    spectra: ndarray, [nstars, 3]
        "3" corresponds to wavelength, flux, flux_err

    x=coefficients of the model: ndarray, [nstars, nlabels]

    scatter: ndarray, [nstars]

    Returns
    ------
    coeff: ndarray
        coefficients of the fit

    xTCinvx: ndarray
        inverse covariance matrix for fit coefficients

    chi: float
        chi-squared at best fit

    logdet_Cinv: float
        inverse of the log determinant of the cov matrix
    """
    Cinv = 1. / (spectra[:, 2] ** 2 + scatter ** 2)
    xTCinvx = np.dot(x.T, Cinv[:, None] * x)
    fluxes = spectra[:, 1]
    xTCinvf = np.dot(x.T, Cinv * fluxes)
    try:
        coeff = np.linalg.solve(xTCinvx, xTCinvf) # this is the model!
    except np.linalg.linalg.LinAlgError:
        print "np.linalg.linalg.LinAlgError, do_one_regression_at_fixed_scatter"
        print MTCinvM, MTCinvx, spectra[:,0], spectra[:,1], spectra[:,2]
        print fluxes
    assert np.all(np.isfinite(coeff))
    chi = np.sqrt(Cinv) * (fluxes - np.dot(x, coeff))
    logdet_Cinv = np.sum(np.log(Cinv))
    return (coeff, xTCinvx, chi, logdet_Cinv)

def do_one_regression(spectra, x):
    """
    Optimizes to find the scatter associated with the best-fit model.

    This scatter is the deviation between the observed spectrum and the model.
    It is wavelength-independent, so we perform this at a single wavelength.

    Input
    -----
    spectra: ndarray, [nstars, 3]
    x = coefficients of the model: ndarray, [nstars, nlabels]

    Output
    -----
    output of do_one_regression_at_fixed_scatter
    """
    # cover the full order of the scatter term, scatter << max term
    ln_scatter_vals = np.arange(np.log(0.0001), 0., 0.5)
    # will minimize over the range of scatter possibilities
    chis_eval = np.zeros_like(ln_scatter_vals)
    for jj, ln_scatter_val in enumerate(ln_scatter_vals):
        coeff, xTCinvx, chi, logdet_Cinv = do_one_regression_at_fixed_scatter(
                spectra, x, scatter = np.exp(ln_scatter_val))
        chis_eval[jj] = np.sum(chi * chi) - logdet_Cinv
    # What do the below two cases *mean*?
    if np.any(np.isnan(chis_eval)):
        best_scatter = np.exp(ln_scatter_vals[-1])
        return do_one_regression_at_fixed_scatter(
                spectra, x, scatter = best_scatter) + (best_scatter, )
    lowest = np.argmin(chis_eval) # the best-fit scatter value?
    # If it was unsuccessful at finding it...
    if lowest == 0 or lowest == len(ln_scatter_vals) + 1:
        best_scatter = np.exp(ln_scatter_vals[lowest])
        return do_one_regression_at_fixed_scatter(spectra, x,
                scatter = best_scatter) + (best_scatter, )
    ln_scatter_vals_short = ln_scatter_vals[np.array(
        [lowest-1, lowest, lowest+1])]
    chis_eval_short = chis_eval[np.array([lowest-1, lowest, lowest+1])]
    z = np.polyfit(ln_scatter_vals_short, chis_eval_short, 2)
    f = np.poly1d(z)
    fit_pder = np.polyder(z)
    fit_pder2 = pylab.polyder(fit_pder)
    best_scatter = np.exp(np.roots(fit_pder)[0])
    return do_one_regression_at_fixed_scatter(
            spectra, x, scatter = best_scatter) + (best_scatter, )

def train_model(training_set):
    """
    This determines the coefficients of the model using the training data

    Input: the training_set, a Dataset object (see dataset.py)
    Returns: the model, which consists of...
    -------
    coefficients: ndarray, (npixels, nstars, 15)
    the covariance matrix
    scatter values
    chi squareds
    the pivot values
    """
    label_names = training_set.label_names
    label_values = training_set.label_values #(nstars, nlabels)
    nlabels = len(label_names)
    spectra = training_set.spectra #(nstars, npixels, 3)
    nstars = spectra.shape[0]
    npixels = spectra.shape[1]

    # Establish label vector
    pivots = np.mean(label_values, axis=0)
    ones = np.ones((nstars, 1))
    linear_offsets = label_values - pivots
    quadratic_offsets = np.array([np.outer(m, m)[np.triu_indices(nlabels)]
        for m in (label_values - pivots)])
    x = np.hstack((ones, linear_offsets, quadratic_offsets))
    x_full = np.array([x,]*npixels)

    # Perform REGRESSIONS
    spectra = spectra.swapaxes(0,1) # for consistency with x_full
    blob = map(do_one_regression, spectra, x_full) # one regression per pixel
    coeffs = np.array([b[0] for b in blob])
    covs = np.array([np.linalg.inv(b[1]) for b in blob])
    chis = np.array([b[2] for b in blob])
    chisqs = np.array([np.dot(b[2],b[2]) - b[3] for b in blob])
    scatters = np.array([b[4] for b in blob])
    model = coeffs, covs, scatters, chis, chisqs, pivots
    return model

def model_diagnostics(training_set, model):
    """Run a set of diagnostics on the model.

    Plot the 0th order coefficients as the baseline spectrum.
    Overplot the continuum pixels.

    Plot each label's leading coefficient as a function of wavelength.
    Color-code by label.
    """
    # Baseline spectrum with continuum
    coeffs_all, covs, scatters, chis, chisqs, pivots = model
    pixels = training_set.spectra[0,:,0]
    baseline_spec = coeffs_all[:,0]
    plt.plot(pixels, baseline_spec)
    contpix = list(np.loadtxt("pixtest4_lambda.txt",
        usecols = (0,), unpack =1))
    y = [1]*len(contpix)
    plt.scatter(contpix, y)
    plt.title("Baseline Spectrum with Continuum Pixels")
    plt.xlabel(r"Wavelength $\lambda (\AA)$")
    plt.ylabel(r"$\theta_0$")
    filename = "baseline_spec_with_cont_pix.png"
    print "Diagnostic plot: baseline spectrum from the model with continuum pixels overlaid. Saved as %s" %filename
    plt.savefig(filename)
    plt.close()

    # Leading coefficients for each label
    nlabels = len(pivots)
    fig, axarr = plt.subplots(nlabels, sharex=True)
    plt.xlabel(r"Wavelength $\lambda (\AA)$")
    for i in range(nlabels):
        ax = axarr[i]
        ax.set_ylabel(r"$\theta_%s$" %i)
        ax.set_title("%s" %training_set.label_names[i])
        ax.plot(pixels, coeffs_all[:,i+1])
    print "Diagnostic plot: leading coefficients as a function of wavelength."
    filename = "leading_coeffs.png"
    print "Saved as %s" %filename
    fig.savefig(filename)
    plt.close(fig)

    # Histogram of the chi squareds of the fits
    plt.hist(chisqs)
    plt.title("Distribution of Chi Squareds of the Model Fit")
    plt.ylabel("Count")
    plt.xlabel("Chi Squared")
    filename = "modelfit_chisqs.png"
    print "Diagnostic plot: histogram of the chi squareds of the fit"
    print "Saved as %s" %filename
    plt.savefig(filename)
    plt.close()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import logging
import datetime

from decimal import Decimal

from django.conf import settings
from django.core.mail import EmailMessage
from django.db import models
from django.template import Context
from django.template.loader import get_template
from django.utils import timezone
from holviapi.utils import barcode as bank_barcode
from members.models import Member
from ndaparser.models import UploadedTransaction

from .models import NotificationSent

logger = logging.getLogger(__name__)

HOLVI_EXCLUDE_KWARGS = {"reference__startswith": "RF"}

class NordeaOverdueInvoicesHandler(object):
    def list_overdue(self):
        """Gets list of overdue Nordea transactions"""
        # TODO: Heuristics to determine which transactions are not paid...
        gd = settings.VELKOJA_NORDEACHECKER_GRACE_DAYS
        if not UploadedTransaction.objects.count():
            logger.warning("No uploaded transactions found defaulting to configured grace period of {} days".format(gd))
            cutoff_date = datetime.datetime.now().date() - datetime.timedelta(days=gd)
        else:
            cutoff_date = UploadedTransaction.objects.order_by('-last_transaction')[0].last_transaction
            today_delta = datetime.datetime.now().date() - cutoff_date
            if today_delta.days > gd:
                logger.warning("Last uploaded transaction was {} days ago".format(today_delta.days))

        ret = []

        # First get members with negative credit in general
        members_qs = Member.objects.all()
        members_qs = members_qs.annotate(credit_annotated=models.Sum('creditor_transactions__amount'))
        for member in members_qs.filter(credit_annotated__lt=0):
            # Then find distinct reference numbers that are not Holvi transactions
            ret += self._list_member_overdue(member, cutoff_date)
        return ret

    def _list_member_overdue(self, member, cutoff_date):
        """Figure out which debits of a given member are still unpaid"""
        base_qs = member.creditor_transactions.exclude(**HOLVI_EXCLUDE_KWARGS).filter(stamp__lte=cutoff_date)
        refnos = [x['reference'] for x in
                  base_qs.order_by('reference').distinct('reference').values('reference')]
        ret = []
        for refno in refnos:
            refno_qs = base_qs.filter(reference=refno)
            refno_credits = refno_qs.filter(amount__gte=0).aggregate(models.Sum('amount'))['amount__sum']
            refno_debits = refno_qs.filter(amount__lt=0).order_by('stamp')
            for transaction in refno_debits:
                refno_credits += transaction.amount  # remember: the amount is negative
                if refno_credits > 0:
                    # While we have not expended all credit, keep spending
                    continue
                # After no more credits are left, put rest of transactions to the unpaid pile.
                ret.append(transaction)

        return ret

    def process_overdue(self, send=False):
        barcode_iban = settings.NORDEA_BARCODE_IBAN
        body_template = get_template('velkoja/nordea_notification_email_body.jinja')
        subject_template = get_template('velkoja/nordea_notification_email_subject.jinja')
        overdue = self.list_overdue()
        ret = []
        for transaction in overdue:
            # If we have already sent notification recently, do not sent one just yet
            if NotificationSent.objects.filter(transaction_unique_id=transaction.unique_id).count():
                notified = NotificationSent.objects.get(transaction_unique_id=transaction.unique_id)
                if (timezone.now() - notified.stamp).days < settings.HOLVI_NOTIFICATION_INTERVAL_DAYS:
                    continue

            barcode = None
            if barcode_iban:
                barcode = bank_barcode(barcode_iban, transaction.reference, -transaction.amount)

            mail = EmailMessage()
            mail.from_email = settings.VELKOJA_FROM_EMAIL
            mail.to = [transaction.owner.email]
            if settings.VELKOJA_CC_EMAIL:
                mail.cc = [settings.VELKOJA_CC_EMAIL]
            mail.subject = subject_template.render(Context({"transaction": transaction, "barcode": barcode})).strip()
            mail.body = body_template.render(Context({"transaction": transaction, "barcode": barcode}))
            if send:
                mail.send()

            try:
                notified = NotificationSent.objects.get(transaction_unique_id=transaction.unique_id)
                notified.notification_no += 1
            except NotificationSent.DoesNotExist:
                notified = NotificationSent()
                notified.transaction_unique_id = transaction.unique_id
            notified.stamp = timezone.now()
            notified.email = transaction.owner.email
            if send:
                notified.save()
            ret.append((notified, transaction))
        return ret
<commit_msg>If we have no credits the value might be None<commit_after># -*- coding: utf-8 -*-
import logging
import datetime

from decimal import Decimal

from django.conf import settings
from django.core.mail import EmailMessage
from django.db import models
from django.template import Context
from django.template.loader import get_template
from django.utils import timezone
from holviapi.utils import barcode as bank_barcode
from members.models import Member
from ndaparser.models import UploadedTransaction

from .models import NotificationSent

logger = logging.getLogger(__name__)

HOLVI_EXCLUDE_KWARGS = {"reference__startswith": "RF"}

class NordeaOverdueInvoicesHandler(object):
    def list_overdue(self):
        """Gets list of overdue Nordea transactions"""
        # TODO: Heuristics to determine which transactions are not paid...
        gd = settings.VELKOJA_NORDEACHECKER_GRACE_DAYS
        if not UploadedTransaction.objects.count():
            logger.warning("No uploaded transactions found defaulting to configured grace period of {} days".format(gd))
            cutoff_date = datetime.datetime.now().date() - datetime.timedelta(days=gd)
        else:
            cutoff_date = UploadedTransaction.objects.order_by('-last_transaction')[0].last_transaction
            today_delta = datetime.datetime.now().date() - cutoff_date
            if today_delta.days > gd:
                logger.warning("Last uploaded transaction was {} days ago".format(today_delta.days))

        ret = []

        # First get members with negative credit in general
        members_qs = Member.objects.all()
        members_qs = members_qs.annotate(credit_annotated=models.Sum('creditor_transactions__amount'))
        for member in members_qs.filter(credit_annotated__lt=0):
            # Then find distinct reference numbers that are not Holvi transactions
            ret += self._list_member_overdue(member, cutoff_date)
        return ret

    def _list_member_overdue(self, member, cutoff_date):
        """Figure out which debits of a given member are still unpaid"""
        base_qs = member.creditor_transactions.exclude(**HOLVI_EXCLUDE_KWARGS).filter(stamp__lte=cutoff_date)
        refnos = [x['reference'] for x in
                  base_qs.order_by('reference').distinct('reference').values('reference')]
        ret = []
        for refno in refnos:
            refno_qs = base_qs.filter(reference=refno)
            refno_credits = refno_qs.filter(amount__gte=0).aggregate(models.Sum('amount'))['amount__sum']
            if refno_credits is None:
                refno_credits = Decimal('0')
            refno_debits = refno_qs.filter(amount__lt=0).order_by('stamp')
            for transaction in refno_debits:
                refno_credits += transaction.amount  # remember: the amount is negative
                if refno_credits >= 0:
                    # While we have not expended all credit, keep spending
                    continue
                # After no more credits are left, put rest of transactions to the unpaid pile.
                ret.append(transaction)

        return ret

    def process_overdue(self, send=False):
        barcode_iban = settings.NORDEA_BARCODE_IBAN
        body_template = get_template('velkoja/nordea_notification_email_body.jinja')
        subject_template = get_template('velkoja/nordea_notification_email_subject.jinja')
        overdue = self.list_overdue()
        ret = []
        for transaction in overdue:
            # If we have already sent notification recently, do not sent one just yet
            if NotificationSent.objects.filter(transaction_unique_id=transaction.unique_id).count():
                notified = NotificationSent.objects.get(transaction_unique_id=transaction.unique_id)
                if (timezone.now() - notified.stamp).days < settings.HOLVI_NOTIFICATION_INTERVAL_DAYS:
                    continue

            barcode = None
            if barcode_iban:
                barcode = bank_barcode(barcode_iban, transaction.reference, -transaction.amount)

            mail = EmailMessage()
            mail.from_email = settings.VELKOJA_FROM_EMAIL
            mail.to = [transaction.owner.email]
            if settings.VELKOJA_CC_EMAIL:
                mail.cc = [settings.VELKOJA_CC_EMAIL]
            mail.subject = subject_template.render(Context({"transaction": transaction, "barcode": barcode})).strip()
            mail.body = body_template.render(Context({"transaction": transaction, "barcode": barcode}))
            if send:
                mail.send()

            try:
                notified = NotificationSent.objects.get(transaction_unique_id=transaction.unique_id)
                notified.notification_no += 1
            except NotificationSent.DoesNotExist:
                notified = NotificationSent()
                notified.transaction_unique_id = transaction.unique_id
            notified.stamp = timezone.now()
            notified.email = transaction.owner.email
            if send:
                notified.save()
            ret.append((notified, transaction))
        return ret
<|endoftext|>


<commit_before>#!/usr/bin/env python

# KMHXcodeTools
# script.py
# Ken M. Haggerty
# CREATED: 2017 Mar 09
# EDITED:  2017 May 05

##### IMPORTS

import glob
from functions import *

##### CODE

print "---------- STARTING PYTHON ----------"

### Obtain .xcodeproj files

projects = glob.glob("*.xcodeproj")

for project in projects:

    ### Input

    filename = project + "/project.pbxproj"
    file = open(filename, "r+")
    text = "".join(file)

    ### Processing

    order = processPBXProjOrder(text)

    text = updatePBXBuildFileSection(text, order)
    text = updatePBXFileReferenceSection(text, order)
    text = updatePBXFrameworksBuildPhaseSection(text, order)
    text = updatePBXGroupSection(text, order)
    text = updatePBXResourcesBuildPhaseSection(text, order)
    text = updatePBXSourcesBuildPhaseSection(text, order)

    ### Save To File

    # file.seek(0)
    # file.write(text)
    # file.truncate()
    # file.close()

print "---------- PYTHON COMPLETE ----------"
<commit_msg>Updates comments in script.py.<commit_after>#!/usr/bin/env python

# KMHXcodeTools
# script.py
# Ken M. Haggerty
# CREATED: 2017 Mar 09
# EDITED:  2017 May 05

##### IMPORTS

import glob
from functions import *

##### CODE

print "---------- STARTING PYTHON ----------"

# Obtain .xcodeproj files

projects = glob.glob("*.xcodeproj")

for project in projects:

    # Input

    filename = project + "/project.pbxproj"
    file = open(filename, "r+")
    text = "".join(file)

    # Processing

    order = processPBXProjOrder(text)

    text = updatePBXBuildFileSection(text, order)
    text = updatePBXFileReferenceSection(text, order)
    text = updatePBXFrameworksBuildPhaseSection(text, order)
    text = updatePBXGroupSection(text, order)
    text = updatePBXResourcesBuildPhaseSection(text, order)
    text = updatePBXSourcesBuildPhaseSection(text, order)

    # Save To File

    # file.seek(0)
    # file.write(text)
    # file.truncate()
    # file.close()

print "---------- PYTHON COMPLETE ----------"
<|endoftext|>


<commit_before>class Node(object):

    def __init__(self, val):
        self.val = val
        self._next = None

        @property
        def next(self):
            return self._next

        @next.setter
        def name(self, value):
            self._next = value

class LinkedList(object):

    def __init__(self, name):
        self.head = None
        self.active = None

    def insert(self, val):
        self.active, self.head = self.head, Node(val)
        self.head.next = self.active
<commit_msg>Constantine: fixed typo.<commit_after>class Node(object):

    def __init__(self, val):
        self.val = val
        self._next = None

        @property
        def next(self):
            return self._next

        @next.setter
        def next(self, value):
            self._next = value

class LinkedList(object):

    def __init__(self, name):
        self.head = None
        self.second = None

    def insert(self, val):
        self.second, self.head = self.head, Node(val)
        self.head.next = self.second

<|endoftext|>


<commit_before>from time import time, strftime
from numpy import savetxt, array
from agilentcounter import AgilentCounter
from agilentmultimeter import AgilentMultimeter

# Config settings
countergpib = 1
multigpib = 6
vrange = 1
vresolution = 0.0001

# Setting up frequency counter
counter = AgilentCounter(gpib = countergpib)
if (counter == None):
    exit
counter.reset()
counter.setupFast()
counter.write(":FUNC 'FREQ 1'")
counter.setupGating(0.1)

# Setting up multimeter Agilent - 34401
multi = AgilentMultimeter(gpib = multigpib)
if (multi == None):
    exit
multi.reset()
# Resistance measurement
multi.write("CONF:FRESISTANCE:DC %d" %(vrange))
multi.write("FRESISTANCE:NPLC 0.2")
multi.write("TRIG:SOUR IMM")
# Need to do one reading to set up Wait-For-Trigger state!
multi.ask("READ?")

# Setting up output file
datafile = "beat_%s.log" %(strftime("%y%m%d_%H%M%S"))
out = file(datafile, 'a')
out.write("#Time(UnixTime) BeatFrequency(Hz) Voltage(V)\n")

# Do logging until stopped by Ctrl-C
while True:
    try:
        start = time()
        counter.initMeasure()
        resistance = float(multi.ask("READ?"))
        freq = counter.getFreq()
        now = time()
        measuretime = (now + start) / 2
        elapsed = now - start
        result = array([[measuretime, freq, resistance]])
        savetxt(out, result)
        print "%f Hz / %f Ohm / %f s" %(freq, resistance, elapsed)
    except (KeyboardInterrupt):
        break
out.close()
<commit_msg>correct parameters<commit_after>from time import time, strftime
from numpy import savetxt, array
from agilentcounter import AgilentCounter
from agilentmultimeter import AgilentMultimeter

# Config settings
countergpib = 1
multigpib = 6
vrange = 1
vresolution = 0.0001

# Setting up frequency counter
counter = AgilentCounter(gpib = countergpib)
if (counter == None):
    exit
counter.reset()
counter.setupFast()
counter.write(":FUNC 'FREQ 1'")
counter.setupGating(0.1)

# Setting up multimeter Agilent - 34401
multi = AgilentMultimeter(gpib = multigpib)
if (multi == None):
    exit
multi.reset()
# Resistance measurement
multi.write("CONF:FRESISTANCE" %(vrange))
multi.write("FRESISTANCE:NPLC 0.2")
multi.write("TRIG:SOUR IMM")
# Need to do one reading to set up Wait-For-Trigger state!
multi.ask("READ?")

# Setting up output file
datafile = "beat_%s.log" %(strftime("%y%m%d_%H%M%S"))
out = file(datafile, 'a')
out.write("#Time(UnixTime) BeatFrequency(Hz) Voltage(V)\n")

# Do logging until stopped by Ctrl-C
while True:
    try:
        start = time()
        counter.initMeasure()
        resistance = float(multi.ask("READ?"))
        freq = counter.getFreq()
        now = time()
        measuretime = (now + start) / 2
        elapsed = now - start
        result = array([[measuretime, freq, resistance]])
        savetxt(out, result)
        print "%f Hz / %f Ohm / %f s" %(freq, resistance, elapsed)
    except (KeyboardInterrupt):
        break
out.close()
<|endoftext|>