<commit_before># -*- encoding: utf-8 -*-

# Dissemin: open access policy enforcement tool
# Copyright (C) 2014 Antonin Delpeuch
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#

import datetime
import unittest
import os

from backend.crossref import convert_to_name_pair
from backend.crossref import CrossRefAPI
from backend.crossref import DOI_PROXY_SUPPORTS_BATCH
from backend.crossref import fetch_dois_by_batch
from backend.crossref import fetch_dois_incrementally
from backend.crossref import fetch_metadata_by_DOI
from backend.crossref import get_publication_date
from backend.crossref import is_oa_license
from backend.crossref import parse_crossref_date
from django.test import TestCase
from papers.errors import MetadataSourceException
from papers.models import Paper

class CrossRefTest(TestCase):

    @classmethod
    def setUpClass(cls):
        super(CrossRefTest, cls).setUpClass()
        cls.testdir = os.path.dirname(os.path.abspath(__file__))

    def setUp(self):
        self.api = CrossRefAPI()

    def test_empty_pubdate(self):
        # This DOI has an empty 'issued' date
        p = self.api.create_paper_by_doi('10.1007/978-1-4020-7884-2_13')
        self.assertEqual(p.pubdate.year, 2006)

    def test_invalid_metadata(self):
        # authors with no family name
        paper = self.api.create_paper_by_doi('10.4156/aiss.vol3.issue9.31')
        self.assertEqual(paper, None)

    def test_affiliations(self):
        p = self.api.create_paper_by_doi('10.4204/eptcs.172.16')
        self.assertEqual(p.authors[0].affiliation,
                         'École Normale Supérieure, Paris')

    def test_dirty_metadata(self):
        # saving a paper with enough metadata to create a paper, but not
        # an OaiRecord.
        p = self.api.save_doi_metadata({
            "DOI": "10.1007/978-1-4020-7884-2_13",
            "subtitle": [],
            "author": [
                {
                    "affiliation": [],
                    "given": "Haowen",
                    "family": "Chan"
                },
                {
                    "affiliation": [],
                    "given": "Adrian",
                    "family": "Perrig"
                },
                {
                    "affiliation": [],
                    "given": "Dawn",
                    "family": "Song"
                }
            ],
            "created": {
                "timestamp": 1166129219000,
                "date-time": "2006-12-14T20:46:59Z",
                "date-parts": [
                    [
                        2006,
                        12,
                        14
                    ]
                ]
            },
            "title": [
                "Key Distribution Techniques for Sensor Networks"
            ],
            "type": "book-chapter"})
        self.assertTrue(p.is_orphan())
        self.assertFalse(p.visible)

    def test_doctype_book(self):
        # Books are ignored
        # (technically, that's because we currently require a
        # 'container-title' in the metadata)
        p = self.api.create_paper_by_doi('10.1385/1592597998')
        self.assertTrue(p.is_orphan())

    def test_doi_open(self):
        self.assertTrue(self.api.create_paper_by_doi('10.15200/winn.145838.88372').pdf_url)
        self.assertFalse(self.api.create_paper_by_doi('10.5061/dryad.b167g').pdf_url)

    def test_fetch_papers(self):
        generator = self.api.fetch_all_records(filters={'issn':'0302-9743'})
        for i in range(30):
            metadata = next(generator)
            self.assertTrue(metadata['DOI'].startswith('10.1007/'))

    def test_import_dump(self):
        self.api.ingest_dump(os.path.join(self.testdir, 'data/sample_crossref_dump.json.bz2'))
        p = Paper.get_by_doi('10.1016/j.jadohealth.2015.10.045')
        self.assertEqual("Sources, Type and Use of Social Support during Early Sexual Development of Black Gay and Bisexual Adolescent Males",
                         p.title)

class CrossRefUnitTest(unittest.TestCase):

    def test_fetch_single_doi(self):
        doi = '10.5380/dp.v1i1.1922'
        metadata = fetch_metadata_by_DOI(doi)
        self.assertEqual(metadata,
                         {'publisher': 'Universidade Federal do Parana',
                          'DOI': '10.5380/dp.v1i1.1922',
                          'subtitle': [],
                          'author': [{'given': 'Frederic', 'family': 'Worms'}],
                          'URL': 'http://dx.doi.org/10.5380/dp.v1i1.1922',
                          'issued': {'date-parts': [[2005, 3, 18]]},
                          'reference-count': 0,
                          'title': 'A concep\xe7\xe3o bergsoniana do tempo',
                          'volume': '1',
                          'source': 'CrossRef',
                          'prefix': 'http://id.crossref.org/prefix/10.5380',
                          'score': 1.0,
                          'deposited': {'timestamp': 1421107200000, 'date-parts': [[2015, 1, 13]]},
                          'type': 'journal-article',
                          'container-title': 'DoisPontos',
                          'indexed': {'timestamp': 1421405831942, 'date-parts': [[2015, 1, 16]]},
                          'issue': '1',
                          'ISSN': ['2179-7412', '1807-3883'],
                          'member': 'http://id.crossref.org/member/3785'})

    def test_parse_crossref_date_incomplete(self):
        self.assertEqual(parse_crossref_date(None), None)
        self.assertEqual(
                parse_crossref_date({'date-parts': [[2015, 7, 6]]}),
                datetime.date(year=2015, month=7, day=6))
        self.assertEqual(
                parse_crossref_date({'date-parts': [[2015, 7]]}),
                datetime.date(year=2015, month=7, day=1))
        self.assertEqual(
                parse_crossref_date({'date-parts': [[2015]]}),
                datetime.date(year=2015, month=1, day=1))

    def test_parse_crossref_date_raw(self):
        self.assertEqual(
                parse_crossref_date({'raw': '2015'}),
                datetime.date(year=2015, month=1, day=1))
        self.assertEqual(
                parse_crossref_date({'raw': '2015-07'}),
                datetime.date(year=2015, month=7, day=1))
        self.assertEqual(
                parse_crossref_date({'raw': '2015-07-06'}),
                datetime.date(year=2015, month=7, day=6))

    def test_get_publication_date(self):
        self.assertEqual(
                get_publication_date(
                    fetch_metadata_by_DOI('10.5281/zenodo.18898')),
                datetime.date(year=2015, month=1, day=1))
        self.assertEqual(
                get_publication_date(
                    fetch_metadata_by_DOI('10.5380/dp.v1i1.1919')),
                datetime.date(year=2005, month=3, day=18))

    def test_batch_queries(self):
        dois = [
            '10.1007/978-3-540-46375-7_2',
            '10.1007/978-3-540-46375-7_9',
            '10.2307/2540916',
            '10.1016/s1169-8330(00)80059-9',
            '10.1017/s0022112009008003',
            '10.1051/proc/2011014',
            '10.1016/0169-5983(88)90079-2',
            '10.1080/14685240600601061',
            '10.1103/physreve.79.026303',
            '10.1103/physreve.66.046307',
            '10.1103/physrevlett.95.244502',
            '10.1017/s0022112089002351',
            '10.1063/1.4738850',
            '10.1103/physrevlett.87.054501',
            '10.1080/14685248.2012.711476',
            '10.1007/978-94-011-4177-2_12',
            '10.1007/978-1-4615-4697-9_2',
            '10.1007/978-1-4612-0137-3_7',
            '10.1007/978-1-4020-6472-2_35']
        incremental = list(fetch_dois_incrementally(dois))
        self.assertEqual(len(incremental), len(dois))
        if DOI_PROXY_SUPPORTS_BATCH:
            batch = fetch_dois_by_batch(dois)
            self.assertEqual([item['DOI'] for item in incremental],
                             [item['DOI'] for item in batch])

    def test_batch_with_comma(self):
        """
        Having a comma in a DOI should not fail the whole batch, just ignore
        that particular DOI
        """
        dois = ['10.1016/j.acta,at.2005.01.046','10.1080/14786430410001678217']
        batch = fetch_dois_by_batch(dois)
        fetched_dois = [ item['DOI'] for item in batch if item ]
        self.assertTrue('10.1080/14786430410001678217' in fetched_dois)

    def test_dirty_batches(self):
        with self.assertRaises(MetadataSourceException):
            fetch_dois_by_batch(['aunirestauniecb898989']
                                )  # definitely not a DOI

        dois = ['10.5281/anuirsetacesecesrbl']  # probably not a DOI
        results = fetch_dois_by_batch(dois)
        self.assertTrue(all([item is None for item in results]))

    def test_mixed_queries(self):
        dois = [
            '10.1016/0169-5983(88)90079-2',  # CrossRef DOI
            '10.5281/zenodo.12826',  # DataCite DOI
            ]
        results = fetch_dois_by_batch(dois)
        self.assertEqual([item['DOI'] for item in results], dois)

    def test_convert_to_name_pair(self):
        self.assertEqual(
                convert_to_name_pair({'family': 'Farge', 'given': 'Marie'}),
                ('Marie', 'Farge'))
        self.assertEqual(
                convert_to_name_pair({'literal': 'Marie Farge'}),
                ('Marie', 'Farge'))
        self.assertEqual(
                convert_to_name_pair({'literal': 'Farge, Marie'}),
                ('Marie', 'Farge'))
        self.assertEqual(
                convert_to_name_pair({'family': 'Arvind'}),
                ('', 'Arvind'))

    def test_is_oa_license(self):
        # Creative Commons licenses
        self.assertTrue(is_oa_license(
            'http://creativecommons.org/licenses/by-nc-nd/2.5/co/'))
        self.assertTrue(is_oa_license(
            'http://creativecommons.org/licenses/by-nc/3.10/'))
        self.assertTrue(is_oa_license(
            'https://creativecommons.org/licenses/by-nc-sa/4.0/'))
        # Other open licenses
        self.assertTrue(is_oa_license(
            'http://www.elsevier.com/open-access/userlicense/1.0/'))
        # Closed licenses
        self.assertFalse(is_oa_license(
            'http://link.aps.org/licenses/aps-default-license'))
        self.assertFalse(is_oa_license(
            'http://www.acs.org/content/acs/en/copyright.html'))
        self.assertFalse(is_oa_license(
            'http://www.elsevier.com/tdm/userlicense/1.0/'))
<commit_msg>Fix backend test after settings cleanup<commit_after># -*- encoding: utf-8 -*-

# Dissemin: open access policy enforcement tool
# Copyright (C) 2014 Antonin Delpeuch
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#

import datetime
import unittest
import os

from backend.crossref import convert_to_name_pair
from backend.crossref import CrossRefAPI
from backend.crossref import fetch_dois_by_batch
from backend.crossref import fetch_dois_incrementally
from backend.crossref import fetch_metadata_by_DOI
from backend.crossref import get_publication_date
from backend.crossref import is_oa_license
from backend.crossref import parse_crossref_date
from django.test import TestCase
from django.conf import settings
from papers.errors import MetadataSourceException
from papers.models import Paper

class CrossRefTest(TestCase):

    @classmethod
    def setUpClass(cls):
        super(CrossRefTest, cls).setUpClass()
        cls.testdir = os.path.dirname(os.path.abspath(__file__))

    def setUp(self):
        self.api = CrossRefAPI()

    def test_empty_pubdate(self):
        # This DOI has an empty 'issued' date
        p = self.api.create_paper_by_doi('10.1007/978-1-4020-7884-2_13')
        self.assertEqual(p.pubdate.year, 2006)

    def test_invalid_metadata(self):
        # authors with no family name
        paper = self.api.create_paper_by_doi('10.4156/aiss.vol3.issue9.31')
        self.assertEqual(paper, None)

    def test_affiliations(self):
        p = self.api.create_paper_by_doi('10.4204/eptcs.172.16')
        self.assertEqual(p.authors[0].affiliation,
                         'École Normale Supérieure, Paris')

    def test_dirty_metadata(self):
        # saving a paper with enough metadata to create a paper, but not
        # an OaiRecord.
        p = self.api.save_doi_metadata({
            "DOI": "10.1007/978-1-4020-7884-2_13",
            "subtitle": [],
            "author": [
                {
                    "affiliation": [],
                    "given": "Haowen",
                    "family": "Chan"
                },
                {
                    "affiliation": [],
                    "given": "Adrian",
                    "family": "Perrig"
                },
                {
                    "affiliation": [],
                    "given": "Dawn",
                    "family": "Song"
                }
            ],
            "created": {
                "timestamp": 1166129219000,
                "date-time": "2006-12-14T20:46:59Z",
                "date-parts": [
                    [
                        2006,
                        12,
                        14
                    ]
                ]
            },
            "title": [
                "Key Distribution Techniques for Sensor Networks"
            ],
            "type": "book-chapter"})
        self.assertTrue(p.is_orphan())
        self.assertFalse(p.visible)

    def test_doctype_book(self):
        # Books are ignored
        # (technically, that's because we currently require a
        # 'container-title' in the metadata)
        p = self.api.create_paper_by_doi('10.1385/1592597998')
        self.assertTrue(p.is_orphan())

    def test_doi_open(self):
        self.assertTrue(self.api.create_paper_by_doi('10.15200/winn.145838.88372').pdf_url)
        self.assertFalse(self.api.create_paper_by_doi('10.5061/dryad.b167g').pdf_url)

    def test_fetch_papers(self):
        generator = self.api.fetch_all_records(filters={'issn':'0302-9743'})
        for i in range(30):
            metadata = next(generator)
            self.assertTrue(metadata['DOI'].startswith('10.1007/'))

    def test_import_dump(self):
        self.api.ingest_dump(os.path.join(self.testdir, 'data/sample_crossref_dump.json.bz2'))
        p = Paper.get_by_doi('10.1016/j.jadohealth.2015.10.045')
        self.assertEqual("Sources, Type and Use of Social Support during Early Sexual Development of Black Gay and Bisexual Adolescent Males",
                         p.title)

class CrossRefUnitTest(unittest.TestCase):

    def test_fetch_single_doi(self):
        doi = '10.5380/dp.v1i1.1922'
        metadata = fetch_metadata_by_DOI(doi)
        self.assertEqual(metadata,
                         {'publisher': 'Universidade Federal do Parana',
                          'DOI': '10.5380/dp.v1i1.1922',
                          'subtitle': [],
                          'author': [{'given': 'Frederic', 'family': 'Worms'}],
                          'URL': 'http://dx.doi.org/10.5380/dp.v1i1.1922',
                          'issued': {'date-parts': [[2005, 3, 18]]},
                          'reference-count': 0,
                          'title': 'A concep\xe7\xe3o bergsoniana do tempo',
                          'volume': '1',
                          'source': 'CrossRef',
                          'prefix': 'http://id.crossref.org/prefix/10.5380',
                          'score': 1.0,
                          'deposited': {'timestamp': 1421107200000, 'date-parts': [[2015, 1, 13]]},
                          'type': 'journal-article',
                          'container-title': 'DoisPontos',
                          'indexed': {'timestamp': 1421405831942, 'date-parts': [[2015, 1, 16]]},
                          'issue': '1',
                          'ISSN': ['2179-7412', '1807-3883'],
                          'member': 'http://id.crossref.org/member/3785'})

    def test_parse_crossref_date_incomplete(self):
        self.assertEqual(parse_crossref_date(None), None)
        self.assertEqual(
                parse_crossref_date({'date-parts': [[2015, 7, 6]]}),
                datetime.date(year=2015, month=7, day=6))
        self.assertEqual(
                parse_crossref_date({'date-parts': [[2015, 7]]}),
                datetime.date(year=2015, month=7, day=1))
        self.assertEqual(
                parse_crossref_date({'date-parts': [[2015]]}),
                datetime.date(year=2015, month=1, day=1))

    def test_parse_crossref_date_raw(self):
        self.assertEqual(
                parse_crossref_date({'raw': '2015'}),
                datetime.date(year=2015, month=1, day=1))
        self.assertEqual(
                parse_crossref_date({'raw': '2015-07'}),
                datetime.date(year=2015, month=7, day=1))
        self.assertEqual(
                parse_crossref_date({'raw': '2015-07-06'}),
                datetime.date(year=2015, month=7, day=6))

    def test_get_publication_date(self):
        self.assertEqual(
                get_publication_date(
                    fetch_metadata_by_DOI('10.5281/zenodo.18898')),
                datetime.date(year=2015, month=1, day=1))
        self.assertEqual(
                get_publication_date(
                    fetch_metadata_by_DOI('10.5380/dp.v1i1.1919')),
                datetime.date(year=2005, month=3, day=18))

    def test_batch_queries(self):
        dois = [
            '10.1007/978-3-540-46375-7_2',
            '10.1007/978-3-540-46375-7_9',
            '10.2307/2540916',
            '10.1016/s1169-8330(00)80059-9',
            '10.1017/s0022112009008003',
            '10.1051/proc/2011014',
            '10.1016/0169-5983(88)90079-2',
            '10.1080/14685240600601061',
            '10.1103/physreve.79.026303',
            '10.1103/physreve.66.046307',
            '10.1103/physrevlett.95.244502',
            '10.1017/s0022112089002351',
            '10.1063/1.4738850',
            '10.1103/physrevlett.87.054501',
            '10.1080/14685248.2012.711476',
            '10.1007/978-94-011-4177-2_12',
            '10.1007/978-1-4615-4697-9_2',
            '10.1007/978-1-4612-0137-3_7',
            '10.1007/978-1-4020-6472-2_35']
        incremental = list(fetch_dois_incrementally(dois))
        self.assertEqual(len(incremental), len(dois))
        if settings.DOI_PROXY_SUPPORTS_BATCH:
            batch = fetch_dois_by_batch(dois)
            self.assertEqual([item['DOI'] for item in incremental],
                             [item['DOI'] for item in batch])

    def test_batch_with_comma(self):
        """
        Having a comma in a DOI should not fail the whole batch, just ignore
        that particular DOI
        """
        dois = ['10.1016/j.acta,at.2005.01.046','10.1080/14786430410001678217']
        batch = fetch_dois_by_batch(dois)
        fetched_dois = [ item['DOI'] for item in batch if item ]
        self.assertTrue('10.1080/14786430410001678217' in fetched_dois)

    def test_dirty_batches(self):
        with self.assertRaises(MetadataSourceException):
            fetch_dois_by_batch(['aunirestauniecb898989']
                                )  # definitely not a DOI

        dois = ['10.5281/anuirsetacesecesrbl']  # probably not a DOI
        results = fetch_dois_by_batch(dois)
        self.assertTrue(all([item is None for item in results]))

    def test_mixed_queries(self):
        dois = [
            '10.1016/0169-5983(88)90079-2',  # CrossRef DOI
            '10.5281/zenodo.12826',  # DataCite DOI
            ]
        results = fetch_dois_by_batch(dois)
        self.assertEqual([item['DOI'] for item in results], dois)

    def test_convert_to_name_pair(self):
        self.assertEqual(
                convert_to_name_pair({'family': 'Farge', 'given': 'Marie'}),
                ('Marie', 'Farge'))
        self.assertEqual(
                convert_to_name_pair({'literal': 'Marie Farge'}),
                ('Marie', 'Farge'))
        self.assertEqual(
                convert_to_name_pair({'literal': 'Farge, Marie'}),
                ('Marie', 'Farge'))
        self.assertEqual(
                convert_to_name_pair({'family': 'Arvind'}),
                ('', 'Arvind'))

    def test_is_oa_license(self):
        # Creative Commons licenses
        self.assertTrue(is_oa_license(
            'http://creativecommons.org/licenses/by-nc-nd/2.5/co/'))
        self.assertTrue(is_oa_license(
            'http://creativecommons.org/licenses/by-nc/3.10/'))
        self.assertTrue(is_oa_license(
            'https://creativecommons.org/licenses/by-nc-sa/4.0/'))
        # Other open licenses
        self.assertTrue(is_oa_license(
            'http://www.elsevier.com/open-access/userlicense/1.0/'))
        # Closed licenses
        self.assertFalse(is_oa_license(
            'http://link.aps.org/licenses/aps-default-license'))
        self.assertFalse(is_oa_license(
            'http://www.acs.org/content/acs/en/copyright.html'))
        self.assertFalse(is_oa_license(
            'http://www.elsevier.com/tdm/userlicense/1.0/'))
<|endoftext|>


<commit_before># Settings for www.djangoproject.com

import os
import platform
from unipath import FSPath as Path

# The full path to the django_website directory.
BASE = Path(__file__).absolute().ancestor(2)

# Far too clever trick to know if we're running on the deployment server.
PRODUCTION = ('DJANGOPROJECT_DEBUG' not in os.environ) and ("djangoproject" in platform.node())

ADMINS = (('Adrian Holovaty','holovaty@gmail.com'),('Jacob Kaplan-Moss', 'jacob@jacobian.org'))
MANAGERS = (('Jacob Kaplan-Moss','jacob@jacobian.org'),)
TIME_ZONE = 'America/Chicago'
SERVER_EMAIL = 'root@djangoproject.com'

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'NAME': 'djangoproject',
        'USER': 'djangoproject'
    }
}

USE_I18N = False
USE_L10N = False

TEMPLATE_DIRS = [BASE.child('templates')]
MEDIA_ROOT = BASE.parent.child('media')

if PRODUCTION:
    DEBUG = False
    PREPEND_WWW = True
    CACHE_BACKEND = 'memcached://127.0.0.1:11211/'
    MEDIA_URL = "https://www.djangoproject.com/m/"
    ADMIN_MEDIA_PREFIX = "https://www.djangoproject.com/m/admin/"
else:
    DEBUG = True
    PREPEND_WWW = False
    CACHE_BACKEND = "dummy:///"
    MEDIA_URL = "/media/"
    ADMIN_MEDIA_PREFIX = '/admin_media/'
    EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'

SITE_ID = 1
ROOT_URLCONF = 'django_website.urls.www'
INSTALLED_APPS = [
    'django.contrib.sites',
    'django.contrib.auth',
    'django.contrib.admin',
    'django.contrib.comments',
    'django.contrib.contenttypes',
    'django.contrib.flatpages',
    'django.contrib.humanize',
    'django.contrib.redirects',
    'django.contrib.sessions',
    'django.contrib.sitemaps',
    'django_website.blog',
    'django_website.aggregator',
    'django_website.docs',
    'registration',
    'south',
]

CACHE_MIDDLEWARE_SECONDS = 60 * 5 # 5 minutes
CACHE_MIDDLEWARE_KEY_PREFIX = 'djangoproject'
CACHE_MIDDLEWARE_GZIP = True
CACHE_MIDDLEWARE_ANONYMOUS_ONLY = True

MIDDLEWARE_CLASSES = [
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.contrib.flatpages.middleware.FlatpageFallbackMiddleware',
    'django.contrib.redirects.middleware.RedirectFallbackMiddleware',
]
if PRODUCTION:
    MIDDLEWARE_CLASSES.insert(0, 'django.middleware.cache.UpdateCacheMiddleware')
    MIDDLEWARE_CLASSES.append('django.middleware.cache.FetchFromCacheMiddleware')

TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.load_template_source',
    'django.template.loaders.app_directories.load_template_source',
)
TEMPLATE_CONTEXT_PROCESSORS = [
    "django.contrib.auth.context_processors.auth",
    "django.core.context_processors.debug",
    "django.core.context_processors.i18n",
    "django.core.context_processors.media",
    "django_website.context_processors.recent_release",
]

DEFAULT_FROM_EMAIL = "noreply@djangoproject.com"

# django-registration settings
ACCOUNT_ACTIVATION_DAYS = 3

# comment_utils settings
AKISMET_API_KEY = "c892e4962244"

# setting for documentation root path
DJANGO_DOCUMENT_ROOT_PATH = "/home/html/djangoproject.com/docs/"
DJANGO_TESTS_PATH = "/home/html/djangoproject.com/tests/"

# XXX What's this for?
DJANGO_SVN_ROOT = "http://code.djangoproject.com/svn/django/"

# If django-debug-toolbar is installed enable it.
if not PRODUCTION:
    try:
        import debug_toolbar
    except ImportError:
        pass
    else:
        # Insert DDT after the common middleware
        common_index = MIDDLEWARE_CLASSES.index('django.middleware.common.CommonMiddleware')
        MIDDLEWARE_CLASSES.insert(common_index+1, 'debug_toolbar.middleware.DebugToolbarMiddleware')
        INTERNAL_IPS = ['127.0.0.1']
        INSTALLED_APPS.append('debug_toolbar')
<commit_msg>Fixed SECRET_KEY to actually be secret.<commit_after># Settings for www.djangoproject.com

import os
import json
import platform
from unipath import FSPath as Path

# The full path to the django_website directory.
BASE = Path(__file__).absolute().ancestor(2)

# Far too clever trick to know if we're running on the deployment server.
PRODUCTION = ('DJANGOPROJECT_DEBUG' not in os.environ) and ("djangoproject" in platform.node())

# It's a secret to everybody
SECRET_KEY = json.load(open(BASE.child('secrets.json')))['secret_key']

ADMINS = (('Adrian Holovaty','holovaty@gmail.com'),('Jacob Kaplan-Moss', 'jacob@jacobian.org'))
MANAGERS = (('Jacob Kaplan-Moss','jacob@jacobian.org'),)
TIME_ZONE = 'America/Chicago'
SERVER_EMAIL = 'root@djangoproject.com'

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'NAME': 'djangoproject',
        'USER': 'djangoproject'
    }
}

USE_I18N = False
USE_L10N = False

TEMPLATE_DIRS = [BASE.child('templates')]
MEDIA_ROOT = BASE.parent.child('media')

if PRODUCTION:
    DEBUG = False
    PREPEND_WWW = True
    CACHE_BACKEND = 'memcached://127.0.0.1:11211/'
    MEDIA_URL = "https://www.djangoproject.com/m/"
    ADMIN_MEDIA_PREFIX = "https://www.djangoproject.com/m/admin/"
else:
    DEBUG = True
    PREPEND_WWW = False
    CACHE_BACKEND = "dummy:///"
    MEDIA_URL = "/media/"
    ADMIN_MEDIA_PREFIX = '/admin_media/'
    EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'

SITE_ID = 1
ROOT_URLCONF = 'django_website.urls.www'
INSTALLED_APPS = [
    'django.contrib.sites',
    'django.contrib.auth',
    'django.contrib.admin',
    'django.contrib.comments',
    'django.contrib.contenttypes',
    'django.contrib.flatpages',
    'django.contrib.humanize',
    'django.contrib.redirects',
    'django.contrib.sessions',
    'django.contrib.sitemaps',
    'django_website.blog',
    'django_website.aggregator',
    'django_website.docs',
    'registration',
    'south',
]

CACHE_MIDDLEWARE_SECONDS = 60 * 5 # 5 minutes
CACHE_MIDDLEWARE_KEY_PREFIX = 'djangoproject'
CACHE_MIDDLEWARE_GZIP = True
CACHE_MIDDLEWARE_ANONYMOUS_ONLY = True

MIDDLEWARE_CLASSES = [
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.contrib.flatpages.middleware.FlatpageFallbackMiddleware',
    'django.contrib.redirects.middleware.RedirectFallbackMiddleware',
]
if PRODUCTION:
    MIDDLEWARE_CLASSES.insert(0, 'django.middleware.cache.UpdateCacheMiddleware')
    MIDDLEWARE_CLASSES.append('django.middleware.cache.FetchFromCacheMiddleware')

TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.load_template_source',
    'django.template.loaders.app_directories.load_template_source',
)
TEMPLATE_CONTEXT_PROCESSORS = [
    "django.contrib.auth.context_processors.auth",
    "django.core.context_processors.debug",
    "django.core.context_processors.i18n",
    "django.core.context_processors.media",
    "django_website.context_processors.recent_release",
]

DEFAULT_FROM_EMAIL = "noreply@djangoproject.com"

# django-registration settings
ACCOUNT_ACTIVATION_DAYS = 3

# comment_utils settings
AKISMET_API_KEY = "c892e4962244"

# setting for documentation root path
DJANGO_DOCUMENT_ROOT_PATH = "/home/html/djangoproject.com/docs/"
DJANGO_TESTS_PATH = "/home/html/djangoproject.com/tests/"

# XXX What's this for?
DJANGO_SVN_ROOT = "http://code.djangoproject.com/svn/django/"

# If django-debug-toolbar is installed enable it.
if not PRODUCTION:
    try:
        import debug_toolbar
    except ImportError:
        pass
    else:
        # Insert DDT after the common middleware
        common_index = MIDDLEWARE_CLASSES.index('django.middleware.common.CommonMiddleware')
        MIDDLEWARE_CLASSES.insert(common_index+1, 'debug_toolbar.middleware.DebugToolbarMiddleware')
        INTERNAL_IPS = ['127.0.0.1']
        INSTALLED_APPS.append('debug_toolbar')
<|endoftext|>


<commit_before>import time
import re
import fnmatch
import ROOT
from ..basictypes import *
from ..core import Object
from ..utils import *
from ..registry import *
from ..io import openFile
from .filtering import *
from ..plotting.core import Plottable

class VarProxy(object):

    def __init__(self, tree, prefix, index, collections=None):

        self.index = index
        self.tree = tree
        self.prefix = prefix
        self.collections = collections

    def __getitem__(self, thing):

        return getattr(self, thing)

    def __getattr__(self, attr):

        if attr.startswith(self.prefix):
            return getattr(self.tree, attr)[self.index]
        return getattr(self.tree, self.prefix + attr)[self.index]

class TreeCollection(object):

    def __init__(self, tree, prefix, size):

        self.tree = tree
        self.prefix = prefix
        self.size = size
        self.subcollections = []
        super(TreeCollection, self).__init__()

    def __getitem__(self, index):

        if index >= len(self):
            raise IndexError()
        return VarProxy(self.tree, self.prefix, index, self.subcollections)

    def __len__(self):

        return getattr(self.tree, self.size).value()

    def collection(self, name, prefix, size):

        self.subcollections.append(TreeCollection(self.tree, self.prefix + prefix, size, self.subcollections))

    def __iter__(self):

        for index in xrange(len(self)):
            yield VarProxy(self.tree, self.prefix, index)

class Tree(Plottable, Object, ROOT.TTree):
    """
    Inherits from TTree so all regular TTree methods are available
    but Draw has been overridden to improve usage in Python
    """
    draw_command = re.compile('^.+>>[\+]?(?P<name>[^(]+).*$')

    def __init__(self, name = None, title = None):

        Object.__init__(self, name, title)
        self._post_init()

    def _post_init(self):

        Plottable.__init__(self)
        self.buffer = TreeBuffer()

    def set_buffer(self, buffer):

        self.buffer.update(buffer)
        for attr, value in buffer.items():
            setattr(self, attr, value)

    def set_branches_from_buffer(self, buffer, variables = None):

        for name, value in buffer.items():
            if variables is not None:
                if name not in variables:
                    continue
            if isinstance(value, Variable):
                self.Branch(name, value, "%s/%s"% (name, value.type()))
            else:
                self.Branch(name, value)
        if variables:
            newbuffer = TreeBuffer()
            for variable in variables:
                if variable in buffer:
                    newbuffer[variable] = buffer[variable]
            buffer = newbuffer
        self.set_buffer(buffer)

    def set_addresses_from_buffer(self, buffer, variables = None):

        for name, value in buffer.items():
            if variables is not None:
                if name not in variables:
                    continue
            if self.GetBranch(name):
                self.SetBranchAddress(name, value)
        self.set_buffer(buffer)

    def get_buffer(self):

        if self.buffer is not None:
            return self.buffer
        buffer = []
        for branch in self.iterbranches():
            if self.GetBranchStatus(branch.GetName()):
                typename = branch.GetClassName()
                if not typename:
                    typename = branch.GetListOfLeaves()[0].GetTypeName()
                buffer.append((branch.GetName(), typename))
        return TreeBuffer(buffer)

    def activate(self, variables, exclusive=True):

        if exclusive:
            self.SetBranchStatus('*',0)
        for variable in variables:
            if self.GetBranch(variable):
                self.SetBranchStatus(variable,1)

    def __getitem__(self, item):

        if isinstance(item, basestring):
            if self.buffer is not None:
                if self.buffer.has_key(item):
                    return self.buffer[item]
            if self.has_branch(item):
                return getattr(self, item)
            raise KeyError("%s"% item)
        if not (0 <= item < len(self)):
            raise IndexError("entry index out of range")
        self.GetEntry(item)
        return self

    def __len__(self):

        return self.GetEntries()

    def __iter__(self):

        i = 0
        while self.GetEntry(i):
            yield self
            i += 1

    def iterbranches(self):

        for branch in self.GetListOfBranches():
            yield branch

    def iterbranchnames(self):

        for branch in self.iterbranches():
            yield branch.GetName()

    def glob(self, pattern, *exclude):

        matches = fnmatch.filter(self.iterbranchnames(), pattern)
        for exc_pattern in exclude:
            matches = [match for match in matches if not fnmatch.fnmatch(match, exc_pattern)]
        return matches

    def has_branch(self, branch):

        return not not self.GetBranch(branch)

    def Scale(self, value):

        self.SetWeight(self.GetWeight() * value)

    def GetEntries(self, cut = None, weighted_cut = None, weighted = False):

        if weighted_cut:
            hist = Hist(1,-1,2)
            branch = self.GetListOfBranches()[0].GetName()
            weight = self.GetWeight()
            self.SetWeight(1)
            self.Draw("%s==%s>>%s"%(branch, branch, hist.GetName()), weighted_cut * cut)
            self.SetWeight(weight)
            entries = hist.Integral()
        elif cut:
            entries = ROOT.TTree.GetEntries(self, str(cut))
        else:
            entries = ROOT.TTree.GetEntries(self)
        if weighted:
            entries *= self.GetWeight()
        return entries

    def GetMaximum(self, expression, cut = None):

        if cut:
            self.Draw(expression, cut, "goff")
        else:
            self.Draw(expression, "", "goff")
        vals = self.GetV1()
        n = self.GetSelectedRows()
        vals = [vals[i] for i in xrange(min(n,10000))]
        return max(vals)

    def GetMinimum(self, expression, cut = None):

        if cut:
            self.Draw(expression, cut, "goff")
        else:
            self.Draw(expression, "", "goff")
        vals = self.GetV1()
        n = self.GetSelectedRows()
        vals = [vals[i] for i in xrange(min(n,10000))]
        return min(vals)

    def Fill(self, *args, **kwargs):

        super(Tree, self).Fill(*args, **kwargs)
        # reset all branches
        if self.buffer:
            self.buffer.reset()

    def Draw(self, *args, **kwargs):
        """
        Draw a TTree with a selection as usual, but return the created histogram.
        """
        hist = kwargs.get("hist", None)
        if len(args) == 0:
            raise TypeError("Draw did not receive any arguments")
        if hist is None:
            match = re.match(Tree.draw_command, args[0])
            histname = None
            if match:
                histname = match.group('name')
                hist_exists = ROOT.gDirectory.Get(histname) is not None
        else:
            args = (args[0] + ">>+%s" % hist.GetName(),) + args[1:]
        super(Tree, self).Draw(self, *args)
        if hist is None:
            if histname is not None:
                hist = asrootpy(ROOT.gDirectory.Get(histname))
                # if the hist already existed then I will
                # not overwrite its plottable features
                if not hist_exists and isinstance(hist, Plottable):
                    hist.decorate(self)
            else:
                hist = asrootpy(ROOT.gPad.GetPrimitive("htemp"))
                if isinstance(hist, Plottable):
                    hist.decorate(self)
        return hist

register(Tree, Tree._post_init)

class TreeChain:
    """
    A replacement for TChain
    """
    def __init__(self, name, files, buffer=None, branches=None, events=-1):

        self.name = name
        if isinstance(files, tuple):
            files = list(files)
        elif not isinstance(files, list):
            files = [files]
        else:
            files = files[:]
        self.files = files
        self.buffer = buffer
        self.branches = branches
        if self.buffer:
            for attr, value in self.userbuffer.items():
                setattr(self, attr, value)
        self.weight = 1.
        self.tree = None
        self.file = None
        self.filters = EventFilterList()
        self.userdata = {}
        self.file_change_hooks = []
        self.events = events

    def init(self):

        if not self.__initialize():
            raise RuntimeError("unable to initialize TreeChain")

    def collection(self, name, prefix, size):

        coll = TreeCollection(self, prefix, size)
        setattr(self, name, coll)

    def add_file_change_hook(self, target, args):

        self.file_change_hooks.append((target, args))

    def __initialize(self):

        if self.tree != None:
            self.tree = None
        if self.file != None:
            self.file.Close()
            self.file = None
        if len(self.files) > 0:
            print "%i files remaining to process"% len(self.files)
            fileName = self.files.pop()
            self.file = openFile(fileName)
            if not self.file:
                print "WARNING: Skipping file. Could not open file %s"%(fileName)
                return self.__initialize()
            self.tree = self.file.Get(self.name)
            if not self.tree:
                print "WARNING: Skipping file. Tree %s does not exist in file %s"%(self.name, fileName)
                return self.__initialize()
            if len(self.tree.GetListOfBranches()) == 0:
                # Try the next file:
                print "WARNING: skipping tree with no branches in file %s"%fileName
                return self.__initialize()
            if self.branches is not None:
                self.tree.activate(self.branches)
            if self.buffer is None:
                buffer = self.tree.get_buffer()
                self.buffer = buffer
                for attr, value in buffer.items():
                    setattr(self, attr, value)
            self.tree.set_addresses_from_buffer(self.buffer)
            self.weight = self.tree.GetWeight()
            for target, args in self.file_change_hooks:
                target(*args, name=self.name, file=self.file)
            return True
        return False

    def set_filters(self, filterlist):

        self.filters = filterlist

    def append_filter(self, filter):

        self.filters.append(filter)

    def prepend_filter(self, filter):

        self.filters.insert(0, filter)

    def __getitem__(self, item):

        return self.tree.__getitem__(item)

    def __iter__(self):

        events = 0
        while True:
            t1 = time.time()
            entries = 0
            for entry in self.tree:
                entries += 1
                self.userdata = {}
                if self.filters(self):
                    yield self
                    events += 1
                    if self.events == events:
                        break
            if self.events == events:
                break
            print "%i entries per second"% int(entries / (time.time() - t1))
            if not self.__initialize():
                break

class TreeBuffer(dict):
    """
    A dictionary mapping variable names to values
    """
    generate("vector<vector<float> >", "<vector>")
    generate("vector<vector<int> >", "<vector>")
    generate("vector<vector<unsigned int> >", "<vector>")
    generate("vector<vector<long> >", "<vector>")
    generate("vector<vector<unsigned long> >", "<vector>")
    generate("vector<vector<double> >", "<vector>")
    generate("vector<vector<string> >")

    demote = {"Bool_t": "B",
              "Float_t":"F",
              "Double_t": "D",
              "Int_t":"I",
              "UInt_t":"UI",
              "Int":"I",
              "Float":"F",
              "F":"F",
              "I":"I",
              "UI":"UI",
              "vector<float>":"F",
              "vector<int>":"I",
              "vector<unsigned int>":"UI",
              "vector<int, allocator<int> >":"I",
              "vector<float, allocator<float> >":"F",
              "VF":"F",
              "VI":"I",
              "VUI":"UI",
              "vector<vector<float> >":"VF",
              "vector<vector<int> >":"VI",
              "vector<vector<unsigned int> >":"VUI",
              "vector<vector<int>, allocator<vector<int> > >":"VI",
              "vector<vector<float>, allocator<vector<float> > >":"VF",
              "VVF":"VF",
              "VVI":"VI"}

    def __init__(self, variables = None, default = -1111, flatten = False):

        self.variables = variables
        if self.variables is None:
            self.variables = []
            data = {}
        else:
            data = self.__process(self.variables, default, flatten)
        dict.__init__(self, data)

    def __process(self, variables, default = -1111, flatten = False):

        data = {}
        methods = dir(self)
        processed = []
        for name, vtype in variables:
            if flatten:
                vtype = TreeBuffer.demote[vtype] #.upper()]
            if name in processed:
                raise ValueError("Duplicate variable name %s"%name)
            else:
                processed.append(name)
            if vtype.upper() in ("B", "BOOL_T"):
                data[name] = Bool(False)
            elif vtype.upper() in ("I", "INT_T"):
                data[name] = Int(default)
            elif vtype.upper() in ("UI", "UINT_T"):
                data[name] = UInt(default)
            elif vtype.upper() in ("F", "FLOAT_T"):
                data[name] = Float(default)
            elif vtype.upper() in ("D", "DOUBLE_T"):
                data[name] = Double(default)
            elif vtype.upper() in ("VS", "VECTOR<SHORT>"):
                data[name] = ROOT.vector("short")()
            elif vtype.upper() in ("VUS", "VECTOR<UNSIGNED SHORT>"):
                data[name] = ROOT.vector("unsigned short")()
            elif vtype.upper() in ("VI", "VECTOR<INT>"):
                data[name] = ROOT.vector("int")()
            elif vtype.upper() in ("VUI", "VECTOR<UNSIGNED INT>"):
                data[name] = ROOT.vector("unsigned int")()
            elif vtype.upper() in ("VL", "VECTOR<LONG>"):
                data[name] = ROOT.vector("long")()
            elif vtype.upper() in ("VF", "VECTOR<FLOAT>"):
                data[name] = ROOT.vector("float")()
            elif vtype.upper() in ("VD", "VECTOR<DOUBLE>"):
                data[name] = ROOT.vector("double")()
            elif vtype.upper() in ("VVI", "VECTOR<VECTOR<INT> >"):
                data[name] = ROOT.vector("vector<int>")()
            elif vtype.upper() in ("VVUI", "VECTOR<VECTOR<UNSIGNED INT> >"):
                data[name] = ROOT.vector("vector<unsigned int>")()
            elif vtype.upper() in ("VVL", "VECTOR<VECTOR<LONG> >"):
                data[name] = ROOT.vector("vector<long>")()
            elif vtype.upper() in ("VVUL", "VECTOR<VECTOR<UNSIGNED LONG> >"):
                data[name] = ROOT.vector("vector<unsigned long>")()
            elif vtype.upper() in ("VVF", "VECTOR<VECTOR<FLOAT> >"):
                data[name] = ROOT.vector("vector<float>")()
            elif vtype.upper() in ("VVD", "VECTOR<VECTOR<DOUBLE> >"):
                data[name] = ROOT.vector("vector<double>")()
            elif vtype.upper() in ("VVSTR", "VECTOR<VECTOR<STRING> >"):
                data[name] = ROOT.vector("vector<string>")()
            elif vtype.upper() in ("VSTR", "VECTOR<STRING>"):
                data[name] = ROOT.vector("string")()
            else:
                raise TypeError("Unsupported variable type: %s"%(vtype.upper()))
            if name not in methods and not name.startswith("_"):
                setattr(self, name, data[name])
            else:
                raise ValueError("Illegal variable name: %s"%name)
        return data

    def reset(self):

        for value in self.values():
            value.clear()

    def flat(self, variables = None):

        if variables is None:
            variables = self.variables
        else:
            variables = filter(lambda a: a[0] in variables, self.variables)
        return TreeBuffer(variables, flatten = True)

    def update(self, variables):

        if not isinstance(variables, TreeBuffer):
            variables = self.__process(variables)
        dict.update(self, variables)

    def __str__(self):

        return self.__repr__()

    def __repr__(self):

        rep = ""
        for var, value in self.items():
            rep += "%s ==> %s\n"%(var, value)
        return rep
<commit_msg>updates<commit_after>import time
import re
import fnmatch
import ROOT
from ..basictypes import *
from ..core import Object
from ..utils import *
from ..registry import *
from ..io import openFile
from .filtering import *
from ..plotting.core import Plottable

class VarProxy(object):

    def __init__(self, tree, prefix, index, collections=None):

        self.index = index
        self.tree = tree
        self.prefix = prefix
        self.collections = collections

    def __getitem__(self, thing):

        return getattr(self, thing)

    def __getattr__(self, attr):

        if attr.startswith(self.prefix):
            return getattr(self.tree, attr)[self.index]
        return getattr(self.tree, self.prefix + attr)[self.index]

class TreeCollection(object):

    def __init__(self, tree, prefix, size):

        self.tree = tree
        self.prefix = prefix
        self.size = size
        self.subcollections = []
        super(TreeCollection, self).__init__()

    def __getitem__(self, index):

        if index >= len(self):
            raise IndexError()
        return VarProxy(self.tree, self.prefix, index, self.subcollections)

    def __len__(self):

        return getattr(self.tree, self.size).value()

    def collection(self, name, prefix, size):

        self.subcollections.append(TreeCollection(self.tree, self.prefix + prefix, size, self.subcollections))

    def __iter__(self):

        for index in xrange(len(self)):
            yield VarProxy(self.tree, self.prefix, index)

class Tree(Plottable, Object, ROOT.TTree):
    """
    Inherits from TTree so all regular TTree methods are available
    but Draw has been overridden to improve usage in Python
    """
    draw_command = re.compile('^.+>>[\+]?(?P<name>[^(]+).*$')

    def __init__(self, name = None, title = None):

        Object.__init__(self, name, title)
        self._post_init()

    def _post_init(self):

        Plottable.__init__(self)
        self.build_buffer()

    def build_buffer(self):

        buffer = []
        for branch in self.iterbranches():
            if self.GetBranchStatus(branch.GetName()):
                typename = branch.GetClassName()
                if not typename:
                    typename = branch.GetListOfLeaves()[0].GetTypeName()
                buffer.append((branch.GetName(), typename))
        self.buffer = TreeBuffer(buffer)

    def set_buffer(self, buffer):

        self.buffer.update(buffer)
        for attr, value in buffer.items():
            setattr(self, attr, value)

    def set_branches_from_buffer(self, buffer, variables = None):

        for name, value in buffer.items():
            if variables is not None:
                if name not in variables:
                    continue
            if isinstance(value, Variable):
                self.Branch(name, value, "%s/%s"% (name, value.type()))
            else:
                self.Branch(name, value)
        if variables:
            newbuffer = TreeBuffer()
            for variable in variables:
                if variable in buffer:
                    newbuffer[variable] = buffer[variable]
            buffer = newbuffer
        self.set_buffer(buffer)

    def set_addresses_from_buffer(self, buffer, variables = None):

        for name, value in buffer.items():
            if variables is not None:
                if name not in variables:
                    continue
            if self.GetBranch(name):
                self.SetBranchAddress(name, value)
        self.set_buffer(buffer)

    def activate(self, variables, exclusive=True):

        if exclusive:
            self.SetBranchStatus('*',0)
        for variable in variables:
            if self.GetBranch(variable):
                self.SetBranchStatus(variable,1)

    def __getitem__(self, item):

        if isinstance(item, basestring):
            if self.buffer is not None:
                if self.buffer.has_key(item):
                    return self.buffer[item]
            if self.has_branch(item):
                return getattr(self, item)
            raise KeyError("%s"% item)
        if not (0 <= item < len(self)):
            raise IndexError("entry index out of range")
        self.GetEntry(item)
        return self

    def __len__(self):

        return self.GetEntries()

    def __iter__(self):

        i = 0
        while self.GetEntry(i):
            yield self
            i += 1

    def iterbranches(self):

        for branch in self.GetListOfBranches():
            yield branch

    def iterbranchnames(self):

        for branch in self.iterbranches():
            yield branch.GetName()

    def glob(self, pattern, *exclude):

        matches = fnmatch.filter(self.iterbranchnames(), pattern)
        for exc_pattern in exclude:
            matches = [match for match in matches if not fnmatch.fnmatch(match, exc_pattern)]
        return matches

    def has_branch(self, branch):

        return not not self.GetBranch(branch)

    def Scale(self, value):

        self.SetWeight(self.GetWeight() * value)

    def GetEntries(self, cut = None, weighted_cut = None, weighted = False):

        if weighted_cut:
            hist = Hist(1,-1,2)
            branch = self.GetListOfBranches()[0].GetName()
            weight = self.GetWeight()
            self.SetWeight(1)
            self.Draw("%s==%s>>%s"%(branch, branch, hist.GetName()), weighted_cut * cut)
            self.SetWeight(weight)
            entries = hist.Integral()
        elif cut:
            entries = ROOT.TTree.GetEntries(self, str(cut))
        else:
            entries = ROOT.TTree.GetEntries(self)
        if weighted:
            entries *= self.GetWeight()
        return entries

    def GetMaximum(self, expression, cut = None):

        if cut:
            self.Draw(expression, cut, "goff")
        else:
            self.Draw(expression, "", "goff")
        vals = self.GetV1()
        n = self.GetSelectedRows()
        vals = [vals[i] for i in xrange(min(n,10000))]
        return max(vals)

    def GetMinimum(self, expression, cut = None):

        if cut:
            self.Draw(expression, cut, "goff")
        else:
            self.Draw(expression, "", "goff")
        vals = self.GetV1()
        n = self.GetSelectedRows()
        vals = [vals[i] for i in xrange(min(n,10000))]
        return min(vals)

    def Fill(self, *args, **kwargs):

        super(Tree, self).Fill(*args, **kwargs)
        # reset all branches
        if self.buffer:
            self.buffer.reset()

    def Draw(self, *args, **kwargs):
        """
        Draw a TTree with a selection as usual, but return the created histogram.
        """
        hist = kwargs.get("hist", None)
        if len(args) == 0:
            raise TypeError("Draw did not receive any arguments")
        if hist is None:
            match = re.match(Tree.draw_command, args[0])
            histname = None
            if match:
                histname = match.group('name')
                hist_exists = ROOT.gDirectory.Get(histname) is not None
        else:
            args = (args[0] + ">>+%s" % hist.GetName(),) + args[1:]
        super(Tree, self).Draw(self, *args)
        if hist is None:
            if histname is not None:
                hist = asrootpy(ROOT.gDirectory.Get(histname))
                # if the hist already existed then I will
                # not overwrite its plottable features
                if not hist_exists and isinstance(hist, Plottable):
                    hist.decorate(self)
            else:
                hist = asrootpy(ROOT.gPad.GetPrimitive("htemp"))
                if isinstance(hist, Plottable):
                    hist.decorate(self)
        return hist

register(Tree, Tree._post_init)

class TreeChain:
    """
    A replacement for TChain
    """
    def __init__(self, name, files, buffer=None, branches=None, events=-1):

        self.name = name
        if isinstance(files, tuple):
            files = list(files)
        elif not isinstance(files, list):
            files = [files]
        else:
            files = files[:]
        self.files = files
        self.buffer = buffer
        self.branches = branches
        if self.buffer:
            for attr, value in self.userbuffer.items():
                setattr(self, attr, value)
        self.weight = 1.
        self.tree = None
        self.file = None
        self.filters = EventFilterList()
        self.userdata = {}
        self.file_change_hooks = []
        self.events = events

    def init(self):

        if not self.__initialize():
            raise RuntimeError("unable to initialize TreeChain")

    def collection(self, name, prefix, size):

        coll = TreeCollection(self, prefix, size)
        setattr(self, name, coll)

    def add_file_change_hook(self, target, args):

        self.file_change_hooks.append((target, args))

    def __initialize(self):

        if self.tree != None:
            self.tree = None
        if self.file != None:
            self.file.Close()
            self.file = None
        if len(self.files) > 0:
            print "%i files remaining to process"% len(self.files)
            fileName = self.files.pop()
            self.file = openFile(fileName)
            if not self.file:
                print "WARNING: Skipping file. Could not open file %s"%(fileName)
                return self.__initialize()
            self.tree = self.file.Get(self.name)
            if not self.tree:
                print "WARNING: Skipping file. Tree %s does not exist in file %s"%(self.name, fileName)
                return self.__initialize()
            if len(self.tree.GetListOfBranches()) == 0:
                # Try the next file:
                print "WARNING: skipping tree with no branches in file %s"%fileName
                return self.__initialize()
            if self.branches is not None:
                self.tree.activate(self.branches)
            if self.buffer is None:
                buffer = self.tree.buffer
                self.buffer = buffer
                for attr, value in buffer.items():
                    setattr(self, attr, value)
            self.tree.set_addresses_from_buffer(self.buffer)
            self.weight = self.tree.GetWeight()
            for target, args in self.file_change_hooks:
                target(*args, name=self.name, file=self.file)
            return True
        return False

    def set_filters(self, filterlist):

        self.filters = filterlist

    def append_filter(self, filter):

        self.filters.append(filter)

    def prepend_filter(self, filter):

        self.filters.insert(0, filter)

    def __getitem__(self, item):

        return self.tree.__getitem__(item)

    def __iter__(self):

        events = 0
        while True:
            t1 = time.time()
            entries = 0
            for entry in self.tree:
                entries += 1
                self.userdata = {}
                if self.filters(self):
                    yield self
                    events += 1
                    if self.events == events:
                        break
            if self.events == events:
                break
            print "%i entries per second"% int(entries / (time.time() - t1))
            if not self.__initialize():
                break

class TreeBuffer(dict):
    """
    A dictionary mapping variable names to values
    """
    generate("vector<vector<float> >", "<vector>")
    generate("vector<vector<int> >", "<vector>")
    generate("vector<vector<unsigned int> >", "<vector>")
    generate("vector<vector<long> >", "<vector>")
    generate("vector<vector<unsigned long> >", "<vector>")
    generate("vector<vector<double> >", "<vector>")
    generate("vector<vector<string> >")

    demote = {"Bool_t": "B",
              "Float_t":"F",
              "Double_t": "D",
              "Int_t":"I",
              "UInt_t":"UI",
              "Int":"I",
              "Float":"F",
              "F":"F",
              "I":"I",
              "UI":"UI",
              "vector<float>":"F",
              "vector<int>":"I",
              "vector<unsigned int>":"UI",
              "vector<int, allocator<int> >":"I",
              "vector<float, allocator<float> >":"F",
              "VF":"F",
              "VI":"I",
              "VUI":"UI",
              "vector<vector<float> >":"VF",
              "vector<vector<int> >":"VI",
              "vector<vector<unsigned int> >":"VUI",
              "vector<vector<int>, allocator<vector<int> > >":"VI",
              "vector<vector<float>, allocator<vector<float> > >":"VF",
              "VVF":"VF",
              "VVI":"VI"}

    def __init__(self, variables = None, default = -1111, flatten = False):

        self.variables = variables
        if self.variables is None:
            self.variables = []
            data = {}
        else:
            data = self.__process(self.variables, default, flatten)
        dict.__init__(self, data)

    def __process(self, variables, default = -1111, flatten = False):

        data = {}
        methods = dir(self)
        processed = []
        for name, vtype in variables:
            if flatten:
                vtype = TreeBuffer.demote[vtype] #.upper()]
            if name in processed:
                raise ValueError("Duplicate variable name %s"%name)
            else:
                processed.append(name)
            if vtype.upper() in ("B", "BOOL_T"):
                data[name] = Bool(False)
            elif vtype.upper() in ("I", "INT_T"):
                data[name] = Int(default)
            elif vtype.upper() in ("UI", "UINT_T"):
                data[name] = UInt(default)
            elif vtype.upper() in ("F", "FLOAT_T"):
                data[name] = Float(default)
            elif vtype.upper() in ("D", "DOUBLE_T"):
                data[name] = Double(default)
            elif vtype.upper() in ("VS", "VECTOR<SHORT>"):
                data[name] = ROOT.vector("short")()
            elif vtype.upper() in ("VUS", "VECTOR<UNSIGNED SHORT>"):
                data[name] = ROOT.vector("unsigned short")()
            elif vtype.upper() in ("VI", "VECTOR<INT>"):
                data[name] = ROOT.vector("int")()
            elif vtype.upper() in ("VUI", "VECTOR<UNSIGNED INT>"):
                data[name] = ROOT.vector("unsigned int")()
            elif vtype.upper() in ("VL", "VECTOR<LONG>"):
                data[name] = ROOT.vector("long")()
            elif vtype.upper() in ("VF", "VECTOR<FLOAT>"):
                data[name] = ROOT.vector("float")()
            elif vtype.upper() in ("VD", "VECTOR<DOUBLE>"):
                data[name] = ROOT.vector("double")()
            elif vtype.upper() in ("VVI", "VECTOR<VECTOR<INT> >"):
                data[name] = ROOT.vector("vector<int>")()
            elif vtype.upper() in ("VVUI", "VECTOR<VECTOR<UNSIGNED INT> >"):
                data[name] = ROOT.vector("vector<unsigned int>")()
            elif vtype.upper() in ("VVL", "VECTOR<VECTOR<LONG> >"):
                data[name] = ROOT.vector("vector<long>")()
            elif vtype.upper() in ("VVUL", "VECTOR<VECTOR<UNSIGNED LONG> >"):
                data[name] = ROOT.vector("vector<unsigned long>")()
            elif vtype.upper() in ("VVF", "VECTOR<VECTOR<FLOAT> >"):
                data[name] = ROOT.vector("vector<float>")()
            elif vtype.upper() in ("VVD", "VECTOR<VECTOR<DOUBLE> >"):
                data[name] = ROOT.vector("vector<double>")()
            elif vtype.upper() in ("VVSTR", "VECTOR<VECTOR<STRING> >"):
                data[name] = ROOT.vector("vector<string>")()
            elif vtype.upper() in ("VSTR", "VECTOR<STRING>"):
                data[name] = ROOT.vector("string")()
            else:
                raise TypeError("Unsupported variable type: %s"%(vtype.upper()))
            if name not in methods and not name.startswith("_"):
                setattr(self, name, data[name])
            else:
                raise ValueError("Illegal variable name: %s"%name)
        return data

    def reset(self):

        for value in self.values():
            value.clear()

    def flat(self, variables = None):

        if variables is None:
            variables = self.variables
        else:
            variables = filter(lambda a: a[0] in variables, self.variables)
        return TreeBuffer(variables, flatten = True)

    def update(self, variables):

        if not isinstance(variables, TreeBuffer):
            variables = self.__process(variables)
        dict.update(self, variables)

    def __str__(self):

        return self.__repr__()

    def __repr__(self):

        rep = ""
        for var, value in self.items():
            rep += "%s ==> %s\n"%(var, value)
        return rep
<|endoftext|>


<commit_before>## Copyright 2013 by Luc Saffre.
## License: BSD, see LICENSE for more details.

"""
An extended `unittest.TestCase` to be run using `setup.py`
in the root of a project which may contain several Django projects.

We cannot import :mod:`djangosite.utils.djangotest` here
because that's designed for unit tests within a particular Django project
(run using `djange-admin test`).

"""
import sys
import doctest

from atelier.test import TestCase

class TestCase(TestCase):
    """
    """

    demo_settings_module = None
    """
    The DJANGO_SETTINGS_MODULE which serves as the central demo
    for this project.
    """

    def build_environment(self):
        env = super(TestCase,self).build_environment()
        if self.demo_settings_module:
            env.update(DJANGO_SETTINGS_MODULE=self.demo_settings_module)
        return env

    def setUp(self):

        if self.demo_settings_module:
            from djangosite.signals import testcase_setup
            #~ from django.test.client import Client
            #~ self.client = Client()
            #~ settings.SITE.never_build_site_cache = self.never_build_site_cache
            #~ settings.SITE.remote_user_header = 'REMOTE_USER'
            testcase_setup.send(self)
        super(TestCase,self).setUp()

    def run_docs_django_tests(self,n,**kw):
        args = ["django-admin.py"]
        args += ["test"]
        args += ["--settings=%s" % n]
        args += ["--failfast"]
        args += ["--verbosity=0"]
        args += ["--pythonpath=%s" % self.project_root.child('docs')]
        self.run_subprocess(args,**kw)

    def run_django_manage_test(self,db,**kw):
        p = self.project_root.child(*db.split('/'))
        args = ["python","manage.py"]
        args += ["test"]
        #~ args += more
        args += ["--noinput"]
        args += ["--failfast"]
        #~ args += ["--settings=settings"]
        args += ["--pythonpath=%s" % p.absolute()]
        kw.update(cwd=p)
        self.run_subprocess(args,**kw)

    def run_django_admin_test(self,settings_module,*args,**kw):
        return self.run_django_admin_command(settings_module,'test',"--verbosity=0",*args,**kw)

    def run_django_admin_command(self,settings_module,*cmdargs,**kw):
        args = ["django-admin.py"]
        args += cmdargs
        args += ["--settings=%s" % settings_module]
        args += ["--noinput"]
        args += ["--failfast"]
        args += ["--traceback"]
        self.run_subprocess(args,**kw)

    def run_docs_doctests(self,filename):
        """
        Run a simple doctest for specified file after importing the
        docs `conf.py` (which causes the demo database to be activated).

        This is used e.g. for testing pages like those below
        :doc:`/tested/index`.

        These tests may fail for the simple reason that the demo database
        has not been initialized (in that case, run `fab initdb`).
        """
        filename = 'docs/' + filename
        #~ p = self.project_root.child(*filename.split('/')).parent
        #~ os.environ['DJANGO_SETTINGS_MODULE']='settings'
        #~ oldcwd = os.getcwd()
        #~ self.project_root.child('docs').chdir()
        #~ p.chdir()
        #~ sys.path.insert(0,'.')
        #~ print p
        sys.path.insert(0,'docs')
        import conf # trigger Django startup

        if False:
            # 20130828 test cases *are* required to restore the language afterwards.
            try:
                from north.dbutils import set_language
                set_language()
                """
                Each test case starts with the site's default language.
                Test cases are not required to restore the language afterwards.
                """
            except ImportError:
                pass # not everybody uses north

        res = doctest.testfile(filename, module_relative=False,encoding='utf-8')

        del sys.path[0]
        #~ os.chdir(oldcwd)

        #~ return res
        if res.failed:
            self.fail("doctest.testfile() failed. See earlier messages.")

<commit_msg>http://code.google.com/p/lino/source/browse/docs/blog/2013/0903.rst<commit_after>## Copyright 2013 by Luc Saffre.
## License: BSD, see LICENSE for more details.

"""
An extended `unittest.TestCase` to be run using `setup.py`
in the root of a project which may contain several Django projects.

We cannot import :mod:`djangosite.utils.djangotest` here
because that's designed for unit tests within a particular Django project
(run using `djange-admin test`).

"""
import sys
import doctest

from atelier.test import TestCase

class TestCase(TestCase):
    """
    """

    demo_settings_module = None
    """
    The DJANGO_SETTINGS_MODULE which serves as the central demo
    for this project.
    """

    def build_environment(self):
        env = super(TestCase,self).build_environment()
        if self.demo_settings_module:
            env.update(DJANGO_SETTINGS_MODULE=self.demo_settings_module)
        return env

    def setUp(self):

        if self.demo_settings_module:
            from djangosite.signals import testcase_setup
            #~ from django.test.client import Client
            #~ self.client = Client()
            #~ settings.SITE.never_build_site_cache = self.never_build_site_cache
            #~ settings.SITE.remote_user_header = 'REMOTE_USER'
            testcase_setup.send(self)
        super(TestCase,self).setUp()

    def run_docs_django_tests(self,n,**kw):
        args = ["django-admin.py"]
        args += ["test"]
        args += ["--settings=%s" % n]
        args += ["--failfast"]
        args += ["--traceback"]
        args += ["--verbosity=0"]
        args += ["--pythonpath=%s" % self.project_root.child('docs')]
        self.run_subprocess(args,**kw)

    def run_django_manage_test(self,db,**kw):
        p = self.project_root.child(*db.split('/'))
        args = ["python","manage.py"]
        args += ["test"]
        #~ args += more
        args += ["--noinput"]
        args += ["--failfast"]
        #~ args += ["--settings=settings"]
        args += ["--pythonpath=%s" % p.absolute()]
        kw.update(cwd=p)
        self.run_subprocess(args,**kw)

    def run_django_admin_test(self,settings_module,*args,**kw):
        return self.run_django_admin_command(settings_module,'test',"--verbosity=0",*args,**kw)

    def run_django_admin_command(self,settings_module,*cmdargs,**kw):
        args = ["django-admin.py"]
        args += cmdargs
        args += ["--settings=%s" % settings_module]
        args += ["--noinput"]
        args += ["--failfast"]
        args += ["--traceback"]
        self.run_subprocess(args,**kw)

    def run_docs_doctests(self,filename):
        """
        Run a simple doctest for specified file after importing the
        docs `conf.py` (which causes the demo database to be activated).

        This is used e.g. for testing pages like those below
        :doc:`/tested/index`.

        These tests may fail for the simple reason that the demo database
        has not been initialized (in that case, run `fab initdb`).
        """
        filename = 'docs/' + filename
        #~ p = self.project_root.child(*filename.split('/')).parent
        #~ os.environ['DJANGO_SETTINGS_MODULE']='settings'
        #~ oldcwd = os.getcwd()
        #~ self.project_root.child('docs').chdir()
        #~ p.chdir()
        #~ sys.path.insert(0,'.')
        #~ print p
        sys.path.insert(0,'docs')
        import conf # trigger Django startup

        if False:
            # 20130828 test cases *are* required to restore the language afterwards.
            try:
                from north.dbutils import set_language
                set_language()
                """
                Each test case starts with the site's default language.
                Test cases are not required to restore the language afterwards.
                """
            except ImportError:
                pass # not everybody uses north

        res = doctest.testfile(filename, module_relative=False,encoding='utf-8')

        del sys.path[0]
        #~ os.chdir(oldcwd)

        #~ return res
        if res.failed:
            self.fail("doctest.testfile() failed. See earlier messages.")

<|endoftext|>


<commit_before>import sys
import asyncio

from discord.ext import commands
from bot_globals import *
from checks import *

#Create the logs and set up the bot
create_log(log_folder, bot_name)
bot = commands.Bot(command_prefix=('|', '$'))

#Change the bot's avatar and name if needed
async def update_profile():
    #Avatar
    if os.path.isfile(bot_pic):
        with open(bot_pic, "rb") as image:
            avatar = image.read()
            log_print("Profile image read.")
            try:
                await bot.edit_profile(avatar=avatar)
                log_print("Avatar changed.")
            except discord.InvalidArgument:
                log_print("Avatar not changed: Invalid image file")
    else:
        log_print("Unable to find bot_pic image: {0}. Picture will not be changed"
        .format(bot_pic))
    #Username
    try:
        await bot.edit_profile(username=bot_name)
    except discord.HTTPException as error:
        log_print("Unable to change bot username: {0}"
        .format(error))

#Start up the bot
@bot.event
async def on_ready():
    #Load extensions
    for e in extensions:
        try:
            bot.load_extension(e)
            extensions_loaded.append(e.split('.')[1])
        except Exception as error:
            exc = "{0}: {1}".format(type(error).__name__, error)
            log_print("Failed to load extension {0}, {1}".format(e, exc))

    #Update bot's profile
    await update_profile()

    log_print("------------------------STATUS------------------------")
    log_print("Ispyra {0} by Ispira using discord.py by Rapptz"
    .format(version))

    log_print("Contributors: {0}"
    .format(' '.join(contributors)))

    log_print("Logged in as: {0}"
    .format(bot.user.name))

    log_print("With ID: {0}"
    .format(bot.user.id))

    log_print("Botmasters: {0}"
    .format(', '.join(bot_masters)))

    log_print("Extensions: {0}"
    .format(', '.join(extensions_loaded)))

    log_print("Connected to:")
    for serv in bot.servers:
        log_print("{0},".format(serv.name))
        server_list.append(serv)

    log_print("------------------------STATUS------------------------")

## EVENTS ##
@bot.event
async def on_message(message):
    #Log the message unless the config says otherwise
    if log_messages:
        log_print("|{0}|[{1}]<{2}>: {3}"
        .format(message.server, message.channel, message.author, message.content))

    #Don't let the bot do anything with itself
    if message.author == bot.user:
        return

    #Handle the command
    await bot.process_commands(message)

@bot.event
async def on_command(command, ctx):
    #Handle logging commands
    destination = None
    if ctx.message.channel.is_private:
        destination = "DM"
    else:
        destination = "[{0.server.name}] #{0.channel.name}".format(ctx.message)

    if log_commands:
        log_print("Command `{0.content}` issued by `{0.author}`."
        .format(ctx.message))
        log_print("Location `{0}`".format(destination))

@bot.event
async def on_server_join(server):
    #Log it
    log_print("[JOINED] Server: {0}."
    .format(server.name))

    #Add to the list of currently connected servers
    if server not in server_list:
        server_list.append(server)

@bot.event
async def on_server_remove(server):
    #Log it
    log_print("[LEFT] Server: {0}"
    .format(server.name))

    #Remove the server from the server_list
    if server in server_list:
        server_list.remove(server)

@bot.event
async def on_server_update(before, after):
    #You know the drill by now
    log_print("[EDIT] Server: {0} was updated. {1}"
    .format(before.name, after.name))

    #Update the server_list
    if before in server_list:
        server_list.remove(before)
        server_list.append(after)

## COMMANDS ##
## These are considered "Base" commands that will always work
## Regardless of loaded extensions
## Completely exit the bot
@commands.command()
@prefix('$')
@is_owner()
async def quit(self):
    """Completely closes the bot."""
    await self.bot.say("Bye...")
    await self.bot.logout()

## Load extensions
@bot.command(pass_context=True)
@prefix('$')
@is_owner()
async def load(ctx, name: str):
    """Load an extension."""
    try:
        bot.load_extension("extensions.{0}".format(name))
        extensions_loaded.append(name)
        await bot.say("Loaded {0}.".format(name))
    except Exception as error:
        exc = "{0}: {1}".format(type(error).__name__, error)
        log_print("Failed to load extension {0}"
        .format(name, exc))

## Unload extensions
@bot.command(pass_context=True)
@prefix('$')
@is_owner()
async def unload(ctx, name: str):
    """Unload an extension."""
    try:
        if name in extensions_loaded:
            bot.unload_extension("extensions.{0}".format(name))
            await bot.say("Unloaded {0}.".format(name))
        else:
            await bot.say("No extension named {0} is loaded."
            .format(name))
    except Exception as error:
        exc = "{0}: {1}".format(type(error).__name__, error)
        log_print("Failed to unload extension {0}"
        .format(name, exc))

## Eval command for debugging
#You'll need to uncomment this if you want to use it.
#But don't do that.
#
#@bot.command(pass_context=True)
#@prefix('$')
#@is_owner()
#async def ev(ctx, *, code: str):
#    """Extremely unsafe eval command."""
#    code = code.strip("` ")
#    python = "```python\n{0}\n```"
#    result = None
#
#    try:
#        result = eval(code)
#    except Exception as error:
#        await bot.say(python.format(type(error).__name__ + ': ' + str(error)))
#        return
#
#    if asyncio.iscoroutine(result):
#        result = await result
#
#    await bot.say(python.format(result))

bot.run(bot_token)
sys.exit(0)
<commit_msg>Improvement to command log format<commit_after>import sys
import asyncio

from discord.ext import commands
from bot_globals import *
from checks import *

#Create the logs and set up the bot
create_log(log_folder, bot_name)
bot = commands.Bot(command_prefix=('|', '$'))

#Change the bot's avatar and name if needed
async def update_profile():
    #Avatar
    if os.path.isfile(bot_pic):
        with open(bot_pic, "rb") as image:
            avatar = image.read()
            log_print("Profile image read.")
            try:
                await bot.edit_profile(avatar=avatar)
                log_print("Avatar changed.")
            except discord.InvalidArgument:
                log_print("Avatar not changed: Invalid image file")
    else:
        log_print("Unable to find bot_pic image: {0}. Picture will not be changed"
        .format(bot_pic))
    #Username
    try:
        await bot.edit_profile(username=bot_name)
    except discord.HTTPException as error:
        log_print("Unable to change bot username: {0}"
        .format(error))

#Start up the bot
@bot.event
async def on_ready():
    #Load extensions
    for e in extensions:
        try:
            bot.load_extension(e)
            extensions_loaded.append(e.split('.')[1])
        except Exception as error:
            exc = "{0}: {1}".format(type(error).__name__, error)
            log_print("Failed to load extension {0}, {1}".format(e, exc))

    #Update bot's profile
    await update_profile()

    log_print("------------------------STATUS------------------------")
    log_print("Ispyra {0} by Ispira using discord.py by Rapptz"
    .format(version))

    log_print("Contributors: {0}"
    .format(' '.join(contributors)))

    log_print("Logged in as: {0}"
    .format(bot.user.name))

    log_print("With ID: {0}"
    .format(bot.user.id))

    log_print("Botmasters: {0}"
    .format(', '.join(bot_masters)))

    log_print("Extensions: {0}"
    .format(', '.join(extensions_loaded)))

    log_print("Connected to:")
    for serv in bot.servers:
        log_print("{0},".format(serv.name))
        server_list.append(serv)

    log_print("------------------------STATUS------------------------")

## EVENTS ##
@bot.event
async def on_message(message):
    #Log the message unless the config says otherwise
    if log_messages:
        log_print("|{0}|[{1}]<{2}>: {3}"
        .format(message.server, message.channel, message.author, message.content))

    #Don't let the bot do anything with itself
    if message.author == bot.user:
        return

    #Handle the command
    await bot.process_commands(message)

@bot.event
async def on_command(command, ctx):
    #Handle logging commands
    destination = None
    if ctx.message.channel.is_private:
        destination = "DM"
    else:
        destination = "[{0.server.name}] #{0.channel.name}".format(ctx.message)

    if log_commands:
        log_print("Command `{0.content}` issued by `{0.author}` in: {1}"
        .format(ctx.message, destination))

@bot.event
async def on_server_join(server):
    #Log it
    log_print("[JOINED] Server: {0}."
    .format(server.name))

    #Add to the list of currently connected servers
    if server not in server_list:
        server_list.append(server)

@bot.event
async def on_server_remove(server):
    #Log it
    log_print("[LEFT] Server: {0}"
    .format(server.name))

    #Remove the server from the server_list
    if server in server_list:
        server_list.remove(server)

@bot.event
async def on_server_update(before, after):
    #You know the drill by now
    log_print("[EDIT] Server: {0} was updated. {1}"
    .format(before.name, after.name))

    #Update the server_list
    if before in server_list:
        server_list.remove(before)
        server_list.append(after)

## COMMANDS ##
## These are considered "Base" commands that will always work
## Regardless of loaded extensions
## Completely exit the bot
@commands.command()
@prefix('$')
@is_owner()
async def quit(self):
    """Completely closes the bot."""
    await self.bot.say("Bye...")
    await self.bot.logout()

## Load extensions
@bot.command(pass_context=True)
@prefix('$')
@is_owner()
async def load(ctx, name: str):
    """Load an extension."""
    try:
        bot.load_extension("extensions.{0}".format(name))
        extensions_loaded.append(name)
        await bot.say("Loaded {0}.".format(name))
    except Exception as error:
        exc = "{0}: {1}".format(type(error).__name__, error)
        log_print("Failed to load extension {0}"
        .format(name, exc))

## Unload extensions
@bot.command(pass_context=True)
@prefix('$')
@is_owner()
async def unload(ctx, name: str):
    """Unload an extension."""
    try:
        if name in extensions_loaded:
            bot.unload_extension("extensions.{0}".format(name))
            await bot.say("Unloaded {0}.".format(name))
        else:
            await bot.say("No extension named {0} is loaded."
            .format(name))
    except Exception as error:
        exc = "{0}: {1}".format(type(error).__name__, error)
        log_print("Failed to unload extension {0}"
        .format(name, exc))

## Eval command for debugging
#You'll need to uncomment this if you want to use it.
#But don't do that.
#
#@bot.command(pass_context=True)
#@prefix('$')
#@is_owner()
#async def ev(ctx, *, code: str):
#    """Extremely unsafe eval command."""
#    code = code.strip("` ")
#    python = "```python\n{0}\n```"
#    result = None
#
#    try:
#        result = eval(code)
#    except Exception as error:
#        await bot.say(python.format(type(error).__name__ + ': ' + str(error)))
#        return
#
#    if asyncio.iscoroutine(result):
#        result = await result
#
#    await bot.say(python.format(result))

bot.run(bot_token)
sys.exit(0)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
This module has functions that generated random values for django fields.
"""
from random import choice
from random import randint

def generate_integer(bits, negative_allowed=True):
    length = randint(1, bits - 1) - 1
    positive = True
    if negative_allowed:
        positive = bool(randint(0, 1))
    if positive:
        low = (1 << length)
        high = 2 * low - 1
        if low == 1:
            low = 0
        return randint(low, high)
    else:
        high = -(1 << length) - 1
        low = 2 * (high + 1)
        if high == -2:
            high = -1
        return randint(low, high)

def generate_big_integer():
    return generate_integer(64)

def generate_int():
    return generate_integer(32)

def generate_small_integer():
    return generate_integer(16)

def generate_positive_integer():
    return generate_integer(32, False)

def generate_positive_small_integer():
    return generate_integer(16, False)
<commit_msg>Added boolean generating function.<commit_after>#!/usr/bin/env python
"""
This module has functions that generated random values for django fields.
"""
from random import choice
from random import randint

def generate_integer(bits, negative_allowed=True):
    length = randint(1, bits - 1) - 1
    positive = True
    if negative_allowed:
        positive = bool(randint(0, 1))
    if positive:
        low = (1 << length)
        high = 2 * low - 1
        if low == 1:
            low = 0
        return randint(low, high)
    else:
        high = -(1 << length) - 1
        low = 2 * (high + 1)
        if high == -2:
            high = -1
        return randint(low, high)

def generate_big_integer():
    return generate_integer(64)

def generate_int():
    return generate_integer(32)

def generate_small_integer():
    return generate_integer(16)

def generate_positive_integer():
    return generate_integer(32, False)

def generate_positive_small_integer():
    return generate_integer(16, False)

def generate_boolean(null_allowed=False):
    res = randint(0, 1 + int(null_allowed))
    if res < 2:
        return bool(res)
<|endoftext|>


<commit_before>r""" In this model I want to test the applicability of efficiency to recovering unbiased
results from a biased set of data.

To this end, I implement the simplest toy model, where we are observing points scattered
from their true value by observational error. The true underlying value is given by :math:`\mu`.

We generate observed data points by adding in noise (our error), which takes the form
of a normal distribution centered around the true value :math:`\mu` with standard
deviation :math:`e`, where :math:`e` is some arbitrary number currently set to
:math:`\mu/5`, where :math:`\mu` is the true value, not the current value in the model.
Denote the observed data point :math:`d`, and consider the reported error :math:`e`
an observed value as well. For notational simplicity, I will be treating :math:`d` as a
single value and not the actual vector that it is. For more rigorous vector treatment,
see the next example.

Now, we bias our sample, by discarding all points which have a signal to noise
(:math:`d/e`) less than some threshold :math:`\alpha`.

Like normal we seek to construct the posterior

.. math::
    P(\mu|d,e) \propto P(d,e|\mu) P(\mu)

Considering a flat prior on :math:`P(\mu)` and dropping the term as a constant
multiplier, we now have to consider the likelihood of observing our data :math:`d`
given our model :math:`\mu`. Given we bias our data, this becomes the probability of
the data being generated at :math:`d` given :math:`\mu`, multiplied by the probability
that the data :math:`d` *also* makes it into our final dataset and is not dropped. As
such we define the likelihood :math:`\mathcal{L} = P(d,e|\mu)` as

.. math::
    \mathcal{L} = \frac{\epsilon(d,e) P_g(d,e|\mu)}
    {\int dR \ \epsilon(R,e) P_g(R,e|\mu)}

where :math:`R` is used to denote a potential realisation of the data, given the underlying
model and the subscript :math:`g` is used to denote "generated".

.. math::
    \mathcal{L} = \frac{\epsilon(d,e) P_g(d,e|\mu)}{\int dR \ \epsilon(R,e) P_g(R,e|\mu) }

So, given a data point :math:`R` with observational error :math:`e`, and
underlying mean :math:`\mu`, we have that

.. math::
    \int dR \ \epsilon(R,e) P_g(R|\mu) = \int_{\alpha e}^{\infty}
    \frac{1}{\sqrt{2\pi}e} \exp\left[ -\frac{(R-\mu)^2}{2 e^2} \right] dR

Evaluating this by transforming coordinate to :math:`x = R-\mu` such that we get

.. math::
    \int dR \ \epsilon(R,e) P_g(R|\mu) = \int_{\alpha e - \mu}^{\infty}
    \frac{1}{\sqrt{2\pi}e} \exp\left[ -\frac{x^2}{2 e^2} \right] dx

gives the answer

.. math::
    \int dR \ \epsilon(R,e) P_g(R|\mu) = \frac{1}{2} {\rm erfc}
    \left[ \frac{\alpha e - \mu}{\sqrt{2} e} \right]

We can not implement this correction, and then implement it, and hopefully see that the
recovered underlying distribution becomes unbiased. I plot three realisations of the data
to confirm that the effect is not by change.

The model PGM:

.. figure::     ../dessn/proofs/efficiency_1/output/pgm.png
    :width:     60%
    :align:     center

The posterior surfaces for both corrected (blue) and uncorrected (red) models.

.. figure::     ../dessn/proofs/efficiency_1/output/surfaces.png
    :align:     center
    :width:     80%

"""<commit_msg>Updating the first efficiency example<commit_after>r""" In this model I want to test the applicability of efficiency to recovering unbiased
results from a biased set of data.

To this end, I implement the simplest toy model, where we are observing points scattered
from their true value by observational error. The true underlying value is given by :math:`\mu`,
and given an error :math:`e`, we have draw our observed value from

.. math::
    d \sim \mathcal{N}(\mu, e)

For notational simplicity, I will be treating :math:`d` as a single value and not
the actual vector that it is. For more rigorous vector treatment, see the next example.

Now, we bias our sample, by discarding all points which have a signal to noise
(:math:`d/e`) less than some threshold :math:`\alpha`. Phrased differently,
all our data samples must now satisfy the condition that :math:`D - \alpha e > 0`. Note
that we denote a data sample here as :math:`D`, which is distinct to the experimental
data :math:`d`. :math:`D` is simply our model stating that *any* input data must
satisfy the selection criterion.

Like normal we seek to construct the posterior

.. math::
    P(\mu|d) \propto P(D = d|D > \alpha e, \mu) P(\mu)

Considering a flat prior on :math:`P(\mu)` and dropping the term as a constant
multiplier, and reducing the expression simply to a likelihood

.. math::
    \mathcal{L} = P(D = d|D > \alpha e, \mu)

In English, this can be read as: *"Given the current model parameters, and our
stipulation that the input data satisfy our selection criterion, what is
the probability of realising the experimental data"*.

Now, performing some mathematics, we can get this to a better state:

.. math::
    \mathcal{L} = P(D = d|D > \alpha e, \mu)

    \mathcal{L} = \frac{P(D=d, D>\alpha e | \mu)}{P(D > \alpha e | \mu)}

It is important to note here that, by itself, :math:`P(D > \alpha e | \mu)` is not
quantifiable until :math:`D` is set to some realisation of the data. In other
words, :math:`D` represents *"the input data"*, and thus requires input data to
evaluate. We do this by introducing an integral over all data realisations. Note that
this is identical to enforcing the normalisation of the likelihood over all data (which
must be true).

.. math::
    \mathcal{L} = \frac{P(D=d, D>\alpha e | \mu) }{\int dR\ P(D > \alpha e, D=R | \mu)}

    \mathcal{L} = \frac{P(d, d>\alpha e | \mu) }{\int dR\ P(R > \alpha e, R | \mu)}

    \mathcal{L} = \frac{P(d>\alpha e | d \mu) P(d|\mu)}{\int dR\ P(R > \alpha e, R | \mu) P(R|\mu)}

We call the term :math:`P(d>\alpha e | d \mu)` the efficiency, for it is the probability
that, given input data and a model parametrisation, our data passes our cuts. We denote this
function :math:`\epsilon(d)`, where we should note that the other arguments on which the
efficiency depends (:math:`\mu`, :math:`\alpha` and :math:`e`) are not written
for convenience as they are either held constant (:math:`\alpha` and :math:`e`)
or :math:`\epsilon` is independent of them (in the case of :math:`\mu`).

.. math::
    \mathcal{L} = \frac{\epsilon(d) P(d|\mu)}{\int dR\ \epsilon(R) P(R|\mu)}

As stated previously, we model :math:`P(d|\mu)` with distribution :math:`\mathcal{N}(\mu, e)`.
The efficiency of all actually observed data must be 1 (as it *did* pass our cuts), so
:math:`\epsilon(d) = 1`. The efficiency of all realisations is more complicated, and
it takes the form of a Heaviside step function :math:`\mathcal{H}(R - \alpha e)`. Putting
this together:

.. math::
    \mathcal{L} = \frac{\mathcal{N}(d;\mu, e)}
    {\int_{-\infty}^\infty dR\ \mathcal{H}(\alpha e) \mathcal{N}(R;\mu, e)}

Using the Heaviside step function to modify the integral limits, we see that the
denominator becomes a truncated cumulative normal distribution function.

.. math::
    \mathcal{L} = \frac{\mathcal{N}(d;\mu,e)}{\int_{\alpha e}^\infty dR\ \mathcal{N}(R;\mu,e)}

Looking only at the denominator:

.. math::
    \int_{\alpha e}^\infty dR\ \mathcal{N}(R;\mu,e) =
    \int_{\alpha e}^{\infty}\frac{1}{\sqrt{2\pi}e} \exp\left[ -\frac{(R-\mu)^2}{2 e^2} \right] dR

Evaluating this by transforming coordinate to :math:`x = R-\mu` such that we get

.. math::
    \int dR \ \epsilon(R,e) P_g(R|\mu) = \int_{\alpha e - \mu}^{\infty}
    \frac{1}{\sqrt{2\pi}e} \exp\left[ -\frac{x^2}{2 e^2} \right] dx

gives the answer

.. math::
     \int_{\alpha e}^{\infty} dR\ \mathcal{N}(R;\mu,e) = \begin{cases}
    \frac{1}{2} - \frac{1}{2}{\rm erf} \left[ \frac{\alpha e - \mu}{\sqrt{2} e} \right] &
    \text{ if } \alpha e - \mu > 0 \\
    \frac{1}{2} + \frac{1}{2}{\rm erf} \left[ \frac{\mu- \alpha e}{\sqrt{2} e} \right] &
    \text{ if } \alpha e - \mu < 0 \\
    \end{cases}

Restricting ourselves to :math:`\mu > \alpha e`, we have

.. math::
     \int_{\alpha e}^{\infty} dR\ \mathcal{N}(R;\mu,e) =
     \frac{1}{2} + \frac{1}{2}{\rm erf} \left[ \frac{\mu- \alpha e}{\sqrt{2} e} \right]

     \int_{\alpha e}^{\infty} dR\ \mathcal{N}(R;\mu,e) =
     \frac{1}{2}{\rm erfc} \left[ \frac{\alpha e - \mu}{\sqrt{2} e} \right]

Giving a final expression for our likelihood (and posterior in this case) of

.. math::
    P(\mu|d) \propto \frac{\mathcal{N}(d;\mu,e)}
    {\frac{1}{2}{\rm erfc} \left[ \frac{\alpha e - \mu}{\sqrt{2} e} \right]}

We can fit biased data (data which has gone through the selection cut) with and without
the :math:`{\rm erfc}` correction, to simulate respectively if we model the bias or
do not model the bias.

The model PGM:

.. figure::     ../dessn/proofs/efficiency_1/output/pgm.png
    :width:     60%
    :align:     center

I plot three realisations of the data to confirm that the effect is not by change,
and show the parameters recovered when correcting for the bias (blue) and the
parameters recovered when you do not correct for the bias (red).

.. figure::     ../dessn/proofs/efficiency_1/output/surfaces.png
    :align:     center
    :width:     80%

"""<|endoftext|>


<commit_before>"""
Test manual user files, made by real humans, with all metrics handcrafted.
"""

import bandicoot as bc
import unittest
from testing_tools import parse_dict, metric_suite
import os

class TestManual(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls._dir_changed = False

    def setUp(self):
        if not TestManual._dir_changed:
            abspath = os.path.abspath(__file__)
            name = abspath.index(os.path.basename(__file__))
            abspath = abspath[:name]
            os.chdir(abspath)
            TestManual._dir_changed = True

        self.user_A = bc.io.read_csv("A", "samples/manual", "samples/towers.csv", describe=False)
        self.user_B = bc.io.read_csv("B", "samples/manual", describe=False)

    def test_A_metrics(self):
        self.assertTrue(*metric_suite(self.user_A, parse_dict("samples/manual/A.json"), groupby=None, decimal=4))

    def test_B_metrics(self):
        self.assertTrue(*metric_suite(self.user_B, parse_dict("samples/manual/B.json"), groupby=None, decimal=3))

    def test_A_orange_metrics(self):
        self.user_A_orange = bc.io.read_orange("samples/manual/A_orange.csv", describe=False)
        self.assertTrue(*metric_suite(self.user_A_orange, parse_dict("samples/manual/A.json"), groupby=None, decimal=4))

    def test_A_telenor_metrics(self):
        self.user_A_telenor = bc.io.read_telenor("samples/manual/A_telenor_incoming","samples/manual/A_telenor_outgoing","samples/manual/A_telenor_cell_towers", describe=False)
        self.assertTrue(*metric_suite(self.user_A_telenor, parse_dict("samples/manual/A.json"), groupby=None, decimal=4))

<commit_msg>Added tests for reading of sample manual user A in telenor and orange formats<commit_after>"""
Test manual user files, made by real humans, with all metrics handcrafted.
"""

import bandicoot as bc
import unittest
from testing_tools import parse_dict, metric_suite
import os

class TestManual(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls._dir_changed = False

    def setUp(self):
        if not TestManual._dir_changed:
            abspath = os.path.abspath(__file__)
            name = abspath.index(os.path.basename(__file__))
            abspath = abspath[:name]
            os.chdir(abspath)
            TestManual._dir_changed = True

        self.user_A = bc.io.read_csv("A", "samples/manual", "samples/towers.csv", describe=False)
        self.user_B = bc.io.read_csv("B", "samples/manual", describe=False)

    def test_A_metrics(self):
        self.assertTrue(*metric_suite(self.user_A, parse_dict("samples/manual/A.json"), groupby=None, decimal=4))

    def test_B_metrics(self):
        self.assertTrue(*metric_suite(self.user_B, parse_dict("samples/manual/B.json"), groupby=None, decimal=3))

    def test_A_orange_metrics(self):
        self.user_A_orange = bc.io.read_orange("samples/manual/A_orange.csv", describe=False)
        self.assertTrue(*metric_suite(self.user_A_orange, parse_dict("samples/manual/A.json"), groupby=None, decimal=4))

    def test_A_telenor_metrics(self):
        self.user_A_telenor = bc.io.read_telenor("samples/manual/A_telenor_incoming","samples/manual/A_telenor_outgoing","samples/manual/A_telenor_cell_towers", describe=False)
        self.assertTrue(*metric_suite(self.user_A_telenor, parse_dict("samples/manual/A.json"), groupby=None, decimal=4))<|endoftext|>


<commit_before>from django.conf import settings

if settings.DEBUG:
    TO_LIST = [settings.SERVER_EMAIL,]
else:
    TO_LIST = [
        settings.CONTINUING_STUDIES_ENRICHMENT_REGISTRATION_EMAIL,
    ]
BCC = settings.MANAGERS
<commit_msg>updated TO_LIST with GPS emails and admin manager<commit_after>from django.conf import settings

if settings.DEBUG:
    #TO_LIST = [settings.SERVER_EMAIL,]
    TO_LIST = [
        settings.CONTINUING_STUDIES_ENRICHMENT_REGISTRATION_EMAIL,
        settings.ADMINS[0][1]
    ]
else:
    TO_LIST = [
        settings.CONTINUING_STUDIES_ENRICHMENT_REGISTRATION_EMAIL,
        settings.ADMINS[0][1]
    ]
BCC = settings.MANAGERS
<|endoftext|>