<commit_before>
"""
    Copyright (c) 2015-2016 Cisco Systems, Inc. and others.  All rights reserved.
    This program and the accompanying materials are made available under the
    terms of the Eclipse Public License v1.0 which accompanies this distribution,
    and is available at http://www.eclipse.org/legal/epl-v10.html
"""

class Message(object):
    """
        Kafka Message class. This class processes header of raw Kafka messages.
    """

    def __init__(self, data):
        """
        Handle the message by parsing header of it.

        :param data: Raw Kafka message as string.
        """

        if not data.strip(): # If "data" is not string, throws error.
            raise "Invalid data!", data

        self.version = float()
        self.collector_hash_id = str()
        self.length = long()
        self.records = long()
        self.router_hash_id = str()
        self.content = str()
        self.content_pos = int()
        self.router_ip = str()

        self.__parse(data)

    def __parse(self, data):
        """
        Parses header of raw Kafka messages and set the version, length, number of records and router hash id.

        :param data: Raw Kafka message as string.
        """

        data_end_pos = data.rfind("\n\n")
        header_data = data[:data_end_pos]

        self.content_pos = data_end_pos + 2;
        self.content = data[self.content_pos:]

        headers = header_data.split("\n");

        for header in headers:
            value = header.split(":")[1].strip();
            attr = header.split(":")[0].strip();

            # Attribute names are from http://openbmp.org/#!docs/MESSAGE_BUS_API.md headers
            if attr == "V":
                self.version = float(value);

            elif attr == "C_HASH_ID":
                self.collector_hash_id = value

            elif attr == "L":
                self.length = long(value)

            elif attr == "R":
                self.records = long(value)

            elif attr == "R_HASH_ID":
                self.router_hash_id = value

            elif attr == "R_IP":
                self.router_ip = value

    # Getters
    def getVersion(self):
        return self.version

    def getCollector_hash_id(self):
        return self.collector_hash_id

    def getLength(self):
        return self.length

    def getRecords(self):
        return self.records

    def getRouter_hash_id(self):
        return self.router_hash_id

    def getRouterIp(self):
        return self.router_ip

    def getContentPos(self):
        return self.content_pos

    def getContent(self):
        return self.content<commit_msg>- In the Message parse function, rfind() function  is fixed to find().<commit_after>
"""
    Copyright (c) 2015-2016 Cisco Systems, Inc. and others.  All rights reserved.
    This program and the accompanying materials are made available under the
    terms of the Eclipse Public License v1.0 which accompanies this distribution,
    and is available at http://www.eclipse.org/legal/epl-v10.html
"""

class Message(object):
    """
        Kafka Message class. This class processes header of raw Kafka messages.
    """

    def __init__(self, data):
        """
        Handle the message by parsing header of it.

        :param data: Raw Kafka message as string.
        """

        if not data.strip(): # If "data" is not string, throws error.
            raise "Invalid data!", data

        self.version = float()
        self.collector_hash_id = str()
        self.length = long()
        self.records = long()
        self.router_hash_id = str()
        self.content = str()
        self.content_pos = int()
        self.router_ip = str()

        self.__parse(data)

    def __parse(self, data):
        """
        Parses header of raw Kafka messages and set the version, length, number of records and router hash id.

        :param data: Raw Kafka message as string.
        """

        data_end_pos = data.find("\n\n")
        header_data = data[:data_end_pos]

        self.content_pos = data_end_pos + 2;
        self.content = data[self.content_pos:]

        headers = header_data.split("\n");

        for header in headers:
            value = header.split(":")[1].strip();
            attr = header.split(":")[0].strip();

            # Attribute names are from http://openbmp.org/#!docs/MESSAGE_BUS_API.md headers
            if attr == "V":
                self.version = float(value);

            elif attr == "C_HASH_ID":
                self.collector_hash_id = value

            elif attr == "L":
                self.length = long(value)

            elif attr == "R":
                self.records = long(value)

            elif attr == "R_HASH_ID":
                self.router_hash_id = value

            elif attr == "R_IP":
                self.router_ip = value

    # Getters
    def getVersion(self):
        return self.version

    def getCollector_hash_id(self):
        return self.collector_hash_id

    def getLength(self):
        return self.length

    def getRecords(self):
        return self.records

    def getRouter_hash_id(self):
        return self.router_hash_id

    def getRouterIp(self):
        return self.router_ip

    def getContentPos(self):
        return self.content_pos

    def getContent(self):
        return self.content<|endoftext|>


<commit_before>import things
from arcade.models import ArcadeCategory, Game

class GameThing(things.Thing):
    game_plays = things.OrderField(
        verbose_name_asc='Play Count',
        verbose_name_desc='Play Count',
        url_asc='least-played',
        url_desc='most-played',
    )
    created = things.OrderField(
        verbose_name_asc='Date Added',
        verbose_name_desc='Date Added',
        url_asc='oldest',
        url_desc='newest'
    )
    name = things.OrderField(
        verbose_name_asc='Alphabetical',
        verbose_name_desc='Alphabetical',
    )
    template_dir = 'games'

class GameThingGroup(things.ThingGroup):
    template_dir = 'games'
    template_name = 'category_list.html'

def group_dicts(thing):
    dicts = [
        {
            'queryset': Game.objects.all(),
            'name': 'All',
            'url_name': 'all',
        },
    ]
    for category in ArcadeCategory.objects.all():
        dicts.append({
            'queryset': category.game_set.all(),
            'name': category.name,
            'url_name': category.slug,
        })
    return dicts<commit_msg>Added support for a default sort ordering field of play count for games.<commit_after>import things
from arcade.models import ArcadeCategory, Game

class GameThing(things.Thing):
    game_plays = things.OrderField(
        verbose_name_asc='Play Count',
        verbose_name_desc='Play Count',
        url_asc='least-played',
        url_desc='most-played',
        default_order_field=things.DESCENDING,
    )
    created = things.OrderField(
        verbose_name_asc='Date Added',
        verbose_name_desc='Date Added',
        url_asc='oldest',
        url_desc='newest'
    )
    name = things.OrderField(
        verbose_name_asc='Alphabetical',
        verbose_name_desc='Alphabetical',
    )
    template_dir = 'games'

class GameThingGroup(things.ThingGroup):
    template_dir = 'games'
    template_name = 'category_list.html'

def group_dicts(thing):
    dicts = [
        {
            'queryset': Game.objects.all(),
            'name': 'All',
            'url_name': 'all',
        },
    ]
    for category in ArcadeCategory.objects.all():
        dicts.append({
            'queryset': category.game_set.all(),
            'name': category.name,
            'url_name': category.slug,
        })
    return dicts<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Execute the tests for the fiona program.

The golden test outputs are generated by the script generate_outputs.sh.

You have to give the root paths to the source and the binaries as arguments to
the program.  These are the paths to the directory that contains the 'projects'
directory.

Usage:  run_tests.py SOURCE_ROOT_PATH BINARY_ROOT_PATH
"""
import logging
import os.path
import subprocess
import sys

# Automagically add util/py_lib to PYTHONPATH environment variable.
path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..',
                                    '..', '..', 'util', 'py_lib'))
sys.path.insert(0, path)

import seqan.app_tests as app_tests

class ResultChecker(object):
    """Check quality with compute_gain program."""

    def __init__(self, compute_gain, path_to_ref, path_to_sam,
                path_to_post, min_gain):
        self.compute_gain = compute_gain
        self.path_to_ref = path_to_ref
        self.path_to_sam = path_to_sam
        self.path_to_post = path_to_post
        self.min_gain = min_gain

    def __call__(self):
        cmd_line = [self.compute_gain,
                    '-g', self.path_to_ref,
                    '--pre', self.path_to_sam,
                    '--post', self.path_to_post]
        print ' '.join(cmd_line)
        process = subprocess.Popen(cmd_line, stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE)
        retcode = process.wait()
        if retcode != 0:
            raise app_tests.BadResultException('compute_gain did not return 0')
        report = process.stdout.read()
        gain = float(report.splitlines()[2].split()[0])
        if gain < self.min_gain:
            fmt = 'Gain too low. Expected >= %f, got %f'
            raise app_tests.BadResultException(fmt % (self.min_gain, gain))
        return True

def main(source_base, binary_base):
    """Main entry point of the script."""

    print 'Executing test for fiona'
    print '========================'
    print

    ph = app_tests.TestPathHelper(
        source_base, binary_base,
        'apps/fiona/tests')  # tests dir

    # ============================================================
    # Auto-detect the binary path.
    # ============================================================

    path_to_fiona = app_tests.autolocateBinary(
      binary_base, 'apps/fiona', 'fiona')
    path_to_fiona_illumina = app_tests.autolocateBinary(
      binary_base, 'apps/fiona', 'fiona_illumina')
    path_to_compute_gain = app_tests.autolocateBinary(
      binary_base, 'apps/fiona', 'compute_gain')

    # ============================================================
    # Built TestConf list.
    # ============================================================

    # Build list with TestConf objects, analoguely to how the output
    # was generated in generate_outputs.sh.
    conf_list = []

    # We prepare a list of transforms to apply to the output files.  This is
    # used to strip the input/output paths from the programs' output to
    # make it more canonical and host independent.
    ph.outFile('-')  # To ensure that the out path is set.
    transforms = [
        app_tests.ReplaceTransform(
            os.path.join(ph.source_base_path,
                         'apps/fiona/tests') + os.sep,
            '', right=True),
        app_tests.ReplaceTransform(ph.temp_dir + os.sep, '', right=True),
        app_tests.NormalizeScientificExponentsTransform(),
        ]

    # ============================================================
    # Run on uniformly random DNA.
    # ============================================================

    # Note that instead of comparing the results with expected results, we
    # use a checker that computes the gain and compares it with a threshold.

    # Illumina Mode

    for i in [1, 2]:
        min_gain = {1: 40.0, 2: 50.0}
        conf = app_tests.TestConf(
            program=path_to_fiona_illumina,
            args=['-nt', '1',
                  '-i', str(i),
                  '-g', '10000',
                  ph.inFile('reads.illumina.fq'),
                  ph.outFile('reads.illumina.corrected.i%d.fa' % i)],
            redir_stdout=ph.outFile('reads.illumina.fq.i%d.stdout' % i),
            redir_stderr=ph.outFile('reads.illumina.fq.i%d.stderr' % i),
            check_callback=ResultChecker(
                path_to_compute_gain, ph.inFile('genome.10k.fa'),
                ph.inFile('reads.illumina.sam'),
                ph.outFile('reads.illumina.corrected.i%d.fa' % i),
                min_gain.get(i, 100.0)),
            to_diff=[(ph.inFile('reads.illumina.fq.i%d.stdout' % i),
                      ph.outFile('reads.illumina.fq.i%d.stdout' % i),
                      transforms),
                     (ph.inFile('reads.illumina.fq.i%d.stderr' % i),
                      ph.outFile('reads.illumina.fq.i%d.stderr' % i),
                      transforms),
                    ])
        conf_list.append(conf)

    # Indel Mode

    for i in [1, 2]:
        min_gain = {1: 70.0, 2: 85.0}
        conf = app_tests.TestConf(
            program=path_to_fiona,
            args=['-nt', '1',
                  '-i', str(i),
                  '-g', '10000',
                  ph.inFile('reads.454.fq'),
                  ph.outFile('reads.454.corrected.i%d.fa' % i)],
            redir_stdout=ph.outFile('reads.454.fq.i%d.stdout' % i),
            redir_stderr=ph.outFile('reads.454.fq.i%d.stderr' % i),
            check_callback=ResultChecker(
                path_to_compute_gain, ph.inFile('genome.10k.fa'),
                ph.inFile('reads.454.sam'),
                ph.outFile('reads.454.corrected.i%d.fa' % i),
                min_gain.get(i, 100.0)),
            to_diff=[(ph.inFile('reads.454.fq.i%d.stdout' % i),
                      ph.outFile('reads.454.fq.i%d.stdout' % i),
                      transforms),
                     (ph.inFile('reads.454.fq.i%d.stderr' % i),
                      ph.outFile('reads.454.fq.i%d.stderr' % i),
                      transforms),
                    ])
        conf_list.append(conf)

    # Execute the tests.
    failures = 0
    for conf in conf_list:
        res = app_tests.runTest(conf)
        # Output to the user.
        print ' '.join(conf.commandLineArgs())
        if res:
             print 'OK'
        else:
            failures += 1
            print 'FAILED'

    # Cleanup.
    ph.deleteTempDir()

    print '=============================='
    print '     total tests: %d' % len(conf_list)
    print '    failed tests: %d' % failures
    print 'successful tests: %d' % (len(conf_list) - failures)
    print '=============================='
    # Compute and return return code.
    return failures != 0

if __name__ == '__main__':
    sys.exit(app_tests.main(main))
<commit_msg>[FIX] Fixes timeout due to deadlock in python runt_test script on windows platforms.<commit_after>#!/usr/bin/env python
"""Execute the tests for the fiona program.

The golden test outputs are generated by the script generate_outputs.sh.

You have to give the root paths to the source and the binaries as arguments to
the program.  These are the paths to the directory that contains the 'projects'
directory.

Usage:  run_tests.py SOURCE_ROOT_PATH BINARY_ROOT_PATH
"""
import logging
import os.path
import subprocess
import sys

# Automagically add util/py_lib to PYTHONPATH environment variable.
path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..',
                                    '..', '..', 'util', 'py_lib'))
sys.path.insert(0, path)

import seqan.app_tests as app_tests

class ResultChecker(object):
    """Check quality with compute_gain program."""

    def __init__(self, compute_gain, path_to_ref, path_to_sam,
                path_to_post, min_gain):
        self.compute_gain = compute_gain
        self.path_to_ref = path_to_ref
        self.path_to_sam = path_to_sam
        self.path_to_post = path_to_post
        self.min_gain = min_gain

    def __call__(self):
        cmd_line = [self.compute_gain,
                    '-g', self.path_to_ref,
                    '--pre', self.path_to_sam,
                    '--post', self.path_to_post]
        print ' '.join(cmd_line)
        process = subprocess.Popen(cmd_line, stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE)
        stdoutbuff, stderrbuff = process.communicate()
        retcode = process.poll()
        if retcode != 0:
            raise app_tests.BadResultException('compute_gain did not return 0')
        gain = float(stdoutbuff.splitlines()[2].split()[0])
        if gain < self.min_gain:
            fmt = 'Gain too low. Expected >= %f, got %f'
            raise app_tests.BadResultException(fmt % (self.min_gain, gain))
        return True

def main(source_base, binary_base):
    """Main entry point of the script."""

    print 'Executing test for fiona'
    print '========================'
    print

    ph = app_tests.TestPathHelper(
        source_base, binary_base,
        'apps/fiona/tests')  # tests dir

    # ============================================================
    # Auto-detect the binary path.
    # ============================================================

    path_to_fiona = app_tests.autolocateBinary(
      binary_base, 'apps/fiona', 'fiona')
    path_to_fiona_illumina = app_tests.autolocateBinary(
      binary_base, 'apps/fiona', 'fiona_illumina')
    path_to_compute_gain = app_tests.autolocateBinary(
      binary_base, 'apps/fiona', 'compute_gain')

    # ============================================================
    # Built TestConf list.
    # ============================================================

    # Build list with TestConf objects, analoguely to how the output
    # was generated in generate_outputs.sh.
    conf_list = []

    # We prepare a list of transforms to apply to the output files.  This is
    # used to strip the input/output paths from the programs' output to
    # make it more canonical and host independent.
    ph.outFile('-')  # To ensure that the out path is set.
    transforms = [
        app_tests.ReplaceTransform(
            os.path.join(ph.source_base_path,
                         'apps/fiona/tests') + os.sep,
            '', right=True),
        app_tests.ReplaceTransform(ph.temp_dir + os.sep, '', right=True),
        app_tests.NormalizeScientificExponentsTransform(),
        ]

    # ============================================================
    # Run on uniformly random DNA.
    # ============================================================

    # Note that instead of comparing the results with expected results, we
    # use a checker that computes the gain and compares it with a threshold.

    # Illumina Mode

    for i in [1, 2]:
        min_gain = {1: 40.0, 2: 50.0}
        conf = app_tests.TestConf(
            program=path_to_fiona_illumina,
            args=['-nt', '1',
                  '-i', str(i),
                  '-g', '10000',
                  ph.inFile('reads.illumina.fq'),
                  ph.outFile('reads.illumina.corrected.i%d.fa' % i)],
            redir_stdout=ph.outFile('reads.illumina.fq.i%d.stdout' % i),
            redir_stderr=ph.outFile('reads.illumina.fq.i%d.stderr' % i),
            check_callback=ResultChecker(
                path_to_compute_gain, ph.inFile('genome.10k.fa'),
                ph.inFile('reads.illumina.sam'),
                ph.outFile('reads.illumina.corrected.i%d.fa' % i),
                min_gain.get(i, 100.0)),
            to_diff=[(ph.inFile('reads.illumina.fq.i%d.stdout' % i),
                      ph.outFile('reads.illumina.fq.i%d.stdout' % i),
                      transforms),
                     (ph.inFile('reads.illumina.fq.i%d.stderr' % i),
                      ph.outFile('reads.illumina.fq.i%d.stderr' % i),
                      transforms),
                    ])
        conf_list.append(conf)

    # Indel Mode

    for i in [1, 2]:
        min_gain = {1: 70.0, 2: 85.0}
        conf = app_tests.TestConf(
            program=path_to_fiona,
            args=['-nt', '1',
                  '-i', str(i),
                  '-g', '10000',
                  ph.inFile('reads.454.fq'),
                  ph.outFile('reads.454.corrected.i%d.fa' % i)],
            redir_stdout=ph.outFile('reads.454.fq.i%d.stdout' % i),
            redir_stderr=ph.outFile('reads.454.fq.i%d.stderr' % i),
            check_callback=ResultChecker(
                path_to_compute_gain, ph.inFile('genome.10k.fa'),
                ph.inFile('reads.454.sam'),
                ph.outFile('reads.454.corrected.i%d.fa' % i),
                min_gain.get(i, 100.0)),
            to_diff=[(ph.inFile('reads.454.fq.i%d.stdout' % i),
                      ph.outFile('reads.454.fq.i%d.stdout' % i),
                      transforms),
                     (ph.inFile('reads.454.fq.i%d.stderr' % i),
                      ph.outFile('reads.454.fq.i%d.stderr' % i),
                      transforms),
                    ])
        conf_list.append(conf)

    # Execute the tests.
    failures = 0
    for conf in conf_list:
        res = app_tests.runTest(conf)
        # Output to the user.
        print ' '.join(conf.commandLineArgs())
        if res:
             print 'OK'
        else:
            failures += 1
            print 'FAILED'

    # Cleanup.
    ph.deleteTempDir()

    print '=============================='
    print '     total tests: %d' % len(conf_list)
    print '    failed tests: %d' % failures
    print 'successful tests: %d' % (len(conf_list) - failures)
    print '=============================='
    # Compute and return return code.
    return failures != 0

if __name__ == '__main__':
    sys.exit(app_tests.main(main))
<|endoftext|>


<commit_before>from django.db import models

import datetime
import logging
from .managers import MetricManager, RawDataManager
from .utils import get_rawdata_for_metric, save_rawdata_for_metric

log = logging.getLogger(__name__)

class UnitCategory(models.Model):
    title = models.CharField(max_length=100, unique=True)

    class Meta:
        verbose_name = "Unit Category"
        verbose_name_plural = "Unit Categories"

    def __unicode__(self):
        return self.title

class Unit(models.Model):
    title = models.CharField(max_length=50, unique=True)
    description = models.TextField()
    unit_category = models.ForeignKey(UnitCategory)

    def __unicode__(self):
        return self.title

class Metric(models.Model):

    objects = MetricManager()

    # Meta Data by User Input
    title = models.CharField(max_length=100, unique=True)
    acronym = models.CharField(max_length=20, unique=True)
    description = models.TextField()
    keywords = models.CharField(max_length=400)

    geo_location = models.CharField(max_length=1000, blank=True)
    publisher = models.CharField(max_length=200, blank=True)
    details_url = models.URLField(max_length=500, blank=True)
    license = models.CharField(max_length=100, blank=True)

    user_id = models.IntegerField()
    language_id = models.IntegerField()
    ext_resource_id = models.IntegerField(blank=True)

    # Auto-Generated Meta Data
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    version = models.IntegerField(editable=False)

    # Data
    formula = models.CharField(max_length=10000, default='1')
    issued = models.DateField()

    unit = models.ForeignKey(Unit)

    _rawdata = None

    @property
    def rawdata(self):
        return get_rawdata_for_metric(self)

    @rawdata.setter
    def rawdata(self, value):
        self._rawdata = value

    @rawdata.deleter
    def rawdata(self):
        pass

    def save(self, *args, **kwargs):

        # Increasing the version on every update
        if self.pk is None:
            self.version = 1
        else:
            self.version += 1

        super(Metric, self).save(*args, **kwargs)
        save_rawdata_for_metric(self, self._rawdata)

    def __unicode__(self):
        return self.title

class MetricInDomain(models.Model):
    domain_id = models.IntegerField()
    metric = models.ForeignKey(Metric)

    class Meta:
        verbose_name = "Metric in Domain"
        verbose_name_plural = "Metrics in Domains"

    def __unicode__(self):
        return str(self.domain_id)

class RawData(models.Model):

    objects = RawDataManager()

    metric = models.ForeignKey(Metric)
    # Saving the row number, not necessary but convenient
    row = models.PositiveIntegerField()
    value = models.FloatField()
    from_date = models.DateField()
    to_date = models.DateField()

    class Meta:
        verbose_name = "Raw Data"
        verbose_name_plural = "Raw Data"
        ordering = ['row']

    def __unicode__(self):
        return str(self.row) + " Raw Data for " + self.metric.title

class RawDataCategory(models.Model):
    title = models.CharField(max_length=100, unique=True)
    description = models.TextField()

    class Meta:
        verbose_name = "Raw Data Category"
        verbose_name_plural = "Raw Data Categories"

    def __unicode__(self):
        return self.title

class RawDataExtra(models.Model):
    metric = models.ForeignKey(Metric)
    category = models.ForeignKey(RawDataCategory)

    class Meta:
        verbose_name = "Raw Data Extra"
        verbose_name_plural = "Raw Data Extras"

    def __unicode__(self):
        return str(self.metric) + " - " + str(self.category)

class RawDataExtraData(models.Model):
    raw_data_extra = models.ForeignKey(RawDataExtra)
    # Saving the row number, not necessary but convenient
    row = models.PositiveIntegerField()
    value = models.CharField(max_length=200)

    class Meta:
        verbose_name = "Raw Data Extra Data"
        verbose_name_plural = "Raw Data Extra Data"
        ordering = ['row']

    def __unicode__(self):
        return str(self.row) + " " + str(self.raw_data_extra)

<commit_msg>Changed Model for Python 3<commit_after>from django.db import models

import datetime
import logging
from .managers import MetricManager, RawDataManager
from .utils import get_rawdata_for_metric, save_rawdata_for_metric

log = logging.getLogger(__name__)

class UnitCategory(models.Model):
    title = models.CharField(max_length=100, unique=True)

    class Meta:
        verbose_name = "Unit Category"
        verbose_name_plural = "Unit Categories"

    def __unicode__(self):
        return self.title

class Unit(models.Model):
    title = models.CharField(max_length=50, unique=True)
    description = models.TextField()
    unit_category = models.ForeignKey(UnitCategory)

    def __str__(self):
        return self.title

class Metric(models.Model):

    objects = MetricManager()

    # Meta Data by User Input
    title = models.CharField(max_length=100, unique=True)
    acronym = models.CharField(max_length=20, unique=True)
    description = models.TextField()
    keywords = models.CharField(max_length=400)

    geo_location = models.CharField(max_length=1000, blank=True)
    publisher = models.CharField(max_length=200, blank=True)
    details_url = models.URLField(max_length=500, blank=True)
    license = models.CharField(max_length=100, blank=True)

    user_id = models.IntegerField()
    language_id = models.IntegerField()
    ext_resource_id = models.IntegerField(blank=True)

    # Auto-Generated Meta Data
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    version = models.IntegerField(editable=False)

    # Data
    formula = models.CharField(max_length=10000, default='1')
    issued = models.DateField()

    unit = models.ForeignKey(Unit)

    _rawdata = None

    @property
    def rawdata(self):
        return get_rawdata_for_metric(self)

    @rawdata.setter
    def rawdata(self, value):
        self._rawdata = value

    @rawdata.deleter
    def rawdata(self):
        pass

    def save(self, *args, **kwargs):

        # Increasing the version on every update
        if self.pk is None:
            self.version = 1
        else:
            self.version += 1

        super(Metric, self).save(*args, **kwargs)
        save_rawdata_for_metric(self, self._rawdata)

    def __str__(self):
        return self.title

class MetricInDomain(models.Model):
    domain_id = models.IntegerField()
    metric = models.ForeignKey(Metric)

    class Meta:
        verbose_name = "Metric in Domain"
        verbose_name_plural = "Metrics in Domains"

    def __str__(self):
        return str(self.domain_id)

class RawData(models.Model):

    objects = RawDataManager()

    metric = models.ForeignKey(Metric)
    # Saving the row number, not necessary but convenient
    row = models.PositiveIntegerField()
    value = models.FloatField()
    from_date = models.DateField()
    to_date = models.DateField()

    class Meta:
        verbose_name = "Raw Data"
        verbose_name_plural = "Raw Data"
        ordering = ['row']

    def __str__(self):
        return str(self.row) + " Raw Data for " + self.metric.title

class RawDataCategory(models.Model):
    title = models.CharField(max_length=100, unique=True)
    description = models.TextField()

    class Meta:
        verbose_name = "Raw Data Category"
        verbose_name_plural = "Raw Data Categories"

    def __str__(self):
        return self.title

class RawDataExtra(models.Model):
    metric = models.ForeignKey(Metric)
    category = models.ForeignKey(RawDataCategory)

    class Meta:
        verbose_name = "Raw Data Extra"
        verbose_name_plural = "Raw Data Extras"

    def __str__(self):
        return str(self.metric) + " - " + str(self.category)

class RawDataExtraData(models.Model):
    raw_data_extra = models.ForeignKey(RawDataExtra)
    # Saving the row number, not necessary but convenient
    row = models.PositiveIntegerField()
    value = models.CharField(max_length=200)

    class Meta:
        verbose_name = "Raw Data Extra Data"
        verbose_name_plural = "Raw Data Extra Data"
        ordering = ['row']

    def __str__(self):
        return str(self.row) + " " + str(self.raw_data_extra)

<|endoftext|>


<commit_before>from models import Users

"""the singleton func"""
def singleton(cls, *args, **kw):
    instances = {}
    def _singleton():
        if cls not in instances:
            instances[cls] = cls(*args, **kw)
        return instances[cls]
    return _singleton

@singleton
class UserUtil(object):
	"""the User Helper class to
		to help Manage the User Entity
	"""
	def __init__(self):
		pass

	def addUser(self,name,lastonLineTime,onlineStatus,ip):
		existUser = Users.objects.filter(name=name)
		if len(existUser) != 0:
			raise Exception("the Users is already exits")

		newUser = Users(name=name,
						ip=ip,
						onlineStatus = True,
						lastonLineTime=lastonLineTime)

		newUser.save()

	def changeUser(self, name, **changeDic):
		existUser = self.getUsers(name)
		changeDic = dict(changeDic)
		print changeDic
		if existUser:
			for attribute, value in changeDic.iteritems():
				print attribute
				print "in set attr " + attribute + "and the value is" + str(value)
				setattr(existUser, attribute, value)
				print attribute + str(getattr(existUser, attribute))

			print "it is saving the user"
			existUser.save()
			print str(existUser.onlineStatus)

	def isInUsers(self, name):
		try:
			existUser = Users.objects.get(name=name)
			return True
		except Exception, e:
			return False

	def getUsers(self, name):
		try:
			existUser = Users.objects.get(name=name)
			return existUser
		except Exception, e:
			return None
<commit_msg>remove the print statements<commit_after>from models import Users

"""the singleton func"""
def singleton(cls, *args, **kw):
    instances = {}
    def _singleton():
        if cls not in instances:
            instances[cls] = cls(*args, **kw)
        return instances[cls]
    return _singleton

@singleton
class UserUtil(object):
	"""the User Helper class to
		to help Manage the User Entity
	"""
	def __init__(self):
		pass

	def addUser(self,name,lastonLineTime,onlineStatus,ip):
		existUser = Users.objects.filter(name=name)
		if len(existUser) != 0:
			raise Exception("the Users is already exits")

		newUser = Users(name=name,
						ip=ip,
						onlineStatus = True,
						lastonLineTime=lastonLineTime)

		newUser.save()

	def changeUser(self, name, **changeDic):
		existUser = self.getUsers(name)
		changeDic = dict(changeDic)
		print changeDic
		if existUser:
			for attribute, value in changeDic.iteritems():
				setattr(existUser, attribute, value)

			existUser.save()

	def isInUsers(self, name):
		try:
			existUser = Users.objects.get(name=name)
			return True
		except Exception, e:
			return False

	def getUsers(self, name):
		try:
			existUser = Users.objects.get(name=name)
			return existUser
		except Exception, e:
			return None
<|endoftext|>


<commit_before>from django.utils import http, importlib
from . import CACHED_VIEW_PREFIX, rcache, COUCH_CACHE_TIMEOUT, CACHE_VIEWS
from django.conf import settings
import simplejson

class GenerationCache(object):
    generation_key = None
    doc_types = []
    views = []

    @staticmethod
    def _get_generational_caches():
        if not getattr(GenerationCache, '_generational_caches', None):
            GenerationCache._generate_caches()
        return getattr(GenerationCache, '_generational_caches')

    @staticmethod
    def view_generation_map():
        view_map = {}
        for gen_model in GenerationCache._get_generational_caches():
            for view_name in gen_model.views:
                view_map[view_name] = gen_model
        return view_map

    @staticmethod
    def doc_type_generation_map():
        doc_type_map = {}
        for gen_model in GenerationCache._get_generational_caches():
            for doc_type in gen_model.doc_types:
                doc_type_map[doc_type] = gen_model
        return doc_type_map

    @staticmethod
    def _generate_caches():
        generational_caches = []

        for cache_str in settings.COUCH_CACHE_BACKENDS:
            mod_path, cache_class_name = cache_str.rsplit('.', 1)
            mod = importlib.import_module(mod_path)
            gen_model = getattr(mod, cache_class_name)
            generational_caches.append(gen_model())
        setattr(GenerationCache, '_generational_caches', generational_caches)

    def _get_generation(self):
        genret = rcache().get(self.generation_key, None)
        if not genret:
            #never seen key before, start from zero
            rcache().set(self.generation_key, 0, timeout=0)
            return str(0)
        else:
            return str(genret)

    def invalidate_all(self):
        """
        Invalidate this cache by incrementing the generation
        """
        current_generation = self._get_generation()
        return rcache().incr(self.generation_key)

    def _mk_view_cache_key(self, view_name, params=None):
        """
        view_name = "design_doc/viewname"
        if params is dict, then make param_string
        if params is '*' it's a wildcard
        """
        param_string = ""
        if isinstance(params, dict):
            param_string = http.urlquote('|'.join(["%s::%s" % (k, v) for k, v in params.items()]))
        elif params == '*':
            param_string = params

        cache_view_key = ':'.join([
            self._get_generation(),
            CACHED_VIEW_PREFIX,
            view_name,
            param_string,
        ])
        return cache_view_key

    def _cached_view_doc(self, doc_or_docid, cache_expire=COUCH_CACHE_TIMEOUT):
        """
        Cache by doc_id a reverse lookup of views that it is mutually dependent on
        """
        from .api import do_cache_doc
        if isinstance(doc_or_docid, dict):
            do_cache_doc(doc_or_docid, cache_expire=cache_expire)

    def cached_view(self, db, view_name, wrapper=None, cache_expire=COUCH_CACHE_TIMEOUT, force_invalidate=False,
                    **params):
        """
        Call a view and cache the results, return cached if it's a cache hit.

        db: couch database to do query on
        view_name, params: couch view call parameters
        force_invalidate: extra param to always hit the db and cache on read.

        Note, a view call with include_docs=True will not be wrapped, you must wrap it on your own.
        """
        from .api import cached_open_doc

        if force_invalidate:
            self.invalidate_all()

        include_docs = params.get('include_docs', False)

        cache_view_key = self._mk_view_cache_key(view_name, params)
        if include_docs:
            #include_docs=True results in couchdbkit remove the 'rows' result and returns just the actual rows in an array
            cached_view = rcache().get(cache_view_key, None)
            if cached_view and CACHE_VIEWS:
                results = simplejson.loads(cached_view)
                final_results = {}

                if include_docs:
                    row_stubs = results['row_stubs']

                    final_results['total_rows'] = results['total_rows']
                    final_results['offset'] = results['offset']

                    rows = []
                    for stub in row_stubs:
                        row = {
                            "id": stub['id'],
                            "value": None,
                            "key": stub["key"],
                            #this feels hacky, but for some reason other views are squashing the master cached doc.
                            #a more true invalidation scheme should have this more readily address this, but for now
                            #do a db call here and cache it. Should be a _cached_doc_only call here
                            "doc": cached_open_doc(db, stub['id'])
                        }
                        rows.append(row)
                    if wrapper:
                        final_results = [wrapper(x['doc']) for x in rows]
                    else:
                        #final_results['rows'] = rows
                        final_results = rows
                return final_results
            else:
                #cache miss, get view, cache it and all docs
                view_obj = db.view(view_name, **params)
                view_obj._fetch_if_needed()
                view_results = view_obj._result_cache
                row_stubs = []

                for row in view_results["rows"]:
                    stub = {
                        "id": row['id'],
                        "value": None,
                        "key": row["key"],
                    }
                    self._cached_view_doc(row["doc"], cache_expire=cache_expire)
                    row_stubs.append(stub)

                cached_results = {
                    "total_rows": view_results['total_rows'],
                    "offset": view_results['offset'],
                    "row_stubs": row_stubs
                }

                if wrapper:
                    retval = [wrapper(x['doc']) for x in view_results['rows']]
                else:
                    retval = view_results['rows']
                rcache().set(cache_view_key, simplejson.dumps(cached_results), timeout=cache_expire)
                return retval

        else:
            ###########################
            # include_docs=False just returns the entire view verbatim
            cached_view = rcache().get(cache_view_key)
            if cached_view and CACHE_VIEWS:
                results = simplejson.loads(cached_view)
                return results
            else:
                view_results = db.view(view_name, **params).all()
                rcache().set(cache_view_key, simplejson.dumps(view_results), timeout=cache_expire)
                for row in view_results:
                    doc_id = row.get('id', None)
                    if doc_id:
                        #a non reduce view will have doc_ids on each row, we want to reverse index these
                        #to know when to invalidate
                        self._cached_view_doc(doc_id)
                return view_results

class GlobalCache(GenerationCache):
    """
    Default cache for those not mapped. No generational tracking.
    """
    generation_key = None
    doc_types = [
    ]
    views = []

    _instance = None

    def _get_generation(self):
        return "#global#"

    def invalidate_all(self):
        raise NotImplementedError("You're trying to call a global cache invalidation - does not support invalidation - please rethink your priorities.")

    @staticmethod
    def nogen():
        if not GlobalCache._instance:
            GlobalCache._instance = GlobalCache()
        return GlobalCache._instance

<commit_msg>making couch_cache_backends lookup safer<commit_after>from django.utils import http, importlib
from . import CACHED_VIEW_PREFIX, rcache, COUCH_CACHE_TIMEOUT, CACHE_VIEWS
from django.conf import settings
import simplejson

class GenerationCache(object):
    generation_key = None
    doc_types = []
    views = []

    @staticmethod
    def _get_generational_caches():
        if not getattr(GenerationCache, '_generational_caches', None):
            GenerationCache._generate_caches()
        return getattr(GenerationCache, '_generational_caches')

    @staticmethod
    def view_generation_map():
        view_map = {}
        for gen_model in GenerationCache._get_generational_caches():
            for view_name in gen_model.views:
                view_map[view_name] = gen_model
        return view_map

    @staticmethod
    def doc_type_generation_map():
        doc_type_map = {}
        for gen_model in GenerationCache._get_generational_caches():
            for doc_type in gen_model.doc_types:
                doc_type_map[doc_type] = gen_model
        return doc_type_map

    @staticmethod
    def _generate_caches():
        generational_caches = []

        for cache_str in getattr(settings, 'COUCH_CACHE_BACKENDS', []):
            mod_path, cache_class_name = cache_str.rsplit('.', 1)
            mod = importlib.import_module(mod_path)
            gen_model = getattr(mod, cache_class_name)
            generational_caches.append(gen_model())
        setattr(GenerationCache, '_generational_caches', generational_caches)

    def _get_generation(self):
        genret = rcache().get(self.generation_key, None)
        if not genret:
            #never seen key before, start from zero
            rcache().set(self.generation_key, 0, timeout=0)
            return str(0)
        else:
            return str(genret)

    def invalidate_all(self):
        """
        Invalidate this cache by incrementing the generation
        """
        current_generation = self._get_generation()
        return rcache().incr(self.generation_key)

    def _mk_view_cache_key(self, view_name, params=None):
        """
        view_name = "design_doc/viewname"
        if params is dict, then make param_string
        if params is '*' it's a wildcard
        """
        param_string = ""
        if isinstance(params, dict):
            param_string = http.urlquote('|'.join(["%s::%s" % (k, v) for k, v in params.items()]))
        elif params == '*':
            param_string = params

        cache_view_key = ':'.join([
            self._get_generation(),
            CACHED_VIEW_PREFIX,
            view_name,
            param_string,
        ])
        return cache_view_key

    def _cached_view_doc(self, doc_or_docid, cache_expire=COUCH_CACHE_TIMEOUT):
        """
        Cache by doc_id a reverse lookup of views that it is mutually dependent on
        """
        from .api import do_cache_doc
        if isinstance(doc_or_docid, dict):
            do_cache_doc(doc_or_docid, cache_expire=cache_expire)

    def cached_view(self, db, view_name, wrapper=None, cache_expire=COUCH_CACHE_TIMEOUT, force_invalidate=False,
                    **params):
        """
        Call a view and cache the results, return cached if it's a cache hit.

        db: couch database to do query on
        view_name, params: couch view call parameters
        force_invalidate: extra param to always hit the db and cache on read.

        Note, a view call with include_docs=True will not be wrapped, you must wrap it on your own.
        """
        from .api import cached_open_doc

        if force_invalidate:
            self.invalidate_all()

        include_docs = params.get('include_docs', False)

        cache_view_key = self._mk_view_cache_key(view_name, params)
        if include_docs:
            #include_docs=True results in couchdbkit remove the 'rows' result and returns just the actual rows in an array
            cached_view = rcache().get(cache_view_key, None)
            if cached_view and CACHE_VIEWS:
                results = simplejson.loads(cached_view)
                final_results = {}

                if include_docs:
                    row_stubs = results['row_stubs']

                    final_results['total_rows'] = results['total_rows']
                    final_results['offset'] = results['offset']

                    rows = []
                    for stub in row_stubs:
                        row = {
                            "id": stub['id'],
                            "value": None,
                            "key": stub["key"],
                            #this feels hacky, but for some reason other views are squashing the master cached doc.
                            #a more true invalidation scheme should have this more readily address this, but for now
                            #do a db call here and cache it. Should be a _cached_doc_only call here
                            "doc": cached_open_doc(db, stub['id'])
                        }
                        rows.append(row)
                    if wrapper:
                        final_results = [wrapper(x['doc']) for x in rows]
                    else:
                        #final_results['rows'] = rows
                        final_results = rows
                return final_results
            else:
                #cache miss, get view, cache it and all docs
                view_obj = db.view(view_name, **params)
                view_obj._fetch_if_needed()
                view_results = view_obj._result_cache
                row_stubs = []

                for row in view_results["rows"]:
                    stub = {
                        "id": row['id'],
                        "value": None,
                        "key": row["key"],
                    }
                    self._cached_view_doc(row["doc"], cache_expire=cache_expire)
                    row_stubs.append(stub)

                cached_results = {
                    "total_rows": view_results['total_rows'],
                    "offset": view_results['offset'],
                    "row_stubs": row_stubs
                }

                if wrapper:
                    retval = [wrapper(x['doc']) for x in view_results['rows']]
                else:
                    retval = view_results['rows']
                rcache().set(cache_view_key, simplejson.dumps(cached_results), timeout=cache_expire)
                return retval

        else:
            ###########################
            # include_docs=False just returns the entire view verbatim
            cached_view = rcache().get(cache_view_key)
            if cached_view and CACHE_VIEWS:
                results = simplejson.loads(cached_view)
                return results
            else:
                view_results = db.view(view_name, **params).all()
                rcache().set(cache_view_key, simplejson.dumps(view_results), timeout=cache_expire)
                for row in view_results:
                    doc_id = row.get('id', None)
                    if doc_id:
                        #a non reduce view will have doc_ids on each row, we want to reverse index these
                        #to know when to invalidate
                        self._cached_view_doc(doc_id)
                return view_results

class GlobalCache(GenerationCache):
    """
    Default cache for those not mapped. No generational tracking.
    """
    generation_key = None
    doc_types = [
    ]
    views = []

    _instance = None

    def _get_generation(self):
        return "#global#"

    def invalidate_all(self):
        raise NotImplementedError("You're trying to call a global cache invalidation - does not support invalidation - please rethink your priorities.")

    @staticmethod
    def nogen():
        if not GlobalCache._instance:
            GlobalCache._instance = GlobalCache()
        return GlobalCache._instance

<|endoftext|>


<commit_before>from ApplianceTrace import ApplianceTrace

import sqlalchemy
import pandas as pd

class PecanStreetDatasetAdapter():
    def __init__(self,db_url):
        '''
        Initialize an adapter using a database url_string.
        Consider the following example:
        db_url="postgresql"+"://"+user_name+":"+ps+"@"+host+":"+port+"/"+db
        '''
        self.eng = sqlalchemy.create_engine(db_url)
        self.source = "PecanStreet"

    def set_table_names(self,schema):
        ''''''
        df = self.get_dataframe('select * from information_schema.tables')
        df=(df.groupby(['table_schema','table_name']))
        d = [k for k in df.groups]
        tables = [l[1] for l in d if l[0]==schema]
        return tables

    def verify_same_range():
        '''check that all data points have the same range'''
        pass

    def get_meta_table(self,schema,table):
        '''would prefer that this be a class that these be attributes'''
        q = 'select distinct dataid from {}.{}'.format(schema,table)
        result = self.eng.execute(q)
        ids = result.fetchall()
        q = 'select * from {}.{} where dataid={}'.format(schema,table,ids[0][0])
        result = self.eng.execute(q)
        apps = result.keys()
        #print apps
        # date_start =
        # date_end =
        # step_size =
        return [ids,apps]

    def get_unique_dataids(self,schema,month,year,group=None):
        '''
        Returns a list of dataids for a specifc schema ("curated","shared", or
        "raw"), month (int), year (int), and group (int).
        '''
        if schema == "curated":
            query = 'select distinct dataid from "PecanStreet_CuratedSets".group{0}_disaggregated_{1}_{2:02d}'.format(group,year,month)
            df = self.get_dataframe(query)
            return list(df["dataid"])
        elif schema == "shared":
            raise NotImplementedError
        elif schema == "raw":
            raise NotImplementedError
        else:
            raise SchemaError(schema)

    def get_month_traces(self,schema,year,month,dataid,group=None,sampling_rate="15T"):
        '''
        Returns a month-long traces for the specified month and sampling rate. Specify
        sampling rate using pd offset aliases (Ex. 15 mins -> "15T")
        '''
        if schema == "curated":
            # Lowest possible sampling rate is 15T
            query = 'select * from "PecanStreet_CuratedSets".group{0}_disaggregated_{1}_{2:02d} where dataid={3}'.format(group,year,month,dataid)
            df = self.get_dataframe(query).fillna(0)
            df.rename(columns={'utc_15min': 'time'}, inplace=True)
            df.index = df['time'].apply(pd.to_datetime)
            df = df.drop(['id','dataid','time'], axis=1)
            if not (sampling_rate == '15T' or sampling_rate == '15Min'):
                how = {col:'sum' for col in dataframe.columns}
                df = df.resample(sampling_rate, how=how)
        elif schema == "shared":
            pass
        elif schema == "raw":
            raise NotImplementedError
        else:
            raise SchemaError(schema)
        traces = []
        for column, series in df.iteritems():
            traces.append(ApplianceTrace(series,self.source))
        return traces

    ##do table work

    def time_align():
        pass

    def clean_dataframe(self,df,schema,drop_cols):
            time_cols = {'\"PecanStreet_CuratedSets\"':'utc_15min','\"PecanStreet_RawData\"':'localminute15minute','\"PecanStreet_SharedData\"':'localminute'}
            print time_cols[schema]
            print time_cols[schema]  in df.columns
            df=df.rename(columns={time_cols[schema]: 'time'})
            #print 'in clean'
            #print type(df)
            #print df.columns
            print df.shape
            df['time']=pd.to_datetime(df['time'], format='%d/%m/%Y %H:%M:%S')
            df.set_index('time', inplace=True)
            #df = df.drop(['id','dataid','time'], axis=1)
            #print df.shape
            dataid = df['dataid'][0]
            df = df.drop(['dataid'], axis=1)
            if schema=='\"PecanStreet_CuratedSets\"':
                df = df.drop(['id'], axis=1)
            if len(drop_cols)!=0:
                df= df.drop(drop_cols,axis=1)
            return [df,dataid]

    def invalid_col(self,col,schema):
        invalids={'\"PecanStreet_CuratedSets\"':['id','utc_15min'],'\"PecanStreet_RawData\"':['localminute15min'], '\"PecanStreet_SharedData\"':['localminute']}
        return col in invalids[schema]

    def get_month_traces_wo_time_align(self,schema,table,dataid):
        ##change this name
        if schema not in ['\"PecanStreet_CuratedSets\"','\"PecanStreet_RawData\"','\"PecanStreet_SharedData\"']:
            raise SchemaError(schema)
        query = 'select * from {0}.{1} where dataid={2}'.format(schema, table,dataid)
        ##NEED TO CHANGE IDS

        ##error checking that query worked

        df = self.get_dataframe(query).fillna(0)

        [df,da] = self.clean_dataframe(df, schema,[])
        traces = []
        for col in df.columns:
            if not self.invalid_col(col,schema):
                meta={'source':self.source,'schema':schema,'table':table ,'dataid':da}
                traces.append(ApplianceTrace(df[col],meta))

        return traces

    def get_app_traces(self,schema,table,app):
        query= 'select {2} from {0}.{1}'.format(schema,table,app)
        df=self.get_dataframe(query)
        print df.shape()

    def get_dataframe(self,query):
        '''Returns a pd dataframe with the query results'''
        eng_object = self.eng.execute(query)

        df = pd.DataFrame.from_records(eng_object.fetchall())
        df.columns = eng_object.keys()
        return df

    def get_list(self,query):
        result  = self.eng.execute(query).fetchall()
        return [result,result.keys()]

class SchemaError(Exception):
    """Exception raised for errors in the schema.

        Attributes:
            schema  -- nonexistent schema
    """
    def __init__(self,schema):
        self.schema = schema

    def __str__(self):
        return "Schema {} not supported or nonexistent.".format(self.schema)
<commit_msg>added files<commit_after>from ApplianceTrace import ApplianceTrace

import sqlalchemy
import pandas as pd

class PecanStreetDatasetAdapter():
    def __init__(self,db_url):
        '''
        Initialize an adapter using a database url_string.
        Consider the following example:
        db_url="postgresql"+"://"+user_name+":"+ps+"@"+host+":"+port+"/"+db
        '''
        self.eng = sqlalchemy.create_engine(db_url)
        self.source = "PecanStreet"

    def set_table_names(self,schema):
        ''''''
        df = self.get_dataframe('select * from information_schema.tables')
        df=(df.groupby(['table_schema','table_name']))
        d = [k for k in df.groups]
        tables = [l[1] for l in d if l[0]==schema]
        return tables

    def verify_same_range():
        '''check that all data points have the same range'''
        pass

    def get_meta_table(self,schema,table):
        '''would prefer that this be a class that these be attributes'''
        q = 'select distinct dataid from {}.{}'.format(schema,table)
        result = self.eng.execute(q)
        ids = result.fetchall()
        q = 'select * from {}.{} where dataid={}'.format(schema,table,ids[0][0])
        result = self.eng.execute(q)
        apps = result.keys()
        ##not totally necessary and should find faster way
        # df = pd.DataFrame.from_records(result.fetchall())
        # df.columns=apps

        #print apps
        # date_start =df[][]
        #date_end =
        #step_size =
        return [ids,apps]

    def get_unique_dataids(self,schema,month,year,group=None):
        '''
        Returns a list of dataids for a specifc schema ("curated","shared", or
        "raw"), month (int), year (int), and group (int).
        '''
        if schema == "curated":
            query = 'select distinct dataid from "PecanStreet_CuratedSets".group{0}_disaggregated_{1}_{2:02d}'.format(group,year,month)
            df = self.get_dataframe(query)
            return list(df["dataid"])
        elif schema == "shared":
            raise NotImplementedError
        elif schema == "raw":
            raise NotImplementedError
        else:
            raise SchemaError(schema)

    def get_month_traces(self,schema,year,month,dataid,group=None,sampling_rate="15T"):
        '''
        Returns a month-long traces for the specified month and sampling rate. Specify
        sampling rate using pd offset aliases (Ex. 15 mins -> "15T")
        '''
        if schema == "curated":
            # Lowest possible sampling rate is 15T
            query = 'select * from "PecanStreet_CuratedSets".group{0}_disaggregated_{1}_{2:02d} where dataid={3}'.format(group,year,month,dataid)
            df = self.get_dataframe(query).fillna(0)
            df.rename(columns={'utc_15min': 'time'}, inplace=True)
            df.index = df['time'].apply(pd.to_datetime)
            df = df.drop(['id','dataid','time'], axis=1)
            if not (sampling_rate == '15T' or sampling_rate == '15Min'):
                how = {col:'sum' for col in dataframe.columns}
                df = df.resample(sampling_rate, how=how)
        elif schema == "shared":
            pass
        elif schema == "raw":
            raise NotImplementedError
        else:
            raise SchemaError(schema)
        traces = []
        for column, series in df.iteritems():
            traces.append(ApplianceTrace(series,self.source))
        return traces

    ##do table work

    def time_align():
        pass

    def clean_dataframe(self,df,schema,drop_cols):
            time_cols = {'\"PecanStreet_CuratedSets\"':'utc_15min','\"PecanStreet_RawData\"':'localminute15minute','\"PecanStreet_SharedData\"':'localminute'}
            print time_cols[schema]
            print time_cols[schema]  in df.columns
            df=df.rename(columns={time_cols[schema]: 'time'})
            #print 'in clean'
            #print type(df)
            #print df.columns
            print df.shape
            df['time']=pd.to_datetime(df['time'], format='%d/%m/%Y %H:%M:%S')
            start_time = df['time'][0]
            end_time = df['time'][len(df['time'])-1]
            step_size = df['time'][1]-start_time
            df.set_index('time', inplace=True)
            #df = df.drop(['id','dataid','time'], axis=1)
            #print df.shape
            dataid = df['dataid'][0]
            df = df.drop(['dataid'], axis=1)
            if schema=='\"PecanStreet_CuratedSets\"':
                df = df.drop(['id'], axis=1)

            if len(drop_cols)!=0:
                df= df.drop(drop_cols,axis=1)
            times = [start_time,end_time, step_size]
            return [df,dataid,times]

    def invalid_col(self,col,schema):
        invalids={'\"PecanStreet_CuratedSets\"':['id','utc_15min'],'\"PecanStreet_RawData\"':['localminute15min'], '\"PecanStreet_SharedData\"':['localminute']}
        return col in invalids[schema]

    def get_month_traces_per_dataid(self,schema,table,dataid):
        ##change this name
        if schema not in ['\"PecanStreet_CuratedSets\"','\"PecanStreet_RawData\"','\"PecanStreet_SharedData\"']:
            raise SchemaError(schema)
        query = 'select * from {0}.{1} where dataid={2}'.format(schema, table,dataid)
        ##NEED TO CHANGE IDS

        ##error checking that query worked

        df = self.get_dataframe(query).fillna(0)

        [df,da,times] = self.clean_dataframe(df, schema,[])
        traces = []
        for col in df.columns:
            if not self.invalid_col(col,schema):
                meta={'source':self.source,'schema':schema,'table':table ,'dataid':da, 'start_time': times[0],'end_time':times[1], 'step_size':times[2] }
                traces.append(ApplianceTrace(df[col],meta))

        return traces

    def get_app_traces(self,schema,table,app):
        query= 'select {2} from {0}.{1}'.format(schema,table,app)
        df=self.get_dataframe(query)
        print df.shape()

    def get_dataframe(self,query):
        '''Returns a pd dataframe with the query results'''
        eng_object = self.eng.execute(query)

        df = pd.DataFrame.from_records(eng_object.fetchall())
        df.columns = eng_object.keys()
        return df

    def get_list(self,query):
        result  = self.eng.execute(query).fetchall()
        return [result,result.keys()]

class SchemaError(Exception):
    """Exception raised for errors in the schema.

        Attributes:
            schema  -- nonexistent schema
    """
    def __init__(self,schema):
        self.schema = schema

    def __str__(self):
        return "Schema {} not supported or nonexistent.".format(self.schema)
<|endoftext|>


<commit_before>"""
The Kullback-Leibler divergence.
"""

import numpy as np

from .cross_entropy import get_pmfs_like
from ..helpers import normalize_rvs
from ..utils import flatten

from .kullback_leibler_divergence import kullback_leibler_divergence

__all__ = ('double_power_sum',
           'hellinger_sum',
           'alpha_divergence',
           'hellinger_divergence',
           'renyi_divergence',
           'tsallis_divergence',
          )

### References for Divergence Formulas ###
## http://arxiv.org/pdf/1105.3259v1.pdf
## http://arxiv.org/pdf/1206.2459.pdf
## http://mitran-lab.amath.unc.edu:8082/subversion/grants/Proposals/2013/DOE-DataCentric/biblio/LieseVajdaDivergencesInforTheory.pdf
## Crooks: http://threeplusone.com/on_information.pdf

def double_power_sum(dist1, dist2, exp1=1, exp2=1, rvs=None, crvs=None, rv_mode=None):
    """
    A common generalization of the sums needed to compute the Hellinger
    and alpha divergences below.

    Parameters
    ----------
    dist1 : Distribution
        The first distribution in the sum.
        The second distribution in the sum.
    exp1 : float, 1
        Fisrt exponent used in the power sum.
    exp2 : float, 1
        Second exponent used in the power sum.
    rvs : list, None
        The indexes of the random variable used to calculate the sum.
        If None, then the sum is calculated over all random variables.
    rv_mode : str, None
        Specifies how to interpret `rvs` and `crvs`. Valid options are:
        {'indices', 'names'}. If equal to 'indices', then the elements of
        `crvs` and `rvs` are interpreted as random variable indices. If equal
        to 'names', the the elements are interpreted as random variable names.
        If `None`, then the value of `dist._rv_mode` is consulted, which
        defaults to 'indices'.

    Returns
    -------
    dkl : float
        The specified sum between `dist1` and `dist2`.

    Raises
    ------
    ditException
        Raised if either `dist1` or `dist2` doesn't have `rvs` or, if `rvs` is
        None, if `dist2` has an outcome length different than `dist1`.

    """

    rvs, crvs, rv_mode = normalize_rvs(dist1, rvs, crvs, rv_mode)
    rvs, crvs = list(flatten(rvs)), list(flatten(crvs))
    normalize_rvs(dist2, rvs, crvs, rv_mode)

    p1s, q1s = get_pmfs_like(dist1, dist2, rvs+crvs, rv_mode)
    div = np.nansum(np.power(p1s, exp1) * np.power(q1s, exp2))

    if crvs:
        p2s, q2s = get_pmfs_like(dist1, dist2, crvs, rv_mode)
        div = np.nansum(np.power(p2s, exp1) * np.power(q2s, exp2))

    return div

def hellinger_sum(dist1, dist2, alpha=1., rvs=None, crvs=None, rv_mode=None):
    """
    The Hellinger sum/integral of `dist1` and `dist2`, used to define other
    divergences.

    Parameters
    ----------
    dist1 : Distribution
        The first distribution in the sum.
        The second distribution in the sum.
    alpha : float, 1
        Exponent used in the sum.
    rvs : list, None
        The indexes of the random variable used to calculate the sum.
        If None, then the sum is calculated over all random variables.
    rv_mode : str, None
        Specifies how to interpret `rvs` and `crvs`. Valid options are:
        {'indices', 'names'}. If equal to 'indices', then the elements of
        `crvs` and `rvs` are interpreted as random variable indices. If equal
        to 'names', the the elements are interpreted as random variable names.
        If `None`, then the value of `dist._rv_mode` is consulted, which
        defaults to 'indices'.

    Returns
    -------
    dkl : float
        The Hellinger sum between `dist1` and `dist2`.

    Raises
    ------
    ditException
        Raised if either `dist1` or `dist2` doesn't have `rvs` or, if `rvs` is
        None, if `dist2` has an outcome length different than `dist1`.

    """

    return double_power_sum(dist1, dist2, alpha, 1.-alpha,
                            rvs=rvs, crvs=crvs, rv_mode=rv_mode)

def hellinger_divergence(dist1, dist2, alpha=1., rvs=None, crvs=None, rv_mode=None):
    # http://mitran-lab.amath.unc.edu:8082/subversion/grants/Proposals/2013/DOE-DataCentric/biblio/LieseVajdaDivergencesInforTheory.pdf
    """
    The Hellinger divergence of `dist1` and `dist2`.

    Parameters
    ----------
    dist1 : Distribution
        The first distribution in the Hellinger divergence.
    dist2 : Distribution
        The second distribution in the Hellinger divergence.
    alpha : float, 1
        The divergence is a one parameter family in alpha.
    rvs : list, None
        The indexes of the random variable used to calculate the
        Hellinger divergence between. If None, then the Hellinger
        divergence is calculated over all random variables.
    rv_mode : str, None
        Specifies how to interpret `rvs` and `crvs`. Valid options are:
        {'indices', 'names'}. If equal to 'indices', then the elements of
        `crvs` and `rvs` are interpreted as random variable indices. If equal
        to 'names', the the elements are interpreted as random variable names.
        If `None`, then the value of `dist._rv_mode` is consulted, which
        defaults to 'indices'.

    Returns
    -------
    dkl : float
        The Hellinger divergence between `dist1` and `dist2`.

    Raises
    ------
    ditException
        Raised if either `dist1` or `dist2` doesn't have `rvs` or, if `rvs` is
        None, if `dist2` has an outcome length different than `dist1`.

    """

    if alpha == 1:
        return kullback_leibler_divergence(dist1, dist2, rvs=rvs,
                                           crvs=crvs, rv_mode=rv_mode)
    s = hellinger_sum(dist1, dist2, rvs=rvs, alpha=alpha,
                      crvs=crvs, rv_mode=rv_mode)
    return (s-1.)/(alpha-1.)

def tsallis_divergence(dist1, dist2, alpha=1., rvs=None, crvs=None, rv_mode=None):
    """
    The Tsallis divergence of `dist1` and `dist2`.

    Parameters
    ----------
    dist1 : Distribution
        The first distribution in the Tsallis divergence.
    dist2 : Distribution
        The second distribution in the Tsallis divergence.
    alpha : float, 1
        The divergence is a one parameter family in alpha.
    rvs : list, None
        The indexes of the random variable used to calculate the
        Tsallis divergence between. If None, then the Tsallis
        divergence is calculated over all random variables.
    rv_mode : str, None
        Specifies how to interpret `rvs` and `crvs`. Valid options are:
        {'indices', 'names'}. If equal to 'indices', then the elements of
        `crvs` and `rvs` are interpreted as random variable indices. If equal
        to 'names', the the elements are interpreted as random variable names.
        If `None`, then the value of `dist._rv_mode` is consulted, which
        defaults to 'indices'.

    Returns
    -------
    dkl : float
        The Tsallis divergence between `dist1` and `dist2`.

    Raises
    ------
    ditException
        Raised if either `dist1` or `dist2` doesn't have `rvs` or, if `rvs` is
        None, if `dist2` has an outcome length different than `dist1`.

    """

    # D_T = (D_alpha -1) / (alpha-1)
    if alpha == 1:
        div = kullback_leibler_divergence(dist1, dist2, rvs=rvs,
                                          crvs=crvs, rv_mode=rv_mode)
    else:
        s = hellinger_sum(dist1, dist2, rvs=rvs, alpha=alpha,
                          crvs=crvs, rv_mode=rv_mode)
        div = (s - 1.) /(alpha - 1.)
    return div

def renyi_divergence(dist1, dist2, alpha=1., rvs=None, crvs=None, rv_mode=None):
    """
    The Renyi divergence of `dist1` and `dist2`.

    Parameters
    ----------
    dist1 : Distribution
        The first distribution in the Renyi divergence.
    dist2 : Distribution
        The second distribution in the Renyi divergence.
    alpha : float, 1
        The divergence is a one parameter family in alpha.
    rvs : list, None
        The indexes of the random variable used to calculate the
        Renyi divergence between. If None, then the Renyi
        divergence is calculated over all random variables.
    rv_mode : str, None
        Specifies how to interpret `rvs` and `crvs`. Valid options are:
        {'indices', 'names'}. If equal to 'indices', then the elements of
        `crvs` and `rvs` are interpreted as random variable indices. If equal
        to 'names', the the elements are interpreted as random variable names.
        If `None`, then the value of `dist._rv_mode` is consulted, which
        defaults to 'indices'.

    Returns
    -------
    dkl : float
        The Renyi divergence between `dist1` and `dist2`.

    Raises
    ------
    ditException
        Raised if either `dist1` or `dist2` doesn't have `rvs` or, if `rvs` is
        None, if `dist2` has an outcome length different than `dist1`.

    """

    # D_R = log D_alpha / (alpha-1)
    if alpha == 1:
        div = kullback_leibler_divergence(dist1, dist2, rvs=rvs,
                                          crvs=crvs, rv_mode=rv_mode)
    else:
        s = hellinger_sum(dist1, dist2, rvs=rvs,
                          alpha=alpha, crvs=crvs, rv_mode=rv_mode)
        div = np.log2(s) / (alpha - 1.)
    return div

def alpha_divergence(dist1, dist2, alpha=1., rvs=None, crvs=None, rv_mode=None):
    """
    The alpha divergence of `dist1` and `dist2`, as used in Information
    Geometry. Note there is more than one inequivalent definition of
    "alpha divergence" in the literature, this one comes
    from http://en.wikipedia.org/wiki/Information_geometry .

    Parameters
    ----------
    dist1 : Distribution
        The first distribution in the alpha divergence.
    dist2 : Distribution
        The second distribution in the alpha divergence.
    alpha : float, 1
        The divergence is a one parameter family in alpha.
    rvs : list, None
        The indexes of the random variable used to calculate the
        alpha divergence between. If None, then the alpha
        divergence is calculated over all random variables.
    rv_mode : str, None
        Specifies how to interpret `rvs` and `crvs`. Valid options are:
        {'indices', 'names'}. If equal to 'indices', then the elements of
        `crvs` and `rvs` are interpreted as random variable indices. If equal
        to 'names', the the elements are interpreted as random variable names.
        If `None`, then the value of `dist._rv_mode` is consulted, which
        defaults to 'indices'.

    Returns
    -------
    dkl : float
        The alpha divergence between `dist1` and `dist2`.

    Raises
    ------
    ditException
        Raised if either `dist1` or `dist2` doesn't have `rvs` or, if `rvs` is
        None, if `dist2` has an outcome length different than `dist1`.

    """

    if alpha == 1:
        return kullback_leibler_divergence(dist1, dist2, rvs=rvs,
                                           crvs=crvs, rv_mode=rv_mode)
    if alpha == -1:
        return kullback_leibler_divergence(dist2, dist1, rvs=rvs,
                                           crvs=crvs, rv_mode=rv_mode)
    s = double_power_sum(dist1, dist2, (1.-alpha)/2, (1.+alpha)/2,
                         rvs=rvs, crvs=crvs, rv_mode=rv_mode)
    return 4*(1.-s)/(1.-alpha*alpha)

def f_divergence(dist1, dist2, f, rvs=None, crvs=None, rv_mode=None):
    """
    The Csiszar f-divergence of `dist1` and `dist2`.

    Parameters
    ----------
    dist1 : Distribution
        The first distribution in the f-divergence.
    dist2 : Distribution
        The second distribution in the f-divergence.
    f : function
        The auxillary function defining the f-divergence
    rvs : list, None
        The indexes of the random variable used to calculate the
        f-divergence between. If None, then the
        f-divergence is calculated over all random variables.
    rv_mode : str, None
        Specifies how to interpret `rvs` and `crvs`. Valid options are:
        {'indices', 'names'}. If equal to 'indices', then the elements of
        `crvs` and `rvs` are interpreted as random variable indices. If equal
        to 'names', the the elements are interpreted as random variable names.
        If `None`, then the value of `dist._rv_mode` is consulted, which
        defaults to 'indices'.

    Returns
    -------
    dkl : float
        The f-divergence between `dist1` and `dist2`.

    Raises
    ------
    ditException
        Raised if either `dist1` or `dist2` doesn't have `rvs` or, if `rvs` is
        None, if `dist2` has an outcome length different than `dist1`.

    """

    rvs, crvs, rv_mode = normalize_rvs(dist1, rvs, crvs, rv_mode)
    rvs, crvs = list(flatten(rvs)), list(flatten(crvs))
    normalize_rvs(dist2, rvs, crvs, rv_mode)

    p1s, q1s = get_pmfs_like(dist1, dist2, rvs+crvs, rv_mode)
    vfunc = np.vectorize(f)
    div = np.nansum(vfunc(np.divide(p1s, q1s)) * q1s)

    if crvs:
        p2s, q2s = get_pmfs_like(dist1, dist2, crvs, rv_mode)
        div = np.nansum(vfunc(np.divide(p2s, q2s)) * q2s)

    return div

<commit_msg>Docstring caveat<commit_after>"""
The Kullback-Leibler divergence.
"""

import numpy as np

from .cross_entropy import get_pmfs_like
from ..helpers import normalize_rvs
from ..utils import flatten

from .kullback_leibler_divergence import kullback_leibler_divergence

__all__ = ('double_power_sum',
           'hellinger_sum',
           'alpha_divergence',
           'hellinger_divergence',
           'renyi_divergence',
           'tsallis_divergence',
          )

### References for Divergence Formulas ###
## http://arxiv.org/pdf/1105.3259v1.pdf
## http://arxiv.org/pdf/1206.2459.pdf
## http://mitran-lab.amath.unc.edu:8082/subversion/grants/Proposals/2013/DOE-DataCentric/biblio/LieseVajdaDivergencesInforTheory.pdf
## Crooks: http://threeplusone.com/on_information.pdf

def double_power_sum(dist1, dist2, exp1=1, exp2=1, rvs=None, crvs=None, rv_mode=None):
    """
    A common generalization of the sums needed to compute the Hellinger
    and alpha divergences below.

    Parameters
    ----------
    dist1 : Distribution
        The first distribution in the sum.
        The second distribution in the sum.
    exp1 : float, 1
        Fisrt exponent used in the power sum.
    exp2 : float, 1
        Second exponent used in the power sum.
    rvs : list, None
        The indexes of the random variable used to calculate the sum.
        If None, then the sum is calculated over all random variables.
    rv_mode : str, None
        Specifies how to interpret `rvs` and `crvs`. Valid options are:
        {'indices', 'names'}. If equal to 'indices', then the elements of
        `crvs` and `rvs` are interpreted as random variable indices. If equal
        to 'names', the the elements are interpreted as random variable names.
        If `None`, then the value of `dist._rv_mode` is consulted, which
        defaults to 'indices'.

    Returns
    -------
    dkl : float
        The specified sum between `dist1` and `dist2`.

    Raises
    ------
    ditException
        Raised if either `dist1` or `dist2` doesn't have `rvs` or, if `rvs` is
        None, if `dist2` has an outcome length different than `dist1`.

    """

    rvs, crvs, rv_mode = normalize_rvs(dist1, rvs, crvs, rv_mode)
    rvs, crvs = list(flatten(rvs)), list(flatten(crvs))
    normalize_rvs(dist2, rvs, crvs, rv_mode)

    p1s, q1s = get_pmfs_like(dist1, dist2, rvs+crvs, rv_mode)
    div = np.nansum(np.power(p1s, exp1) * np.power(q1s, exp2))

    if crvs:
        p2s, q2s = get_pmfs_like(dist1, dist2, crvs, rv_mode)
        div = np.nansum(np.power(p2s, exp1) * np.power(q2s, exp2))

    return div

def hellinger_sum(dist1, dist2, alpha=1., rvs=None, crvs=None, rv_mode=None):
    """
    The Hellinger sum/integral of `dist1` and `dist2`, used to define other
    divergences.

    Parameters
    ----------
    dist1 : Distribution
        The first distribution in the sum.
        The second distribution in the sum.
    alpha : float, 1
        Exponent used in the sum.
    rvs : list, None
        The indexes of the random variable used to calculate the sum.
        If None, then the sum is calculated over all random variables.
    rv_mode : str, None
        Specifies how to interpret `rvs` and `crvs`. Valid options are:
        {'indices', 'names'}. If equal to 'indices', then the elements of
        `crvs` and `rvs` are interpreted as random variable indices. If equal
        to 'names', the the elements are interpreted as random variable names.
        If `None`, then the value of `dist._rv_mode` is consulted, which
        defaults to 'indices'.

    Returns
    -------
    dkl : float
        The Hellinger sum between `dist1` and `dist2`.

    Raises
    ------
    ditException
        Raised if either `dist1` or `dist2` doesn't have `rvs` or, if `rvs` is
        None, if `dist2` has an outcome length different than `dist1`.

    """

    return double_power_sum(dist1, dist2, alpha, 1.-alpha,
                            rvs=rvs, crvs=crvs, rv_mode=rv_mode)

def hellinger_divergence(dist1, dist2, alpha=1., rvs=None, crvs=None, rv_mode=None):
    # http://mitran-lab.amath.unc.edu:8082/subversion/grants/Proposals/2013/DOE-DataCentric/biblio/LieseVajdaDivergencesInforTheory.pdf
    """
    The Hellinger divergence of `dist1` and `dist2`.

    Parameters
    ----------
    dist1 : Distribution
        The first distribution in the Hellinger divergence.
    dist2 : Distribution
        The second distribution in the Hellinger divergence.
    alpha : float, 1
        The divergence is a one parameter family in alpha.
    rvs : list, None
        The indexes of the random variable used to calculate the
        Hellinger divergence between. If None, then the Hellinger
        divergence is calculated over all random variables.
    rv_mode : str, None
        Specifies how to interpret `rvs` and `crvs`. Valid options are:
        {'indices', 'names'}. If equal to 'indices', then the elements of
        `crvs` and `rvs` are interpreted as random variable indices. If equal
        to 'names', the the elements are interpreted as random variable names.
        If `None`, then the value of `dist._rv_mode` is consulted, which
        defaults to 'indices'.

    Returns
    -------
    dkl : float
        The Hellinger divergence between `dist1` and `dist2`.

    Raises
    ------
    ditException
        Raised if either `dist1` or `dist2` doesn't have `rvs` or, if `rvs` is
        None, if `dist2` has an outcome length different than `dist1`.

    """

    if alpha == 1:
        return kullback_leibler_divergence(dist1, dist2, rvs=rvs,
                                           crvs=crvs, rv_mode=rv_mode)
    s = hellinger_sum(dist1, dist2, rvs=rvs, alpha=alpha,
                      crvs=crvs, rv_mode=rv_mode)
    return (s-1.)/(alpha-1.)

def tsallis_divergence(dist1, dist2, alpha=1., rvs=None, crvs=None, rv_mode=None):
    """
    The Tsallis divergence of `dist1` and `dist2`.

    Parameters
    ----------
    dist1 : Distribution
        The first distribution in the Tsallis divergence.
    dist2 : Distribution
        The second distribution in the Tsallis divergence.
    alpha : float, 1
        The divergence is a one parameter family in alpha.
    rvs : list, None
        The indexes of the random variable used to calculate the
        Tsallis divergence between. If None, then the Tsallis
        divergence is calculated over all random variables.
    rv_mode : str, None
        Specifies how to interpret `rvs` and `crvs`. Valid options are:
        {'indices', 'names'}. If equal to 'indices', then the elements of
        `crvs` and `rvs` are interpreted as random variable indices. If equal
        to 'names', the the elements are interpreted as random variable names.
        If `None`, then the value of `dist._rv_mode` is consulted, which
        defaults to 'indices'.

    Returns
    -------
    dkl : float
        The Tsallis divergence between `dist1` and `dist2`.

    Raises
    ------
    ditException
        Raised if either `dist1` or `dist2` doesn't have `rvs` or, if `rvs` is
        None, if `dist2` has an outcome length different than `dist1`.

    """

    # D_T = (D_alpha -1) / (alpha-1)
    if alpha == 1:
        div = kullback_leibler_divergence(dist1, dist2, rvs=rvs,
                                          crvs=crvs, rv_mode=rv_mode)
    else:
        s = hellinger_sum(dist1, dist2, rvs=rvs, alpha=alpha,
                          crvs=crvs, rv_mode=rv_mode)
        div = (s - 1.) /(alpha - 1.)
    return div

def renyi_divergence(dist1, dist2, alpha=1., rvs=None, crvs=None, rv_mode=None):
    """
    The Renyi divergence of `dist1` and `dist2`.

    Parameters
    ----------
    dist1 : Distribution
        The first distribution in the Renyi divergence.
    dist2 : Distribution
        The second distribution in the Renyi divergence.
    alpha : float, 1
        The divergence is a one parameter family in alpha.
    rvs : list, None
        The indexes of the random variable used to calculate the
        Renyi divergence between. If None, then the Renyi
        divergence is calculated over all random variables.
    rv_mode : str, None
        Specifies how to interpret `rvs` and `crvs`. Valid options are:
        {'indices', 'names'}. If equal to 'indices', then the elements of
        `crvs` and `rvs` are interpreted as random variable indices. If equal
        to 'names', the the elements are interpreted as random variable names.
        If `None`, then the value of `dist._rv_mode` is consulted, which
        defaults to 'indices'.

    Returns
    -------
    dkl : float
        The Renyi divergence between `dist1` and `dist2`.

    Raises
    ------
    ditException
        Raised if either `dist1` or `dist2` doesn't have `rvs` or, if `rvs` is
        None, if `dist2` has an outcome length different than `dist1`.

    """

    # D_R = log D_alpha / (alpha-1)
    if alpha == 1:
        div = kullback_leibler_divergence(dist1, dist2, rvs=rvs,
                                          crvs=crvs, rv_mode=rv_mode)
    else:
        s = hellinger_sum(dist1, dist2, rvs=rvs,
                          alpha=alpha, crvs=crvs, rv_mode=rv_mode)
        div = np.log2(s) / (alpha - 1.)
    return div

def alpha_divergence(dist1, dist2, alpha=1., rvs=None, crvs=None, rv_mode=None):
    """
    The alpha divergence of `dist1` and `dist2`, as used in Information
    Geometry. Note there is more than one inequivalent definition of
    "alpha divergence" in the literature, this one comes
    from http://en.wikipedia.org/wiki/Information_geometry .

    Parameters
    ----------
    dist1 : Distribution
        The first distribution in the alpha divergence.
    dist2 : Distribution
        The second distribution in the alpha divergence.
    alpha : float, 1
        The divergence is a one parameter family in alpha.
    rvs : list, None
        The indexes of the random variable used to calculate the
        alpha divergence between. If None, then the alpha
        divergence is calculated over all random variables.
    rv_mode : str, None
        Specifies how to interpret `rvs` and `crvs`. Valid options are:
        {'indices', 'names'}. If equal to 'indices', then the elements of
        `crvs` and `rvs` are interpreted as random variable indices. If equal
        to 'names', the the elements are interpreted as random variable names.
        If `None`, then the value of `dist._rv_mode` is consulted, which
        defaults to 'indices'.

    Returns
    -------
    dkl : float
        The alpha divergence between `dist1` and `dist2`.

    Raises
    ------
    ditException
        Raised if either `dist1` or `dist2` doesn't have `rvs` or, if `rvs` is
        None, if `dist2` has an outcome length different than `dist1`.

    """

    if alpha == 1:
        return kullback_leibler_divergence(dist1, dist2, rvs=rvs,
                                           crvs=crvs, rv_mode=rv_mode)
    if alpha == -1:
        return kullback_leibler_divergence(dist2, dist1, rvs=rvs,
                                           crvs=crvs, rv_mode=rv_mode)
    s = double_power_sum(dist1, dist2, (1.-alpha)/2, (1.+alpha)/2,
                         rvs=rvs, crvs=crvs, rv_mode=rv_mode)
    return 4*(1.-s)/(1.-alpha*alpha)

def f_divergence(dist1, dist2, f, rvs=None, crvs=None, rv_mode=None):
    """
    The Csiszar f-divergence of `dist1` and `dist2`. Note that it is typically
    more accurate to use a specialized divergence function when available
    due to roundoff errors and small probability effects.

    Parameters
    ----------
    dist1 : Distribution
        The first distribution in the f-divergence.
    dist2 : Distribution
        The second distribution in the f-divergence.
    f : function
        The auxillary function defining the f-divergence
    rvs : list, None
        The indexes of the random variable used to calculate the
        f-divergence between. If None, then the
        f-divergence is calculated over all random variables.
    rv_mode : str, None
        Specifies how to interpret `rvs` and `crvs`. Valid options are:
        {'indices', 'names'}. If equal to 'indices', then the elements of
        `crvs` and `rvs` are interpreted as random variable indices. If equal
        to 'names', the the elements are interpreted as random variable names.
        If `None`, then the value of `dist._rv_mode` is consulted, which
        defaults to 'indices'.

    Returns
    -------
    dkl : float
        The f-divergence between `dist1` and `dist2`.

    Raises
    ------
    ditException
        Raised if either `dist1` or `dist2` doesn't have `rvs` or, if `rvs` is
        None, if `dist2` has an outcome length different than `dist1`.

    """

    rvs, crvs, rv_mode = normalize_rvs(dist1, rvs, crvs, rv_mode)
    rvs, crvs = list(flatten(rvs)), list(flatten(crvs))
    normalize_rvs(dist2, rvs, crvs, rv_mode)

    p1s, q1s = get_pmfs_like(dist1, dist2, rvs+crvs, rv_mode)
    vfunc = np.vectorize(f)
    div = np.nansum(vfunc(np.divide(p1s, q1s)) * q1s)

    if crvs:
        p2s, q2s = get_pmfs_like(dist1, dist2, crvs, rv_mode)
        div = np.nansum(vfunc(np.divide(p2s, q2s)) * q2s)

    return div

<|endoftext|>


<commit_before>"""
Wrapper for loading templates from "template" directories in INSTALLED_APPS
packages.
"""

import os
import sys

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.template import TemplateDoesNotExist
from django.utils._os import safe_join
from django.utils.importlib import import_module

# At compile time, cache the directories to search.
fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
app_template_dirs = []
for app in settings.INSTALLED_APPS:
    try:
        mod = import_module(app)
    except ImportError, e:
        raise ImproperlyConfigured, 'ImportError %s: %s' % (app, e.args[0])
    template_dir = os.path.join(os.path.dirname(mod.__file__), 'templates')
    if os.path.isdir(template_dir):
        app_template_dirs.append(template_dir.decode(fs_encoding))

# It won't change, so convert it to a tuple to save memory.
app_template_dirs = tuple(app_template_dirs)

def get_template_sources(template_name, template_dirs=None):
    """
    Returns the absolute paths to "template_name", when appended to each
    directory in "template_dirs". Any paths that don't lie inside one of the
    template dirs are excluded from the result set, for security reasons.
    """
    if not template_dirs:
        template_dirs = app_template_dirs
    for template_dir in template_dirs:
        try:
            yield safe_join(template_dir, template_name)
        except UnicodeDecodeError:
            # The template dir name was a bytestring that wasn't valid UTF-8.
            raise
        except ValueError:
            # The joined path was located outside of template_dir.
            pass

def load_template_source(template_name, template_dirs=None):
    for filepath in get_template_sources(template_name, template_dirs):
        try:
            return (open(filepath).read().decode(settings.FILE_CHARSET), filepath)
        except IOError:
            pass
    raise TemplateDoesNotExist, template_name
load_template_source.is_usable = True
<commit_msg>Fixed #10053: the most exciting commit you'll ever see.<commit_after>"""
Wrapper for loading templates from "templates" directories in INSTALLED_APPS
packages.
"""

import os
import sys

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.template import TemplateDoesNotExist
from django.utils._os import safe_join
from django.utils.importlib import import_module

# At compile time, cache the directories to search.
fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
app_template_dirs = []
for app in settings.INSTALLED_APPS:
    try:
        mod = import_module(app)
    except ImportError, e:
        raise ImproperlyConfigured, 'ImportError %s: %s' % (app, e.args[0])
    template_dir = os.path.join(os.path.dirname(mod.__file__), 'templates')
    if os.path.isdir(template_dir):
        app_template_dirs.append(template_dir.decode(fs_encoding))

# It won't change, so convert it to a tuple to save memory.
app_template_dirs = tuple(app_template_dirs)

def get_template_sources(template_name, template_dirs=None):
    """
    Returns the absolute paths to "template_name", when appended to each
    directory in "template_dirs". Any paths that don't lie inside one of the
    template dirs are excluded from the result set, for security reasons.
    """
    if not template_dirs:
        template_dirs = app_template_dirs
    for template_dir in template_dirs:
        try:
            yield safe_join(template_dir, template_name)
        except UnicodeDecodeError:
            # The template dir name was a bytestring that wasn't valid UTF-8.
            raise
        except ValueError:
            # The joined path was located outside of template_dir.
            pass

def load_template_source(template_name, template_dirs=None):
    for filepath in get_template_sources(template_name, template_dirs):
        try:
            return (open(filepath).read().decode(settings.FILE_CHARSET), filepath)
        except IOError:
            pass
    raise TemplateDoesNotExist, template_name
load_template_source.is_usable = True
<|endoftext|>