<commit_before>import unittest
import json
import pivportal.security
import pivportal.rest

class TestCli(unittest.TestCase):

    def test_dn_is_valid_withinvalidchars(self):
        assert pivportal.security.dn_is_valid("%#DW;$%&*") == False

    def test_dn_is_valid(self):
        assert pivportal.security.dn_is_valid("test_user-123") == True

    def test_username_is_valid_withinvalidchars(self):
        assert pivportal.security.username_is_valid("%#DW;") == False

    def test_username_is_valid(self):
        pivportal.security.dn_to_username = {'test_dn1': "test_user-123"}
        assert pivportal.security.username_is_valid("test_user-123") == True

    def test_requestid_is_valid(self):
        assert pivportal.security.requestid_is_valid("1234567890123456") == True

    def test_requestid_is_valid_bad(self):
        assert pivportal.security.requestid_is_valid("11234567890123456") == False

    def test_requestid_is_valid_badchar(self):
        assert pivportal.security.requestid_is_valid("^1234567890123456") == False

    def test_ip_is_valid(self):
        assert pivportal.security.ip_is_valid("127.0.0.1") == True

    def test_ip_is_valid_badchar(self):
        assert pivportal.security.ip_is_valid(".127.0.0.1") == False

    def test_is_duplicate_register(self):
        username = "user1"
        requestid = "1234567890123456"
        authorized = False
        auth_requests = [{ "username": username, "requestid": "2234567890123456", "authorized": authorized},]
        assert pivportal.security.is_duplicate_register(username, requestid, auth_requests) == False

    def test_is_duplicate_register_samerequestid(self):
        username = "user1"
        requestid = "1234567890123456"
        authorized = False
        auth_requests = [{ "username": username, "requestid": requestid, "authorized": authorized},]
        assert pivportal.security.is_duplicate_register(username, requestid, auth_requests) == True

    def test_is_duplicate_register_exact(self):
        username = "user1"
        requestid = "1234567890123456"
        authorized = False
        auth_requests = [{ "username": username, "requestid": requestid, "authorized": authorized},]
        assert pivportal.security.is_duplicate_register(username, requestid, auth_requests) == True

    def test_user_auth(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        result = pivportal.rest.app.test_client().post("/api/rest/user/login", headers={'SSL_CLIENT_S_DN': 'test_dn1'})
        assert result.status_code == 200

    def test_user_info(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        token_raw = pivportal.rest.app.test_client().post("/api/rest/user/login", headers={'SSL_CLIENT_S_DN': 'test_dn1'})
        token = "%s %s" % ("Authorization", json.loads(token_raw.get_data())["token"])
        result = pivportal.rest.app.test_client().post("/api/rest/user/info", headers={'SSL_CLIENT_S_DN': 'test_dn1', "Authorization": token})
        assert result.status_code == 200

    def test_request_list_invaliduser(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        result = pivportal.rest.app.test_client().post("/api/rest/request/list", headers={'SSL_CLIENT_S_DN': 'test_dn9'})
        assert result.status_code == 401

    def test_request_list(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        token_raw = pivportal.rest.app.test_client().post("/api/rest/user/login", headers={'SSL_CLIENT_S_DN': 'test_dn1'})
        token = "%s %s" % ("Authorization", json.loads(token_raw.get_data())["token"])
        result = pivportal.rest.app.test_client().post("/api/rest/request/list", headers={'SSL_CLIENT_S_DN': 'test_dn1', "Authorization": token})
        assert result.status_code == 200

    def test_request_list_with_requests(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        result = pivportal.rest.app.test_client().post("/api/client/request/register", data={'username': 'testuser1', 'requestid': '0204067090123451'})
        token_raw = pivportal.rest.app.test_client().post("/api/rest/user/login", headers={'SSL_CLIENT_S_DN': 'test_dn1'})
        token = "%s %s" % ("Authorization", json.loads(token_raw.get_data())["token"])
        result = pivportal.rest.app.test_client().post("/api/rest/request/list", headers={'SSL_CLIENT_S_DN': 'test_dn1', "Authorization": token})
        assert result.status_code == 200 and len(json.loads(result.get_data())) == 1

    def test_request_register_usernotfound(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        result = pivportal.rest.app.test_client().post("/api/client/request/register", data={'username': 'unknownuser', 'requestid': '234567890123456'})
        print(result)
        assert result.status_code == 400

    def test_request_register_badrequestid(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        result = pivportal.rest.app.test_client().post("/api/client/request/register", data={'username': 'testuser1', 'requestid': '234567890123456'})
        print(result)
        assert result.status_code == 400

    def test_request_register_nousername(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        result = pivportal.rest.app.test_client().post("/api/client/request/register", data={'requestid': '234567890123456'})
        print(result)
        assert result.status_code == 400

    def test_request_register_norequestid(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        result = pivportal.rest.app.test_client().post("/api/client/request/register", data={'username': 'testuser1'})
        assert result.status_code == 400

    def test_request_register(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        result = pivportal.rest.app.test_client().post("/api/client/request/register", data={'username': 'testuser1', 'requestid': '1234567890123456'})
        assert result.status_code == 200

    def test_request_status_notauthed(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        pivportal.rest.app.test_client().post("/api/client/request/register", data={'username': 'testuser1', 'requestid': '1234567890123456'})
        result = pivportal.rest.app.test_client().post("/api/client/request/status", data={'username': 'testuser1', 'requestid': '1234567890123456'})
        assert result.status_code == 401

    def test_request_status(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        # Register request
        pivportal.rest.app.test_client().post("/api/client/request/register", data={'username': 'testuser1', 'requestid': '1234567890123456'})

        # Login and auth request
        token_raw = pivportal.rest.app.test_client().post("/api/rest/user/login", headers={'SSL_CLIENT_S_DN': 'test_dn1'})
        print(token_raw)
        token = "%s %s" % ("Authorization", json.loads(token_raw.get_data())["token"])
        result = pivportal.rest.app.test_client().post("/api/rest/request/auth", headers={'SSL_CLIENT_S_DN': 'test_dn1', "Authorization": token}, data=json.dumps({'username': 'testuser1', 'requestid': '1234567890123456', 'authorized': True, 'client_ip': None}), content_type='application/json')

        # Status on request
        result = pivportal.rest.app.test_client().post("/api/client/request/status", data={'username': 'testuser1', 'requestid': '1234567890123456'})
        assert result.status_code == 200

    def test_request_auth(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        token_raw = pivportal.rest.app.test_client().post("/api/rest/user/login", headers={'SSL_CLIENT_S_DN': 'test_dn1'})
        print(token_raw)
        token = "%s %s" % ("Authorization", json.loads(token_raw.get_data())["token"])
        result = pivportal.rest.app.test_client().post("/api/rest/request/auth", headers={'SSL_CLIENT_S_DN': 'test_dn1', "Authorization": token}, data=json.dumps({'username': 'testuser1', 'requestid': '1234567890123456', 'authorized': True, 'client_ip': None}), content_type='application/json')
        assert result.status_code == 200
<commit_msg>#24. even more unit testing<commit_after>import unittest
import json
import pivportal.security
import pivportal.rest

class TestCli(unittest.TestCase):

    def test_dn_is_valid_withinvalidchars(self):
        assert pivportal.security.dn_is_valid("%#DW;$%&*") == False

    def test_dn_is_valid(self):
        assert pivportal.security.dn_is_valid("test_user-123") == True

    def test_username_is_valid_withinvalidchars(self):
        assert pivportal.security.username_is_valid("%#DW;") == False

    def test_username_is_valid(self):
        pivportal.security.dn_to_username = {'test_dn1': "test_user-123"}
        assert pivportal.security.username_is_valid("test_user-123") == True

    def test_requestid_is_valid(self):
        assert pivportal.security.requestid_is_valid("1234567890123456") == True

    def test_requestid_is_valid_bad(self):
        assert pivportal.security.requestid_is_valid("11234567890123456") == False

    def test_requestid_is_valid_badchar(self):
        assert pivportal.security.requestid_is_valid("^1234567890123456") == False

    def test_ip_is_valid(self):
        assert pivportal.security.ip_is_valid("127.0.0.1") == True

    def test_ip_is_valid_badchar(self):
        assert pivportal.security.ip_is_valid(".127.0.0.1") == False

    def test_is_duplicate_register(self):
        username = "user1"
        requestid = "1234567890123456"
        authorized = False
        auth_requests = [{ "username": username, "requestid": "2234567890123456", "authorized": authorized},]
        assert pivportal.security.is_duplicate_register(username, requestid, auth_requests) == False

    def test_is_duplicate_register_samerequestid(self):
        username = "user1"
        requestid = "1234567890123456"
        authorized = False
        auth_requests = [{ "username": username, "requestid": requestid, "authorized": authorized},]
        assert pivportal.security.is_duplicate_register(username, requestid, auth_requests) == True

    def test_is_duplicate_register_exact(self):
        username = "user1"
        requestid = "1234567890123456"
        authorized = False
        auth_requests = [{ "username": username, "requestid": requestid, "authorized": authorized},]
        assert pivportal.security.is_duplicate_register(username, requestid, auth_requests) == True

    def test_user_auth(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        result = pivportal.rest.app.test_client().post("/api/rest/user/login", headers={'SSL_CLIENT_S_DN': 'test_dn1'})
        assert result.status_code == 200

    def test_user_info(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        token_raw = pivportal.rest.app.test_client().post("/api/rest/user/login", headers={'SSL_CLIENT_S_DN': 'test_dn1'})
        token = "%s %s" % ("Authorization", json.loads(token_raw.get_data())["token"])
        result = pivportal.rest.app.test_client().post("/api/rest/user/info", headers={'SSL_CLIENT_S_DN': 'test_dn1', "Authorization": token})
        assert result.status_code == 200

    def test_request_list_invaliduser(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        result = pivportal.rest.app.test_client().post("/api/rest/request/list", headers={'SSL_CLIENT_S_DN': 'test_dn9'})
        assert result.status_code == 401

    def test_request_list(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        token_raw = pivportal.rest.app.test_client().post("/api/rest/user/login", headers={'SSL_CLIENT_S_DN': 'test_dn1'})
        token = "%s %s" % ("Authorization", json.loads(token_raw.get_data())["token"])
        result = pivportal.rest.app.test_client().post("/api/rest/request/list", headers={'SSL_CLIENT_S_DN': 'test_dn1', "Authorization": token})
        assert result.status_code == 200

    def test_request_list_with_requests(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        result = pivportal.rest.app.test_client().post("/api/client/request/register", data={'username': 'testuser1', 'requestid': '0204067090123451'})
        token_raw = pivportal.rest.app.test_client().post("/api/rest/user/login", headers={'SSL_CLIENT_S_DN': 'test_dn1'})
        token = "%s %s" % ("Authorization", json.loads(token_raw.get_data())["token"])
        result = pivportal.rest.app.test_client().post("/api/rest/request/list", headers={'SSL_CLIENT_S_DN': 'test_dn1', "Authorization": token})
        assert result.status_code == 200 and len(json.loads(result.get_data())) == 1

    def test_request_register_usernotfound(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        result = pivportal.rest.app.test_client().post("/api/client/request/register", data={'username': 'unknownuser', 'requestid': '234567890123456'})
        print(result)
        assert result.status_code == 400

    def test_request_register_badrequestid(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        result = pivportal.rest.app.test_client().post("/api/client/request/register", data={'username': 'testuser1', 'requestid': '234567890123456'})
        print(result)
        assert result.status_code == 400

    def test_request_register_nousername(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        result = pivportal.rest.app.test_client().post("/api/client/request/register", data={'requestid': '234567890123456'})
        print(result)
        assert result.status_code == 400

    def test_request_register_norequestid(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        result = pivportal.rest.app.test_client().post("/api/client/request/register", data={'username': 'testuser1'})
        assert result.status_code == 400

    def test_request_register(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        result = pivportal.rest.app.test_client().post("/api/client/request/register", data={'username': 'testuser1', 'requestid': '1234567890123456'})
        assert result.status_code == 200

    def test_request_status_notauthed(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        pivportal.rest.app.test_client().post("/api/client/request/register", data={'username': 'testuser1', 'requestid': '1234567890123456'})
        result = pivportal.rest.app.test_client().post("/api/client/request/status", data={'username': 'testuser1', 'requestid': '1234567890123456'})
        assert result.status_code == 401

    def test_request_status(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        # Register request
        pivportal.rest.app.test_client().post("/api/client/request/register", data={'username': 'testuser1', 'requestid': '1234567890123456'})

        # Login and auth request
        token_raw = pivportal.rest.app.test_client().post("/api/rest/user/login", headers={'SSL_CLIENT_S_DN': 'test_dn1'})
        print(token_raw)
        token = "%s %s" % ("Authorization", json.loads(token_raw.get_data())["token"])
        result = pivportal.rest.app.test_client().post("/api/rest/request/auth", headers={'SSL_CLIENT_S_DN': 'test_dn1', "Authorization": token}, data=json.dumps({'username': 'testuser1', 'requestid': '1234567890123456', 'authorized': True, 'client_ip': None}), content_type='application/json')

        # Status on request
        result = pivportal.rest.app.test_client().post("/api/client/request/status", data={'username': 'testuser1', 'requestid': '1234567890123456'})
        assert result.status_code == 200

    def test_request_auth(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        token_raw = pivportal.rest.app.test_client().post("/api/rest/user/login", headers={'SSL_CLIENT_S_DN': 'test_dn1'})
        print(token_raw)
        token = "%s %s" % ("Authorization", json.loads(token_raw.get_data())["token"])
        result = pivportal.rest.app.test_client().post("/api/rest/request/auth", headers={'SSL_CLIENT_S_DN': 'test_dn1', "Authorization": token}, data=json.dumps({'username': 'testuser1', 'requestid': '1234567890123456', 'authorized': True, 'client_ip': None}), content_type='application/json')
        assert result.status_code == 200

    def test_token_required_invalidtoken(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        token = "%s %s" % ("Authorization", "badtoken123")
        result = pivportal.rest.app.test_client().post("/api/rest/request/auth", headers={'SSL_CLIENT_S_DN': 'test_dn1', "Authorization": token}, data=json.dumps({'username': 'testuser1', 'requestid': '1234567890123456', 'authorized': True, 'client_ip': None}), content_type='application/json')
        assert result.status_code == 401

    def test_valid_client_cert_required_missingclientdnheader(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1"}
        token_raw = pivportal.rest.app.test_client().post("/api/rest/user/login", headers={'SSL_CLIENT_S_DN': 'test_dn1'})
        print(token_raw)
        token = "%s %s" % ("Authorization", json.loads(token_raw.get_data())["token"])
        result = pivportal.rest.app.test_client().post("/api/rest/request/auth", headers={"Authorization": token}, data=json.dumps({'username': 'testuser1', 'requestid': '1234567890123456', 'authorized': True, 'client_ip': None}), content_type='application/json')
        assert result.status_code == 401

    def test_valid_client_cert_required_notvaliduser(self):
        pivportal.security.dn_to_username = {}
        result = pivportal.rest.app.test_client().post("/api/rest/user/login", headers={'SSL_CLIENT_S_DN': 'test_dn1'})
        assert result.status_code == 401

    def test_valid_client_cert_required_invalidusername(self):
        pivportal.security.dn_to_username = {'test_dn1': "testuser1.$#@#"}
        result = pivportal.rest.app.test_client().post("/api/rest/user/login", headers={'SSL_CLIENT_S_DN': 'testuser1.$#@#'})
        assert result.status_code == 400<|endoftext|>


<commit_before>
import unittest
import os

from PySide.QtCore import QSettings
from QtMobility.ServiceFramework import QServiceManager

class InterfaceDefault(unittest.TestCase):

    def testNullInterfaceDefault(self):
        manager = QServiceManager()

        self.assertFalse(manager.interfaceDefault('').isValid())

class AddRemoveService(unittest.TestCase):

    def setUp(self):
        self.settings = QSettings('com.nokia.qt.serviceframework.tests',
                                  'SampleServicePlugin')
        self.manager = QServiceManager()

        self.serviceXml = os.path.join(os.path.dirname(__file__),
                                  'sampleservice.xml')

    def testAddRemoveService(self):
        self.assertTrue(self.manager.addService(self.serviceXml))
        self.assertEqual(self.manager.findServices(), ['SampleService'])
        self.assertTrue(self.settings.value('installed'))

        self.assertEqual(self.manager.findServices('com.nokia.qt.TestInterfaceA'),
                         ['SampleService'])
        self.assertTrue(self.manager.removeService('SampleService'))
        self.assertEqual(self.manager.findServices(), [])

        self.assertEqual(self.manager.findServices('com.nokia.qt.TestInterfaceA'), [])
        self.assertFalse(self.settings.value('installed'))

class RemoveService(unittest.TestCase):

    def testRemoveNonExistent(self):
        manager = QServiceManager()

        self.assertFalse(manager.removeService('NonExistentService'))

if __name__ == '__main__':
    unittest.main()
<commit_msg>Improve QServiceManager test<commit_after>
import unittest
import os

from PySide.QtCore import QSettings
from QtMobility.ServiceFramework import QServiceManager

class InterfaceDefault(unittest.TestCase):

    def testNullInterfaceDefault(self):
        manager = QServiceManager()

        self.assertFalse(manager.interfaceDefault('').isValid())

class AddRemoveService(unittest.TestCase):

    def setUp(self):
        self.settings = QSettings('com.nokia.qt.serviceframework.tests',
                                  'SampleServicePlugin')
        self.manager = QServiceManager()

        self.serviceXml = os.path.join(os.path.dirname(__file__),
                                  'sampleservice.xml')

    def testAddRemoveService(self):
        self.assertTrue(self.manager.addService(self.serviceXml))
        self.assertTrue('SampleService' in self.manager.findServices())
        self.assertTrue(self.settings.value('installed'))

        self.assertEqual(self.manager.findServices('com.nokia.qt.TestInterfaceA'),
                         ['SampleService'])
        self.assertTrue(self.manager.removeService('SampleService'))
        self.assertTrue('SampleService' not in self.manager.findServices())

        self.assertEqual(self.manager.findServices('com.nokia.qt.TestInterfaceA'), [])
        self.assertFalse(self.settings.value('installed'))

class RemoveService(unittest.TestCase):

    def testRemoveNonExistent(self):
        manager = QServiceManager()

        self.assertFalse(manager.removeService('NonExistentService'))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>def is_permutation(s, t):
    """
    Returns True iff s is a permutation of t.

    Clarifications for interviewer:
    - How are the strings encoded? ASCII? Unicode?
    - What kinds of characters are used? Alphanumeric? Punctuation?

    Here, we assume that all strings are encoded with ASCII (256 chars).
    """

    #---------------------------------------------------------------------------
    # Algorithm 1: Cross off matching characters from strings.
    #              O(n^2) time, O(n) space.
    # Algorithm 2: Sort both strings and compare one-by-one.
    #              O(n log n) time, O(n) space.
    # Algorithm 3: Compare character counts (implemented below).
    #              O(n) time, O(n) space.
    #---------------------------------------------------------------------------

    char_counts = [0] * 256
    for c in s:
        char_counts[ord(c)] += 1
    for c in t:
        char_counts[ord(c)] -= 1
    return not any(char_counts)  # True iff all zeroes.

if __name__ == "__main__":
    testStringPairs = [
        ("", ""),
        ("stackoverflow", "flowstackover"),
        ("12345", "5xy1234"),
        ("stackoverflaw", "overflowstack"),
        ("!!?!", "!?!")
    ]
    for (index, pair) in enumerate(testStringPairs):
        s, t = pair
        print("Test String #{}:".format(index + 1))
        print("{} {} a permutation of {}.".format(
            repr(s), "is" if is_permutation(s, t) else "is not", repr(t)
        ))<commit_msg>Added more explanatory comments and formatting.<commit_after>def is_permutation(s, t):
    """
    Returns True iff s is a permutation of t.

    Clarifications to ask the interviewer:
    - How are the strings encoded? ASCII? Unicode?
    - What kinds of characters are used? Alphanumeric? Punctuation?

    Here, we assume that all strings are encoded with ASCII (256 chars). Recall
    that the ord() function takes as input an 8-bit ASCII string of length one
    and returns the integer value of the byte in [0, 255]. Otherwise, if the
    string is encoded with Unicode (2^16 = 65536 chars), then the ord() function
    will return the integer representing the Unicode code point of the character
    in [0, 65535].
    """

    #---------------------------------------------------------------------------
    # Algorithm 1: Cross off matching characters from strings.
    #              O(n^2) time, O(n) space.
    #---------------------------------------------------------------------------
    # Algorithm 2: Sort both strings and compare one-by-one.
    #              O(n log n) time, O(n) space.
    #---------------------------------------------------------------------------
    # Algorithm 3: Compare character counts (implemented below).
    #              O(n) time, O(n) space.
    #---------------------------------------------------------------------------

    MAX_CHARS = 256
    char_counts = [0] * MAX_CHARS
    for c in s:
        char_counts[ord(c)] += 1
    for c in t:
        char_counts[ord(c)] -= 1
    return not any(char_counts)  # True iff all zeroes.

if __name__ == "__main__":
    testStringPairs = [
        ("", ""),
        ("stackoverflow", "flowstackover"),
        ("12345", "5xy1234"),
        ("stackoverflaw", "overflowstack"),
        ("!!?!", "!?!")
    ]
    for (index, pair) in enumerate(testStringPairs):
        s, t = pair
        print("Test String #{}:".format(index + 1))
        print("{} {} a permutation of {}.".format(
            repr(s), "is" if is_permutation(s, t) else "is not", repr(t)
        ))<|endoftext|>


<commit_before>#
# Copyright (c) 2014 QuarksLab.
# This file is part of IRMA project.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License in the top-level directory
# of this distribution and at:
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# No part of the project, including this file, may be copied,
# modified, propagated, or distributed except according to the
# terms contained in the LICENSE file.

from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
from ..common.exceptions import IrmaValueError, IrmaDatabaseResultNotFound
from ..common.exceptions import IrmaDatabaseError

class SQLDatabaseObject(object):
    """Mother class for the SQL tables
    """

    __tablename__ = None
    _idname = None

    # Fields
    # In the subclasses, the variables names must be the same has fields
    # names without the suffix except for the foreign keys and PFKs
    # Ex for a PK or field : var name: some_field
    #                        field name: some_field_suffix
    # Ex for a FK or PFK : var name: some_key
    #                      key name: some_key
    id = None

    def __init__(self):
        if type(self) is SQLDatabaseObject:
            reason = "The SQLDatabaseObject class has to be overloaded"
            raise IrmaValueError(reason)

    def to_dict(self, include_pks=True, include_fks=True, columns_list=None):
        """Converts object to dict.
        :rtype: dict
        """
        res = {}
        if columns_list is None:
            columns_list = []
            for column in self.__table__.columns:
                # table name removal (fixed length)
                item = str(column)[len(self.__tablename__ + '.'):]
                columns_list.append(item)

        pk_names_list = []
        if not include_pks:
            for pk in self.__mapper__.primary_key:
                pk_names_list.append(pk.name)

        fk_names_list = []
        if not include_fks:
            for fk in self.__table__.foreign_keys:
                # table name removal (various length)
                fk_names_list.append(fk.target_fullname.rsplit('.', 1)[1])

        for key in columns_list:
            if getattr(self, key) is not None:
                if (not include_pks and key in pk_names_list) or\
                        (not include_fks and key in fk_names_list):
                    continue
                res[key] = getattr(self, key)
        return res

    def update(self, columns_list=None, session=None):
        """Save the new state of the current object in the database
        :param update_dict: the fields to update (all fields are being
            updated if not provided)
        :param session: the session to use
        """
        update_dict = self.to_dict(include_pks=False,
                                   columns_list=columns_list)
        session.query(self.__class__).\
            filter(self.__class__.id == self.id).\
            update(update_dict)

    def save(self, session):
        """Save the current object in the database
        :param session: the session to use
        """
        session.add(self)

    @classmethod
    def query_fields(cls):
        fields = {}
        for (name, column) in cls.__table__.columns.items():
            if column.primary_key:
                continue
            if len(column.foreign_keys) > 0:
                continue
            fields[name] = column
        return fields

    @classmethod
    def load(cls, id, session):
        """Load an object from the database
        :param id: the id to look for
        :param session: the session to use
        :rtype: cls
        :return: the object that corresponds to the id
        :raise IrmaDatabaseResultNotFound: if the object doesn't exist
        """
        try:
            return cls.find_by_id(id, session=session)
        except Exception:
            raise IrmaDatabaseResultNotFound(
                "The given id ({0}) doesn't exist in {1}".format(
                    id, cls.__tablename__
                )
            )

    def remove(self, session):
        """Remove the current object from the database
        :param session: the session to use
        """
        session.delete(self)

    @classmethod
    def find_by_id(cls, id, session):
        """Find the object in the database
        :param id: the id to look for
        :param session: the session to use
        :rtype: cls
        :return: the object that corresponds to the id
        :raise IrmaDatabaseResultNotFound, IrmaDatabaseError
        """
        try:
            return session.query(cls).filter(
                cls.id == id
            ).one()
        except NoResultFound as e:
            raise IrmaDatabaseResultNotFound(e)
        except MultipleResultsFound as e:
            raise IrmaDatabaseError(e)

    @classmethod
    def paginate(cls, query, page=None, page_size=None, order_by=None):
        """Paginate query
        :param query a sqlalchemy query with all filtering done
        :param page skip page * page_size items in results
        :param page_size nb of items returned
        :param order_by results are sorted by this column
               due to join not known, should be checked by caller
        :rtype: cls
        :return: a key:value dict with returned objects
        :raise IrmaDatabaseError
        """
        res = dict()
        res['total'] = query.count()
        if order_by is not None:
            query = query.order_by(order_by)
        if page_size is not None:
            page_size = int(page_size)
            query = query.limit(page_size)
        if page is not None and page > 0:
            page = int(page) - 1
            query = query.offset(page * page_size)
        res['items'] = []
        for row in query.all():
            dict_row = dict((k, row[i]) for i, k in enumerate(row.keys()))
            res['items'].append(dict_row)
        return res

    def __repr__(self):
        return str(self.to_dict())

    def __str__(self):
        return str(self.to_dict())
<commit_msg>add desc parameter for paginate and default pagesize to 25<commit_after>#
# Copyright (c) 2014 QuarksLab.
# This file is part of IRMA project.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License in the top-level directory
# of this distribution and at:
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# No part of the project, including this file, may be copied,
# modified, propagated, or distributed except according to the
# terms contained in the LICENSE file.

import sqlalchemy
from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
from ..common.exceptions import IrmaValueError, IrmaDatabaseResultNotFound
from ..common.exceptions import IrmaDatabaseError

class SQLDatabaseObject(object):
    """Mother class for the SQL tables
    """

    __tablename__ = None
    _idname = None

    # Fields
    # In the subclasses, the variables names must be the same has fields
    # names without the suffix except for the foreign keys and PFKs
    # Ex for a PK or field : var name: some_field
    #                        field name: some_field_suffix
    # Ex for a FK or PFK : var name: some_key
    #                      key name: some_key
    id = None

    def __init__(self):
        if type(self) is SQLDatabaseObject:
            reason = "The SQLDatabaseObject class has to be overloaded"
            raise IrmaValueError(reason)

    def to_dict(self, include_pks=True, include_fks=True, columns_list=None):
        """Converts object to dict.
        :rtype: dict
        """
        res = {}
        if columns_list is None:
            columns_list = []
            for column in self.__table__.columns:
                # table name removal (fixed length)
                item = str(column)[len(self.__tablename__ + '.'):]
                columns_list.append(item)

        pk_names_list = []
        if not include_pks:
            for pk in self.__mapper__.primary_key:
                pk_names_list.append(pk.name)

        fk_names_list = []
        if not include_fks:
            for fk in self.__table__.foreign_keys:
                # table name removal (various length)
                fk_names_list.append(fk.target_fullname.rsplit('.', 1)[1])

        for key in columns_list:
            if getattr(self, key) is not None:
                if (not include_pks and key in pk_names_list) or\
                        (not include_fks and key in fk_names_list):
                    continue
                res[key] = getattr(self, key)
        return res

    def update(self, columns_list=None, session=None):
        """Save the new state of the current object in the database
        :param update_dict: the fields to update (all fields are being
            updated if not provided)
        :param session: the session to use
        """
        update_dict = self.to_dict(include_pks=False,
                                   columns_list=columns_list)
        session.query(self.__class__).\
            filter(self.__class__.id == self.id).\
            update(update_dict)

    def save(self, session):
        """Save the current object in the database
        :param session: the session to use
        """
        session.add(self)

    @classmethod
    def query_fields(cls):
        fields = {}
        for (name, column) in cls.__table__.columns.items():
            if column.primary_key:
                continue
            if len(column.foreign_keys) > 0:
                continue
            fields[name] = column
        return fields

    @classmethod
    def load(cls, id, session):
        """Load an object from the database
        :param id: the id to look for
        :param session: the session to use
        :rtype: cls
        :return: the object that corresponds to the id
        :raise IrmaDatabaseResultNotFound: if the object doesn't exist
        """
        try:
            return cls.find_by_id(id, session=session)
        except Exception:
            raise IrmaDatabaseResultNotFound(
                "The given id ({0}) doesn't exist in {1}".format(
                    id, cls.__tablename__
                )
            )

    def remove(self, session):
        """Remove the current object from the database
        :param session: the session to use
        """
        session.delete(self)

    @classmethod
    def find_by_id(cls, id, session):
        """Find the object in the database
        :param id: the id to look for
        :param session: the session to use
        :rtype: cls
        :return: the object that corresponds to the id
        :raise IrmaDatabaseResultNotFound, IrmaDatabaseError
        """
        try:
            return session.query(cls).filter(
                cls.id == id
            ).one()
        except NoResultFound as e:
            raise IrmaDatabaseResultNotFound(e)
        except MultipleResultsFound as e:
            raise IrmaDatabaseError(e)

    @classmethod
    def paginate(cls, query, page=None, page_size=None,
                 order_by=None, desc=False):
        """Paginate query
        :param query a sqlalchemy query with all filtering done
        :param page skip page * page_size items in results
        :param page_size nb of items returned
        :param order_by results are sorted by this column
               due to join not known, should be checked by caller
        :param rev_order boolean to switch order_by ordering
        :rtype: dict
        :return: a key:value dict with returned objects
        :raise IrmaDatabaseError
        """
        res = dict()
        res['total'] = query.count()
        if order_by is not None:
            if desc:
                order_by = sqlalchemy.desc(order_by)
            query = query.order_by(order_by)
        if page_size is not None:
            try:
                page_size = int(page_size)
            except ValueError:
                raise IrmaValueError("wrong argument for page_size")
        else:
            page_size = 25
        query = query.limit(page_size)
        if page is not None and page > 0:
            try:
                page = int(page) - 1
            except ValueError:
                raise IrmaValueError("wrong argument for page")
            query = query.offset(page * page_size)
        res['items'] = []
        for row in query.all():
            dict_row = dict((k, row[i]) for i, k in enumerate(row.keys()))
            res['items'].append(dict_row)
        return res

    def __repr__(self):
        return str(self.to_dict())

    def __str__(self):
        return str(self.to_dict())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import logging, random, traceback
l = logging.getLogger(__name__)

from datetime import date

from django.conf import settings
from django.contrib.auth.models import User
from django.db import models

class AnonymousVisitor(models.Model):
    """An anonymous visitor"""
    created = models.DateTimeField(auto_now_add=True, db_index=True)

class GoalType(models.Model):
    """Defines a type of goal."""
    name = models.CharField(max_length=128, unique=True)

    def __unicode__(self):
        return self.name

class GoalRecord(models.Model):
    """Records a discreet goal achievement."""
    created = models.DateTimeField(auto_now_add=True, db_index=True)
    anonymous_visitor = models.ForeignKey(AnonymousVisitor)
    goal_type = models.ForeignKey(GoalType)

    @classmethod
    def _record(cls, goal_name, experiment_user):
        """
        Records a goal achievement for the experiment user.
        If the user does not have an anonymous visitor ID, does nothing.
        If the goal name is not known, throws an Exception.
        """
        anonymous_id = experiment_user.get_anonymous_id()
        if anonymous_id:
            anonymous_visitor = AnonymousVisitor.objects.get(id=anonymous_id)
            goal_type = GoalType.objects.get(name=goal_name)
            GoalRecord(goal_type=goal_type,
                       anonymous_visitor=anonymous_visitor).save()

    @classmethod
    def record(cls, goal_name, experiment_user):
        try:
            return cls._record(goal_name, experiment_user)
        except GoalType.DoesNotExist:
            raise
        except Exception, e:
            l.error("Unexpected exception in GoalRecord.record:\n"
                    "%s" % traceback.format_exc)

class Experiment(models.Model):
    """ Defines a split testing experiment"""
    class __UnverifiedUser(object):
        def __init__(self, experiment_user):
            self.experiment_user = experiment_user

        def get_enrollment(self, experiment):
            return self.experiment_user.get_temporary_enrollment(
                experiment.name)

        def set_enrollment(self, experiment, group_id):
            self.experiment_user.store_temporary_enrollment(experiment.name,
                                                            group_id)

    class __RegisteredUser(object):
        def __init__(self, experiment_user):
            self.experiment_user = experiment_user

        def get_enrollment(self, experiment):
            participants = Participant.objects.filter(
                user=self.experiment_user.get_registered_user(),
                experiment=experiment)
            if participants.count() == 1:
                return participants[0].group
            else:
                return None

        def set_enrollment(self, experiment, group_id):
            Participant.objects.create(
                user=self.experiment_user.get_registered_user(),
                experiment=experiment, group=group_id)

    class __AnonymousUser(object):
        def __init__(self, experiment_user):
            self.experiment_user = experiment_user

        def __get_anonymous_visitor(self):
            anonymous_id = self.experiment_user.get_anonymous_id()
            if anonymous_id == None:
                return None
            anonymous_visitors = AnonymousVisitor.objects.filter(
                id=anonymous_id)
            if anonymous_visitors.count() != 1:
                return None
            return anonymous_visitors[0]

        def get_enrollment(self, experiment):
            anonymous_visitor = self.__get_anonymous_visitor()

            if anonymous_visitor:
                participants = Participant.objects.filter(
                    anonymous_visitor=anonymous_visitor,
                    experiment=experiment)
                if participants.count() == 1:
                    return participants[0].group
            return None

        def set_enrollment(self, experiment, group_id):
            anonymous_visitor = self.__get_anonymous_visitor()
            if not anonymous_visitor:
                anonymous_visitor = AnonymousVisitor()
                anonymous_visitor.save()
                self.experiment_user.set_anonymous_id(anonymous_visitor.id)

            Participant.objects.create(
                anonymous_visitor=anonymous_visitor,
                experiment=experiment, group=group_id)

    @classmethod
    def __create_user(cls, experiment_user):
        if not experiment_user.is_anonymous():
            return cls.__RegisteredUser(experiment_user)
        if not experiment_user.is_verified_human():
            return cls.__UnverifiedUser(experiment_user)
        else:
            return cls.__AnonymousUser(experiment_user)

    DISABLED_STATE = 0
    ENABLED_STATE = 1
    PROMOTED_STATE = 2

    STATES = (
        (DISABLED_STATE, 'Disabled'),
        (ENABLED_STATE, 'Enabled'),
        (PROMOTED_STATE, 'Promoted'))

    name = models.CharField(unique=True, max_length=128)
    state = models.IntegerField(default=DISABLED_STATE, choices=STATES)
    start_date = models.DateField(blank=True, null=True, db_index=True)
    end_date = models.DateField(blank=True, null=True)

    def __unicode__(self):
        return self.name

    def save(self, *args, **kwargs):
        """
        The save override's goal is to save the start or end date when
        the state changes
        """
        #do nothing for new ones
        if self.id:
            old_self = None

            try:
                old_self = Experiment.objects.get(id=self.id)
            except Experiment.DoesNotExist:
                raise Exception("Can't find the existing Experiment.")

            if old_self.state != self.state:
                if (old_self.state == Experiment.DISABLED_STATE
                    and self.state == Experiment.ENABLED_STATE
                    and not old_self.start_date):
                    # enabling
                    self.start_date = date.today()
                elif (old_self.state == Experiment.ENABLED_STATE
                      and self.state == Experiment.DISABLED_STATE
                      and not old_self.end_date):
                    # disabling
                    self.end_date = date.today()
                elif (old_self.state == Experiment.ENABLED_STATE
                      and self.state == Experiment.PROMOTED_STATE
                      and not old_self.end_date):
                    #promoting
                    self.end_date = date.today()
        return super(Experiment, self).save(*args, **kwargs)

    @staticmethod
    def control(experiment_name, experiment_user):
        """
        Will return True when user is part of the control group in
        the passed experiment. If the user is not enrolled in this
        experiment, and the experiment is enabled, it will enroll the user.
        """
        return Experiment.__test_group(experiment_name, experiment_user,
                                       Participant.CONTROL_GROUP)

    @staticmethod
    def test(experiment_name, experiment_user):
        """
        Will return True when user is part of the test group in
        the passed experiment. If the user is not enrolled in this
        experiment, and the experiment is enabled, it will enroll the user.
        """
        return Experiment.__test_group(experiment_name, experiment_user,
                                       Participant.TEST_GROUP)

    @classmethod
    def __test_group(cls, experiment_name, experiment_user, queried_group):
        """does the real work"""
        experiment = None
        try:
            experiment = Experiment.objects.get(name=experiment_name)
        except Experiment.DoesNotExist:
            if settings.DEBUG:
                raise Exception("Can't find the Experiment named %s" %
                                experiment_name)
            else:
                l.warning("Can't find the Experiment named %s" %
                          experiment_name)
                return queried_group == Participant.CONTROL_GROUP
        if experiment.state == Experiment.DISABLED_STATE:
            return queried_group == Participant.CONTROL_GROUP
        elif experiment.state == Experiment.PROMOTED_STATE:
            return queried_group == Participant.TEST_GROUP

        if experiment.state != Experiment.ENABLED_STATE:
            raise Exception("Invalid experiment state !")

        user = cls.__create_user(experiment_user)

        assigned_group = user.get_enrollment(experiment)

        if assigned_group == None:
            assigned_group = random.choice((Participant.CONTROL_GROUP,
                                            Participant.TEST_GROUP))
            user.set_enrollment(experiment, assigned_group)

        return queried_group == assigned_group

class Participant(models.Model):
    """A participant in a split testing experiment """

    class Meta:
        unique_together= (('user', 'experiment'),
                          ('anonymous_visitor', 'experiment'))

    CONTROL_GROUP = 0
    TEST_GROUP = 1

    GROUPS = (
        (CONTROL_GROUP, "Control"),
        (TEST_GROUP, "Test"))

    user = models.ForeignKey(User, null=True)
    experiment = models.ForeignKey(Experiment)
    enrollment_date = models.DateField(db_index=True, auto_now_add=True)
    group = models.IntegerField(choices=GROUPS)
    anonymous_visitor = models.ForeignKey(AnonymousVisitor, null=True, blank=True)

    def __unicode__(self):
        if self.user: # can be null
            username = self.user.username
        else:
            username = 'anonymous#%d' % self.anonymous_visitor.id
        return "%s %s" % (username, self.group)

    def __init__(self, *args, **kwargs):
        super(Participant, self).__init__(*args, **kwargs)
        if not self.id:
            if (not self.anonymous_visitor) == (not self.user):
                raise Exception("Participants require exactly one of "
                                "`anonymous_visitor` or `user`.")

class DailyEngagementReport(models.Model):
    """Hold the scores for a given experiment on a given day"""
    date = models.DateField(db_index=True)
    experiment = models.ForeignKey(Experiment)
    test_score = models.FloatField(null=True)
    control_score = models.FloatField(null=True)
    test_group_size = models.IntegerField()
    control_group_size = models.IntegerField()
    confidence = models.FloatField(null=True)

class DailyConversionReport(models.Model):
    """Stores the daily conversion scores."""
    date = models.DateField(db_index=True)
    experiment = models.ForeignKey(Experiment)
    overall_test_conversion = models.IntegerField()
    overall_control_conversion = models.IntegerField()
    test_group_size = models.IntegerField()
    control_group_size = models.IntegerField()
    confidence = models.FloatField(null=True)

class  DailyConversionReportGoalData(models.Model):
    """Stores the daily conversion report goal data."""
    report = models.ForeignKey(DailyConversionReport, related_name="goal_data")
    goal_type = models.ForeignKey(GoalType)
    test_conversion = models.IntegerField()
    control_conversion = models.IntegerField()
    confidence = models.FloatField(null=True)
<commit_msg>do not propagate GoalType.DoesNotExist exception in GoalRecord.record if DEBUG is False<commit_after># -*- coding: utf-8 -*-
import logging, random, traceback
l = logging.getLogger(__name__)

from datetime import date

from django.conf import settings
from django.contrib.auth.models import User
from django.db import models

class AnonymousVisitor(models.Model):
    """An anonymous visitor"""
    created = models.DateTimeField(auto_now_add=True, db_index=True)

class GoalType(models.Model):
    """Defines a type of goal."""
    name = models.CharField(max_length=128, unique=True)

    def __unicode__(self):
        return self.name

class GoalRecord(models.Model):
    """Records a discreet goal achievement."""
    created = models.DateTimeField(auto_now_add=True, db_index=True)
    anonymous_visitor = models.ForeignKey(AnonymousVisitor)
    goal_type = models.ForeignKey(GoalType)

    @classmethod
    def _record(cls, goal_name, experiment_user):
        """
        Records a goal achievement for the experiment user.
        If the user does not have an anonymous visitor ID, does nothing.
        If the goal name is not known, throws an Exception.
        """
        anonymous_id = experiment_user.get_anonymous_id()
        if anonymous_id:
            anonymous_visitor = AnonymousVisitor.objects.get(id=anonymous_id)
            goal_type = GoalType.objects.get(name=goal_name)
            GoalRecord(goal_type=goal_type,
                       anonymous_visitor=anonymous_visitor).save()

    @classmethod
    def record(cls, goal_name, experiment_user):
        try:
            return cls._record(goal_name, experiment_user)
        except GoalType.DoesNotExist:
            if settings.DEBUG:
                raise
            l.warning("Can't find the GoalType named %s" % goal_name)
        except Exception, e:
            l.error("Unexpected exception in GoalRecord.record:\n"
                    "%s" % traceback.format_exc)

class Experiment(models.Model):
    """ Defines a split testing experiment"""
    class __UnverifiedUser(object):
        def __init__(self, experiment_user):
            self.experiment_user = experiment_user

        def get_enrollment(self, experiment):
            return self.experiment_user.get_temporary_enrollment(
                experiment.name)

        def set_enrollment(self, experiment, group_id):
            self.experiment_user.store_temporary_enrollment(experiment.name,
                                                            group_id)

    class __RegisteredUser(object):
        def __init__(self, experiment_user):
            self.experiment_user = experiment_user

        def get_enrollment(self, experiment):
            participants = Participant.objects.filter(
                user=self.experiment_user.get_registered_user(),
                experiment=experiment)
            if participants.count() == 1:
                return participants[0].group
            else:
                return None

        def set_enrollment(self, experiment, group_id):
            Participant.objects.create(
                user=self.experiment_user.get_registered_user(),
                experiment=experiment, group=group_id)

    class __AnonymousUser(object):
        def __init__(self, experiment_user):
            self.experiment_user = experiment_user

        def __get_anonymous_visitor(self):
            anonymous_id = self.experiment_user.get_anonymous_id()
            if anonymous_id == None:
                return None
            anonymous_visitors = AnonymousVisitor.objects.filter(
                id=anonymous_id)
            if anonymous_visitors.count() != 1:
                return None
            return anonymous_visitors[0]

        def get_enrollment(self, experiment):
            anonymous_visitor = self.__get_anonymous_visitor()

            if anonymous_visitor:
                participants = Participant.objects.filter(
                    anonymous_visitor=anonymous_visitor,
                    experiment=experiment)
                if participants.count() == 1:
                    return participants[0].group
            return None

        def set_enrollment(self, experiment, group_id):
            anonymous_visitor = self.__get_anonymous_visitor()
            if not anonymous_visitor:
                anonymous_visitor = AnonymousVisitor()
                anonymous_visitor.save()
                self.experiment_user.set_anonymous_id(anonymous_visitor.id)

            Participant.objects.create(
                anonymous_visitor=anonymous_visitor,
                experiment=experiment, group=group_id)

    @classmethod
    def __create_user(cls, experiment_user):
        if not experiment_user.is_anonymous():
            return cls.__RegisteredUser(experiment_user)
        if not experiment_user.is_verified_human():
            return cls.__UnverifiedUser(experiment_user)
        else:
            return cls.__AnonymousUser(experiment_user)

    DISABLED_STATE = 0
    ENABLED_STATE = 1
    PROMOTED_STATE = 2

    STATES = (
        (DISABLED_STATE, 'Disabled'),
        (ENABLED_STATE, 'Enabled'),
        (PROMOTED_STATE, 'Promoted'))

    name = models.CharField(unique=True, max_length=128)
    state = models.IntegerField(default=DISABLED_STATE, choices=STATES)
    start_date = models.DateField(blank=True, null=True, db_index=True)
    end_date = models.DateField(blank=True, null=True)

    def __unicode__(self):
        return self.name

    def save(self, *args, **kwargs):
        """
        The save override's goal is to save the start or end date when
        the state changes
        """
        #do nothing for new ones
        if self.id:
            old_self = None

            try:
                old_self = Experiment.objects.get(id=self.id)
            except Experiment.DoesNotExist:
                raise Exception("Can't find the existing Experiment.")

            if old_self.state != self.state:
                if (old_self.state == Experiment.DISABLED_STATE
                    and self.state == Experiment.ENABLED_STATE
                    and not old_self.start_date):
                    # enabling
                    self.start_date = date.today()
                elif (old_self.state == Experiment.ENABLED_STATE
                      and self.state == Experiment.DISABLED_STATE
                      and not old_self.end_date):
                    # disabling
                    self.end_date = date.today()
                elif (old_self.state == Experiment.ENABLED_STATE
                      and self.state == Experiment.PROMOTED_STATE
                      and not old_self.end_date):
                    #promoting
                    self.end_date = date.today()
        return super(Experiment, self).save(*args, **kwargs)

    @staticmethod
    def control(experiment_name, experiment_user):
        """
        Will return True when user is part of the control group in
        the passed experiment. If the user is not enrolled in this
        experiment, and the experiment is enabled, it will enroll the user.
        """
        return Experiment.__test_group(experiment_name, experiment_user,
                                       Participant.CONTROL_GROUP)

    @staticmethod
    def test(experiment_name, experiment_user):
        """
        Will return True when user is part of the test group in
        the passed experiment. If the user is not enrolled in this
        experiment, and the experiment is enabled, it will enroll the user.
        """
        return Experiment.__test_group(experiment_name, experiment_user,
                                       Participant.TEST_GROUP)

    @classmethod
    def __test_group(cls, experiment_name, experiment_user, queried_group):
        """does the real work"""
        experiment = None
        try:
            experiment = Experiment.objects.get(name=experiment_name)
        except Experiment.DoesNotExist:
            if settings.DEBUG:
                raise Exception("Can't find the Experiment named %s" %
                                experiment_name)
            else:
                l.warning("Can't find the Experiment named %s" %
                          experiment_name)
                return queried_group == Participant.CONTROL_GROUP
        if experiment.state == Experiment.DISABLED_STATE:
            return queried_group == Participant.CONTROL_GROUP
        elif experiment.state == Experiment.PROMOTED_STATE:
            return queried_group == Participant.TEST_GROUP

        if experiment.state != Experiment.ENABLED_STATE:
            raise Exception("Invalid experiment state !")

        user = cls.__create_user(experiment_user)

        assigned_group = user.get_enrollment(experiment)

        if assigned_group == None:
            assigned_group = random.choice((Participant.CONTROL_GROUP,
                                            Participant.TEST_GROUP))
            user.set_enrollment(experiment, assigned_group)

        return queried_group == assigned_group

class Participant(models.Model):
    """A participant in a split testing experiment """

    class Meta:
        unique_together= (('user', 'experiment'),
                          ('anonymous_visitor', 'experiment'))

    CONTROL_GROUP = 0
    TEST_GROUP = 1

    GROUPS = (
        (CONTROL_GROUP, "Control"),
        (TEST_GROUP, "Test"))

    user = models.ForeignKey(User, null=True)
    experiment = models.ForeignKey(Experiment)
    enrollment_date = models.DateField(db_index=True, auto_now_add=True)
    group = models.IntegerField(choices=GROUPS)
    anonymous_visitor = models.ForeignKey(AnonymousVisitor, null=True, blank=True)

    def __unicode__(self):
        if self.user: # can be null
            username = self.user.username
        else:
            username = 'anonymous#%d' % self.anonymous_visitor.id
        return "%s %s" % (username, self.group)

    def __init__(self, *args, **kwargs):
        super(Participant, self).__init__(*args, **kwargs)
        if not self.id:
            if (not self.anonymous_visitor) == (not self.user):
                raise Exception("Participants require exactly one of "
                                "`anonymous_visitor` or `user`.")

class DailyEngagementReport(models.Model):
    """Hold the scores for a given experiment on a given day"""
    date = models.DateField(db_index=True)
    experiment = models.ForeignKey(Experiment)
    test_score = models.FloatField(null=True)
    control_score = models.FloatField(null=True)
    test_group_size = models.IntegerField()
    control_group_size = models.IntegerField()
    confidence = models.FloatField(null=True)

class DailyConversionReport(models.Model):
    """Stores the daily conversion scores."""
    date = models.DateField(db_index=True)
    experiment = models.ForeignKey(Experiment)
    overall_test_conversion = models.IntegerField()
    overall_control_conversion = models.IntegerField()
    test_group_size = models.IntegerField()
    control_group_size = models.IntegerField()
    confidence = models.FloatField(null=True)

class  DailyConversionReportGoalData(models.Model):
    """Stores the daily conversion report goal data."""
    report = models.ForeignKey(DailyConversionReport, related_name="goal_data")
    goal_type = models.ForeignKey(GoalType)
    test_conversion = models.IntegerField()
    control_conversion = models.IntegerField()
    confidence = models.FloatField(null=True)
<|endoftext|>


<commit_before>from django.shortcuts import render_to_response
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
from core.models import *
from django.forms.models import modelformset_factory, inlineformset_factory
from django.views.decorators.csrf import csrf_protect
from django.template import RequestContext
from django.http import HttpResponseRedirect
from core.forms import *

@login_required(login_url='/login/')
@csrf_protect
def projects(request):

    project = Project.objects.all()

    # TODO check if this is the best way to get the current user in session.
    user = User.objects.get(username=request.META["USER"])

    ProjectFormSet = inlineformset_factory(User, Project, max_num=10, extra=1, \
        fields = ('name', 'version', 'release_date'))

    if request.method == 'POST':
        formset = ProjectFormSet(request.POST, instance=user)
        if formset.is_valid():
            formset.save()
            return HttpResponseRedirect('/projects/')
    else:
        formset = ProjectFormSet(instance=user)

    return render_to_response("projects.html", {
        'formset' : formset
    }, context_instance=RequestContext(request))

<commit_msg>Added callback handler for issues.<commit_after>from django.shortcuts import render_to_response
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
from core.models import *
from django.forms.models import modelformset_factory, inlineformset_factory
from django.views.decorators.csrf import csrf_protect
from django.template import RequestContext
from django.http import HttpResponseRedirect
from core.forms import *

@login_required(login_url='/login/')
@csrf_protect
def projects(request):

    project = Project.objects.all()

    # TODO check if this is the best way to get the current user in session.
    user = User.objects.get(username=request.META["USER"])

    ProjectFormSet = inlineformset_factory(User, Project, max_num=10, extra=1, \
        fields = ('name', 'version', 'release_date'))

    if request.method == 'POST':
        formset = ProjectFormSet(request.POST, instance=user)
        if formset.is_valid():
            formset.save()
            return HttpResponseRedirect('/projects/')
    else:
        formset = ProjectFormSet(instance=user)

    return render_to_response("projects.html", {
        'formset' : formset
    }, context_instance=RequestContext(request))

@login_required(login_url='/login/')
@csrf_protect
def issues(request, project_id):
    # TODO check if this is the best way to get the current user in session.
    user = User.objects.get(username=request.META["USER"])
    project = Project.objects.get(id=project_id)

    IssuesFormSet = inlineformset_factory(Project, Issue, max_num=10, extra=1, \
        fields = ('status', 'level', 'title', 'description'))

    if request.method == 'POST':
        formset = IssuesFormSet(request.POST, instance=project)
        if formset.is_valid():
            formset.save()
            return HttpResponseRedirect('/issues/%s' % project_id)
    else:
        formset = IssuesFormSet(instance=project)

    return render_to_response("issues.html", {
        'formset' : formset,
        'project_id' : project_id
    }, context_instance=RequestContext(request))

<|endoftext|>


<commit_before>import json
import requests

from flask import Flask
from numpy import mean, median

app = Flask(__name__)

@app.route("/<album_id>")
def get_track_data(album_id):
  url = "http://ws.spotify.com/lookup/1/.json" + \
          "?uri=spotify:album:%s&extras=trackdetail" % album_id
  resp = requests.get(url).json()
  artist = resp['album']['artist']
  album_name = resp['album']['name']
  tracks = resp['album']['tracks']

  pops = [float(t['popularity']) for t in tracks]
  pop_max = max(pops)
  pop_min = min(pops)
  pop_range = pop_max - pop_min

  track_data = []
  for t in tracks:
    pop = float(t['popularity'])
    track_data.append({
      'name':     t['name'],
      'num':      int(t['track-number']),
      'pop':      pop,
      'rel_pop':  round((pop - pop_min) / pop_range, 2)
    })

  return json.dumps({
    'tracks': track_data,
    'max':    pop_max,
    'min':    pop_min,
    'mean':   mean(pops),
    'median': median(pops),
    'name':   album_name,
    'artist': artist,
  })

@app.after_request
def add_cors_header(response):
  from flask import request
  origin = request.headers.get('Origin', '')
  if origin in ['null', 'http://explorify.tomfleischer.com']:
    response.headers['Access-Control-Allow-Origin'] = origin
    response.headers['Access-Control-Allow-Credentials'] = 'true'
    response.headers['Access-Control-Allow-Headers'] = \
      'Content-Type, Pragma, Cache-Control, *'
    response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, DELETE'
  return response

if __name__ == "__main__":
  app.run(host='0.0.0.0', port=5001)
<commit_msg>Updated CORS URL again<commit_after>import json
import requests

from flask import Flask
from numpy import mean, median

app = Flask(__name__)

@app.route("/<album_id>")
def get_track_data(album_id):
  url = "http://ws.spotify.com/lookup/1/.json" + \
          "?uri=spotify:album:%s&extras=trackdetail" % album_id
  resp = requests.get(url).json()
  artist = resp['album']['artist']
  album_name = resp['album']['name']
  tracks = resp['album']['tracks']

  pops = [float(t['popularity']) for t in tracks]
  pop_max = max(pops)
  pop_min = min(pops)
  pop_range = pop_max - pop_min

  track_data = []
  for t in tracks:
    pop = float(t['popularity'])
    track_data.append({
      'name':     t['name'],
      'num':      int(t['track-number']),
      'pop':      pop,
      'rel_pop':  round((pop - pop_min) / pop_range, 2)
    })

  return json.dumps({
    'tracks': track_data,
    'max':    pop_max,
    'min':    pop_min,
    'mean':   mean(pops),
    'median': median(pops),
    'name':   album_name,
    'artist': artist,
  })

@app.after_request
def add_cors_header(response):
  from flask import request
  origin = request.headers.get('Origin', '')
  if origin in ['null', 'http://explorify.jawsapps.com']:
    response.headers['Access-Control-Allow-Origin'] = origin
    response.headers['Access-Control-Allow-Credentials'] = 'true'
    response.headers['Access-Control-Allow-Headers'] = \
      'Content-Type, Pragma, Cache-Control, *'
    response.headers['Access-Control-Allow-Methods'] = 'GET, POST, PUT, DELETE'
  return response

if __name__ == "__main__":
  app.run(host='0.0.0.0', port=5001)
<|endoftext|>


<commit_before>"""
Tools for mapping names to user identifiers.
"""
from axiom import attributes, errors as ae, item
from exponent._util import synchronous
from exponent.auth import errors as eae, user
from zope import interface

class INameResolver(interface.Interface):
    """
    Resolves users' names to user identifiers.
    """
    def resolve(name):
        """
        Resolve a name.

        :param name: The name to resolve.
        :type name: ``unicode``

        :return: A ``Deferred`` that will fire with the user identifier or
            ``BadCredentials`` if the name could not be resolved.
        """

@interface.implementer(INameResolver)
class _NameReferenceResolver(item.Item):
    """
    A name resolve that uses local items.
    """
    _dummy = attributes.boolean()

    @synchronous
    def resolve(self, name):
        _UUR = _UserIdentifierNameReference

        try:
            self.store.findUnique(_UUR, _UUR.name == name)
        except ae.ItemNotFound:
            raise eae.BadCredentials()

        reference = self.store.findUnique(_UUR, _UUR.name == name)
        return user.User.findUniqueChild(self.store, reference.identifier)

class _UserIdentifierNameReference(item.Item):
    """
    A reference to match a user identifier to a name.
    """
    identifier = attributes.bytes(allowNone=False, indexed=True)
    name = attributes.text(allowNone=False, indexed=True)

<commit_msg>user -> common<commit_after>"""
Tools for mapping names to user identifiers.
"""
from axiom import attributes, errors as ae, item
from exponent._util import synchronous
from exponent.auth import errors as eae, common
from zope import interface

class INameResolver(interface.Interface):
    """
    Resolves users' names to user identifiers.
    """
    def resolve(name):
        """
        Resolve a name.

        :param name: The name to resolve.
        :type name: ``unicode``

        :return: A ``Deferred`` that will fire with the user identifier or
            ``BadCredentials`` if the name could not be resolved.
        """

@interface.implementer(INameResolver)
class _NameReferenceResolver(item.Item):
    """
    A name resolve that uses local items.
    """
    _dummy = attributes.boolean()

    @synchronous
    def resolve(self, name):
        _UUR = _UserIdentifierNameReference

        try:
            self.store.findUnique(_UUR, _UUR.name == name)
        except ae.ItemNotFound:
            raise eae.BadCredentials()

        identifier = self.store.findUnique(_UUR, _UUR.name == name).identifier
        return common.User.findUniqueChild(self.store, identifier)

class _UserIdentifierNameReference(item.Item):
    """
    A reference to match a user identifier to a name.
    """
    identifier = attributes.bytes(allowNone=False, indexed=True)
    name = attributes.text(allowNone=False, indexed=True)

<|endoftext|>


<commit_before>"""
Automatically builds conda packages from pypi.
"""
from __future__ import print_function
from os.path import isdir
import argparse
import subprocess
import yaml
import shlex

parser = argparse.ArgumentParser()
parser.add_argument("--init",
                    help="Initialize packages.yaml",
                    action="store_true")
parser.add_argument("-n",
                    help="Number of packages (only works with --init)",
                    type=int)
parser.add_argument("--anaconda",
                    help="Build pacakges included in Anaconda")
parser.add_argument("--commit-and-push",
                    help="Commit the reports and logs and push to gh-pages",
                    action="store_true")
args = parser.parse_args()

def init_packages_yaml(n):
    sorted_file = open('sorted_packages', 'r')
    package_list = [package.strip() for package in sorted_file.readlines()][:n]

    anaconda = set([package.strip() for package in
                    open('anaconda', 'r').readlines()])

    package_list = [dict([('name', name), ('recipe', None), ('build', None),
                    ('requirements', []), ('anaconda', name.lower() in anaconda)])
                    for name in package_list]

    open('packages.yaml', 'w').writelines(yaml.dump(package_list))

def create_recipe(package):
    log_file_name = log_dir + "%s_recipe.log" % (package['name'])
    log_file = open(log_file_name, 'w')

    msg = "Creating Conda recipe for %s\n" % (package['name'])
    print(msg)

    err = 0
    cond = package['recipe'] is None and package['anaconda'] is False
    if cond:
        if not isdir(recipes_dir + package['name']):
            # XXX: the normalization of package names comes into way of
            # directory detection
            cmd = "conda skeleton pypi %s --output-dir %s --recursive"
            cmd = cmd % (package['name'], recipes_dir)
            err = subprocess.call(shlex.split(cmd), stdout=log_file,
                                  stderr=subprocess.STDOUT)
        else:
            err = 0

    if err is 0:
        msg = "Succesfully created conda recipe for %s\n" % (package['name'])
        package['recipe'] = True
    else:
        msg = "Failed to create conda recipe for %s\n" % (package['name'])
        package['recipe'] = False
        print(msg)
    log_file.close()

    return package['recipe'], log_file_name

def build_recipe(package):
    log_file_name = log_dir + "%s_build.log" % (package['name'])
    log_file = open(log_file_name, 'w')

    msg = "Building Conda recipe for %s\n" % (package['name'])
    print(msg)

    err = 0
    if package['build'] is None and package['anaconda'] is False:
        cmd = "conda build %s" % (recipes_dir + package['name'])
        err = subprocess.call(shlex.split(cmd), stdout=log_file,
                              stderr=subprocess.STDOUT)

    if err is 0:
        msg = "Succesfully build conda package for %s\n" % (package['name'])
        package['build'] = True
    else:
        msg = "Failed to build conda package for %s\n" % (package['name'])
        package['build'] = False
    print(msg)
    log_file.close()

    return package['build'], log_file_name

def compile_report():
    # Add score
    recipe_score = sum([1 for package in packages if package['recipe'] is True])
    build_score = sum([1 for package in packages if package['build'] is True])

    n = len(packages)

    report_lines.append("\nrecipe score: %s/%s\n" % (recipe_score, n))
    report_lines.append("\nbuild score: %s/%s\n" % (build_score, n))

    # Write to file and convert to html
    open("report.md", "w").writelines("\n".join(report_lines))
    cmd = "pandoc report.md -o report.html"
    subprocess.call(shlex.split(cmd))

def commit_and_push():
    cmd1 = "git checkout gh-pages"
    cmd2 = "git add --all logs"
    cmd3 = "git add report.md report.html"
    cmd4 = "git commit -m \"Auto commit\""
    cmd5 = "git push origin gh-pages -f"
    cmd6 = "git checkout master"

    subprocess.call(shlex.split(cmd1))
    subprocess.call(shlex.split(cmd2))
    subprocess.call(shlex.split(cmd3))
    subprocess.call(shlex.split(cmd4))
    subprocess.call(shlex.split(cmd5))
    subprocess.call(shlex.split(cmd6))

if args.init:
    init_packages_yaml(args.n)

packages = yaml.load(file('packages.yaml', 'r'))
log_dir = "./logs/"
recipes_dir = "./recipes/"

report_lines = ["|package|recipe|build|anaconda|",
                "|-------|:-----|:----|:-------|"]

for package in packages:
    recipe_status, recipe_log = create_recipe(package)
    build_status, build_log = build_recipe(package)

    report = "|%s|[%s](%s)|[%s](%s)|%s" % (package['name'], str(recipe_status),
                                           recipe_log, str(build_status),
                                           build_log, package['anaconda'])

    report_lines.append(report)

open('packages.yaml', 'w').writelines(yaml.dump(packages))
compile_report()

if args.commit_and_push:
    commit_and_push()
<commit_msg>Better organize report generation<commit_after>"""
Automatically builds conda packages from pypi.
"""
from __future__ import print_function
from os.path import isdir
import argparse
import subprocess
import yaml
import shlex

parser = argparse.ArgumentParser()
parser.add_argument("--init",
                    help="Initialize packages.yaml",
                    action="store_true")
parser.add_argument("-n",
                    help="Number of packages (only works with --init)",
                    type=int)
parser.add_argument("--anaconda",
                    help="Build pacakges included in Anaconda")
parser.add_argument("--commit-and-push",
                    help="Commit the reports and logs and push to gh-pages",
                    action="store_true")
args = parser.parse_args()

def init_packages_yaml(n):
    sorted_file = open('sorted_packages', 'r')
    package_list = [package.strip() for package in sorted_file.readlines()][:n]

    anaconda = set([package.strip() for package in
                    open('anaconda', 'r').readlines()])

    package_list = [dict([('name', name), ('recipe', None), ('build', None),
                    ('requirements', []), ('anaconda', name.lower() in anaconda)])
                    for name in package_list]

    open('packages.yaml', 'w').writelines(yaml.dump(package_list))

def create_recipe(package):
    log_file_name = log_dir + "%s_recipe.log" % (package['name'])
    log_file = open(log_file_name, 'w')

    msg = "Creating Conda recipe for %s\n" % (package['name'])
    print(msg)

    err = 0
    cond = package['recipe'] is None and package['anaconda'] is False
    if cond:
        if not isdir(recipes_dir + package['name']):
            # XXX: the normalization of package names comes into way of
            # directory detection
            cmd = "conda skeleton pypi %s --output-dir %s --recursive"
            cmd = cmd % (package['name'], recipes_dir)
            err = subprocess.call(shlex.split(cmd), stdout=log_file,
                                  stderr=subprocess.STDOUT)
        else:
            err = 0

    if err is 0:
        msg = "Succesfully created conda recipe for %s\n" % (package['name'])
        package['recipe'] = True
    else:
        msg = "Failed to create conda recipe for %s\n" % (package['name'])
        package['recipe'] = False
        print(msg)
    log_file.close()

    return package['recipe'], log_file_name

def build_recipe(package):
    log_file_name = log_dir + "%s_build.log" % (package['name'])
    log_file = open(log_file_name, 'w')

    msg = "Building Conda recipe for %s\n" % (package['name'])
    print(msg)

    err = 0
    if package['build'] is None and package['anaconda'] is False:
        cmd = "conda build %s" % (recipes_dir + package['name'])
        err = subprocess.call(shlex.split(cmd), stdout=log_file,
                              stderr=subprocess.STDOUT)

    if err is 0:
        msg = "Succesfully build conda package for %s\n" % (package['name'])
        package['build'] = True
    else:
        msg = "Failed to build conda package for %s\n" % (package['name'])
        package['build'] = False
    print(msg)
    log_file.close()

    return package['build'], log_file_name

def compile_report():
    report_lines = ["|package|recipe|build|anaconda|",
                    "|-------|:-----|:----|:-------|"]

    for package in packages:
        recipe_log = log_dir + "%s_recipe.log" % (package['name'])
        build_log = log_dir + "%s_build.log" % (package['name'])
        report = "|%s|[%s](%s)|[%s](%s)|%s" % (package['name'], package['recipe'],
                                               recipe_log, package['build'],
                                               build_log, package['anaconda'])

        report_lines.append(report)

    # Add score
    recipe_score = sum([1 for package in packages if package['recipe'] is True])
    build_score = sum([1 for package in packages if package['build'] is True])

    n = len(packages)

    report_lines.append("\nrecipe score: %s/%s\n" % (recipe_score, n))
    report_lines.append("\nbuild score: %s/%s\n" % (build_score, n))

    # Write to file and convert to html
    open("report.md", "w").writelines("\n".join(report_lines))
    cmd = "pandoc report.md -o report.html"
    subprocess.call(shlex.split(cmd))

def commit_and_push():
    cmd1 = "git checkout gh-pages"
    cmd2 = "git add --all logs"
    cmd3 = "git add report.md report.html"
    cmd4 = "git commit -m \"Auto commit\""
    cmd5 = "git push origin gh-pages -f"
    cmd6 = "git checkout master"

    subprocess.call(shlex.split(cmd1))
    subprocess.call(shlex.split(cmd2))
    subprocess.call(shlex.split(cmd3))
    subprocess.call(shlex.split(cmd4))
    subprocess.call(shlex.split(cmd5))
    subprocess.call(shlex.split(cmd6))

if args.init:
    init_packages_yaml(args.n)

packages = yaml.load(file('packages.yaml', 'r'))
log_dir = "./logs/"
recipes_dir = "./recipes/"

for package in packages:
    create_recipe(package)
    build_recipe(package)
open('packages.yaml', 'w').writelines(yaml.dump(packages))

compile_report()

if args.commit_and_push:
    commit_and_push()
<|endoftext|>


<commit_before>#!/usr/bin/env python
""" Extracts RES-Europe data to examine forecasting issues.
"""

import numpy as np
import pandas as pd
import networkx as nx
import os

from load_network import load_network
from scipy.stats import norm, beta

__copyright__ = "Copyright 2016, Tue Vissing Jensen"
__credits__ = ["Tue Vissing Jensen"]
__license__ = "MIT"
__version__ = "1.0"
__maintainer__ = "Tue Vissing Jensen"
__email__ = "tvjens@elektro.dtu.dk"
__status__ = "Prototype"

# CATEGORY = 'wind'
CATEGORY = 'solar'
NUM_SCENARIOS = 100

scenarios = ['s' + str(i) for i in np.arange(NUM_SCENARIOS)]
store = pd.HDFStore('covariance.h5')
cov = store['/'.join((CATEGORY, 'empirical'))]
store.close()

store = pd.HDFStore('data/TSVault.h5')
windmean = store['windmean']
solarmean = store['solarmean']
store.close()

# Point forecasts [n,k] -> pfc
# EXAMPLE: assume 0.5 pfc
# pfcs = pd.DataFrame(data=np.array([windmean]*2).T, index=cov.index, columns=[pd.Timedelta('1d'), pd.Timedelta('2d')])
# EXAMPLE: Extract from time series
tsfile = pd.read_csv('RE-Europe_dataset_package/Nodal_TS/wind_signal_COSMO.csv', index_col=0, parse_dates=True)
tsfile = pd.read_csv('RE-Europe_dataset_package/Nodal_TS/solar_signal_COSMO.csv', index_col=0, parse_dates=True)
tsfile.columns = tsfile.columns.astype(int)
pfcs = tsfile['2013-06-24']
pfcs.index = pfcs.index - pd.to_datetime('2013-06-23 12:00')

store = pd.HDFStore('data/marginalstore.h5')
meanpanel = store['/'.join((CATEGORY, 'mean'))]
varpanel = store['/'.join((CATEGORY, 'var'))]
scalefactors = store['/'.join((CATEGORY, 'scalefactors'))]
store.close()

outpanel = {}
for k, pfc in pfcs.iterrows():
    print k
    # Generate NUM_SCENARIOS samples with marginal normal distribution and the measured covariance.
    vs = np.random.multivariate_normal([0]*len(cov), cov.values, NUM_SCENARIOS)
    # Convert these samples to uniformly distributed values
    unfvs = pd.DataFrame(
        data=norm.cdf(vs),
        columns=cov.columns,
        index=scenarios)
    outpanel[k] = {}
    for n, col in unfvs.iteritems():
        meancol = meanpanel[n, k]
        varcol = varpanel[n, k]
        mean = np.interp(pfc[n], meancol.index, meancol.values)
        var = np.interp(pfc[n], varcol.index, varcol.values)
        outcol = beta.ppf(
            col,
            mean*(mean*(1-mean)/var - 1),
            (1 - mean)*(mean*(1-mean)/var - 1)
        )
        outpanel[k][n] = pd.Series(data=outcol*scalefactors[n], index=col.index)

scenariopanel = pd.Panel(outpanel)
outscenarios = scenariopanel
outscenarios.items = outscenarios.items + pd.to_datetime('2013-06-23 12:00')
store = pd.HDFStore('data/scenariostore.h5')
store['/'.join(CATEGORY, 'scenarios')] = outscenarios
store.close()
<commit_msg>update for solar<commit_after>#!/usr/bin/env python
""" Extracts RES-Europe data to examine forecasting issues.
"""

import numpy as np
import pandas as pd
import networkx as nx
import os

from load_network import load_network
from scipy.stats import norm, beta

__copyright__ = "Copyright 2016, Tue Vissing Jensen"
__credits__ = ["Tue Vissing Jensen"]
__license__ = "MIT"
__version__ = "1.0"
__maintainer__ = "Tue Vissing Jensen"
__email__ = "tvjens@elektro.dtu.dk"
__status__ = "Prototype"

# CATEGORY = 'wind'
CATEGORY = 'solar'
NUM_SCENARIOS = 100

scenarios = ['s' + str(i) for i in np.arange(NUM_SCENARIOS)]
store = pd.HDFStore('covariance.h5')
cov = store['/'.join((CATEGORY, 'empirical'))]
store.close()

# Point forecasts [n,k] -> pfc
# EXAMPLE: Use mean as point forecast
# store = pd.HDFStore('data/TSVault.h5')
# if CATEGORY == 'wind':
#     windmean = store['windmean']
# elif CATEGORY == 'solar':
#     solarmean = store['solarmean']
# store.close()
# pfcs = pd.DataFrame(data=np.array([windmean]*2).T, index=cov.index, columns=[pd.Timedelta('1d'), pd.Timedelta('2d')])

# EXAMPLE: Extract from time series
if CATEGORY == 'wind':
    tsfile = pd.read_csv('RE-Europe_dataset_package/Nodal_TS/wind_signal_COSMO.csv', index_col=0, parse_dates=True)
elif CATEGORY == 'solar':
    tsfile = pd.read_csv('RE-Europe_dataset_package/Nodal_TS/solar_signal_COSMO.csv', index_col=0, parse_dates=True)
tsfile.columns = tsfile.columns.astype(int)
pfcs = tsfile['2013-06-24']
pfcs.index = pfcs.index - pd.to_datetime('2013-06-23 12:00')

# Load marginal distributions
store = pd.HDFStore('data/marginalstore.h5')
meanpanel = store['/'.join((CATEGORY, 'mean'))]
varpanel = store['/'.join((CATEGORY, 'var'))]
scalefactors = store['/'.join((CATEGORY, 'scalefactors'))]
store.close()

outpanel = {}
for k, pfc in pfcs.iterrows():
    print k
    # Generate NUM_SCENARIOS samples with marginal normal distribution and the measured covariance.
    vs = np.random.multivariate_normal([0]*len(cov), cov.values, NUM_SCENARIOS)
    # Convert these samples to uniformly distributed values
    unfvs = pd.DataFrame(
        data=norm.cdf(vs),
        columns=cov.columns,
        index=scenarios)
    outpanel[k] = {}
    for n, col in unfvs.iteritems():
        meancol = meanpanel[n, k]
        varcol = varpanel[n, k]
        mean = np.interp(pfc[n], meancol.index, meancol.values)
        var = np.interp(pfc[n], varcol.index, varcol.values)
        outcol = beta.ppf(
            col,
            mean*(mean*(1-mean)/var - 1),
            (1 - mean)*(mean*(1-mean)/var - 1)
        )
        outpanel[k][n] = pd.Series(data=outcol*scalefactors[n], index=col.index)

scenariopanel = pd.Panel(outpanel)
outscenarios = scenariopanel
outscenarios.items = outscenarios.items + pd.to_datetime('2013-06-23 12:00')
store = pd.HDFStore('data/scenariostore.h5')
store['/'.join(CATEGORY, 'scenarios')] = outscenarios
store.close()
<|endoftext|>