<commit_before>import os
import unittest
from urllib.parse import urlparse

from scrapy.http import Response, TextResponse, HtmlResponse
from scrapy.utils.python import to_bytes
from scrapy.utils.response import (response_httprepr, open_in_browser,
                                   get_meta_refresh, get_base_url, response_status_message)

__doctests__ = ['scrapy.utils.response']

class ResponseUtilsTest(unittest.TestCase):
    dummy_response = TextResponse(url='http://example.org/', body=b'dummy_response')

    def test_response_httprepr(self):
        r1 = Response("http://www.example.com")
        self.assertEqual(response_httprepr(r1), b'HTTP/1.1 200 OK\r\n\r\n')

        r1 = Response("http://www.example.com", status=404, headers={"Content-type": "text/html"}, body=b"Some body")
        self.assertEqual(response_httprepr(r1), b'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body')

        r1 = Response("http://www.example.com", status=6666, headers={"Content-type": "text/html"}, body=b"Some body")
        self.assertEqual(response_httprepr(r1), b'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body')

    def test_open_in_browser(self):
        url = "http:///www.example.com/some/page.html"
        body = b"  <title>test page</title>  test body "

        def browser_open(burl):
            path = urlparse(burl).path
            if not os.path.exists(path):
                path = burl.replace('file://', '')
            with open(path, "rb") as f:
                bbody = f.read()
            self.assertIn(b'<base href="' + to_bytes(url) + b'">', bbody)
            return True
        response = HtmlResponse(url, body=body)
        assert open_in_browser(response, _openfunc=browser_open), "Browser not called"

        resp = Response(url, body=body)
        self.assertRaises(TypeError, open_in_browser, resp, debug=True)

    def test_get_meta_refresh(self):
        r1 = HtmlResponse("http://www.example.com", body=b"""

        <title>Dummy</title><meta http-equiv="refresh" content="5;url=http://example.org/newpage" />
        blahablsdfsal&
        """)
        r2 = HtmlResponse("http://www.example.com", body=b"""

        <title>Dummy</title>
        <meta http-equiv="refresh" content="5;url=http://example.org/newpage" />

        blahablsdfsal&
        """)
        r3 = HtmlResponse("http://www.example.com", body=b"""
    <meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage

        """)
        self.assertEqual(get_meta_refresh(r1), (5.0, 'http://example.org/newpage'))
        self.assertEqual(get_meta_refresh(r2), (None, None))
        self.assertEqual(get_meta_refresh(r3), (None, None))

    def test_get_base_url(self):
        resp = HtmlResponse("http://www.example.com", body=b"""

        <base href="http://www.example.com/img/" target="_blank">
        blahablsdfsal&
        """)
        self.assertEqual(get_base_url(resp), "http://www.example.com/img/")

        resp2 = HtmlResponse("http://www.example.com", body=b"""
        blahablsdfsal&""")
        self.assertEqual(get_base_url(resp2), "http://www.example.com")

    def test_response_status_message(self):
        self.assertEqual(response_status_message(200), '200 OK')
        self.assertEqual(response_status_message(404), '404 Not Found')
        self.assertEqual(response_status_message(573), "573 Unknown Status")
<commit_msg>Test coverage for open_in_browser base tag injection (#5319)<commit_after>import os
import unittest
from urllib.parse import urlparse

from scrapy.http import Response, TextResponse, HtmlResponse
from scrapy.utils.python import to_bytes
from scrapy.utils.response import (response_httprepr, open_in_browser,
                                   get_meta_refresh, get_base_url, response_status_message)

__doctests__ = ['scrapy.utils.response']

class ResponseUtilsTest(unittest.TestCase):
    dummy_response = TextResponse(url='http://example.org/', body=b'dummy_response')

    def test_response_httprepr(self):
        r1 = Response("http://www.example.com")
        self.assertEqual(response_httprepr(r1), b'HTTP/1.1 200 OK\r\n\r\n')

        r1 = Response("http://www.example.com", status=404, headers={"Content-type": "text/html"}, body=b"Some body")
        self.assertEqual(response_httprepr(r1), b'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body')

        r1 = Response("http://www.example.com", status=6666, headers={"Content-type": "text/html"}, body=b"Some body")
        self.assertEqual(response_httprepr(r1), b'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body')

    def test_open_in_browser(self):
        url = "http:///www.example.com/some/page.html"
        body = b"  <title>test page</title>  test body "

        def browser_open(burl):
            path = urlparse(burl).path
            if not os.path.exists(path):
                path = burl.replace('file://', '')
            with open(path, "rb") as f:
                bbody = f.read()
            self.assertIn(b'<base href="' + to_bytes(url) + b'">', bbody)
            return True
        response = HtmlResponse(url, body=body)
        assert open_in_browser(response, _openfunc=browser_open), "Browser not called"

        resp = Response(url, body=body)
        self.assertRaises(TypeError, open_in_browser, resp, debug=True)

    def test_get_meta_refresh(self):
        r1 = HtmlResponse("http://www.example.com", body=b"""

        <title>Dummy</title><meta http-equiv="refresh" content="5;url=http://example.org/newpage" />
        blahablsdfsal&
        """)
        r2 = HtmlResponse("http://www.example.com", body=b"""

        <title>Dummy</title>
        <meta http-equiv="refresh" content="5;url=http://example.org/newpage" />

        blahablsdfsal&
        """)
        r3 = HtmlResponse("http://www.example.com", body=b"""
    <meta http-equiv="REFRESH" content="0;url=http://www.example.com/newpage

        """)
        self.assertEqual(get_meta_refresh(r1), (5.0, 'http://example.org/newpage'))
        self.assertEqual(get_meta_refresh(r2), (None, None))
        self.assertEqual(get_meta_refresh(r3), (None, None))

    def test_get_base_url(self):
        resp = HtmlResponse("http://www.example.com", body=b"""

        <base href="http://www.example.com/img/" target="_blank">
        blahablsdfsal&
        """)
        self.assertEqual(get_base_url(resp), "http://www.example.com/img/")

        resp2 = HtmlResponse("http://www.example.com", body=b"""
        blahablsdfsal&""")
        self.assertEqual(get_base_url(resp2), "http://www.example.com")

    def test_response_status_message(self):
        self.assertEqual(response_status_message(200), '200 OK')
        self.assertEqual(response_status_message(404), '404 Not Found')
        self.assertEqual(response_status_message(573), "573 Unknown Status")

    def test_inject_base_url(self):
        url = "http://www.example.com"

        def check_base_url(burl):
            path = urlparse(burl).path
            if not os.path.exists(path):
                path = burl.replace('file://', '')
            with open(path, "rb") as f:
                bbody = f.read()
            self.assertEqual(bbody.count(b'<base href="' + to_bytes(url) + b'">'), 1)
            return True

        r1 = HtmlResponse(url, body=b"""

            <title>Dummy</title>

Hello world.

        """)
        r2 = HtmlResponse(url, body=b"""

            <title>Dummy</title>
            Hello world.
        """)
        r3 = HtmlResponse(url, body=b"""

            <title>Dummy</title>

                Hello header

Hello world.

        """)
        r4 = HtmlResponse(url, body=b"""

            <title>Dummy</title>

Hello world.

        """)
        r5 = HtmlResponse(url, body=b"""

            <title>Standard head</title>

Hello world.

        """)

        assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url"
        assert open_in_browser(r2, _openfunc=check_base_url), "Inject base url with argumented head"
        assert open_in_browser(r3, _openfunc=check_base_url), "Inject unique base url with misleading tag"
        assert open_in_browser(r4, _openfunc=check_base_url), "Inject unique base url with misleading comment"
        assert open_in_browser(r5, _openfunc=check_base_url), "Inject unique base url with conditional comment"
<|endoftext|>


<commit_before>#!/usr/bin/env python

import rospy
import time
import scapy.all as sca
import numpy as np
from msg import Rssi

class WifiSensor():
  def __init__(self):
    self.pub = rospy.Publisher("rssi_mean", Rssi, queue_size=10)
    while not rospy.is_shutdown():
      rssi = self.mesRaw()
      msg = Rssi()
      msg.rssi = rssi
      msg.variance = 0.0
      self.pub.publish(msg)
      time.sleep(0.1)
  def parsePacket(self, pkt):
    if pkt.haslayer(sca.Dot11) :
      if pkt.addr2 != None:
        return pkt.addr2, -(256-ord(pkt.notdecoded[-4:-3]))
    return None, None
  def mesRaw(self, n=10):
    data = []
    while(len(data) < n):
      packets = sca.sniff(iface="wlan0", count = n/5)
      for pkt in packets:
        addr, rssi = self.parsePacket(pkt)
        # eduroam
        if addr == "20:b3:99:1b:f7:58":
          data.append(rssi)
    return data
<commit_msg>* sniffing is now in its own thread * conforms to new msg standard * some minor improvments<commit_after>#!/usr/bin/env python

import rospy
import time
import scapy.all as sca
import numpy as np
from msg import Rssi
import thread

class WifiSensor():
  def __init__(self):
    # setup shared data for threads and start
    self.data = {}
    self.dataMutex = thread.allocate_lock()
    thread.start_new_thread(self.mesRaw, ())
    # setup main loop
    rospy.init_node("wifisensor")
    self.pub = rospy.Publisher("rssi", Rssi, queue_size=10)
    r = rospy.Rate(10) # 10Hz
    while not rospy.is_shutdown():
      data = {}
      with self.dataMutex:
        data = dict(self.data)
        self.data = {}
      for addr in data.keys():
        msg = Rssi()
        msg.macaddr = addr
        msg.rssi = data[addr]
        self.pub.publish(msg)
      r.sleep()
  def mesRaw(self):
    while not rospy.is_shutdown():
      packets = sca.sniff(iface="wlan0", count = 10)
      for pkt in packets:
        addr, rssi = self.parsePacket(pkt)
        if addr is not None:
          with self.dataMutex:
            if addr in self.data.keys():
              self.data[addr].append(rssi)
            else:
              self.data[addr] = [rssi]
  def parsePacket(self, pkt):
    if pkt.haslayer(sca.Dot11):
      if pkt.addr2 is not None:
        return pkt.addr2, ord(pkt.notdecoded[-4:-3])-256
    return None, None
<|endoftext|>


<commit_before>#!/usr/bin/python

# This tests a particular tricky case: the interplay of "black" wrap mode
# with fill color.  Outside the s,t [0,1] range, it should be black, NOT
# fill color.

# Make an RGB grid for our test
command += (oiio_app("oiiotool")
            + parent + "/oiio-images/grid.tif"
            + " -ch R,G,B -o grid3.tif >> out.txt ;\n")

# And a 1-channel grid for our test
command += (oiio_app("oiiotool")
            + parent + "/oiio-images/grid.tif"
            + " -ch R -o grid1.tif >> out.txt ;\n")

command += testtex_command ("grid3.tif",
                            " -res 256 256 -automip " +
                            " -wrap black -fill 0.5 -d uint8 -o out3.tif >> out.txt ;\n")

command += testtex_command ("grid1.tif",
                            " -res 200 200 -automip --graytorgb " +
                            " -wrap black -fill 0.5 -d uint8 -o out1.tif >> out.txt ;\n")

outputs = [ "out1.tif" ]
<commit_msg>Fix the run.py commands for the texture-wrapfill test.<commit_after>#!/usr/bin/python

# This tests a particular tricky case: the interplay of "black" wrap mode
# with fill color.  Outside the s,t [0,1] range, it should be black, NOT
# fill color.

# Make an RGB grid for our test
command += (oiio_app("oiiotool")
            + parent + "/oiio-images/grid.tif"
            + " -ch R,G,B -o grid3.tif >> out.txt ;\n")

# And a 1-channel grid for our test
command += (oiio_app("oiiotool")
            + parent + "/oiio-images/grid.tif"
            + " -ch R -o grid1.tif >> out.txt ;\n")

command += testtex_command ("grid3.tif",
                            " -res 256 256 -automip " +
                            " -wrap black -fill 0.5 -d uint8 -o out3.tif")

command += testtex_command ("grid1.tif",
                            " -res 200 200 -automip --graytorgb " +
                            " -wrap black -fill 0.5 -d uint8 -o out1.tif")

outputs = [ "out3.tif", "out1.tif" ]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    :codauthor: :email:`Mike Place <mp@saltstack.com>`
'''

# Import Salt Testing libs
from salttesting import skipIf, TestCase
from salttesting.helpers import ensure_in_syspath
from salttesting.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch, mock_open, call
ensure_in_syspath('../')

# Import salt libs
import salt.overstate

opts = {'file_roots': {},
        'conf_file': ''}

overstate_sls = {
    'mysql': {
        'match': 'db*',
        'sls': {
            'mysql.server', 'drbd'
        }
    },
    'webservers': {
        'match': 'web*',
        'require': ['mysql']
    },
    'all': {
        'match': '*',
        'require': {
            'mysql', 'webservers'
        }
    }

}

@skipIf(NO_MOCK, NO_MOCK_REASON)
class OverstateTestCase(TestCase):
    @patch('salt.client.LocalClient.cmd')
    def test__stage_list(self, local_client_mock):
        overstate = salt.overstate.OverState(opts)
        overstate._stage_list(['test1', 'test2'])
        local_client_mock.assert_called_with('test1 or test2', 'test.ping', expr_form='compound')

    # @skipIf(True, "Do not run this test! It exposes a memory leak inside pyyaml: https://bitbucket.org/xi/pyyaml/issue/24")
    # def test__names(self):
    #     y = 'a'
    #     mopen = mock_open(read_data=y)
    #     with patch('salt.utils.fopen', mopen, create=True):
    #         overstate = salt.overstate.OverState(opts, overstate='a')

    def test__names(self):
        overstate = salt.overstate.OverState(opts)
        overstate.over = overstate._OverState__sort_stages(overstate_sls)
        self.assertEqual({'webservers', 'all', 'mysql'}, overstate._names())

    def test_get_stage(self):
        overstate = salt.overstate.OverState(opts)
        overstate.over = overstate._OverState__sort_stages(overstate_sls)
        ret = overstate.get_stage('mysql')
        self.assertDictEqual({'mysql': {'match': 'db*', 'sls': {'drbd', 'mysql.server'}}}, ret)

    @patch('salt.overstate.OverState.call_stage')
    def test_stages(self, call_stage_mock):
        '''
        This is a very basic test and needs expansion, since call_stage is mocked!
        '''
        overstate = salt.overstate.OverState(opts)
        overstate.over = overstate._OverState__sort_stages(overstate_sls)
        overstate.stages()
        expected_calls = [call('all', {'require': {'webservers', 'mysql'}, 'match': '*'}),
                          call('mysql', {'match': 'db*', 'sls': {'drbd', 'mysql.server'}}),
                          call('webservers', {'require': ['mysql'], 'match': 'web*'})]
        call_stage_mock.assert_has_calls(expected_calls, any_order=False)

    # @patch('salt.overstate.OverState.call_stage')
    # def test_call_stage(self, call_stage_mock):
    #     overstate = salt.overstate.OverState(opts)
    #     overstate.over = overstate._OverState__sort_stages(overstate_sls)
    #     overstate.call_stage('all', {'require': {'webservers', 'mysql'}, 'match': '*'})
    #     overstate.call_stage('mysql', {'match': 'db*', 'sls': {'drbd', 'mysql.server'}})
    #     overstate.call_stage({'require': ['mysql'], 'match': 'web*'})
<commit_msg>More tests.<commit_after># -*- coding: utf-8 -*-
'''
    :codauthor: :email:`Mike Place <mp@saltstack.com>`
'''

# Import Salt Testing libs
from salttesting import skipIf, TestCase
from salttesting.helpers import ensure_in_syspath
from salttesting.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch, mock_open, call
ensure_in_syspath('../')

# Import salt libs
import salt.overstate

opts = {'file_roots': {},
        'conf_file': ''}

overstate_sls = {
    'mysql': {
        'match': 'db*',
        'sls': {
            'mysql.server', 'drbd'
        }
    },
    'webservers': {
        'match': 'web*',
        'require': ['mysql']
    },
    'all': {
        'match': '*',
        'require': {
            'mysql', 'webservers'
        }
    }

}

@skipIf(NO_MOCK, NO_MOCK_REASON)
class OverstateTestCase(TestCase):
    @patch('salt.client.LocalClient.cmd')
    def test__stage_list(self, local_client_mock):
        overstate = salt.overstate.OverState(opts)
        overstate._stage_list(['test1', 'test2'])
        local_client_mock.assert_called_with('test1 or test2', 'test.ping', expr_form='compound')

    # @skipIf(True, "Do not run this test! It exposes a memory leak inside pyyaml: https://bitbucket.org/xi/pyyaml/issue/24")
    # def test__names(self):
    #     y = 'a'
    #     mopen = mock_open(read_data=y)
    #     with patch('salt.utils.fopen', mopen, create=True):
    #         overstate = salt.overstate.OverState(opts, overstate='a')

    def test__names(self):
        overstate = salt.overstate.OverState(opts)
        overstate.over = overstate._OverState__sort_stages(overstate_sls)
        self.assertEqual({'webservers', 'all', 'mysql'}, overstate._names())

    def test_get_stage(self):
        overstate = salt.overstate.OverState(opts)
        overstate.over = overstate._OverState__sort_stages(overstate_sls)
        ret = overstate.get_stage('mysql')
        self.assertDictEqual({'mysql': {'match': 'db*', 'sls': {'drbd', 'mysql.server'}}}, ret)

    @patch('salt.overstate.OverState.call_stage')
    def test_stages(self, call_stage_mock):
        '''
        This is a very basic test and needs expansion, since call_stage is mocked!
        '''
        overstate = salt.overstate.OverState(opts)
        overstate.over = overstate._OverState__sort_stages(overstate_sls)
        overstate.stages()
        expected_calls = [call('all', {'require': {'webservers', 'mysql'}, 'match': '*'}),
                          call('mysql', {'match': 'db*', 'sls': {'drbd', 'mysql.server'}}),
                          call('webservers', {'require': ['mysql'], 'match': 'web*'})]
        call_stage_mock.assert_has_calls(expected_calls, any_order=False)

    def test_verify_stage(self):
        overstate = salt.overstate.OverState(opts)
        test_stage = {'require': {'webservers', 'mysql'}, 'match': '*'}
        ret = overstate.verify_stage(test_stage)
        self.assertDictEqual(test_stage, test_stage)

    def test_verify_fail(self):
        overstate = salt.overstate.OverState(opts)
        test_stage = {'require': {'webservers', 'mysql'}}
        ret = overstate.verify_stage(test_stage)
        self.assertIn('No "match" argument in stage.', ret)

    # @patch('salt.overstate.OverState.call_stage')
    # def test_call_stage(self, call_stage_mock):
    #     overstate = salt.overstate.OverState(opts)
    #     overstate.over = overstate._OverState__sort_stages(overstate_sls)
    #     overstate.call_stage('all', {'require': {'webservers', 'mysql'}, 'match': '*'})
    #     overstate.call_stage('mysql', {'match': 'db*', 'sls': {'drbd', 'mysql.server'}})
    #     overstate.call_stage({'require': ['mysql'], 'match': 'web*'})
<|endoftext|>


<commit_before>from contextlib import contextmanager
from itertools import chain
import logging
import os
from ming.odm import mapper
from pymongo.errors import DuplicateKeyError
from tgext.ecommerce.model import DBSession, Cart, Product, Setting

log = logging.getLogger('tgext.ecommerce')

@contextmanager
def cleanup_session(session):
    yield
    session.flush_all()
    session.close_all()

def clean_expired_carts(clear_all=False):
    with cleanup_session(DBSession):
        if clear_all:
            expired_carts = Cart.query.find().all()
        else:
            expired_carts = Cart.expired_carts().all()

        expired_items = chain(*[c.items.iteritems() for c in expired_carts])
        for sku, item in expired_items:
            DBSession.update(Product, {'configurations.sku': sku},
                             {'$inc': {'configurations.$.qty': item['qty']}})
        [c.delete() for c in expired_carts]

def cart_locked_by_me():
    locked_by_me = Setting.query.find({'setting': 'cart_locked', 'value': os.getpid()}).first()
    return locked_by_me is not None

def lock_carts():
    settings_collection = mapper(Setting).collection

    # Ensure there is the lock row
    try:
        DBSession.impl.insert(settings_collection({'setting': 'cart_locked', 'value': 0}))
    except DuplicateKeyError:
        pass

    # Try to get the lock
    mypid = os.getpid()
    locked = DBSession.impl.update_partial(settings_collection,
                                           {'setting': 'cart_locked',
                                            'value': 0},
                                           {'$set': {'value': mypid}})

    if locked.get('updatedExisting', False):
        log.warn('Cart Locked by %s...' % mypid)
        clean_expired_carts(clear_all=True)
    else:
        log.warn('Cart already locked!')

def unlock_carts():
    mypid = os.getpid()
    settings_collection = mapper(Setting).collection
    locked = DBSession.impl.update_partial(settings_collection,
                                           {'setting': 'cart_locked',
                                            'value': mypid},
                                           {'$set': {'value': 0}})

    if locked.get('updatedExisting', False):
        log.warn('Cart Unlocked by %s...' % mypid)<commit_msg>Fix carts expiration in concurrent environments<commit_after>from contextlib import contextmanager
from datetime import datetime, timedelta
from itertools import chain
import logging
import os
from ming.odm import mapper
from pymongo.errors import DuplicateKeyError
from tgext.ecommerce.model import DBSession, Cart, Product, Setting

log = logging.getLogger('tgext.ecommerce')

@contextmanager
def cleanup_session(session):
    yield
    session.flush_all()
    session.close_all()

def clean_expired_cart(expired_cart):
    log.warn('Expiring Cart %s for user %s', expired_cart._id, expired_cart.user_id)

    for sku, item in expired_cart.items.iteritems():
        DBSession.update(Product, {'configurations.sku': sku},
                         {'$inc': {'configurations.$.qty': item['qty']}})

    expired_cart.delete()

def clean_expired_carts(clear_all=False):
    with cleanup_session(DBSession):
        if clear_all:
            expired_carts = Cart.query.find().all()
            for c in expired_carts:
                clean_expired_cart(c)
        else:
            now = datetime.utcnow()
            shifted_expire = now + timedelta(minutes=5)
            while True:
                expired_cart = Cart.query.find_and_modify(query={'expires_at': {'$lte': now}},
                                                          update={'$set': {'expires_at': shifted_expire}})
                if expired_cart is None:
                    break

                clean_expired_cart(expired_cart)

def cart_locked_by_me():
    locked_by_me = Setting.query.find({'setting': 'cart_locked', 'value': os.getpid()}).first()
    return locked_by_me is not None

def lock_carts():
    settings_collection = mapper(Setting).collection

    # Ensure there is the lock row
    try:
        DBSession.impl.insert(settings_collection({'setting': 'cart_locked', 'value': 0}))
    except DuplicateKeyError:
        pass

    # Try to get the lock
    mypid = os.getpid()
    locked = DBSession.impl.update_partial(settings_collection,
                                           {'setting': 'cart_locked',
                                            'value': 0},
                                           {'$set': {'value': mypid}})

    if locked.get('updatedExisting', False):
        log.warn('Cart Locked by %s...' % mypid)
        clean_expired_carts(clear_all=True)
    else:
        log.warn('Cart already locked!')

def unlock_carts():
    mypid = os.getpid()
    settings_collection = mapper(Setting).collection
    locked = DBSession.impl.update_partial(settings_collection,
                                           {'setting': 'cart_locked',
                                            'value': mypid},
                                           {'$set': {'value': 0}})

    if locked.get('updatedExisting', False):
        log.warn('Cart Unlocked by %s...' % mypid)<|endoftext|>


<commit_before>#
# (c) Simon Marlow 2002
#

# ToDo:
#   GHCi tests
#   expect failure for some ways only

import sys
import os
import string
import getopt
import platform
import time

from testutil import *
from testglobals import *

# Readline sometimes spews out ANSI escapes for some values of TERM,
# which result in test failures. Thus set TERM to a nice, simple, safe
# value.
os.environ['TERM'] = 'vt100'

global config
config = getConfig() # get it from testglobals

# -----------------------------------------------------------------------------
# cmd-line options

long_options = [
  "config=",  		# config file
  "rootdir=", 		# root of tree containing tests (default: .)
  "output-summary=", 	# file in which to save the (human-readable) summary
  "only=",		# just this test (can be give multiple --only= flags)
  "way=",		# just this way
  "skipway=",		# skip this way
  "threads=",           # threads to run simultaneously
  ]

opts, args = getopt.getopt(sys.argv[1:], "e:", long_options)

for opt,arg in opts:
    if opt == '--config':
        execfile(arg)

    # -e is a string to execute from the command line.  For example:
    # testframe -e 'config.compiler=ghc-5.04'
    if opt == '-e':
        exec arg

    if opt == '--rootdir':
        config.rootdirs.append(arg)

    if opt == '--output-summary':
        config.output_summary = arg

    if opt == '--times-file':
        config.times_file = arg

    if opt == '--only':
        config.only.append(arg)

    if opt == '--way':
        if (arg not in config.run_ways and arg not in config.compile_ways and arg not in config.other_ways):
            sys.stderr.write("ERROR: requested way \'" +
                             arg + "\' does not exist\n")
            sys.exit(1)
        config.cmdline_ways = [arg] + config.cmdline_ways
        if (arg in config.other_ways):
            config.run_ways = [arg] + config.run_ways
            config.compile_ways = [arg] + config.compile_ways

    if opt == '--skipway':
        if (arg not in config.run_ways and arg not in config.compile_ways and arg not in config.other_ways):
            sys.stderr.write("ERROR: requested way \'" +
                             arg + "\' does not exist\n")
            sys.exit(1)
        config.other_ways = filter(neq(arg), config.other_ways)
        config.run_ways = filter(neq(arg), config.run_ways)
        config.compile_ways = filter(neq(arg), config.compile_ways)

    if opt == '--threads':
        # Trac #1558 says threads don't work in python 2.4.4, but do
        # in 2.5.2. Probably >= 2.5 is sufficient, but let's be
        # conservative here.
        (maj, min, pat) = platform.python_version_tuple()
        maj = int(maj)
        min = int(min)
        pat = int(pat)
        if (maj, min, pat) >= (2, 5, 2):
            config.threads = int(arg)
            config.use_threads = 1
        else:
            print "Warning: Ignoring request to use threads as python version < 2.5.2"

# This has to come after arg parsing as the args can change the compiler
get_compiler_info()

# Can't import this earlier as we need to know if threading will be
# enabled or not
from testlib import *

global testopts_local
testopts_local.x = TestOptions()

global thisdir_testopts
thisdir_testopts = getThisDirTestOpts()

if config.use_threads:
    t.lock = threading.Lock()
    t.thread_pool = threading.Condition(t.lock)
    t.running_threads = 0

# if timeout == -1 then we try to calculate a sensible value
if config.timeout == -1:
    config.timeout = int(read_no_crs(config.top + '/timeout/calibrate.out'))

print 'Timeout is ' + str(config.timeout)

# -----------------------------------------------------------------------------
# The main dude

if config.rootdirs == []:
    config.rootdirs = ['.']

t_files = findTFiles(config.rootdirs)

print 'Found', len(t_files), '.T files...'

t = getTestRun()

t.start_time = chop(os.popen('date').read())
print 'Beginning test run at', t.start_time

# set stdout to unbuffered (is this the best way to do it?)
sys.stdout.flush()
sys.stdout = os.fdopen(sys.__stdout__.fileno(), "w", 0)

times = {}

for file in t_files:
    print '====> Running', file
    newTestDir(os.path.dirname(file))
    try:
        if config.use_threads:
            t.running_threads=0
        start = time.time()
        execfile(file)
        end = time.time()
        times[file] = end - start
        if config.use_threads:
            t.thread_pool.acquire()
            while t.running_threads>0:
                t.thread_pool.wait()
            t.thread_pool.release()
    except:
        print '*** framework failure: found an error while executing ', file, ':'
        t.n_framework_failures = t.n_framework_failures + 1
        traceback.print_exc()

summary(t, sys.stdout)

if config.output_summary != '':
    summary(t, open(config.output_summary, 'w'))

if config.times_file != '':
    h = open(config.times_file, 'w')
    for (k, v) in times.items():
        h.write(k + ',' + str(v) + '\n')
    h.close()

sys.exit(0)

<commit_msg>Fix trac #3091: the driver was choking on python versions containing letters<commit_after>#
# (c) Simon Marlow 2002
#

# ToDo:
#   GHCi tests
#   expect failure for some ways only

import sys
import os
import string
import getopt
import platform
import time
import re

from testutil import *
from testglobals import *

# Readline sometimes spews out ANSI escapes for some values of TERM,
# which result in test failures. Thus set TERM to a nice, simple, safe
# value.
os.environ['TERM'] = 'vt100'

global config
config = getConfig() # get it from testglobals

# -----------------------------------------------------------------------------
# cmd-line options

long_options = [
  "config=",  		# config file
  "rootdir=", 		# root of tree containing tests (default: .)
  "output-summary=", 	# file in which to save the (human-readable) summary
  "only=",		# just this test (can be give multiple --only= flags)
  "way=",		# just this way
  "skipway=",		# skip this way
  "threads=",           # threads to run simultaneously
  ]

opts, args = getopt.getopt(sys.argv[1:], "e:", long_options)

for opt,arg in opts:
    if opt == '--config':
        execfile(arg)

    # -e is a string to execute from the command line.  For example:
    # testframe -e 'config.compiler=ghc-5.04'
    if opt == '-e':
        exec arg

    if opt == '--rootdir':
        config.rootdirs.append(arg)

    if opt == '--output-summary':
        config.output_summary = arg

    if opt == '--times-file':
        config.times_file = arg

    if opt == '--only':
        config.only.append(arg)

    if opt == '--way':
        if (arg not in config.run_ways and arg not in config.compile_ways and arg not in config.other_ways):
            sys.stderr.write("ERROR: requested way \'" +
                             arg + "\' does not exist\n")
            sys.exit(1)
        config.cmdline_ways = [arg] + config.cmdline_ways
        if (arg in config.other_ways):
            config.run_ways = [arg] + config.run_ways
            config.compile_ways = [arg] + config.compile_ways

    if opt == '--skipway':
        if (arg not in config.run_ways and arg not in config.compile_ways and arg not in config.other_ways):
            sys.stderr.write("ERROR: requested way \'" +
                             arg + "\' does not exist\n")
            sys.exit(1)
        config.other_ways = filter(neq(arg), config.other_ways)
        config.run_ways = filter(neq(arg), config.run_ways)
        config.compile_ways = filter(neq(arg), config.compile_ways)

    if opt == '--threads':
        # Trac #1558 says threads don't work in python 2.4.4, but do
        # in 2.5.2. Probably >= 2.5 is sufficient, but let's be
        # conservative here.
        # Some versions of python have things like '1c1' for some of
        # these components (see trac #3091), but int() chokes on the
        # 'c1', so we drop it.
        (maj, min, pat) = platform.python_version_tuple()
        maj = int(re.sub('[^0-9].*', '', maj))
        min = int(re.sub('[^0-9].*', '', min))
        pat = int(re.sub('[^0-9].*', '', pat))
        if (maj, min, pat) >= (2, 5, 2):
            config.threads = int(arg)
            config.use_threads = 1
        else:
            print "Warning: Ignoring request to use threads as python version < 2.5.2"

# This has to come after arg parsing as the args can change the compiler
get_compiler_info()

# Can't import this earlier as we need to know if threading will be
# enabled or not
from testlib import *

global testopts_local
testopts_local.x = TestOptions()

global thisdir_testopts
thisdir_testopts = getThisDirTestOpts()

if config.use_threads:
    t.lock = threading.Lock()
    t.thread_pool = threading.Condition(t.lock)
    t.running_threads = 0

# if timeout == -1 then we try to calculate a sensible value
if config.timeout == -1:
    config.timeout = int(read_no_crs(config.top + '/timeout/calibrate.out'))

print 'Timeout is ' + str(config.timeout)

# -----------------------------------------------------------------------------
# The main dude

if config.rootdirs == []:
    config.rootdirs = ['.']

t_files = findTFiles(config.rootdirs)

print 'Found', len(t_files), '.T files...'

t = getTestRun()

t.start_time = chop(os.popen('date').read())
print 'Beginning test run at', t.start_time

# set stdout to unbuffered (is this the best way to do it?)
sys.stdout.flush()
sys.stdout = os.fdopen(sys.__stdout__.fileno(), "w", 0)

times = {}

for file in t_files:
    print '====> Running', file
    newTestDir(os.path.dirname(file))
    try:
        if config.use_threads:
            t.running_threads=0
        start = time.time()
        execfile(file)
        end = time.time()
        times[file] = end - start
        if config.use_threads:
            t.thread_pool.acquire()
            while t.running_threads>0:
                t.thread_pool.wait()
            t.thread_pool.release()
    except:
        print '*** framework failure: found an error while executing ', file, ':'
        t.n_framework_failures = t.n_framework_failures + 1
        traceback.print_exc()

summary(t, sys.stdout)

if config.output_summary != '':
    summary(t, open(config.output_summary, 'w'))

if config.times_file != '':
    h = open(config.times_file, 'w')
    for (k, v) in times.items():
        h.write(k + ',' + str(v) + '\n')
    h.close()

sys.exit(0)

<|endoftext|>


<commit_before>from datetime import date

from . import base
from . import mixins

class TransformedRecord(
    mixins.GenericCompensationMixin,
    mixins.GenericDepartmentMixin, mixins.GenericIdentifierMixin,
    mixins.GenericJobTitleMixin, mixins.GenericPersonMixin,
    mixins.MembershipMixin, mixins.OrganizationMixin, mixins.PostMixin,
    mixins.RaceMixin, mixins.LinkMixin, base.BaseTransformedRecord):

    MAP = {
        'full_name': 'Employee Name',
        'department': 'Department',
        'job_title': 'Job Title',
        'hire_date': 'CONTINUOUS EMPLOYMENT DATE',
        'employee_type': 'Full or Part Time',
        'gender': 'Gender',
        'given_race': 'Race',
        'compensation': 'Salary',
    }

    gender_map = {u'FEMALE': u'F', u'MALE': u'M', u'': u'Unknown'}

    NAME_FIELDS = ('first_name', 'middle_name', 'last_name', )

    ORGANIZATION_NAME = 'University of Texas at Austin'

    ORGANIZATION_CLASSIFICATION = 'University'

    DATE_PROVIDED = date(2015, 6, 29)

    URL = 'http://s3.amazonaws.com/raw.texastribune.org/ut_austin/salaries/2015-06/ut_austin.xlsx'

    @property
    def is_valid(self):
        # Adjust to return False on invalid fields.  For example:
        return self.last_name.strip() != ''

    @property
    def hire_date(self):
        hire_date = self.get_mapped_value('hire_date')
        if hire_date.strip() == "":
            return ""
        year = hire_date[0:4]
        month = hire_date[4:6]
        day = hire_date[6:8]
        return "-".join([year, month, day])

    @property
    def gender(self):
        sex = self.gender_map[self.get_mapped_value('gender')]
        if sex.strip() == "":
            return ""
        return sex.strip()

    @property
    def compensation_type(self):
        employee_type = self.employee_type

        if employee_type == 'FULL TIME':
            return 'FT'

        if employee_type == 'PART TIME':
            return 'PT'

    @property
    def description(self):
        employee_type = self.employee_type

        if employee_type == 'FULL TIME':
            return "Fiscal year allocation"

        if employee_type == 'PART TIME':
            return "Part-time compensation"

    @property
    def person(self):
        name = self.get_name()
        r = {
            'family_name': name.last,
            'given_name': name.first,
            'additional_name': name.middle,
            'name': unicode(name),
            'gender': self.gender,
        }

        return r

    @property
    def race(self):
        race = self.given_race.strip()
        if race == '':
            race = 'UNKNOWN'
        return {'name': race}

    def calculate_tenure(self):
        hire_date_data = map(int, self.hire_date.split('-'))
        hire_date = date(hire_date_data[0], hire_date_data[1],
                         hire_date_data[2])
        tenure = float((self.DATE_PROVIDED - hire_date).days) / float(360)
        if tenure < 0:
            tenure = 0
        return tenure

transform = base.transform_factory(TransformedRecord)
<commit_msg>got transformer organized a bit while waiting on hire dates<commit_after>from datetime import date

from . import base
from . import mixins

class TransformedRecord(
    mixins.GenericCompensationMixin,
    mixins.GenericDepartmentMixin, mixins.GenericIdentifierMixin,
    mixins.GenericJobTitleMixin, mixins.GenericPersonMixin,
    mixins.MembershipMixin, mixins.OrganizationMixin, mixins.PostMixin,
    mixins.RaceMixin, mixins.LinkMixin, base.BaseTransformedRecord):

    MAP = {
        'full_name': 'Employee Name',
        'department': 'Department',
        'job_title': 'Job Title',
        'hire_date': '',
        'employee_type': 'Full or Part Time',
        'gender': 'Gender',
        'given_race': 'Race',
        'compensation': 'Salary',
    }

    gender_map = {u'Female': u'F', u'Male': u'M'}

    ORGANIZATION_NAME = 'Texas Tech University Health Sciences Center'

    ORGANIZATION_CLASSIFICATION = 'University'

    DATE_PROVIDED = date()

    URL = ''

    @property
    def is_valid(self):
        # Adjust to return False on invalid fields.  For example:
        return self.full_name.strip() != ''

    @property
    def compensation_type(self):
        employee_type = self.employee_type

        if employee_type == 'F':
            return 'FT'

        if employee_type == 'P':
            return 'PT'

    @property
    def description(self):
        employee_type = self.employee_type

        if employee_type == 'F':
            return "Salary"

        if employee_type == 'P':
            return "Part-time salary"

    @property
    def person(self):
        name = self.get_name()
        r = {
            'family_name': name.last,
            'given_name': name.first,
            'additional_name': name.middle,
            'name': unicode(name),
            'gender': self.gender_map[self.get_mapped_value('gender')],
        }

        return r

    def get_raw_name(self):
        split_name = self.full_name.split(', ')
        last_name = split_name[0]
        split_firstname = split_name[1].split(' ')
        first_name = split_firstname[0]
        if len(split_firstname) == 2 and len(split_firstname[1]) == 1:
            middle_name = split_firstname[1]
        else:
            first_name = split_name[1]
            middle_name = ''

        return u' '.join([first_name, middle_name, last_name])

transform = base.transform_factory(TransformedRecord)
<|endoftext|>


<commit_before>import os

# If we are not given a path to list, use /tmp
root='/'
for dir_name, sub_dirs, files in os.walk(root): #dir_name is the current directory, sub_dirs are subs and files....
    #print '\n', dir_name
    # Make the subdirectory names stand out with /
    sub_dirs = [ '%s/' % n for n in sub_dirs ]
    # Mix the directory contents together
    contents = sub_dirs + files
    contents.sort()
    # Show the contents
    #for c in contents:
        #print '\t%s' % c
<commit_msg>modification to contents<commit_after>import os

# If we are not given a path to list, use /tmp
root='/'
for dir_name, sub_dirs, files in os.walk(root): #dir_name is the current directory, sub_dirs are subs and files....
    #print '\n', dir_name
    # Make the subdirectory names stand out with /
    sub_dirs = [ '%s/' % n for n in sub_dirs ]
    # Mix the directory contents together
    contents = files #originally sub_dirs + files
    contents.sort()
    for c in contents:

<|endoftext|>


<commit_before>python = Runtime.start("python","Python")
sr04 = Runtime.start("sr04", "UltrasonicSensor")
sr04.attach("COM15", 12, 11)
sr04.addRangeListener(python)

def onRange(distance):
  print "distance ", distance, " cm"

# event driven ranging
# start ranging for 5 seconds - the publishRange(distance) will be
# called for every attempt to range - this SHOULD NOT INTERFERE WITH SERVOS
# YAY !
# the even ranging DOES NOT USE Arduino's pulseIn() method -
# at the moment range() & ping() do
sr04.startRanging()
sleep(5)
sr04.stopRanging()

# range can also be retreieved in a blocking call
print "on demand range is ", sr04.range()

# you can also ping - ping does not do any calculations
# it simply returns the duration in microseconds of the ping
print "ping ", sr04.ping(), " mircoseconds"
<commit_msg>Update UltrasonicSensor.py<commit_after>python = Runtime.start("python","Python")
sr04 = Runtime.start("sr04", "UltrasonicSensor")
arduino = Runtime.start("arduino", "Arduino")
arduino.connect("COM15")
sr04.attach(arduino, 12, 11)
sr04.addRangeListener(python)

def onRange(distance):
  print "distance ", distance, " cm"

# event driven ranging
# start ranging for 5 seconds - the publishRange(distance) will be
# called for every attempt to range - this SHOULD NOT INTERFERE WITH SERVOS
# YAY !
# the even ranging DOES NOT USE Arduino's pulseIn() method -
# at the moment range() & ping() do
sr04.startRanging()
sleep(5)
sr04.stopRanging()

# range can also be retreieved in a blocking call
print "on demand range is ", sr04.range()

# you can also ping - ping does not do any calculations
# it simply returns the duration in microseconds of the ping
print "ping ", sr04.ping(), " mircoseconds"
<|endoftext|>


<commit_before><commit_msg>Easy way to import Lua examples<commit_after>import os

def _read(fn):
    fullpath = os.path.join(os.path.dirname(__file__), '%s.lua' % fn)
    return open(fullpath, 'rb').read()

player1 = _read('player1')
err_syntax = _read('err_syntax')

__all__ = ['player1', 'err_syntax']
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from ..common import *

class Youku(VideoExtractor):
    name = "优酷 (Youku)"

    stream_types = [
        # FIXME: Does not work for 1080P
        # {'id': 'hd3', 'container': 'flv', 'video_profile': '1080P'},
        {'id': 'hd2', 'container': 'flv', 'video_profile': '超清'},
        {'id': 'mp4', 'container': 'mp4', 'video_profile': '高清'},
        {'id': 'flvhd', 'container': 'flv', 'video_profile': '高清'},
        {'id': 'flv', 'container': 'flv', 'video_profile': '标清'},
        {'id': '3gphd', 'container': '3gp', 'video_profile': '高清（3GP）'},
    ]

    def get_vid_from_url(url):
        """Extracts video ID from URL.
        """
        patterns = [
            'youku.com/v_show/id_([\w=]+)',
            'player.youku.com/player.php/sid/([\w=]+)/v.swf',
            'loader\.swf\?VideoIDS=([\w=]+)',
        ]
        matches = match1(url, *patterns)
        if matches:
            return matches[0]
        else:
            return None

    def parse_m3u8(m3u8):
        return re.findall(r'(http://[^?]+)\?ts_start=0', m3u8)

    def prepare(self, **kwargs):
        assert self.url or self.vid
        if self.url and not self.vid:
            self.vid = __class__.get_vid_from_url(self.url)

        meta = json.loads(get_html('http://v.youku.com/player/getPlayList/VideoIDS/%s' % self.vid))
        if not meta['data']:
            log.wtf('[Failed] Video not found.')
        metadata0 = meta['data'][0]

        self.title = metadata0['title']

        for stream_type in self.stream_types:
            if stream_type['id'] in metadata0['streamsizes']:
                stream_id = stream_type['id']
                stream_size = int(metadata0['streamsizes'][stream_id])
                self.streams[stream_id] = {'container': stream_type['container'], 'video_profile': stream_type['video_profile'], 'size': stream_size}

        if not self.streams:
            for stream_type in self.stream_types:
                if stream_type['id'] in metadata0['streamtypes_o']:
                    stream_id = stream_type['id']
                    self.streams[stream_id] = {'container': stream_type['container'], 'video_profile': stream_type['video_profile']}

    def extract(self, **kwargs):
        if 'stream_id' in kwargs and kwargs['stream_id']:
            # Extract the stream
            stream_id = kwargs['stream_id']

            if stream_id not in self.streams:
                log.e('[Error] Invalid video format.')
                log.e('Use without specifying any video format to check all available formats.')
                exit(2)
        else:
            # Extract stream with the best quality
            stream_id = self.streams_sorted[0]['id']

        m3u8_url = "http://v.youku.com/player/getM3U8/vid/{vid}/type/{stream_id}/video.m3u8".format(vid=self.vid, stream_id=stream_id)
        m3u8 = get_html(m3u8_url)
        if not m3u8:
            log.w('[Warning] This video can only be streamed within Mainland China!')
            log.w('Use \'-y\' to specify a proxy server for extracting stream data.\n')

        self.streams[stream_id]['src'] = __class__.parse_m3u8(m3u8)

site = Youku()
download = site.download_by_url
download_playlist = playlist_not_supported('youku')

youku_download_by_vid = site.download_by_vid
# Used by: acfun.py bilibili.py miomio.py tudou.py
<commit_msg>Youku: playlist support<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from ..common import *

class Youku(VideoExtractor):
    name = "优酷 (Youku)"

    stream_types = [
        # FIXME: Does not work for 1080P
        # {'id': 'hd3', 'container': 'flv', 'video_profile': '1080P'},
        {'id': 'hd2', 'container': 'flv', 'video_profile': '超清'},
        {'id': 'mp4', 'container': 'mp4', 'video_profile': '高清'},
        {'id': 'flvhd', 'container': 'flv', 'video_profile': '高清'},
        {'id': 'flv', 'container': 'flv', 'video_profile': '标清'},
        {'id': '3gphd', 'container': '3gp', 'video_profile': '高清（3GP）'},
    ]

    def parse_m3u8(m3u8):
        return re.findall(r'(http://[^?]+)\?ts_start=0', m3u8)

    def get_vid_from_url(url):
        """Extracts video ID from URL.
        """
        return match1(url, r'youku\.com/v_show/id_([\w=]+)') or \
          match1(url, r'player\.youku\.com/player\.php/sid/([\w=]+)/v\.swf') or \
          match1(url, r'loader\.swf\?VideoIDS=([\w=]+)')

    def get_playlist_id_from_url(url):
        """Extracts playlist ID from URL.
        """
        return match1(url, r'youku\.com/playlist_show/id_([\w=]+)')

    def download_playlist_by_url(self, url, **kwargs):
        self.url = url

        playlist_id = __class__.get_playlist_id_from_url(self.url)
        if playlist_id is None:
            log.wtf('[Failed] Unsupported URL pattern.')

        video_page = get_content('http://www.youku.com/playlist_show/id_%s' % playlist_id)
        videos = set(re.findall(r'href="(http://v\.youku\.com/[^?"]+)', video_page))
        self.title = re.search(r'<meta name="title" content="([^"]+)"', video_page).group(1)
        self.p_playlist()
        for video in videos:
            index = parse_query_param(video, 'f')
            __class__().download_by_url(video, index=index, **kwargs)

    def prepare(self, **kwargs):
        assert self.url or self.vid

        if self.url and not self.vid:
            self.vid = __class__.get_vid_from_url(self.url)

            if self.vid is None:
                self.download_playlist_by_url(self.url, **kwargs)
                exit(0)

        meta = json.loads(get_html('http://v.youku.com/player/getPlayList/VideoIDS/%s' % self.vid))
        if not meta['data']:
            log.wtf('[Failed] Video not found.')
        metadata0 = meta['data'][0]

        self.title = metadata0['title']

        for stream_type in self.stream_types:
            if stream_type['id'] in metadata0['streamsizes']:
                stream_id = stream_type['id']
                stream_size = int(metadata0['streamsizes'][stream_id])
                self.streams[stream_id] = {'container': stream_type['container'], 'video_profile': stream_type['video_profile'], 'size': stream_size}

        if not self.streams:
            for stream_type in self.stream_types:
                if stream_type['id'] in metadata0['streamtypes_o']:
                    stream_id = stream_type['id']
                    self.streams[stream_id] = {'container': stream_type['container'], 'video_profile': stream_type['video_profile']}

    def extract(self, **kwargs):
        if 'stream_id' in kwargs and kwargs['stream_id']:
            # Extract the stream
            stream_id = kwargs['stream_id']

            if stream_id not in self.streams:
                log.e('[Error] Invalid video format.')
                log.e('Use without specifying any video format to check all available formats.')
                exit(2)
        else:
            # Extract stream with the best quality
            stream_id = self.streams_sorted[0]['id']

        m3u8_url = "http://v.youku.com/player/getM3U8/vid/{vid}/type/{stream_id}/video.m3u8".format(vid=self.vid, stream_id=stream_id)
        m3u8 = get_html(m3u8_url)
        if not m3u8:
            log.w('[Warning] This video can only be streamed within Mainland China!')
            log.w('Use \'-y\' to specify a proxy server for extracting stream data.\n')

        self.streams[stream_id]['src'] = __class__.parse_m3u8(m3u8)

site = Youku()
download = site.download_by_url
download_playlist = site.download_playlist_by_url

youku_download_by_vid = site.download_by_vid
# Used by: acfun.py bilibili.py miomio.py tudou.py
<|endoftext|>


<commit_before># coding: utf8
from __future__ import unicode_literals

from .._pickle_api import pickle_dumps, pickle_loads

def test_pickle_dumps():
    data = {"hello": "world", "test": 123}
    expected = [
        b"\x80\x04\x95\x1e\x00\x00\x00\x00\x00\x00\x00}\x94(\x8c\x05hello\x94\x8c\x05world\x94\x8c\x04test\x94K{u.",
        b"\x80\x02}q\x00(X\x04\x00\x00\x00testq\x01K{X\x05\x00\x00\x00helloq\x02X\x05\x00\x00\x00worldq\x03u.",
    ]
    msg = pickle_dumps(data)
    assert msg in expected

def test_pickle_loads():
    msg = pickle_dumps({"hello": "world", "test": 123})
    data = pickle_loads(msg)
    assert len(data) == 2
    assert data["hello"] == "world"
    assert data["test"] == 123
<commit_msg>Fix test<commit_after># coding: utf8
from __future__ import unicode_literals

from .._pickle_api import pickle_dumps, pickle_loads

def test_pickle_dumps():
    data = {"hello": "world", "test": 123}
    expected = [
        b"\x80\x04\x95\x1e\x00\x00\x00\x00\x00\x00\x00}\x94(\x8c\x05hello\x94\x8c\x05world\x94\x8c\x04test\x94K{u.",
        b"\x80\x04\x95\x1e\x00\x00\x00\x00\x00\x00\x00}\x94(\x8c\x04test\x94K{\x8c\x05hello\x94\x8c\x05world\x94u.",
        b"\x80\x02}q\x00(X\x04\x00\x00\x00testq\x01K{X\x05\x00\x00\x00helloq\x02X\x05\x00\x00\x00worldq\x03u.",
    ]
    msg = pickle_dumps(data)
    assert msg in expected

def test_pickle_loads():
    msg = pickle_dumps({"hello": "world", "test": 123})
    data = pickle_loads(msg)
    assert len(data) == 2
    assert data["hello"] == "world"
    assert data["test"] == 123
<|endoftext|>


<commit_before>from ...api import layerize
from ..util import get_array_module
from .model import Model

def begin_stepwise_tanh(X, nG):
    xp = get_array_module(X)
    Y = xp.zeros(X.shape, dtype='f')
    def tanh_fwd(t):
        Y[t] = xp.tanh(X[t])
        return Y[t]
    def tanh_bwd(dY):
        return (1-Y**2) * dY
    return Y, tanh_fwd, tanh_bwd

def begin_stepwise_relu(X, nG):
    xp = get_array_module(X)
    Y = xp.zeros(X.shape, dtype='f')
    def relu_fwd(t):
        Y[t] = X[t] * (X[t] > 0)
        return Y[t]
    def relu_bwd(dY):
        return dY * (X>0)
    return Y, relu_fwd, relu_bwd

def begin_stepwise_LSTM(gates, nG):
    xp = get_array_module(gates)
    nN = gates.shape[0]
    nO = gates.shape[1]//nG
    gates = gates.reshape((nN, nO, nG))
    Hout = xp.zeros((nN, nO), dtype='f')
    cells = xp.zeros((nN, nO), dtype='f')
    pad = xp.zeros((nO,), dtype='f')
    d_pad = xp.zeros((nO,), dtype='f')
    def lstm_nonlin_fwd(t):
        ops.lstm(Hout[t], cells[t],
            gates[t], cells[t-1] if t >= 1 else pad)
        return Hout[t]

    def lstm_nonlin_bwd(d_output, Wh):
        d_gates = xp.zeros(gates.shape, dtype='f')
        d_cells = xp.zeros(cells.shape, dtype='f')
        if d_output.shape[0] >= 2:
            d_gates[:-1] += xp.tensordot(d_output[1:], Wh,
                            axes=[[1], [1]]).reshape((nN-1, nO, nG))
        for t in range(d_output.shape[0]-1, 0, -1):
            ops.backprop_lstm(d_cells[t], d_cells[t-1], d_gates[t],
                d_output[t], gates[t], cells[t], cells[t-1])
        ops.backprop_lstm(d_cells[0], d_pad, d_gates[0], d_output[0], gates[0],
            cells[0], pad)
        return d_gates.reshape((nN, nO*nG))

    return Hout, lstm_nonlin_fwd, lstm_nonlin_bwd

def LSTM(width, residual=False, xp=None):
    alloc, params = xp_params(xp)
    model = _ResidualLSTM(alloc, width)
    def lstm_fwd(X, drop=0.):
        y, bp_y = model(X)

        def lstm_bwd(dy, sgd=None):
            dX = bp_y(dy)
            for param, grad in params:
                sgd(param.ravel(), grad.ravel(), key=id(param))
                grad.fill(0)
            return dX
        return y, lstm_bwd
    return layerize(lstm_fwd)

def BiLSTM(width, residual=False, xp=None):
    alloc, params = xp_params(xp)
    model = _BiLSTM(alloc, width, width, residual=residual)
    def lstm_fwd(X, drop=0.):
        y, bp_y = model(X)

        def lstm_bwd(dy, sgd=None):
            dX = bp_y(dy)
            for param, grad in params:
                sgd(param.ravel(), grad.ravel(), key=id(param))
                grad.fill(0)
            return dX
        return y, lstm_bwd
    return layerize(lstm_fwd)

def BiRNN(width, residual=False, xp=None):
    alloc, params = xp_params(xp)
    model = _BiRNN(alloc, width, width, nonlinearity=begin_stepwise_selu,
                   residual=residual)
    def rnn_fwd(X, drop=0.):
        y, bp_y = model(X)

        def rnn_bwd(dy, sgd=None):
            dX = bp_y(dy)
            for param, grad in params:
                sgd(param.ravel(), grad.ravel(), key=id(param))
                grad.fill(0)
            return dX
        return y, rnn_bwd
    return layerize(rnn_fwd)

def RNN(width, residual=True, xp=None):
    alloc, params = xp_params(xp)
    model = _RNN(alloc, width, width, nonlinearity=begin_stepwise_relu,
                 residual=residual)
    def rnn_fwd(X, drop=0.):
        y, bp_y = model(X)

        def rnn_bwd(dy, sgd=None):
            dX = bp_y(dy)
            for param, grad in params:
                sgd(param.ravel(), grad.ravel(), key=id(param))
                grad.fill(0)
            return dX
        return y, rnn_bwd
    return layerize(rnn_fwd)

def xp_params(xp=None):
    if xp is None:
        xp = Model.Ops.xp
    params = []
    def allocate(shape, gradient=False):
        param = xp.zeros(shape, dtype='f')
        if not gradient:
            return param
        else:
            d_param = xp.zeros(shape, dtype='f')
            params.append([param, d_param])
            return param, d_param
    return allocate, params

def _BiRNN(alloc, nO, nI, nG=1, nonlinearity=begin_stepwise_tanh, residual=False):
    #l2r_model = _RNN(alloc, nO, nI, nonlinearity, nG=nG, residual=residual)
    #r2l_model = _RNN(alloc, nO, nI, nonlinearity, nG=nG, residual=residual)
    assert nO == nI
    l2r_model = _ResidualLSTM(alloc, nI)
    r2l_model = _ResidualLSTM(alloc, nI)
    def birnn_fwd(Xs):
        xp = get_array_module(Xs[0])
        l2r_Zs, bp_l2r_Zs = l2r_model(Xs)
        r2l_Zs, bp_r2l_Zs = r2l_model([xp.ascontiguousarray(X[::-1]) for X in Xs])
        def birnn_bwd(dZs):
            d_l2r_Zs = []
            d_r2l_Zs = []
            for dZ in dZs:
                l2r = dZ[:, :nO]
                r2l = dZ[:, nO:]
                d_l2r_Zs.append(xp.ascontiguousarray(l2r))
                d_r2l_Zs.append(xp.ascontiguousarray(r2l[::-1]))
            dXs_l2r = bp_l2r_Zs(d_l2r_Zs)
            dXs_r2l = bp_r2l_Zs(d_r2l_Zs)
            dXs = [dXf+dXb[::-1] for dXf, dXb in zip(dXs_l2r, dXs_r2l)]
            return dXs
        Zs = [xp.hstack((Zf, Zb[::-1])) for Zf, Zb in zip(l2r_Zs, r2l_Zs)]
        return Zs, birnn_bwd
    return birnn_fwd

def _BiLSTM(alloc, nO, nI, residual=False):
    return _BiRNN(alloc, nO, nI, nG=4, nonlinearity=begin_stepwise_LSTM,
                 residual=residual)

def _RNN(alloc, nO, nI, nonlinearity=begin_stepwise_tanh, nG=1,
        residual=False):
    begin_nonlin = nonlinearity
    if not residual:
        Wx, dWx    = alloc((nO*nG, nI), gradient=True)
    Wh, dWh    = alloc((nO*nG, nO), gradient=True)
    b, db      = alloc((nO*nG,),    gradient=True)
    pad, d_pad = alloc((nO,),       gradient=True)
    xp = get_array_module(Wh)
    if not residual:
        Wx += xp.random.normal(scale=xp.sqrt(1./nI), size=Wx.size).reshape(Wx.shape)
    Wh += xp.random.normal(scale=xp.sqrt(1./nI), size=Wh.size).reshape(Wh.shape)
    # Initialize forget gates' bias
    if nG == 4:
        b = b.reshape((nO, nG))
        b[:, 0] = 3.
        b = b.reshape((nO * nG,))

    nonlocals = [[], d_pad, dWx, dWh, db]

    def rnn_fwd(Xs):
        nonlocals[0] = []
        Zs = nonlocals[0]
        Ys = []
        backprops = []
        for X in Xs:
            if residual:
                Y = xp.zeros((X.shape[0], nO*nG), dtype='f')
            else:
                Y = xp.tensordot(X, Wx, axes=[[1], [1]])
            Y += b
            Z, nonlin_fwd, bp_nonlin = begin_nonlin(Y, nG)
            state = pad
            for t in range(Y.shape[0]):
                Y[t] += Wh.dot(state)
                state = nonlin_fwd(t)
                if residual:
                    Z[t] += X[t]
                    state += X[t]
            backprops.append(bp_nonlin)
            Ys.append(Y)
            Zs.append(Z)

        def rnn_bwd(dZs):
            Zs, d_pad, dWx, dWh, db = nonlocals
            dXs = []
            for X, Z, dZ, bp_Z in zip(Xs, Zs, dZs, backprops):
                dY = bp_Z(dZ, Wh)
                if residual:
                    dX = dZ.copy()
                else:
                    dX   = xp.tensordot(dY,     Wx,     axes=[[1], [0]])
                    dWx += xp.tensordot(dY,     X,      axes=[[0], [0]])
                if dY.shape[0] >= 2:
                    dWh += xp.tensordot(dY[1:], Z[:-1], axes=[[0], [0]])
                db  += dY.sum(axis=0)
                dXs.append(dX)
            return dXs
        return Zs, rnn_bwd
    return rnn_fwd

def _ResidualLSTM(alloc, nI):
    nO = nI
    nG = 4
    W, dW      = alloc((nO*nG, nO), gradient=True)
    b, db      = alloc((nO*nG,),    gradient=True)
    pad = alloc((nO,))
    xp = get_array_module(W)
    W += xp.random.normal(scale=xp.sqrt(1./nI), size=W.size).reshape(W.shape)
    # Initialize forget gates' bias
    b = b.reshape((nO, nG))
    b[:, 0] = 3.
    b = b.reshape((nO * nG,))

    nonlocals = [dW, db]
    ops = Model.ops

    def lstm_fwd(Xs):
        batch_gates = []
        batch_cells = []
        batch_Houts = []
        for X in Xs:
            nN = X.shape[0]
            gates = xp.zeros((nN, nO * nG), dtype='f')
            Hout = xp.zeros((nN, nO), dtype='f')
            cells = xp.zeros((nN, nO), dtype='f')

            gates += b
            for t in range(nN):
                gates[t] += W.dot(Hout[t-1] if t >= 1 else pad)
                ops.lstm(Hout[t], cells[t], gates[t],
                    cells[t-1] if t >= 1 else pad)
                Hout[t] += X[t]
            batch_gates.append(gates)
            batch_cells.append(cells)
            batch_Houts.append(Hout)

        def lstm_bwd(d_Houts):
            dW, db = nonlocals
            dXs = []
            for X, gates, cells, dH in zip(Xs, batch_gates, batch_cells, d_Houts):
                nN = X.shape[0]
                d_gates = xp.zeros((nN, nO * nG), dtype='f')
                d_cells = xp.zeros((nN, nO), dtype='f')
                for t in range(nN-1, 0, -1):
                    ops.backprop_lstm(d_cells[t], d_cells[t-1], d_gates[t],
                        dH[t], gates[t], cells[t], cells[t-1])
                    dH[t-1] += xp.tensordot(d_gates[t], W, axes=[[0], [0]])
                if nN >= 2:
                    dW += xp.tensordot(d_gates[1:], dH[:-1], axes=[[0], [0]])
                db  += d_gates.sum(axis=0)
                dXs.append(dH.copy())
            return dXs
        return batch_Houts, lstm_bwd
    return lstm_fwd

def lstm_fwd(Xs_lengths, W, b, drop_input=None, drop_hidden=None, drop_cells=None):
    Xs, lengths = Xs_lengths
    timesteps = []
    Hp = pad
    Cp = pad
    for t in max(lengths):
        Xt = _make_timestep(Xs, lengths, t)
        Gt = xp.zeros((nB, nO, nG), dtype='f')
        Ht = xp.zeros((nB, nO), dtype='f')
        Ct = xp.zeros((nN, nO), dtype='f')

        if drop_hidden is not None:
            Hp *= drop_hidden
        if drop_cells is not None:
            Cp *= drop_cells

        Gt += b
        Gt += W.dot(Hp)
        ops.lstm(Ht, Ct, Gt,
            Cp)
        if drop_input is not None:
            Xt *= drop_input
        Ht += Xt
        timesteps.append((Xt, Gt, Ct))
        _write_timestep(Hs, lengths, t, Ht)
        Cp = Ct
        Hp = Ht

    def lstm_bwd(dHs):
        dXs = []
        Cp = pad
        Hp = pad
        dHp = xp.zeros((nB, nO), dtype='f')
        dXs = xp.zeros(Xs.shape, dtype='f')
        for t, (Xt, Gt, Ct) in reversed(enumerate(timesteps)):
            dHt = dHp + _make_timestep(Hs, lengths, t)
            dGt.fill(0)
            dCt.fill(0)
            ops.backprop_lstm(dCt, dCp, dGt,
                dHt, Gt, Ct, Cp)
            dHp = dG.dot(W.T)
            dW += xp.tensordot(dGn, dHt, axes=[[0], [0]])
            db += dGt.sum(axis=0)
            _write_timestep(dXs, lengths, t, dHt)
            dCt, dCp = dCp, dCt
        return dXs
    return Hs, lstm_bwd

def _make_timestep(Xs, lengths, t):
    xp = get_array_module(Xs)
    n = 0
    for i, length in enumerate(lengths):
        n += length < t
    output = xp.zeros((n,) + Xs.shape[1:], dtype=Xs.dtype)
    start = 0
    i = 0
    for length in lengths:
        if t < length:
            output[i] = Xs[start + t]
            i += 1
        start += length
    return output

def _write_timestep(Xs, lengths, t, timestep):
    xp = get_array_module(Xs)
    start = 0
    i = 0
    for length in lengths:
        if t < length:
            Xs[start + t] = timestep[i]
            i += 1
        start += length

#
#def begin_stepwise_LSTM(gates, nG):
#    ops = NumpyOps()
#    xp = ops.xp
#    nN = gates.shape[0]
#    nO = gates.shape[1]//nG
#    gates = gates.reshape((nN, nO, nG))
#    Hout = numpy.zeros((nN, nO), dtype='f')
#    cells = numpy.zeros((nN, nO), dtype='f')
#    pad = numpy.zeros((nO,), dtype='f')
#    d_pad = numpy.zeros((nO,), dtype='f')
#    def lstm_nonlin_fwd(t):
#        ops.lstm(Hout[t], cells[t],
#            gates[t], cells[t-1] if t >= 1 else pad)
#        return Hout[t]
#
#    def lstm_nonlin_bwd(d_output, Wh):
#        d_gates = numpy.zeros(gates.shape, dtype='f')
#        d_cells = numpy.zeros(cells.shape, dtype='f')
#        if d_output.shape[0] >= 2:
#            d_gates[:-1] += xp.tensordot(d_output[1:], Wh,
#                            axes=[[1], [1]]).reshape((nN-1, nO, nG))
#        for t in range(d_output.shape[0]-1, 0, -1):
#            ops.backprop_lstm(d_cells[t], d_cells[t-1], d_gates[t],
#                d_output[t], gates[t], cells[t], cells[t-1])
#        ops.backprop_lstm(d_cells[0], d_pad, d_gates[0], d_output[0], gates[0],
#            cells[0], pad)
#        return d_gates.reshape((nN, nO*nG))
#
#    return Hout, lstm_nonlin_fwd, lstm_nonlin_bwd
#
#
<commit_msg>Remove broken LSTM code<commit_after><|endoftext|>


<commit_before>
# Author: J. Michael Burgess

# Provides some universal statistical utilities and stats comparison tools

import numpy as np
import pandas as pd
from threeML.io.rich_display import display
import copy

import warnings

def li_and_ma(total, background, alpha=1.):
    """"Li and Ma (1983) signal to noise significance"""

    a = total / (total + background)
    b = background / (total + background)
    S = np.sqrt(2) * np.sqrt((total * np.log(((1. + alpha) / alpha) * a) + background * np.log((1 + alpha) * b)))

    return S

def aic(log_like, n_parameters, n_data_points):
    """
    The Aikake information criterion.
    A model comparison tool based of infomormation theory. It assumes that N is large i.e.,
    that the model is approaching the CLT.

    """

    val = -2. * log_like + 2 * n_parameters
    val += 2 * n_parameters * (n_parameters + 1) / (n_data_points - n_parameters - 1)

    return val

def bic(log_like, n_parameters, n_data_points):
    """
    The Bayesian information criterion.

    Returns:

    """
    val = -2. * log_like + n_parameters * np.log(n_data_points)

    return val

def waic(bayesian_trace):
    raise NotImplementedError("Coming soon to a theater near you.")

def effective_number_of_parameters(bayesian_trace):
    raise NotImplementedError("Coming soon to a theater near you.")

def dic(bayesian_trace):
    """
    The Deviance information criteria derived from MCMC traces
    Read more:  dx.doi.org/10.1111/1467-9868.00353

    Args:
        bayesian_trace: an instance of Bayesian Analysis

    Returns: deviance information criteria

    """

    mean_deviance = -2. * np.mean(bayesian_trace.log_probability_values)

    mean_of_free_parameters = np.mean(bayesian_trace.raw_samples, axis=0)

    deviance_at_mean = -2. * bayesian_trace.log_probability(mean_of_free_parameters)[0]

    return 2 * mean_deviance - deviance_at_mean

class ModelComparison(object):
    def __init__(self, *analyses):
        self._analysis_container = analyses

        # First make sure that it is all bayesian or all MLE
        if (np.unique([a._analysis_type for a in analyses])).shape[0] > 1:
            raise RuntimeError("Only all Bayesian or all MLE analyses are allowed. Not a mixture!")

        self._analysis_type = analyses[0]._analysis_type

        if self._analysis_type == 'mle':

            self._stat_df = self._compute_mle_statistics()

        elif self._analysis_type == 'bayesian':

            self._stat_df = self._compute_bayes_statistics()

    def report(self, sort=None, normalized=True, precision=1):

        pd.options.display.float_format = ('{:.%df}' % (precision)).format

        this_df = self._stat_df.copy()

        if self._analysis_type == 'bayesian':

            # Remember to add WAIC in later

            display_order = ['Model',
                             '-2 ln(like)',
                             'AIC',
                             'BIC',
                             'DIC',
                             'log10 (Z)',
                             'N. Free Parameters',
           #                  'Eff. N. Free Parameters',
                             'dof']

            min_stat = ['-2 ln(like)',
                        'AIC',
                        'BIC',
                        'DIC'
                        ]

            max_stat = ['log10 (Z)']

        else:

            display_order = ['Model',
                             '-2 ln(like)',
                             'AIC',
                             'BIC',
                             'N. Free Parameters',
                             'dof']

            min_stat = ['-2 ln(like)',
                        'AIC',
                        'BIC'
                        ]

            max_stat = []

        if normalized:

            # Normalize the statistics to the 'worst' fit.
            # MLE type stats have lowest value as best
            # while Bayesian evidence has highest as best

            for key in min_stat:
                this_df[key] = this_df[key] - this_df[key].max()

            for key in max_stat:
                this_df[key] = this_df[key] - this_df[key].max()

        if sort is not None:

            if sort in min_stat:

                ascend = True

            elif sort in max_stat:

                ascend = False

            else:

                warnings.warn('%s is not a valid statistic' % sort)

                display(this_df[display_order])

                return self._stat_df

            this_df = this_df.sort_values(by=sort, ascending=ascend, inplace=False)

            # for col in format_columns:

            #    self._float_format(this_df[col])

            display(this_df[display_order])

            return self._stat_df

        else:

            #   for col in format_columns:
            #      self._float_format(self._stat_df[col])

            display(this_df[display_order])

            return self._stat_df

        pd.reset_option('float_format')

    @property
    def statistical_dataframe(self):

        return self._stat_df

    def _compute_mle_statistics(self):

        stat_table = {}
        stat_table['AIC'] = []
        stat_table['BIC'] = []
        stat_table['-2 ln(like)'] = []
        stat_table['dof'] = []
        stat_table['N. Free Parameters'] = []
        stat_table['Model'] = []

        for analysis in self._analysis_container:
            n_data_points = np.sum([data.get_number_of_data_points() for data in analysis.data_list.values()])
            n_free_params = len(analysis._free_parameters.values())  # should add a property
            dof = n_data_points - n_free_params
            model_name = \
            [parameter_name.split('.')[-2] for (parameter_name, parameter) in analysis._free_parameters.iteritems()][0]
            loglike = - analysis.current_minimum

            this_aic = aic(loglike, n_free_params, n_data_points)
            this_bic = bic(loglike, n_free_params, n_data_points)

            stat_table['AIC'].append(this_aic)
            stat_table['BIC'].append(this_bic)
            stat_table['-2 ln(like)'].append(-2. * loglike)
            stat_table['dof'].append(dof)
            stat_table['N. Free Parameters'].append(n_free_params)
            stat_table['Model'].append(model_name)

        stat_df = pd.DataFrame(stat_table)

        return stat_df

    def _compute_bayes_statistics(self):

        stat_table = {}
        stat_table['AIC'] = []
        stat_table['BIC'] = []
        stat_table['DIC'] = []
        stat_table['log10 (Z)'] = []
        # stat_table['WAIC'] = []
        stat_table['-2 ln(like)'] = []
        stat_table['dof'] = []
        stat_table['N. Free Parameters'] = []
        #stat_table['Eff. N. Free Parameters'] = []
        stat_table['Model'] = []

        for analysis in self._analysis_container:
            n_data_points = np.sum([data.get_number_of_data_points() for data in analysis.data_list.values()])
            n_free_params = len(analysis._free_parameters.values())  # should add a property
            dof = n_data_points - n_free_params

            #eff_n_params = analysis.get_effective_free_parameters()  # change this to local function later

            model_name = \
            [parameter_name.split('.')[-2] for (parameter_name, parameter) in analysis._free_parameters.iteritems()][0]

            if analysis.log_like_values is None:
                stat_table['AIC'].append(None)
                stat_table['BIC'].append(None)
                stat_table['DIC'].append(None)
                stat_table['log10 (Z)'].append(analysis.log10_evidence)
                # stat_table['WAIC'].append(this_waic)
                stat_table['-2 ln(like)'].append(None)
                stat_table['N. Free Parameters'].append(n_free_params)
                # stat_table['Eff. N. Free Parameters'].append(eff_n_params)
                stat_table['dof'].append(dof)
                stat_table['Model'].append(model_name)

            else:

                this_dic = dic(analysis)
                # this_waic = waic(analysis)

                # We will now compute the AIC/BIC/ etc. at the max of the posterior likelihood

                loglike = analysis.log_like_values.max()

                this_aic = aic(loglike, n_free_params, n_data_points)
                this_bic = bic(loglike, n_free_params, n_data_points)

                # Create the dataframe

                stat_table['AIC'].append(this_aic)
                stat_table['BIC'].append(this_bic)
                stat_table['DIC'].append(this_dic)
                stat_table['log10 (Z)'].append(analysis.log10_evidence)
                # stat_table['WAIC'].append(this_waic)
                stat_table['-2 ln(like)'].append(-2. * loglike)
                stat_table['N. Free Parameters'].append(n_free_params)
                #stat_table['Eff. N. Free Parameters'].append(eff_n_params)
                stat_table['dof'].append(dof)
                stat_table['Model'].append(model_name)

        stat_df = pd.DataFrame(stat_table)

        return stat_df

<commit_msg>added back a property that calculates the number of data points. this is required for AIC/BIC statistics.<commit_after>
# Author: J. Michael Burgess

# Provides some universal statistical utilities and stats comparison tools

import numpy as np
import pandas as pd
from threeML.io.rich_display import display
import copy

import warnings

def li_and_ma(total, background, alpha=1.):
    """"Li and Ma (1983) signal to noise significance"""

    a = total / (total + background)
    b = background / (total + background)
    S = np.sqrt(2) * np.sqrt((total * np.log(((1. + alpha) / alpha) * a) + background * np.log((1 + alpha) * b)))

    return S

def aic(log_like, n_parameters, n_data_points):
    """
    The Aikake information criterion.
    A model comparison tool based of infomormation theory. It assumes that N is large i.e.,
    that the model is approaching the CLT.

    """

    val = -2. * log_like + 2 * n_parameters
    val += 2 * n_parameters * (n_parameters + 1) / (n_data_points - n_parameters - 1)

    return val

def bic(log_like, n_parameters, n_data_points):
    """
    The Bayesian information criterion.

    Returns:

    """
    val = -2. * log_like + n_parameters * np.log(n_data_points)

    return val

def waic(bayesian_trace):
    raise NotImplementedError("Coming soon to a theater near you.")

def effective_number_of_parameters(bayesian_trace):
    raise NotImplementedError("Coming soon to a theater near you.")

def dic(bayesian_trace):
    """
    The Deviance information criteria derived from MCMC traces
    Read more:  dx.doi.org/10.1111/1467-9868.00353

    Args:
        bayesian_trace: an instance of Bayesian Analysis

    Returns: deviance information criteria

    """

    mean_deviance = -2. * np.mean(bayesian_trace.log_probability_values)

    mean_of_free_parameters = np.mean(bayesian_trace.raw_samples, axis=0)

    deviance_at_mean = -2. * bayesian_trace.log_probability(mean_of_free_parameters)[0]

    return 2 * mean_deviance - deviance_at_mean

class ModelComparison(object):
    def __init__(self, *analyses):
        self._analysis_container = analyses

        # First make sure that it is all bayesian or all MLE
        if (np.unique([a._analysis_type for a in analyses])).shape[0] > 1:
            raise RuntimeError("Only all Bayesian or all MLE analyses are allowed. Not a mixture!")

        self._analysis_type = analyses[0]._analysis_type

        if self._analysis_type == 'mle':

            self._stat_df = self._compute_mle_statistics()

        elif self._analysis_type == 'bayesian':

            self._stat_df = self._compute_bayes_statistics()

    def report(self, sort=None, normalized=True, precision=1):

        pd.options.display.float_format = ('{:.%df}' % (precision)).format

        this_df = self._stat_df.copy()

        if self._analysis_type == 'bayesian':

            # Remember to add WAIC in later

            display_order = ['Model',
                             '-2 ln(like)',
                             'AIC',
                             'BIC',
                             'DIC',
                             'log10 (Z)',
                             'N. Free Parameters',
           #                  'Eff. N. Free Parameters',
                             'dof']

            min_stat = ['-2 ln(like)',
                        'AIC',
                        'BIC',
                        'DIC'
                        ]

            max_stat = ['log10 (Z)']

        else:

            display_order = ['Model',
                             '-2 ln(like)',
                             'AIC',
                             'BIC',
                             'N. Free Parameters',
                             'dof']

            min_stat = ['-2 ln(like)',
                        'AIC',
                        'BIC'
                        ]

            max_stat = []

        if normalized:

            # Normalize the statistics to the 'worst' fit.
            # MLE type stats have lowest value as best
            # while Bayesian evidence has highest as best

            for key in min_stat:
                this_df[key] = this_df[key] - this_df[key].max()

            for key in max_stat:
                this_df[key] = this_df[key] - this_df[key].max()

        if sort is not None:

            if sort in min_stat:

                ascend = True

            elif sort in max_stat:

                ascend = False

            else:

                warnings.warn('%s is not a valid statistic' % sort)

                display(this_df[display_order])

                return self._stat_df

            this_df = this_df.sort_values(by=sort, ascending=ascend, inplace=False)

            # for col in format_columns:

            #    self._float_format(this_df[col])

            display(this_df[display_order])

            return self._stat_df

        else:

            #   for col in format_columns:
            #      self._float_format(self._stat_df[col])

            display(this_df[display_order])

            return self._stat_df

        pd.reset_option('float_format')

    @property
    def statistical_dataframe(self):

        return self._stat_df

    def _compute_mle_statistics(self):

        stat_table = {}
        stat_table['AIC'] = []
        stat_table['BIC'] = []
        stat_table['-2 ln(like)'] = []
        stat_table['dof'] = []
        stat_table['N. Free Parameters'] = []
        stat_table['Model'] = []

        for analysis in self._analysis_container:
            n_data_points = np.sum([data.n_data_points for data in analysis.data_list.values()])
            n_free_params = len(analysis._free_parameters.values())  # should add a property
            dof = n_data_points - n_free_params
            model_name = \
            [parameter_name.split('.')[-2] for (parameter_name, parameter) in analysis._free_parameters.iteritems()][0]
            loglike = - analysis.current_minimum

            this_aic = aic(loglike, n_free_params, n_data_points)
            this_bic = bic(loglike, n_free_params, n_data_points)

            stat_table['AIC'].append(this_aic)
            stat_table['BIC'].append(this_bic)
            stat_table['-2 ln(like)'].append(-2. * loglike)
            stat_table['dof'].append(dof)
            stat_table['N. Free Parameters'].append(n_free_params)
            stat_table['Model'].append(model_name)

        stat_df = pd.DataFrame(stat_table)

        return stat_df

    def _compute_bayes_statistics(self):

        stat_table = {}
        stat_table['AIC'] = []
        stat_table['BIC'] = []
        stat_table['DIC'] = []
        stat_table['log10 (Z)'] = []
        # stat_table['WAIC'] = []
        stat_table['-2 ln(like)'] = []
        stat_table['dof'] = []
        stat_table['N. Free Parameters'] = []
        #stat_table['Eff. N. Free Parameters'] = []
        stat_table['Model'] = []

        for analysis in self._analysis_container:
            n_data_points = np.sum([data.n_data_points for data in analysis.data_list.values()])
            n_free_params = len(analysis._free_parameters.values())  # should add a property
            dof = n_data_points - n_free_params

            #eff_n_params = analysis.get_effective_free_parameters()  # change this to local function later

            model_name = \
            [parameter_name.split('.')[-2] for (parameter_name, parameter) in analysis._free_parameters.iteritems()][0]

            if analysis.log_like_values is None:
                stat_table['AIC'].append(None)
                stat_table['BIC'].append(None)
                stat_table['DIC'].append(None)
                stat_table['log10 (Z)'].append(analysis.log10_evidence)
                # stat_table['WAIC'].append(this_waic)
                stat_table['-2 ln(like)'].append(None)
                stat_table['N. Free Parameters'].append(n_free_params)
                # stat_table['Eff. N. Free Parameters'].append(eff_n_params)
                stat_table['dof'].append(dof)
                stat_table['Model'].append(model_name)

            else:

                this_dic = dic(analysis)
                # this_waic = waic(analysis)

                # We will now compute the AIC/BIC/ etc. at the max of the posterior likelihood

                loglike = analysis.log_like_values.max()

                this_aic = aic(loglike, n_free_params, n_data_points)
                this_bic = bic(loglike, n_free_params, n_data_points)

                # Create the dataframe

                stat_table['AIC'].append(this_aic)
                stat_table['BIC'].append(this_bic)
                stat_table['DIC'].append(this_dic)
                stat_table['log10 (Z)'].append(analysis.log10_evidence)
                # stat_table['WAIC'].append(this_waic)
                stat_table['-2 ln(like)'].append(-2. * loglike)
                stat_table['N. Free Parameters'].append(n_free_params)
                #stat_table['Eff. N. Free Parameters'].append(eff_n_params)
                stat_table['dof'].append(dof)
                stat_table['Model'].append(model_name)

        stat_df = pd.DataFrame(stat_table)

        return stat_df

<|endoftext|>


<commit_before>from __future__ import unicode_literals

import re

from django.utils.translation import ugettext_lazy as _
from django.utils import formats

from rapidsms.contrib.handlers.handlers.keyword import KeywordHandler

class AppointmentHandler(KeywordHandler):
    "Base keyword handler for the APPT prefix."

    prefix = 'APPT|REPORT|REP'
    form = None
    success_text = ''

    @classmethod
    def _keyword(cls):
        if hasattr(cls, "keyword"):
            pattern = r"^\s*(?:%s)\s*(?:%s)(?:[\s,;:]+(.+))?$"\
                % (cls.prefix, cls.keyword)
        else:
            pattern = r"^\s*(?:%s)\s*?$" % cls.prefix
        return re.compile(pattern, re.IGNORECASE)

    def handle(self, text):
        "Parse text, validate data, and respond."
        parsed = self.parse_message(text)
        form = self.form(data=parsed, connection=self.msg.connection)
        if form.is_valid():
            params = form.save()
            print params
            if 'date' in params:
                params['date'] = formats.date_format(params['date'],
                                                     'SHORT_DATE_FORMAT')
            # TODO separate birth handler!
            if 'patient' in params and params.get('patient') is not None:
                self.respond('Congratulations! Please use id %s for the'
                             ' new child' % params['patient']['id'])
            self.respond(self.success_text % params)
        else:
            error = form.error()
            if error is None:
                self.unknown()
            else:
                self.respond(error)
        return True

    def help(self):
        "Return help mesage."
        if self.help_text:
            keyword = self.keyword.split('|')[0].upper()
            help_text = self.help_text % {'prefix': self.prefix,
                                          'keyword': keyword}
            self.respond(help_text)

    def unknown(self):
        "Common fallback for unknown errors."
        keyword = self.keyword.split('|')[0].upper()
        params = {'prefix': self.prefix, 'keyword': keyword}
        self.respond(_('Sorry, we cannot understand that message. '
                       'For additional help send: %(prefix)s %(keyword)s')
                     % params)
<commit_msg>update base keywords<commit_after>from __future__ import unicode_literals

import re

from django.utils.translation import ugettext_lazy as _
from django.utils import formats

from rapidsms.contrib.handlers.handlers.keyword import KeywordHandler

class AppointmentHandler(KeywordHandler):
    "Base keyword handler for the APPT prefix."

    prefix = 'APPT|REPORT|REP|REGISTER|REG'
    form = None
    success_text = ''

    @classmethod
    def _keyword(cls):
        if hasattr(cls, "keyword"):
            pattern = r"^\s*(?:%s)\s*(?:%s)(?:[\s,;:]+(.+))?$"\
                % (cls.prefix, cls.keyword)
        else:
            pattern = r"^\s*(?:%s)\s*?$" % cls.prefix
        return re.compile(pattern, re.IGNORECASE)

    def handle(self, text):
        "Parse text, validate data, and respond."
        parsed = self.parse_message(text)
        form = self.form(data=parsed, connection=self.msg.connection)
        if form.is_valid():
            params = form.save()
            print params
            if 'date' in params:
                params['date'] = formats.date_format(params['date'],
                                                     'SHORT_DATE_FORMAT')
            # TODO separate birth handler!
            if 'patient' in params and params.get('patient') is not None:
                self.respond('Congratulations! Please use id %s for the'
                             ' new child' % params['patient']['id'])
            self.respond(self.success_text % params)
        else:
            error = form.error()
            if error is None:
                self.unknown()
            else:
                self.respond(error)
        return True

    def help(self):
        "Return help mesage."
        if self.help_text:
            keyword = self.keyword.split('|')[0].upper()
            help_text = self.help_text % {'prefix': self.prefix,
                                          'keyword': keyword}
            self.respond(help_text)

    def unknown(self):
        "Common fallback for unknown errors."
        keyword = self.keyword.split('|')[0].upper()
        params = {'prefix': self.prefix, 'keyword': keyword}
        self.respond(_('Sorry, we cannot understand that message. '
                       'For additional help send: %(prefix)s %(keyword)s')
                     % params)
<|endoftext|>


<commit_before>from datetime import timedelta

from django.db import models
from django.core.mail import send_mail
from django.conf import settings
from django.utils import timezone

from slugify import slugify

from django.core.urlresolvers import reverse

from dashboard.models import SiteOptions
from comics.email import COMIC_EMAIL, COMIC_EMAIL_HTML
import random_name

from .email import VERIFICATION_EMAIL, VERIFICATION_EMAIL_HTML

class SpamSpamSpamSpam(Exception):
    """
    Thrown when you try to email the EmailSubscriber more than once a day.
    """
    pass

class EmailSubscriber(models.Model):
    """
    One e-mail, subscribed to your comic.
    """
    email = models.CharField(max_length=100, null=False, blank=False, unique=True)
    verification_code = models.CharField(max_length=200, null=False, blank=False)
    verified = models.BooleanField(default=False)
    created = models.DateTimeField()
    last_email_sent = models.DateTimeField()

    def save(self):
        if not self.id:
            self.created = timezone.now()
            self.last_email_sent = timezone.now()
        if not self.verification_code:
            self.verification_code = slugify(random_name.special_thing())
        super().save()

    def send_verification_email(self):
        site_options = SiteOptions.get()

        site_url = settings.SITE_URL
        verification_relative_url = reverse('publish.views.verify',
                                            kwargs={'email':self.email,
                                                    'verification_code':self.verification_code})
        verification_absolute_url = "{}{}".format(site_url, verification_relative_url)

        subject = "Welcome to {}!".format(site_options.title)
        message = VERIFICATION_EMAIL.format(verification_absolute_url)
        messsage_html = VERIFICATION_EMAIL_HTML.format(verification_absolute_url)
        self.send_mail(subject, message, message_html)

    def send_promo_email(self, comic):
        site_options = SiteOptions.get()

        comic_absolute_url = comic.absolute_url()

        subject = "{} - {}".format(site_options.title, comic.title)
        message = COMIC_EMAIL.format(promo_text=comic.promo_text,
                                     alt_text=comic.alt_text,
                                     comic_absolute_url=comic_absolute_url)
        message_html = COMIC_EMAIL_HTML.format(promo_text=comic.promo_text,
                                     alt_text=comic.alt_text,
                                     secret_text=comic.secret_text,
                                     image_url=comic.image_url,
                                     comic_absolute_url=comic_absolute_url)
        self.send_mail(subject, message, message_html)

    def send_mail(self, subject, message, html_message=None):
        one_day_ago = timezone.now() - timedelta(hours=23)
        if self.last_email_sent > one_day_ago:
            raise SpamSpamSpamSpam("You're sending e-mail too often.")

        site_options = SiteOptions.get()

        site_url = settings.SITE_URL
        unsubscribe_relative_url = reverse('publish.views.unsubscribe_email',
                                           kwargs={'email':self.email})
        unsubscribe_absolute_url = "{}{}".format(site_url, unsubscribe_relative_url)

        message = message + "\nTo unsubscribe forever from these messages, go to {}".format(unsubscribe_absolute_url)
        if html_message:
            html_message = html_message + """

To unsubscribe forever from these messages, go to Unsubscribe
            """.format(unsubscribe_absolute_url)

        self.last_email_sent = timezone.now()
        self.save()
        send_mail(subject=subject,
                  message=message,
                  from_email=settings.SERVER_EMAIL,
                  recipient_list=[self.email],
                  html_message=html_message,
                  fail_silently=False)

    def __str__(self):
        return "<Subscriber: {}>".format(self.email)

    @classmethod
    def tidy(cls):
        two_days_old = timezone.now() - timedelta(days=2)
        unverified = EmailSubscriber.objects.filter(verified=False,
                                                    created__lt=two_days_old)
        if not unverified:
            return []
        for subscriber in unverified:
            subscriber.delete()
        return unverified

    @classmethod
    def subscribers(cls):
        return EmailSubscriber.objects.filter(verified=True)
<commit_msg>fix to subscribing<commit_after>from datetime import timedelta

from django.db import models
from django.core.mail import send_mail
from django.conf import settings
from django.utils import timezone

from slugify import slugify

from django.core.urlresolvers import reverse

from dashboard.models import SiteOptions
from comics.email import COMIC_EMAIL, COMIC_EMAIL_HTML
import random_name

from .email import VERIFICATION_EMAIL, VERIFICATION_EMAIL_HTML

class SpamSpamSpamSpam(Exception):
    """
    Thrown when you try to email the EmailSubscriber more than once a day.
    """
    pass

class EmailSubscriber(models.Model):
    """
    One e-mail, subscribed to your comic.
    """
    email = models.CharField(max_length=100, null=False, blank=False, unique=True)
    verification_code = models.CharField(max_length=200, null=False, blank=False)
    verified = models.BooleanField(default=False)
    created = models.DateTimeField()
    last_email_sent = models.DateTimeField()

    def save(self):
        if not self.id:
            self.created = timezone.now()
            self.last_email_sent = timezone.now()
        if not self.verification_code:
            self.verification_code = slugify(random_name.special_thing())
        super().save()

    def send_verification_email(self):
        site_options = SiteOptions.get()

        site_url = settings.SITE_URL
        verification_relative_url = reverse('publish.views.verify',
                                            kwargs={'email':self.email,
                                                    'verification_code':self.verification_code})
        verification_absolute_url = "{}{}".format(site_url, verification_relative_url)

        subject = "Welcome to {}!".format(site_options.title)
        message = VERIFICATION_EMAIL.format(verification_absolute_url)
        message_html = VERIFICATION_EMAIL_HTML.format(verification_absolute_url)
        self.send_mail(subject, message, message_html)

    def send_promo_email(self, comic):
        site_options = SiteOptions.get()

        comic_absolute_url = comic.absolute_url()

        subject = "{} - {}".format(site_options.title, comic.title)
        message = COMIC_EMAIL.format(promo_text=comic.promo_text,
                                     alt_text=comic.alt_text,
                                     comic_absolute_url=comic_absolute_url)
        message_html = COMIC_EMAIL_HTML.format(promo_text=comic.promo_text,
                                     alt_text=comic.alt_text,
                                     secret_text=comic.secret_text,
                                     image_url=comic.image_url,
                                     comic_absolute_url=comic_absolute_url)
        self.send_mail(subject, message, message_html)

    def send_mail(self, subject, message, html_message=None):
        one_day_ago = timezone.now() - timedelta(hours=23)
        if self.last_email_sent > one_day_ago:
            raise SpamSpamSpamSpam("You're sending e-mail too often.")

        site_options = SiteOptions.get()

        site_url = settings.SITE_URL
        unsubscribe_relative_url = reverse('publish.views.unsubscribe_email',
                                           kwargs={'email':self.email})
        unsubscribe_absolute_url = "{}{}".format(site_url, unsubscribe_relative_url)

        message = message + "\nTo unsubscribe forever from these messages, go to {}".format(unsubscribe_absolute_url)
        if html_message:
            html_message = html_message + """

To unsubscribe forever from these messages, go to Unsubscribe
            """.format(unsubscribe_absolute_url)

        self.last_email_sent = timezone.now()
        self.save()
        send_mail(subject=subject,
                  message=message,
                  from_email=settings.SERVER_EMAIL,
                  recipient_list=[self.email],
                  html_message=html_message,
                  fail_silently=False)

    def __str__(self):
        return "<Subscriber: {}>".format(self.email)

    @classmethod
    def tidy(cls):
        two_days_old = timezone.now() - timedelta(days=2)
        unverified = EmailSubscriber.objects.filter(verified=False,
                                                    created__lt=two_days_old)
        if not unverified:
            return []
        for subscriber in unverified:
            subscriber.delete()
        return unverified

    @classmethod
    def subscribers(cls):
        return EmailSubscriber.objects.filter(verified=True)
<|endoftext|>