<commit_before>#!/usr/bin/env python
import rospy, os
import numpy as np
# Optitrack
import socket
import optirx as rx
from optitrack.utils import get_ip_address, read_parameter
# Transformations
import tf
import tf.transformations as tr
from hrl_geom.pose_converter import PoseConv
# Messages
from geometry_msgs.msg import Pose, Point, Quaternion
from optitrack.msg import RigidBody, RigidBodyArray

class RigidBodiesPublisher(object):
  """
  Naming convention for the transformations:
    - wTo: Transformation between C{parent_frame} and C{optitrack_frame}
    - oTr: Transformation between C{optitrack_frame} and the rigid body
    - wTr: Transformation between C{parent_frame} and the rigid body
  """
  def __init__(self):
    # Read parameters to configure the node
    tf_publish_rate = read_parameter('~tf_publish_rate', 50.)
    tf_period = 1./tf_publish_rate if tf_publish_rate > 0 else float('inf')
    parent_frame = read_parameter('~parent_frame', 'world')
    optitrack_frame = read_parameter('~optitrack_frame', 'optitrack')
    rigid_bodies = read_parameter('~rigid_bodies', dict())
    names = []
    ids = []
    for name,value in rigid_bodies.items():
      names.append(name)
      ids.append(value)
    # Setup Publishers
    pose_pub = rospy.Publisher('/optitrack/rigid_bodies', RigidBodyArray, queue_size=3)
    # Setup TF listener and broadcaster
    tf_listener = tf.TransformListener()
    tf_broadcaster = tf.TransformBroadcaster()
    # Connect to the optitrack system
    iface = read_parameter('~iface', 'eth1')
    version = (2, 9, 0, 0)  # the latest SDK version
    optitrack = rx.mkdatasock(ip_address=get_ip_address(iface))
    rospy.loginfo('Successfully connected to optitrack')
    # Get transformation from the world to optitrack frame
    (parent, child) = (parent_frame, optitrack_frame)
    try:
      now = rospy.Time.now() + rospy.Duration(1.0)
      tf_listener.waitForTransform(parent, child, now, rospy.Duration(5.0))
      (pos,rot) = tf_listener.lookupTransform(parent, child, now)
      wTo = PoseConv.to_homo_mat(pos, rot)
    except (tf.Exception, tf.LookupException, tf.ConnectivityException):
      rospy.logwarn('Failed to get transformation from %r to %r frame' % (parent, child))
      parent_frame = optitrack_frame
      wTo = np.eye(4)
    # Track up to 24 rigid bodies
    prevtime = np.ones(24)*rospy.get_time()
    while not rospy.is_shutdown():
      try:
        data = optitrack.recv(rx.MAX_PACKETSIZE)
      except socket.error:
        if rospy.is_shutdown():  # exit gracefully
          return
        else:
          rospy.logwarn('Failed to receive packet from optitrack')
      packet = rx.unpack(data, version=version)
      if type(packet) is rx.SenderData:
        version = packet.natnet_version
        rospy.loginfo('NatNet version received: ' + str(version))
      if type(packet) in [rx.SenderData, rx.ModelDefs, rx.FrameOfData]:
        # Optitrack gives the position of the centroid.
        array_msg = RigidBodyArray()
        for i, rigid_body in enumerate(packet.rigid_bodies):
          body_id = rigid_body.id
          pos_opt = np.array(rigid_body.position)
          rot_opt = np.array(rigid_body.orientation)
          oTr = PoseConv.to_homo_mat(pos_opt, rot_opt)
          # Transformation between world frame and the rigid body
          wTr = np.dot(wTo, oTr)
          array_msg.header.stamp = rospy.Time.now()
          array_msg.header.frame_id = parent_frame
          body_msg = RigidBody()
          pose = Pose()
          pose.position = Point(*wTr[:3,3])
          pose.orientation = Quaternion(*tr.quaternion_from_matrix(wTr))
          body_msg.id = body_id
          body_msg.tracking_valid = rigid_body.tracking_valid
          body_msg.mrk_mean_error = rigid_body.mrk_mean_error
          body_msg.pose = pose
          for marker in rigid_body.markers:
            # TODO: Should transform the markers as well
            body_msg.markers.append(Point(*marker))
          array_msg.bodies.append(body_msg)
          # Control the publish rate for the TF broadcaster
          if rigid_body.tracking_valid and (rospy.get_time()-prevtime[body_id] >= tf_period):
            body_name = 'rigid_body_%d' % (body_id)
            if body_id in ids:
              idx = ids.index(body_id)
              body_name = names[idx]
            tf_broadcaster.sendTransform(pos_opt, rot_opt, rospy.Time.now(), body_name, optitrack_frame)
            prevtime[body_id] = rospy.get_time()
        pose_pub.publish(array_msg)

if __name__ == '__main__':
  node_name = os.path.splitext(os.path.basename(__file__))[0]
  rospy.init_node(node_name)
  rospy.loginfo('Starting [%s] node' % node_name)
  opti_node = RigidBodiesPublisher()
  rospy.loginfo('Shuting down [%s] node' % node_name)
<commit_msg>Fixed a bug where ids list was populated with dictionaries instead of id values.<commit_after>#!/usr/bin/env python
import rospy, os
import numpy as np
# Optitrack
import socket
import optirx as rx
from optitrack.utils import get_ip_address, read_parameter
# Transformations
import tf
import tf.transformations as tr
from hrl_geom.pose_converter import PoseConv
# Messages
from geometry_msgs.msg import Pose, Point, Quaternion
from optitrack.msg import RigidBody, RigidBodyArray

class RigidBodiesPublisher(object):
  """
  Naming convention for the transformations:
    - wTo: Transformation between C{parent_frame} and C{optitrack_frame}
    - oTr: Transformation between C{optitrack_frame} and the rigid body
    - wTr: Transformation between C{parent_frame} and the rigid body
  """
  def __init__(self):
    # Read parameters to configure the node
    tf_publish_rate = read_parameter('~tf_publish_rate', 50.)
    tf_period = 1./tf_publish_rate if tf_publish_rate > 0 else float('inf')
    parent_frame = read_parameter('~parent_frame', 'world')
    optitrack_frame = read_parameter('~optitrack_frame', 'optitrack')
    rigid_bodies = read_parameter('~rigid_bodies', dict())
    names = []
    ids = []
    for name,value in rigid_bodies.items():
      names.append(name)
      ids.append(value["id"])
    # Setup Publishers
    pose_pub = rospy.Publisher('/optitrack/rigid_bodies', RigidBodyArray, queue_size=3)
    # Setup TF listener and broadcaster
    tf_listener = tf.TransformListener()
    tf_broadcaster = tf.TransformBroadcaster()
    # Connect to the optitrack system
    iface = read_parameter('~iface', 'eth1')
    version = (2, 9, 0, 0)  # the latest SDK version
    optitrack = rx.mkdatasock(ip_address=get_ip_address(iface))
    rospy.loginfo('Successfully connected to optitrack')
    # Get transformation from the world to optitrack frame
    (parent, child) = (parent_frame, optitrack_frame)
    try:
      now = rospy.Time.now() + rospy.Duration(1.0)
      tf_listener.waitForTransform(parent, child, now, rospy.Duration(5.0))
      (pos,rot) = tf_listener.lookupTransform(parent, child, now)
      wTo = PoseConv.to_homo_mat(pos, rot)
    except (tf.Exception, tf.LookupException, tf.ConnectivityException):
      rospy.logwarn('Failed to get transformation from %r to %r frame' % (parent, child))
      parent_frame = optitrack_frame
      wTo = np.eye(4)
    # Track up to 24 rigid bodies
    prevtime = np.ones(24)*rospy.get_time()
    while not rospy.is_shutdown():
      try:
        data = optitrack.recv(rx.MAX_PACKETSIZE)
      except socket.error:
        if rospy.is_shutdown():  # exit gracefully
          return
        else:
          rospy.logwarn('Failed to receive packet from optitrack')
      packet = rx.unpack(data, version=version)
      if type(packet) is rx.SenderData:
        version = packet.natnet_version
        rospy.loginfo('NatNet version received: ' + str(version))
      if type(packet) in [rx.SenderData, rx.ModelDefs, rx.FrameOfData]:
        # Optitrack gives the position of the centroid.
        array_msg = RigidBodyArray()
        for i, rigid_body in enumerate(packet.rigid_bodies):
          body_id = rigid_body.id
          pos_opt = np.array(rigid_body.position)
          rot_opt = np.array(rigid_body.orientation)
          oTr = PoseConv.to_homo_mat(pos_opt, rot_opt)
          # Transformation between world frame and the rigid body
          wTr = np.dot(wTo, oTr)
          array_msg.header.stamp = rospy.Time.now()
          array_msg.header.frame_id = parent_frame
          body_msg = RigidBody()
          pose = Pose()
          pose.position = Point(*wTr[:3,3])
          pose.orientation = Quaternion(*tr.quaternion_from_matrix(wTr))
          body_msg.id = body_id
          body_msg.tracking_valid = rigid_body.tracking_valid
          body_msg.mrk_mean_error = rigid_body.mrk_mean_error
          body_msg.pose = pose
          for marker in rigid_body.markers:
            # TODO: Should transform the markers as well
            body_msg.markers.append(Point(*marker))
          array_msg.bodies.append(body_msg)
          # Control the publish rate for the TF broadcaster
          if rigid_body.tracking_valid and (rospy.get_time()-prevtime[body_id] >= tf_period):
            body_name = 'rigid_body_%d' % (body_id)
            if body_id in ids:
              idx = ids.index(body_id)
              body_name = names[idx]
            tf_broadcaster.sendTransform(pos_opt, rot_opt, rospy.Time.now(), body_name, optitrack_frame)
            prevtime[body_id] = rospy.get_time()
        pose_pub.publish(array_msg)

if __name__ == '__main__':
  node_name = os.path.splitext(os.path.basename(__file__))[0]
  rospy.init_node(node_name)
  rospy.loginfo('Starting [%s] node' % node_name)
  opti_node = RigidBodiesPublisher()
  rospy.loginfo('Shuting down [%s] node' % node_name)
<|endoftext|>


<commit_before>"""
Webhooks for FOIA app
"""

# Django
from django.conf import settings
from django.http.response import (
    HttpResponse,
    HttpResponseBadRequest,
    HttpResponseForbidden,
)
from django.views.decorators.csrf import csrf_exempt

# Standard Library
import hashlib
import hmac
import json
import logging
import time

# Third Party
import dateutil.parser

# MuckRock
from muckrock.communication.models import MailCommunication
from muckrock.core.utils import new_action

logger = logging.getLogger(__name__)

@csrf_exempt
def lob_webhook(request):
    """Handle Lob Webhook"""

    logger.info("Lob webhook")

    if not _validate_lob(
        request.META["HTTP_LOB_SIGNATURE"],
        request.META["HTTP_LOB_SIGNATURE_TIMESTAMP"],
        request.body,
    ):
        logger.warning("Lob webhook failed verification: %s", request.META)
        return HttpResponseForbidden()

    try:
        data = json.loads(request.body)
    except ValueError:
        logger.error("Lob webhook JSON decode error")
        return HttpResponseBadRequest("JSON decode error")

    try:
        mail_id = data["body"]["metadata"]["mail_id"]
    except KeyError:
        logger.error("Lob webhook JSON missing data")
        return HttpResponseBadRequest("Missing JSON data")

    mail = MailCommunication.objects.filter(pk=mail_id).first()

    if mail is None:
        logger.error("Missing mail communication for mail_id: %s", mail_id)
    elif mail is not None:
        event = mail.events.create(
            datetime=dateutil.parser.parse(data["date_created"]),
            event=data["event_type"]["id"],
        )
        if (
            event.event == "check.processed_for_delivery"
            and settings.CHECK_NOTIFICATIONS
        ):
            foia = mail.communication.foia
            action = new_action(
                foia.agency, "check processed for delivery", target=foia, public=False
            )
            foia.notify(action)

    return HttpResponse("OK")

def _validate_lob(signature, timestamp, body):
    """Verify the message is from Lob"""
    digest = hmac.new(
        key=settings.LOB_WEBHOOK_KEY.encode("utf8"),
        # timestamp is a string, body is already bytes
        msg=timestamp.encode("utf8") + b"." + body,
        digestmod=hashlib.sha256,
    ).hexdigest()
    match = hmac.compare_digest(signature, digest)
    logger.info(
        "Lob webhook validate debug: "
        "signature: %s timestamp: %s digest: %s match: %s time: %s",
        signature,
        timestamp,
        digest,
        match,
        time.time(),
    )
    return match and int(timestamp) / 1000 + 300 > time.time()
<commit_msg>lob webhook update validate timestamp<commit_after>"""
Webhooks for FOIA app
"""

# Django
from django.conf import settings
from django.http.response import (
    HttpResponse,
    HttpResponseBadRequest,
    HttpResponseForbidden,
)
from django.views.decorators.csrf import csrf_exempt

# Standard Library
import hashlib
import hmac
import json
import logging
import time

# Third Party
import dateutil.parser

# MuckRock
from muckrock.communication.models import MailCommunication
from muckrock.core.utils import new_action

logger = logging.getLogger(__name__)

@csrf_exempt
def lob_webhook(request):
    """Handle Lob Webhook"""

    logger.info("Lob webhook")

    if not _validate_lob(
        request.META["HTTP_LOB_SIGNATURE"],
        request.META["HTTP_LOB_SIGNATURE_TIMESTAMP"],
        request.body,
    ):
        logger.warning("Lob webhook failed verification: %s", request.META)
        return HttpResponseForbidden()

    try:
        data = json.loads(request.body)
    except ValueError:
        logger.error("Lob webhook JSON decode error")
        return HttpResponseBadRequest("JSON decode error")

    try:
        mail_id = data["body"]["metadata"]["mail_id"]
    except KeyError:
        logger.error("Lob webhook JSON missing data")
        return HttpResponseBadRequest("Missing JSON data")

    mail = MailCommunication.objects.filter(pk=mail_id).first()

    if mail is None:
        logger.error("Missing mail communication for mail_id: %s", mail_id)
    elif mail is not None:
        event = mail.events.create(
            datetime=dateutil.parser.parse(data["date_created"]),
            event=data["event_type"]["id"],
        )
        if (
            event.event == "check.processed_for_delivery"
            and settings.CHECK_NOTIFICATIONS
        ):
            foia = mail.communication.foia
            action = new_action(
                foia.agency, "check processed for delivery", target=foia, public=False
            )
            foia.notify(action)

    return HttpResponse("OK")

def _validate_lob(signature, timestamp, body):
    """Verify the message is from Lob"""
    digest = hmac.new(
        key=settings.LOB_WEBHOOK_KEY.encode("utf8"),
        # timestamp is a string, body is already bytes
        msg=timestamp.encode("utf8") + b"." + body,
        digestmod=hashlib.sha256,
    ).hexdigest()
    match = hmac.compare_digest(signature, digest)
    logger.info(
        "Lob webhook validate debug: "
        "signature: %s timestamp: %s digest: %s match: %s time: %s",
        signature,
        timestamp,
        digest,
        match,
        time.time(),
    )
    return match and int(timestamp) + 300 > time.time()
<|endoftext|>


<commit_before>from unittest import TestCase
import os
from jose import jwt, jws
from jose.constants import ALGORITHMS
from funsize.utils import properties_to_dict, sign_task
from hypothesis import given
import hypothesis.strategies as st
from . import PVT_KEY, PUB_KEY, OTHER_PUB_KEY

class TestPropertiesToDict(TestCase):

    @given(st.lists(elements=st.text(), min_size=3),
           st.lists(elements=st.text(), min_size=3))
    def test_generic(self, l1, l2):
        props = [l1, l2]
        self.assertDictEqual({l1[0]: l1[1], l2[0]: l2[1]},
                             properties_to_dict(props))

    @given(st.lists(elements=st.text(), min_size=2),
           st.lists(elements=st.text(), min_size=2))
    def test_short(self, l1, l2):
        props = [l1, l2]
        self.assertDictEqual({l1[0]: l1[1], l2[0]: l2[1]},
                             properties_to_dict(props))

    @given(st.lists(elements=st.text(), max_size=1),
           st.lists(elements=st.text(), min_size=2))
    def test_truncated(self, l1, l2):
        props = [l1, l2]
        self.assertDictEqual({l2[0]: l2[1]},
                             properties_to_dict(props))

class TestTaskSigning(TestCase):

    def test_task_id(self):
        token = sign_task("xyz", pvt_key=PVT_KEY)
        claims = jwt.decode(token, PUB_KEY, algorithms=[ALGORITHMS.RS512])
        assert claims["taskId"] == "xyz"

    def test_exp(self):
        token = sign_task("xyz", pvt_key=PVT_KEY)
        claims = jwt.decode(token, PUB_KEY, algorithms=[ALGORITHMS.RS512])
        assert "exp" in claims

    def test_exp_int(self):
        token = sign_task("xyz", pvt_key=PVT_KEY)
        claims = jwt.decode(token, PUB_KEY, algorithms=[ALGORITHMS.RS512])
        assert isinstance(claims["exp"], int)

    def test_verify(self):
        token = sign_task("xyz", pvt_key=PVT_KEY)
        claims = jws.verify(token, PUB_KEY, algorithms=[ALGORITHMS.RS512])
        assert claims["taskId"] == "xyz"

    def test_verify_bad_signature(self):
        token = sign_task("xyz", pvt_key=PVT_KEY)
        self.assertRaises(jws.JWSError, jws.verify, token, OTHER_PUB_KEY,
                          [ALGORITHMS.RS512])
<commit_msg>Remove unused import<commit_after>from unittest import TestCase
from jose import jwt, jws
from jose.constants import ALGORITHMS
from funsize.utils import properties_to_dict, sign_task
from hypothesis import given
import hypothesis.strategies as st
from . import PVT_KEY, PUB_KEY, OTHER_PUB_KEY

class TestPropertiesToDict(TestCase):

    @given(st.lists(elements=st.text(), min_size=3),
           st.lists(elements=st.text(), min_size=3))
    def test_generic(self, l1, l2):
        props = [l1, l2]
        self.assertDictEqual({l1[0]: l1[1], l2[0]: l2[1]},
                             properties_to_dict(props))

    @given(st.lists(elements=st.text(), min_size=2),
           st.lists(elements=st.text(), min_size=2))
    def test_short(self, l1, l2):
        props = [l1, l2]
        self.assertDictEqual({l1[0]: l1[1], l2[0]: l2[1]},
                             properties_to_dict(props))

    @given(st.lists(elements=st.text(), max_size=1),
           st.lists(elements=st.text(), min_size=2))
    def test_truncated(self, l1, l2):
        props = [l1, l2]
        self.assertDictEqual({l2[0]: l2[1]},
                             properties_to_dict(props))

class TestTaskSigning(TestCase):

    def test_task_id(self):
        token = sign_task("xyz", pvt_key=PVT_KEY)
        claims = jwt.decode(token, PUB_KEY, algorithms=[ALGORITHMS.RS512])
        assert claims["taskId"] == "xyz"

    def test_exp(self):
        token = sign_task("xyz", pvt_key=PVT_KEY)
        claims = jwt.decode(token, PUB_KEY, algorithms=[ALGORITHMS.RS512])
        assert "exp" in claims

    def test_exp_int(self):
        token = sign_task("xyz", pvt_key=PVT_KEY)
        claims = jwt.decode(token, PUB_KEY, algorithms=[ALGORITHMS.RS512])
        assert isinstance(claims["exp"], int)

    def test_verify(self):
        token = sign_task("xyz", pvt_key=PVT_KEY)
        claims = jws.verify(token, PUB_KEY, algorithms=[ALGORITHMS.RS512])
        assert claims["taskId"] == "xyz"

    def test_verify_bad_signature(self):
        token = sign_task("xyz", pvt_key=PVT_KEY)
        self.assertRaises(jws.JWSError, jws.verify, token, OTHER_PUB_KEY,
                          [ALGORITHMS.RS512])
<|endoftext|>


<commit_before>#!/usr/bin/python3
import threading
import logging
import sqlite3
import time
import os
import sys
from datetime import timedelta
from threading import Lock

g_db_count = 0
g_params = {}
g_lock = Lock();

# This is a way to get the column names after grabbing everything
# I guess it's also good practice
_SCHEMA = {
  'kv': [
     ('id', 'INTEGER PRIMARY KEY'),
     ('key', 'TEXT UNIQUE'),
     ('value', 'TEXT'),
     ('namespace', 'TEXT'),
     ('created_at', 'TIMESTAMP DEFAULT CURRENT_TIMESTAMP'),
   ],
  'streams': [
     ('id', 'INTEGER PRIMARY KEY'),
     ('name', 'TEXT UNIQUE'),
     ('start_unix', 'TIMESTAMP DEFAULT CURRENT_TIMESTAMP'),
     ('end_unix', 'TIMESTAMP DEFAULT CURRENT_TIMESTAMP'),
     ('start_minute', 'REAL DEFAULT 0'),
     ('end_minute', 'REAL DEFAULT 0'),
     ('week_number', 'INTEGER DEFAULT 0'),
     ('size', 'INTEGER DEFAULT 0'),
     # This is the meta-information that links
     # this file to the next one in the stream
     # (Presuming that there's only one of course)
     ('current_block', 'INTEGER DEFAULT 0'),
     ('next_file_name', 'TEXT'),
     ('next_file_block', 'INTEGER DEFAULT 0')
   ]
}

# Ok so if column order or type changes, this isn't found ... nor
# are we doing formal migrations where you can roll back or whatever
# because way too fancy ...
def upgrade():
  my_set = __builtins__['set']
  db = connect()

  for table, schema in list(_SCHEMA.items()):
    existing_schema = db['c'].execute('pragma table_info(%s)' % table).fetchall()
    existing_column_names = [str(row[1]) for row in existing_schema]

    our_column_names = [row[0] for row in schema]

    # print table, existing_column_names, our_column_names

    to_add = my_set(our_column_names).difference(my_set(existing_column_names))

    # These are the things we should add ... this can be an empty set, that's fine.
    for key in to_add:
      #
      # sqlite doesn't support adding things into positional places (add column after X)
      # they just get tacked on at the end ... which is fine - you'd have to rebuild
      # everything to achieve positional columns - that's not worth it - we just always
      # tack on at the end as a policy in our schema and we'll be fine.
      #
      # However, given all of that, we still need the schema
      #
      our_schema = schema[our_column_names.index(key)][1]
      # print 'alter table %s add column %s %s' % (table, key, our_schema)
      db['c'].execute('alter table %s add column %s %s' % (table, key, our_schema))
      db['conn'].commit()

    to_remove = my_set(existing_column_names).difference(my_set(our_column_names))

    if len(to_remove) > 0:
      our_schema = ','.join(["%s %s" % (key, klass) for key, klass in schema])
      our_columns = ','.join(our_column_names)

      drop_column_sql = """
      CREATE TEMPORARY TABLE my_backup(%s);
      INSERT INTO my_backup SELECT %s FROM %s;
      DROP TABLE %s;
      CREATE TABLE %s(%s);
      INSERT INTO %s SELECT %s FROM my_backup;
      DROP TABLE my_backup;
      """ % (our_schema, our_columns, table, table,    table, our_schema, table, our_columns)

      for sql_line in drop_column_sql.strip().split('\n'):
        db['c'].execute(sql_line)

      db['conn'].commit()

def map(row_list, table, db=None):
  # Using the schema of a table, map the row_list to a list of dicts.
  mapped = []
  my_schema = schema(table, db)

  if not row_list:
    return row_list

  if type(row_list[0]) is str:
    row_list = [row_list]

  for row in row_list:
    mapped_row = {}
    for ix in range(len(my_schema)):
      mapped_row[my_schema[ix]] = row[ix]

    mapped.append(mapped_row)

  return mapped

def all(table, field_list='*', sort_by='id'):
  # Returns all entries from the sqlite3 database for a given table.

  column_count = 1
  if type(field_list) is not str:
    column_count = len(field_list)
    field_list = ','.join(field_list)

  query = run('select %s from %s order by %s asc' % (field_list, table, sort_by))
  if column_count is 1 and field_list != '*':
    return [record[0] for record in query.fetchall()]

  else:
    return [record for record in query.fetchall()]

def schema(table, db=None):
  existing_schema = run('pragma table_info(%s)' % table, db=db).fetchall()
  if existing_schema:
    return [str(row[1]) for row in existing_schema]

  return None

def disconnect(what):
  what['conn'].close()
  pass

def connect(db_file=None):
  # A "singleton pattern" or some other fancy $10-world style of maintaining
  # the database connection throughout the execution of the script.
  # Returns the database instance.
  global g_db_count

  if not db_file:
    db_file = 'config.db'

  #
  # We need to have one instance per thread, as this is what
  # sqlite's driver dictates ... so we do this based on thread id.
  #
  # We don't have to worry about the different memory sharing models here.
  # Really, just think about it ... it's totally irrelevant.
  #

  instance = {}

  if not os.path.exists(db_file):
    sys.stderr.write("Info: Creating db file %s\n" % db_file)

  conn = sqlite3.connect(db_file)
  instance.update({
    'conn': conn,
    'c': conn.cursor()
  })

  if db_file == 'config.db' and g_db_count == 0:

    for table, schema in list(_SCHEMA.items()):
      dfn = ','.join(["%s %s" % (key, klass) for key, klass in schema])
      instance['c'].execute("CREATE TABLE IF NOT EXISTS %s(%s)" % (table, dfn))

    instance['conn'].commit()

  g_db_count += 1

  return instance

def incr(key, value=1):
  # Increments some key in the database by some value.  It is used
  # to maintain statistical counters.

  try:
    run('update kv set value = value + ? where key = ?', args=(value, key, ))

  except Exception as exc:
    try:
      run('insert into kv(value, key) values(?, ?)', args=(value, key, ))
    except sqlite3.OperationalError as exc:
      logging.warn("Unable to increment key: %s", exc)
      pass

def set(key, value):
  # Sets (or replaces) a given key to a specific value.
  # Returns the value that was sent.
  global g_params

  try:
    if value is None:
      res = run('delete from kv where key = ?', (key, ))

    else:
      # From http://stackoverflow.com/questions/418898/sqlite-upsert-not-insert-or-replace
      res = run('''
        INSERT OR REPLACE INTO kv (key, value, created_at)
          VALUES (
            COALESCE((SELECT key FROM kv WHERE key = ?), ?),
            ?,
            current_timestamp
        )''', (key, key, value, ))

  except:
    logging.warn("Couldn't set %s to %s" % (key, value))

  g_params[key] = value

  return value

def clear_cache():
  global g_params
  g_params = {}

def get(key, expiry=0, use_cache=True, default=None):
  # Retrieves a value from the database, tentative on the expiry.
  # If the cache is set to true then it retrieves it from in-memory if available, otherwise
  # it goes out to the db. Other than directly hitting up the g_params parameter which is
  # used internally, there is no way to invalidate the cache.
  global g_params

  # only use the cache if the expiry is not set.
  if use_cache and key in g_params and expiry == 0:
    if default and type(default) is int:
      g_params[key] = int(g_params[key])

    return g_params[key]

  if expiry > 0:
    # If we let things expire, we first sweep for it
    res = run("select value, created_at from kv where key = '%s' and created_at >= date(current_timestamp, '-%d second')" % (key, expiry)).fetchone()
  else:
    res = run('select value, created_at from kv where key = ?', (key, )).fetchone()

  if res:
    if default and type(default) is int:
      res = list(res)
      res[0] = int(res[0])

    g_params[key] = res[0]
    return res[0]

  else:
    # It's ok if this doesn't exist. SQLite likes to throw an error here
    try:
      run("delete from kv where key = '%s' and created_at < date(current_timestamp, '-%d second')" % (key, expiry))
    except Exception as inst:
      pass

  return default

def run(query, args=None, with_last=False, db=None):
  start = time.time()
  """
  if args is None:
    print "%d: %s" % (g_db_count, query)
  else:
    $print "%d: %s (%s)" % (g_db_count, query, ', '.join([str(m) for m in args]))
  """

  g_lock.acquire()
  if db is None:
    db = connect()

  res = None

  try:
    if args is None:
      res = db['c'].execute(query)
    else:
      res = db['c'].execute(query, args)

    db['conn'].commit()
    last = db['c'].lastrowid

    if db['c'].rowcount == 0:
      raise Exception("0 rows")

  except Exception as exc:
    raise exc

  finally:
    g_lock.release()

  if with_last:
    return (res, last)

  return res

def unregister_stream(name, do_all=False):
  # Deletes a stream by name, contingent on it existing only once

  res = run('select id from streams where name = ?', (name, )).fetchall()

  if res and (len(res) == 1 or do_all):
    logging.debug("Removing our reference of %s" % name)
    res = run('delete from streams where id = %d' % res[0][0])

  else:
    logging.warn("Requested to remove reference of %s but couldn't find it." % name)

  return True

def register_stream(info):
  # Registers a stream as existing to be found later when trying to stitch and slice files together.
  # This is all that ought to be needed to know if the streams should attempt to be stitched. The input
  # info is grabbed from audio.stream_info(filename)
  name = info['name']
  start_unix = info['start_date']
  end_unix = info['start_date'] + timedelta(seconds=info['duration_sec'])
  start_minute = float(info['start_minute'])
  end_minute = float(info['end_minute'])
  week_number = info['week_number']
  size = info['size']

  res = run('select id from streams where name = ?', (name, )).fetchone()

  # If something exists then we remove it and reinsert ... this is not as effecient
  # as an upsert with coalesce, but this is done on rare occasions in heavy workloads ...
  # so we don't really need to be entirely efficient about it.
  if res:
    unregister_stream(name, do_all=True)

  last = False
  try:
    res, last = run("""insert into streams
    (name, start_unix, end_unix, start_minute, end_minute, week_number, size) values
    (   ?,          ?,        ?,            ?,          ?,           ?,    ?) """,
    (name, start_unix, end_unix, start_minute, end_minute, week_number, size), with_last=True)

  except:
    logging.warn("Unable to insert a record with a name %s" % name)

  return last

<commit_msg>fixing a fucking stupid expiry fucking bug. fuck you, sqlite, fuck you. I mean really, fuck you. Fuck you fuck fuck you.<commit_after>#!/usr/bin/python3
import threading
import logging
import sqlite3
import time
import os
import sys
from datetime import timedelta
from threading import Lock

g_db_count = 0
g_params = {}
g_lock = Lock();

# This is a way to get the column names after grabbing everything
# I guess it's also good practice
_SCHEMA = {
  'kv': [
     ('id', 'INTEGER PRIMARY KEY'),
     ('key', 'TEXT UNIQUE'),
     ('value', 'TEXT'),
     ('namespace', 'TEXT'),
     ('created_at', 'TIMESTAMP DEFAULT CURRENT_TIMESTAMP'),
   ],
  'streams': [
     ('id', 'INTEGER PRIMARY KEY'),
     ('name', 'TEXT UNIQUE'),
     ('start_unix', 'TIMESTAMP DEFAULT CURRENT_TIMESTAMP'),
     ('end_unix', 'TIMESTAMP DEFAULT CURRENT_TIMESTAMP'),
     ('start_minute', 'REAL DEFAULT 0'),
     ('end_minute', 'REAL DEFAULT 0'),
     ('week_number', 'INTEGER DEFAULT 0'),
     ('size', 'INTEGER DEFAULT 0'),
     # This is the meta-information that links
     # this file to the next one in the stream
     # (Presuming that there's only one of course)
     ('current_block', 'INTEGER DEFAULT 0'),
     ('next_file_name', 'TEXT'),
     ('next_file_block', 'INTEGER DEFAULT 0')
   ]
}

# Ok so if column order or type changes, this isn't found ... nor
# are we doing formal migrations where you can roll back or whatever
# because way too fancy ...
def upgrade():
  my_set = __builtins__['set']
  db = connect()

  for table, schema in list(_SCHEMA.items()):
    existing_schema = db['c'].execute('pragma table_info(%s)' % table).fetchall()
    existing_column_names = [str(row[1]) for row in existing_schema]

    our_column_names = [row[0] for row in schema]

    # print table, existing_column_names, our_column_names

    to_add = my_set(our_column_names).difference(my_set(existing_column_names))

    # These are the things we should add ... this can be an empty set, that's fine.
    for key in to_add:
      #
      # sqlite doesn't support adding things into positional places (add column after X)
      # they just get tacked on at the end ... which is fine - you'd have to rebuild
      # everything to achieve positional columns - that's not worth it - we just always
      # tack on at the end as a policy in our schema and we'll be fine.
      #
      # However, given all of that, we still need the schema
      #
      our_schema = schema[our_column_names.index(key)][1]
      # print 'alter table %s add column %s %s' % (table, key, our_schema)
      db['c'].execute('alter table %s add column %s %s' % (table, key, our_schema))
      db['conn'].commit()

    to_remove = my_set(existing_column_names).difference(my_set(our_column_names))

    if len(to_remove) > 0:
      our_schema = ','.join(["%s %s" % (key, klass) for key, klass in schema])
      our_columns = ','.join(our_column_names)

      drop_column_sql = """
      CREATE TEMPORARY TABLE my_backup(%s);
      INSERT INTO my_backup SELECT %s FROM %s;
      DROP TABLE %s;
      CREATE TABLE %s(%s);
      INSERT INTO %s SELECT %s FROM my_backup;
      DROP TABLE my_backup;
      """ % (our_schema, our_columns, table, table,    table, our_schema, table, our_columns)

      for sql_line in drop_column_sql.strip().split('\n'):
        db['c'].execute(sql_line)

      db['conn'].commit()

def map(row_list, table, db=None):
  # Using the schema of a table, map the row_list to a list of dicts.
  mapped = []
  my_schema = schema(table, db)

  if not row_list:
    return row_list

  if type(row_list[0]) is str:
    row_list = [row_list]

  for row in row_list:
    mapped_row = {}
    for ix in range(len(my_schema)):
      mapped_row[my_schema[ix]] = row[ix]

    mapped.append(mapped_row)

  return mapped

def all(table, field_list='*', sort_by='id'):
  # Returns all entries from the sqlite3 database for a given table.

  column_count = 1
  if type(field_list) is not str:
    column_count = len(field_list)
    field_list = ','.join(field_list)

  query = run('select %s from %s order by %s asc' % (field_list, table, sort_by))
  if column_count is 1 and field_list != '*':
    return [record[0] for record in query.fetchall()]

  else:
    return [record for record in query.fetchall()]

def schema(table, db=None):
  existing_schema = run('pragma table_info(%s)' % table, db=db).fetchall()
  if existing_schema:
    return [str(row[1]) for row in existing_schema]

  return None

def disconnect(what):
  what['conn'].close()
  pass

def connect(db_file=None):
  # A "singleton pattern" or some other fancy $10-world style of maintaining
  # the database connection throughout the execution of the script.
  # Returns the database instance.
  global g_db_count

  if not db_file:
    db_file = 'config.db'

  #
  # We need to have one instance per thread, as this is what
  # sqlite's driver dictates ... so we do this based on thread id.
  #
  # We don't have to worry about the different memory sharing models here.
  # Really, just think about it ... it's totally irrelevant.
  #

  instance = {}

  if not os.path.exists(db_file):
    sys.stderr.write("Info: Creating db file %s\n" % db_file)

  conn = sqlite3.connect(db_file)
  instance.update({
    'conn': conn,
    'c': conn.cursor()
  })

  if db_file == 'config.db' and g_db_count == 0:

    for table, schema in list(_SCHEMA.items()):
      dfn = ','.join(["%s %s" % (key, klass) for key, klass in schema])
      instance['c'].execute("CREATE TABLE IF NOT EXISTS %s(%s)" % (table, dfn))

    instance['conn'].commit()

  g_db_count += 1

  return instance

def incr(key, value=1):
  # Increments some key in the database by some value.  It is used
  # to maintain statistical counters.

  try:
    run('update kv set value = value + ? where key = ?', args=(value, key, ))

  except Exception as exc:
    try:
      run('insert into kv(value, key) values(?, ?)', args=(value, key, ))
    except sqlite3.OperationalError as exc:
      logging.warn("Unable to increment key: %s", exc)
      pass

def set(key, value):
  # Sets (or replaces) a given key to a specific value.
  # Returns the value that was sent.
  global g_params

  try:
    if value is None:
      res = run('delete from kv where key = ?', (key, ))

    else:
      # From http://stackoverflow.com/questions/418898/sqlite-upsert-not-insert-or-replace
      res = run('''
        INSERT OR REPLACE INTO kv (key, value, created_at)
          VALUES (
            COALESCE((SELECT key FROM kv WHERE key = ?), ?),
            ?,
            current_timestamp
        )''', (key, key, value, ))

  except:
    logging.warn("Couldn't set %s to %s" % (key, value))

  g_params[key] = value

  return value

def clear_cache():
  global g_params
  g_params = {}

def get(key, expiry=0, use_cache=True, default=None):
  # Retrieves a value from the database, tentative on the expiry.
  # If the cache is set to true then it retrieves it from in-memory if available, otherwise
  # it goes out to the db. Other than directly hitting up the g_params parameter which is
  # used internally, there is no way to invalidate the cache.
  global g_params

  # only use the cache if the expiry is not set.
  if use_cache and key in g_params and expiry == 0:
    if default and type(default) is int:
      g_params[key] = int(g_params[key])

    return g_params[key]

  if expiry > 0:
    # If we let things expire, we first sweep for it
    res = run("select value, created_at from kv where key = '%s' and created_at >= datetime(current_timestamp, '-%d second')" % (key, expiry)).fetchone()
  else:
    res = run('select value, created_at from kv where key = ?', (key, )).fetchone()

  if res:
    if default and type(default) is int:
      res = list(res)
      res[0] = int(res[0])

    g_params[key] = res[0]
    return res[0]

  else:
    # It's ok if this doesn't exist. SQLite likes to throw an error here
    try:
      run("delete from kv where key = '%s' and created_at < datetime(current_timestamp, '-%d second')" % (key, expiry))
    except Exception as inst:
      pass

  return default

def run(query, args=None, with_last=False, db=None):
  start = time.time()
  """
  if args is None:
    print "%d: %s" % (g_db_count, query)
  else:
    $print "%d: %s (%s)" % (g_db_count, query, ', '.join([str(m) for m in args]))
  """

  g_lock.acquire()
  if db is None:
    db = connect()

  res = None

  try:
    if args is None:
      res = db['c'].execute(query)
    else:
      res = db['c'].execute(query, args)

    db['conn'].commit()
    last = db['c'].lastrowid

    if db['c'].rowcount == 0:
      raise Exception("0 rows")

  except Exception as exc:
    raise exc

  finally:
    g_lock.release()

  if with_last:
    return (res, last)

  return res

def unregister_stream(name, do_all=False):
  # Deletes a stream by name, contingent on it existing only once

  res = run('select id from streams where name = ?', (name, )).fetchall()

  if res and (len(res) == 1 or do_all):
    logging.debug("Removing our reference of %s" % name)
    res = run('delete from streams where id = %d' % res[0][0])

  else:
    logging.warn("Requested to remove reference of %s but couldn't find it." % name)

  return True

def register_stream(info):
  # Registers a stream as existing to be found later when trying to stitch and slice files together.
  # This is all that ought to be needed to know if the streams should attempt to be stitched. The input
  # info is grabbed from audio.stream_info(filename)
  name = info['name']
  start_unix = info['start_date']
  end_unix = info['start_date'] + timedelta(seconds=info['duration_sec'])
  start_minute = float(info['start_minute'])
  end_minute = float(info['end_minute'])
  week_number = info['week_number']
  size = info['size']

  res = run('select id from streams where name = ?', (name, )).fetchone()

  # If something exists then we remove it and reinsert ... this is not as effecient
  # as an upsert with coalesce, but this is done on rare occasions in heavy workloads ...
  # so we don't really need to be entirely efficient about it.
  if res:
    unregister_stream(name, do_all=True)

  last = False
  try:
    res, last = run("""insert into streams
    (name, start_unix, end_unix, start_minute, end_minute, week_number, size) values
    (   ?,          ?,        ?,            ?,          ?,           ?,    ?) """,
    (name, start_unix, end_unix, start_minute, end_minute, week_number, size), with_last=True)

  except:
    logging.warn("Unable to insert a record with a name %s" % name)

  return last

<|endoftext|>


<commit_before>import numpy as np

from mushroom_rl.environments import Environment, MDPInfo
from mushroom_rl.utils import spaces

class LQR(Environment):
    """
    This class implements a Linear-Quadratic Regulator.
    This task aims to minimize the undesired deviations from nominal values of
    some controller settings in control problems.
    The system equations in this task are:

    .. math::
        x_{t+1} = Ax_t + Bu_t

    where x is the state and u is the control signal.

    The reward function is given by:

    .. math::
        r_t = -\\left( x_t^TQx_t + u_t^TRu_t \\right)

    "Policy gradient approaches for multi-objective sequential decision making".
    Parisi S., Pirotta M., Smacchia N., Bascetta L., Restelli M.. 2014

    """
    def __init__(self, A, B, Q, R, max_pos=np.inf, max_action=np.inf,
                 random_init=False, episodic=False, gamma=0.9, horizon=50):
        """
        Constructor.

            Args:
                A (np.ndarray): the state dynamics matrix;
                B (np.ndarray): the action dynamics matrix;
                Q (np.ndarray): reward weight matrix for state;
                R (np.ndarray): reward weight matrix for action;
                max_pos (float, np.inf): maximum value of the state;
                max_action (float, np.inf): maximum value of the action;
                random_init (bool, False): start from a random state;
                episodic (bool, False): end the episode when the state goes over
                the threshold;
                gamma (float, 0.9): discount factor;
                horizon (int, 50): horizon of the mdp.

        """
        self.A = A
        self.B = B
        self.Q = Q
        self.R = R
        self._max_pos = max_pos
        self._max_action = max_action
        self._episodic = episodic
        self.random_init = random_init

        # MDP properties
        high_x = self._max_pos * np.ones(A.shape[0])
        low_x = -high_x

        high_u = self._max_action * np.ones(B.shape[1])
        low_u = -high_u

        observation_space = spaces.Box(low=low_x, high=high_x)
        action_space = spaces.Box(low=low_u, high=high_u)
        mdp_info = MDPInfo(observation_space, action_space, gamma, horizon)

        super().__init__(mdp_info)

    @staticmethod
    def generate(dimensions, max_pos=np.inf, max_action=np.inf, eps=.1,
                 index=0, scale=1.0, random_init=False, episodic=False,
                 gamma=.9, horizon=50):
        """
        Factory method that generates an lqr with identity dynamics and
        symmetric reward matrices.

        Args:
            dimensions (int): number of state-action dimensions;
            max_pos (float, np.inf): maximum value of the state;
            max_action (float, np.inf): maximum value of the action;
            eps (double, .1): reward matrix weights specifier;
            index (int, 0): selector for the principal state;
            scale (float, 1.0): scaling factor for the reward function;
            random_init (bool, False): start from a random state;
            episodic (bool, False): end the episode when the state goes over the
                threshold;
            gamma (float, .9): discount factor;
            horizon (int, 50): horizon of the mdp.

        """
        assert dimensions >= 1

        A = np.eye(dimensions)
        B = np.eye(dimensions)
        Q = eps * np.eye(dimensions) * scale
        R = (1. - eps) * np.eye(dimensions) * scale

        Q[index, index] = 1. - eps
        R[index, index] = eps

        return LQR(A, B, Q, R, max_pos, max_action, random_init, episodic,
                   gamma, horizon)

    def reset(self, state=None):
        if state is None:
            if self.random_init:
                self._state = self._bound(
                    np.random.uniform(-3, 3, size=self.A.shape[0]),
                    self.info.observation_space.low,
                    self.info.observation_space.high
                )
            else:
                init_value = .9 * self._max_pos if np.isfinite(
                    self._max_pos) else 10
                self._state = init_value * np.ones(self.A.shape[0])
        else:
            self._state = state

        return self._state

    def step(self, action):
        x = self._state
        u = self._bound(action, self.info.action_space.low,
                        self.info.action_space.high)

        reward = -(x.dot(self.Q).dot(x) + u.dot(self.R).dot(u))
        self._state = self.A.dot(x) + self.B.dot(u)

        absorbing = False

        if np.any(np.abs(self._state) > self._max_pos):
            if self._episodic:
                reward = -self._max_pos ** 2 * 10
                absorbing = True
            else:
                self._state = self._bound(self._state,
                                          self.info.observation_space.low,
                                          self.info.observation_space.high)

        return self._state, reward, absorbing, {}
<commit_msg>Fixed scale in LQR generate<commit_after>import numpy as np

from mushroom_rl.environments import Environment, MDPInfo
from mushroom_rl.utils import spaces

class LQR(Environment):
    """
    This class implements a Linear-Quadratic Regulator.
    This task aims to minimize the undesired deviations from nominal values of
    some controller settings in control problems.
    The system equations in this task are:

    .. math::
        x_{t+1} = Ax_t + Bu_t

    where x is the state and u is the control signal.

    The reward function is given by:

    .. math::
        r_t = -\\left( x_t^TQx_t + u_t^TRu_t \\right)

    "Policy gradient approaches for multi-objective sequential decision making".
    Parisi S., Pirotta M., Smacchia N., Bascetta L., Restelli M.. 2014

    """
    def __init__(self, A, B, Q, R, max_pos=np.inf, max_action=np.inf,
                 random_init=False, episodic=False, gamma=0.9, horizon=50):
        """
        Constructor.

            Args:
                A (np.ndarray): the state dynamics matrix;
                B (np.ndarray): the action dynamics matrix;
                Q (np.ndarray): reward weight matrix for state;
                R (np.ndarray): reward weight matrix for action;
                max_pos (float, np.inf): maximum value of the state;
                max_action (float, np.inf): maximum value of the action;
                random_init (bool, False): start from a random state;
                episodic (bool, False): end the episode when the state goes over
                the threshold;
                gamma (float, 0.9): discount factor;
                horizon (int, 50): horizon of the mdp.

        """
        self.A = A
        self.B = B
        self.Q = Q
        self.R = R
        self._max_pos = max_pos
        self._max_action = max_action
        self._episodic = episodic
        self.random_init = random_init

        # MDP properties
        high_x = self._max_pos * np.ones(A.shape[0])
        low_x = -high_x

        high_u = self._max_action * np.ones(B.shape[1])
        low_u = -high_u

        observation_space = spaces.Box(low=low_x, high=high_x)
        action_space = spaces.Box(low=low_u, high=high_u)
        mdp_info = MDPInfo(observation_space, action_space, gamma, horizon)

        super().__init__(mdp_info)

    @staticmethod
    def generate(dimensions, max_pos=np.inf, max_action=np.inf, eps=.1,
                 index=0, scale=1.0, random_init=False, episodic=False,
                 gamma=.9, horizon=50):
        """
        Factory method that generates an lqr with identity dynamics and
        symmetric reward matrices.

        Args:
            dimensions (int): number of state-action dimensions;
            max_pos (float, np.inf): maximum value of the state;
            max_action (float, np.inf): maximum value of the action;
            eps (double, .1): reward matrix weights specifier;
            index (int, 0): selector for the principal state;
            scale (float, 1.0): scaling factor for the reward function;
            random_init (bool, False): start from a random state;
            episodic (bool, False): end the episode when the state goes over the
                threshold;
            gamma (float, .9): discount factor;
            horizon (int, 50): horizon of the mdp.

        """
        assert dimensions >= 1

        A = np.eye(dimensions)
        B = np.eye(dimensions)
        Q = eps * np.eye(dimensions) * scale
        R = (1. - eps) * np.eye(dimensions) * scale

        Q[index, index] = (1. - eps) * scale
        R[index, index] = eps * scale

        return LQR(A, B, Q, R, max_pos, max_action, random_init, episodic,
                   gamma, horizon)

    def reset(self, state=None):
        if state is None:
            if self.random_init:
                self._state = self._bound(
                    np.random.uniform(-3, 3, size=self.A.shape[0]),
                    self.info.observation_space.low,
                    self.info.observation_space.high
                )
            else:
                init_value = .9 * self._max_pos if np.isfinite(
                    self._max_pos) else 10
                self._state = init_value * np.ones(self.A.shape[0])
        else:
            self._state = state

        return self._state

    def step(self, action):
        x = self._state
        u = self._bound(action, self.info.action_space.low,
                        self.info.action_space.high)

        reward = -(x.dot(self.Q).dot(x) + u.dot(self.R).dot(u))
        self._state = self.A.dot(x) + self.B.dot(u)

        absorbing = False

        if np.any(np.abs(self._state) > self._max_pos):
            if self._episodic:
                reward = -self._max_pos ** 2 * 10
                absorbing = True
            else:
                self._state = self._bound(self._state,
                                          self.info.observation_space.low,
                                          self.info.observation_space.high)

        return self._state, reward, absorbing, {}
<|endoftext|>


<commit_before># inspired from:

# https://python-packaging-user-guide.readthedocs.org/en/latest/single_source_version/
# https://github.com/pypa/warehouse/blob/master/warehouse/__about__.py

__title__ = 'spacy'
__version__ = '1.4.0'
__summary__ = 'Industrial-strength NLP'
__uri__ = 'https://spacy.io'
__author__ = 'Matthew Honnibal'
__email__ = 'matt@spacy.io'
__license__ = 'MIT'
__models__ = {
    'en': 'en>=1.1.0,<1.2.0',
    'de': 'de>=1.0.0,<1.1.0',
}
<commit_msg>Increment version<commit_after># inspired from:

# https://python-packaging-user-guide.readthedocs.org/en/latest/single_source_version/
# https://github.com/pypa/warehouse/blob/master/warehouse/__about__.py

__title__ = 'spacy'
__version__ = '1.5.0'
__summary__ = 'Industrial-strength NLP'
__uri__ = 'https://spacy.io'
__author__ = 'Matthew Honnibal'
__email__ = 'matt@spacy.io'
__license__ = 'MIT'
__models__ = {
    'en': 'en>=1.1.0,<1.2.0',
    'de': 'de>=1.0.0,<1.1.0',
}
<|endoftext|>


<commit_before><commit_msg>OPT: no need to copy the header anymore (correctly set in sct_resample) Fix #596<commit_after><|endoftext|>


<commit_before>import logging
logger = logging.getLogger('multilingual_model')

import re

from django.utils.translation import ugettext_lazy as _
from django.utils.translation import ugettext

from django.db import models
from django.utils.translation import get_language
from django.core.exceptions import ObjectDoesNotExist

# What does this line of code do?
models.options.DEFAULT_NAMES += ('translation', 'multilingual')

from . import settings

# Match something like en, but also en_us
LANGUAGE_CODE_RE = re.compile(
    r'_(?P<base_code>[a-z]{2,5})(_(?P<ext_code>[a-z]{2,5})){0,1}$'
)

class MultilingualTranslation(models.Model):
    """ Abstract base class for translations. """

    class Meta:
        abstract = True

    language_code = models.CharField(
        _('language'), max_length=5, choices=settings.LANGUAGES,
        blank=False, null=False
    )

class MultilingualModel(models.Model):
    """Provides support for multilingual fields. """

    class Meta:
        abstract = True

    def __init__(self, *args, **kwargs):
        super(MultilingualModel, self).__init__(*args, **kwargs)
        self._language = get_language()
        self._translation_cache = {}

    def _get_translation(self, field, code):
        """
        Gets the translation of a specific field for a specific language code.

        This raises ObjectDoesNotExist if the lookup was unsuccesful. As of
        today, this stuff is cached. As the cache is rather aggressive it
        might cause rather strange effects. However, we would see the same
        effects when an ordinary object is changed which is already in memory:
        the old state would remain.
        """

        if not code in self._translation_cache:
            translations = self.translations.select_related()

            logger.debug(
                u'Matched with field %s for language %s. Attempting lookup.',
                field, code
            )

            try:
                translation_obj = translations.get(language_code=code)

            except ObjectDoesNotExist:
                translation_obj = None

            self._translation_cache[code] = translation_obj

            logger.debug(u'Translation not found in cache.')

        else:
            logger.debug(u'Translation found in cache.')
            # Get the translation from the cache
            translation_obj = self._translation_cache.get(code)

        # If this is none, it means that a translation does not exist
        # It is important to cache this one as well
        if not translation_obj:
            raise ObjectDoesNotExist

        field_value = getattr(translation_obj, field)

        logger.debug(
            u'Found translation object %s, returning value %s.',
            translation_obj, field_value
        )

        return field_value

    def __getattr__(self, attr):
        # If an attribute is defined in the current model, don't
        # look any further.
        if attr in self.__dict__:
            return self.__dict__[attr]

        #logger.debug(u'Looking for a translated field for: %s', attr)

        # See whether we can find a translation for the field
        translated_fields = self.translations.model._meta.get_all_field_names()
        for field in translated_fields:
            code = None

            # Only consider attributes starting with this field name
            if attr.startswith(field):
                # If we have a match, see if we can re-match the language
                # code in the remaining string.
                match = LANGUAGE_CODE_RE.match(attr[len(field):])
                if match:
                    base_code = match.group('base_code')
                    ext_code = match.group('ext_code')

                    # TODO: CLEANUP
                    # This is ugly code and redundant.

                    if ext_code:
                        code = '%s-%s' % (base_code, ext_code)
                    else:
                        code = base_code

                    logger.debug(
                        u'Regular expression match, resulting code: %s', code
                    )

                elif attr in translated_fields:
                    code = self._language
                    base_code = None
                    field = attr

                    logger.debug(
                        u'Regular expression not matched but translated'
                        u' field detected.'
                    )

            if code:
                try:
                    return self._get_translation(field, code)

                except ObjectDoesNotExist:
                    # TODO: CLEANUP
                    # This is ugly code and redundant.
                    if not base_code:
                        base_pos = code.find('-')
                        if base_pos > 0:
                            base_code = code[:base_pos]

                    if base_code:

                        logger.debug(
                            u'Attempting a match for the base \'%s\'',
                            base_code
                        )

                        try:
                            return self._get_translation(field, base_code)
                        except ObjectDoesNotExist:
                            pass

                    logger.debug(
                        u'Lookup failed, attempting fallback or '
                        u'failing silently.'
                    )

                    # If we're using a default language and the current
                    # language is not the default language (which has already
                    # been checked), lookup the value for the default language.
                    if (
                        settings.FALL_BACK_TO_DEFAULT and
                        settings.DEFAULT_LANGUAGE and
                        code != settings.DEFAULT_LANGUAGE
                    ):

                        try:
                            return self._get_translation(
                                field, settings.DEFAULT_LANGUAGE
                            )

                        except ObjectDoesNotExist:
                            pass

                    if settings.FAIL_SILENTLY:
                        return None

                    raise ValueError(
                        u"'%s' object with pk '%s' has no"
                        u" translation to '%s'" % (
                            self._meta.object_name, self.pk, code
                        )
                    )

        raise AttributeError(
            u"'%s' object has no attribute '%s'" % (
                self._meta.object_name, str(attr)
            )
        )

    def for_language(self, code):
        """Sets the language for the translation fields of this object"""

        if code is not None and len(code) >= 2 and len(code) <= 5:
            self._language = code

    def unicode_wrapper(self, property, default=ugettext('Untitled')):
        """
        Wrapper to allow for easy unicode representation of an object by
        the specified property. If this wrapper is not able to find the
        right translation of the specified property, it will return the
        default value instead.

        Example::
            def __unicode__(self):
                return unicode_wrapper('name', default='Unnamed')

        """

        try:
            value = getattr(self, property)
        except ValueError:
            logger.warn(
                u'ValueError rendering unicode for %s object.',
                self._meta.object_name
            )

            value = None

        if not value:
            value = default

        return value
<commit_msg>Slight cleanup.<commit_after>import logging
logger = logging.getLogger('multilingual_model')

import re

from django.utils.translation import ugettext_lazy as _
from django.utils.translation import ugettext

from django.db import models
from django.utils.translation import get_language
from django.core.exceptions import ObjectDoesNotExist

# What does this line of code do?
models.options.DEFAULT_NAMES += ('translation', 'multilingual')

from . import settings

# Match something like en, but also en_us
LANGUAGE_CODE_RE = re.compile(
    r'_(?P<base_code>[a-z]{2,5})(_(?P<ext_code>[a-z]{2,5})){0,1}$'
)

class MultilingualTranslation(models.Model):
    """ Abstract base class for translations. """

    class Meta:
        abstract = True

    language_code = models.CharField(
        _('language'), max_length=5, choices=settings.LANGUAGES,
        blank=False, null=False
    )

class MultilingualModel(models.Model):
    """ Provides support for multilingual fields. """

    class Meta:
        abstract = True

    def __init__(self, *args, **kwargs):
        super(MultilingualModel, self).__init__(*args, **kwargs)
        self._language = get_language()
        self._translation_cache = {}

    def _get_translation(self, field, code):
        """
        Gets the translation of a specific field for a specific language code.

        This raises ObjectDoesNotExist if the lookup was unsuccesful. As of
        today, this stuff is cached. As the cache is rather aggressive it
        might cause rather strange effects. However, we would see the same
        effects when an ordinary object is changed which is already in memory:
        the old state would remain.
        """

        if not code in self._translation_cache:
            translations = self.translations.select_related()

            logger.debug(
                u'Matched with field %s for language %s. Attempting lookup.',
                field, code
            )

            try:
                translation_obj = translations.get(language_code=code)

            except ObjectDoesNotExist:
                translation_obj = None

            self._translation_cache[code] = translation_obj

            logger.debug(u'Translation not found in cache.')

        else:
            logger.debug(u'Translation found in cache.')
            # Get the translation from the cache
            translation_obj = self._translation_cache.get(code)

        # If this is none, it means that a translation does not exist
        # It is important to cache this one as well
        if not translation_obj:
            raise ObjectDoesNotExist

        field_value = getattr(translation_obj, field)

        logger.debug(
            u'Found translation object %s, returning value %s.',
            translation_obj, field_value
        )

        return field_value

    def __getattr__(self, attr):
        # If an attribute is defined in the current model, don't
        # look any further.
        if attr in self.__dict__:
            return self.__dict__[attr]

        #logger.debug(u'Looking for a translated field for: %s', attr)

        # See whether we can find a translation for the field
        translated_fields = self.translations.model._meta.get_all_field_names()
        for field in translated_fields:
            code = None

            # Only consider attributes starting with this field name
            if attr.startswith(field):
                # If we have a match, see if we can re-match the language
                # code in the remaining string.
                match = LANGUAGE_CODE_RE.match(attr[len(field):])
                if match:
                    base_code = match.group('base_code')
                    ext_code = match.group('ext_code')

                    # TODO: CLEANUP
                    # This is ugly code and redundant.

                    if ext_code:
                        code = '%s-%s' % (base_code, ext_code)
                    else:
                        code = base_code

                    logger.debug(
                        u'Regular expression match, resulting code: %s', code
                    )

                elif attr in translated_fields:
                    code = self._language
                    base_code = None
                    field = attr

                    logger.debug(
                        u'Regular expression not matched but translated'
                        u' field detected.'
                    )

            if code:
                try:
                    return self._get_translation(field, code)

                except ObjectDoesNotExist:
                    # TODO: CLEANUP
                    # This is ugly code and redundant.
                    if not base_code:
                        base_pos = code.find('-')
                        if base_pos > 0:
                            base_code = code[:base_pos]

                    if base_code:

                        logger.debug(
                            u'Attempting a match for the base \'%s\'',
                            base_code
                        )

                        try:
                            return self._get_translation(field, base_code)
                        except ObjectDoesNotExist:
                            pass

                    logger.debug(
                        u'Lookup failed, attempting fallback or '
                        u'failing silently.'
                    )

                    # If we're using a default language and the current
                    # language is not the default language (which has already
                    # been checked), lookup the value for the default language.
                    if (
                        settings.FALL_BACK_TO_DEFAULT and
                        settings.DEFAULT_LANGUAGE and
                        code != settings.DEFAULT_LANGUAGE
                    ):

                        try:
                            return self._get_translation(
                                field, settings.DEFAULT_LANGUAGE
                            )

                        except ObjectDoesNotExist:
                            pass

                    if settings.FAIL_SILENTLY:
                        return None

                    raise ValueError(
                        u"'%s' object with pk '%s' has no"
                        u" translation to '%s'" % (
                            self._meta.object_name, self.pk, code
                        )
                    )

        raise AttributeError(
            u"'%s' object has no attribute '%s'" % (
                self._meta.object_name, str(attr)
            )
        )

    def for_language(self, code):
        """Sets the language for the translation fields of this object"""

        if code is not None and len(code) >= 2 and len(code) <= 5:
            self._language = code

    def unicode_wrapper(self, property, default=ugettext('Untitled')):
        """
        Wrapper to allow for easy unicode representation of an object by
        the specified property. If this wrapper is not able to find the
        right translation of the specified property, it will return the
        default value instead.

        Example::
            def __unicode__(self):
                return unicode_wrapper('name', default='Unnamed')

        """

        try:
            value = getattr(self, property)
        except ValueError:
            logger.warn(
                u'ValueError rendering unicode for %s object.',
                self._meta.object_name
            )

            value = None

        if not value:
            value = default

        return value
<|endoftext|>


<commit_before># inspired from:
# https://python-packaging-user-guide.readthedocs.org/en/latest/single_source_version/
# https://github.com/pypa/warehouse/blob/master/warehouse/__about__.py
# fmt: off

__title__ = "spacy-nightly"
__version__ = "2.1.0a7.dev2"
__summary__ = "Industrial-strength Natural Language Processing (NLP) with Python and Cython"
__uri__ = "https://spacy.io"
__author__ = "Explosion AI"
__email__ = "contact@explosion.ai"
__license__ = "MIT"
__release__ = False

__download_url__ = "https://github.com/explosion/spacy-models/releases/download"
__compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json"
__shortcuts__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/shortcuts-v2.json"
<commit_msg>Set version to v2.1.0a7.dev3<commit_after># inspired from:
# https://python-packaging-user-guide.readthedocs.org/en/latest/single_source_version/
# https://github.com/pypa/warehouse/blob/master/warehouse/__about__.py
# fmt: off

__title__ = "spacy-nightly"
__version__ = "2.1.0a7.dev3"
__summary__ = "Industrial-strength Natural Language Processing (NLP) with Python and Cython"
__uri__ = "https://spacy.io"
__author__ = "Explosion AI"
__email__ = "contact@explosion.ai"
__license__ = "MIT"
__release__ = False

__download_url__ = "https://github.com/explosion/spacy-models/releases/download"
__compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json"
__shortcuts__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/shortcuts-v2.json"
<|endoftext|>


<commit_before>from collections import OrderedDict
import re
from pathlib import Path
from typing import Sequence, Mapping

from pywikibot import Site, Page
import yaml
import yamlordereddictloader

from scripts.service.ws_re.data_types import ReVolumes
from tools import path_or_str
from tools.bots import CanonicalBot

class ReImporter(CanonicalBot):
    _register_folder = "register"

    def __init__(self, wiki: Site = None, debug: bool = True,
                 log_to_screen: bool = True, log_to_wiki: bool = True):
        CanonicalBot.__init__(self, wiki, debug, log_to_screen, log_to_wiki)

    def task(self):
        re_volumes = ReVolumes()
        for volume in re_volumes.all_volumes:
            self.logger.info("Dumping Register for {}".format(volume.name))
            old_register = Page(self.wiki, "Paulys Realencyclopädie der classischen "
                                           "Altertumswissenschaft/Register/{}".format(volume.name))
            self._dump_register(volume.file_name, old_register.text)
        return True

    def _dump_register(self, volume: str, old_register: str):
        new_register = self._build_register(old_register)
        file = path_or_str(Path(__file__).parent
                           .joinpath(self._register_folder).joinpath("{}.yaml".format(volume)))
        with open(file, mode="w", encoding="utf-8") as yaml_file:
            yaml.dump(new_register, yaml_file, Dumper=yamlordereddictloader.Dumper,
                      default_flow_style=False, allow_unicode=True)

    def _split_line(self, register_line: str) -> Sequence[str]:
        splitted_lines = re.split(r"\n\|", register_line)
        for idx, line in enumerate(splitted_lines):
            splitted_lines[idx] = line.strip("|")
        if not len(splitted_lines) == 4:
            self.logger.critical(register_line)
            raise ValueError("The split of the line went wrong")
        return splitted_lines

    @staticmethod
    def _split_table(table: str) -> Sequence[str]:
        match = re.search(r"\{\|\n\|-(.*)\|\}", table, re.S)
        lines = match.group(1).split("|-\n")
        for idx, line in enumerate(lines):
            lines[idx] = line.strip()
        return lines

    @staticmethod
    def _analyse_first_column(content: str) -> Mapping:
        mapping = dict()
        match = re.search(r"\[\[RE:(.*?)\]\]", content)
        mapping["lemma"] = match.group(1)
        return mapping

    def _analyse_second_column(self, content: str) -> Mapping:
        mapping = dict()
        match = re.search(r"\[\[Spe.*?\|[SR AIXV1234,]*?(\d{1,4})\]\](?:.*?\])?(?:-(\d{1,4}))?",
                          content)
        try:
            mapping["start"] = int(match.group(1))
            end = match.group(2)
            mapping["end"] = end
        except AttributeError:
            self.logger.error("There is a problem with the second column.")
            self.logger.error(content)
            raise AttributeError
        if end:
            mapping["end"] = int(end)
        else:
            mapping["end"] = mapping["start"]
        return mapping

    def _build_lemma_from_line(self, line: str) -> Mapping:
        splitted_line = self._split_line(line)
        mapping_1 = self._analyse_first_column(splitted_line[0])
        mapping_2 = self._analyse_second_column(splitted_line[1])
        author = splitted_line[2]
        lemma_dict = OrderedDict()
        lemma_dict["lemma"] = mapping_1["lemma"]
        lemma_dict["next"] = ""
        lemma_dict["previous"] = ""
        lemma_dict["redirect"] = False
        chapter_dict = OrderedDict([("start", mapping_2["start"]),
                                    ("end", mapping_2["end"]),
                                    ("author", author)])
        if author == "X":
            lemma_dict["redirect"] = True
            chapter_dict["author"] = ""
        if author == "?":
            chapter_dict["author"] = ""
        lemma_dict["chapters"] = list()
        lemma_dict["chapters"].append(chapter_dict)
        return lemma_dict

    def _build_register(self, register_text: str) -> Sequence:
        register = list()
        splitted_lines = self._split_table(register_text)
        for line in splitted_lines:
            lemma = self._build_lemma_from_line(line)
            register.append(lemma)
        return register

if __name__ == "__main__":  # pragma: no cover
    WS_WIKI = Site(code='de', fam='wikisource', user='THEbotIT')
    with ReImporter(wiki=WS_WIKI, debug=True, log_to_wiki=False) as bot:
        bot.run()
<commit_msg>more log lines for better time handling<commit_after>from collections import OrderedDict
import re
from pathlib import Path
from typing import Sequence, Mapping

from pywikibot import Site, Page
import yaml
import yamlordereddictloader

from scripts.service.ws_re.data_types import ReVolumes
from tools import path_or_str
from tools.bots import CanonicalBot

class ReImporter(CanonicalBot):
    _register_folder = "register"

    def __init__(self, wiki: Site = None, debug: bool = True,
                 log_to_screen: bool = True, log_to_wiki: bool = True):
        CanonicalBot.__init__(self, wiki, debug, log_to_screen, log_to_wiki)

    def task(self):
        re_volumes = ReVolumes()
        for volume in re_volumes.all_volumes:
            self.logger.info("Reading Register for {}".format(volume.name))
            old_register = Page(self.wiki, "Paulys Realencyclopädie der classischen "
                                           "Altertumswissenschaft/Register/{}".format(volume.name))
            self.logger.info("Dumping Register for {}".format(volume.name))
            self._dump_register(volume.file_name, old_register.text)
        return True

    def _dump_register(self, volume: str, old_register: str):
        new_register = self._build_register(old_register)
        file = path_or_str(Path(__file__).parent
                           .joinpath(self._register_folder).joinpath("{}.yaml".format(volume)))
        with open(file, mode="w", encoding="utf-8") as yaml_file:
            yaml.dump(new_register, yaml_file, Dumper=yamlordereddictloader.Dumper,
                      default_flow_style=False, allow_unicode=True)

    def _split_line(self, register_line: str) -> Sequence[str]:
        splitted_lines = re.split(r"\n\|", register_line)
        for idx, line in enumerate(splitted_lines):
            splitted_lines[idx] = line.strip("|")
        if not len(splitted_lines) == 4:
            self.logger.critical(register_line)
            raise ValueError("The split of the line went wrong")
        return splitted_lines

    @staticmethod
    def _split_table(table: str) -> Sequence[str]:
        match = re.search(r"\{\|\n\|-(.*)\|\}", table, re.S)
        lines = match.group(1).split("|-\n")
        for idx, line in enumerate(lines):
            lines[idx] = line.strip()
        return lines

    @staticmethod
    def _analyse_first_column(content: str) -> Mapping:
        mapping = dict()
        match = re.search(r"\[\[RE:(.*?)\]\]", content)
        mapping["lemma"] = match.group(1)
        return mapping

    def _analyse_second_column(self, content: str) -> Mapping:
        mapping = dict()
        match = re.search(r"\[\[Spe.*?\|[SR AIXV1234,]*?(\d{1,4})\]\](?:.*?\])?(?:-(\d{1,4}))?",
                          content)
        try:
            mapping["start"] = int(match.group(1))
            end = match.group(2)
            mapping["end"] = end
        except AttributeError:
            self.logger.error("There is a problem with the second column.")
            self.logger.error(content)
            raise AttributeError
        if end:
            mapping["end"] = int(end)
        else:
            mapping["end"] = mapping["start"]
        return mapping

    def _build_lemma_from_line(self, line: str) -> Mapping:
        splitted_line = self._split_line(line)
        mapping_1 = self._analyse_first_column(splitted_line[0])
        mapping_2 = self._analyse_second_column(splitted_line[1])
        author = splitted_line[2]
        lemma_dict = OrderedDict()
        lemma_dict["lemma"] = mapping_1["lemma"]
        lemma_dict["next"] = ""
        lemma_dict["previous"] = ""
        lemma_dict["redirect"] = False
        chapter_dict = OrderedDict([("start", mapping_2["start"]),
                                    ("end", mapping_2["end"]),
                                    ("author", author)])
        if author == "X":
            lemma_dict["redirect"] = True
            chapter_dict["author"] = ""
        if author == "?":
            chapter_dict["author"] = ""
        lemma_dict["chapters"] = list()
        lemma_dict["chapters"].append(chapter_dict)
        return lemma_dict

    def _build_register(self, register_text: str) -> Sequence:
        register = list()
        splitted_lines = self._split_table(register_text)
        for line in splitted_lines:
            lemma = self._build_lemma_from_line(line)
            register.append(lemma)
        return register

if __name__ == "__main__":  # pragma: no cover
    WS_WIKI = Site(code='de', fam='wikisource', user='THEbotIT')
    with ReImporter(wiki=WS_WIKI, debug=True, log_to_wiki=False) as bot:
        bot.run()
<|endoftext|>


<commit_before>import random

def guess():
	n= raw_input("Number:")
	target= random.randint(1,100)
	p= int(n)

	print target

	if p == target:
		print "good"
	if p < target :
		print "too low"
	if p > target:
		print "too high"
	guess()

guess()
<commit_msg>NOt done<commit_after>import random

def guess():
	n= raw_input("Number:")
	target= random.randint(1,100)
	p= int(n)

	print target

	if p == target:
		print "good"
	if p < target :
		print "too low"
	if p > target:
		print "too high"
	guess()

def number_round():
	if number_round == 0:
		return "Out of round"
	else:
		return guess()

def main():
	number_round = 3
	number_tries = 5

<|endoftext|>


<commit_before>from django.test import TestCase
from django.core.validators import URLValidator
from django.core.exceptions import ValidationError
from myuw.dao.category_links import _get_links_by_category_and_campus, \
    _get_category_id, Res_Links
import re

class TestCategoryLinks(TestCase):

    def test_get_all_likes(self):
        all_links = Res_Links.get_all_links()
        self.assertEquals(len(all_links), 159)
        val = URLValidator()
        for link in all_links:
            try:
                link.url.decode('ascii')
            except UnicodeDecodeError:
                self.fail("%s url has non-ASCII text: %s" %
                          (link.title, link.url))

            try:
                val(link.url)
            except ValidationError:
                # allow relative references
                if not re.match('^/[\w/]+$', link.url):
                    self.fail("Invalid url:" + link.url)

            try:
                link.title.decode('ascii')
            except UnicodeDecodeError:
                self.fail("Link title has non-ASCII text:" + link.title)

    def test_undergrad_category(self):
        category_id = _get_category_id("Student & Campus Life")
        self.assertEquals(category_id, "studentcampuslife")
        affi = {"grad": False,
                "undergrad": True,
                "pce": False}
        links = _get_links_by_category_and_campus(category_id,
                                                  "seattle",
                                                  affi)
        self.assertEquals(len(links), 25)

        links = _get_links_by_category_and_campus(category_id,
                                                  "bothell",
                                                  affi)
        self.assertEquals(len(links), 23)

        links = _get_links_by_category_and_campus(category_id,
                                                  "tacoma",
                                                  affi)
        self.assertEquals(len(links), 23)

    def test_grad_category(self):
        category_id = _get_category_id("Student & Campus Life")
        self.assertEquals(category_id, "studentcampuslife")
        affi = {"grad": True,
                "undergrad": False,
                "pce": False}
        links = _get_links_by_category_and_campus(category_id,
                                                  "seattle",
                                                  affi)
        self.assertEquals(len(links), 26)

        links = _get_links_by_category_and_campus(category_id,
                                                  "bothell",
                                                  affi)
        self.assertEquals(len(links), 24)

        links = _get_links_by_category_and_campus(category_id,
                                                  "tacoma",
                                                  affi)
        self.assertEquals(len(links), 24)

    def test_pce_links(self):
        category_id = _get_category_id("Student & Campus Life")
        self.assertEquals(category_id, "studentcampuslife")
        affi = {"grad": False,
                "undergrad": False,
                "pce": True}
        links = _get_links_by_category_and_campus(category_id,
                                                  "",
                                                  affi)
        self.assertEquals(len(links), 6)
<commit_msg>update unit-test<commit_after>from django.test import TestCase
from django.core.validators import URLValidator
from django.core.exceptions import ValidationError
from myuw.dao.category_links import _get_links_by_category_and_campus, \
    _get_category_id, Res_Links
import re

class TestCategoryLinks(TestCase):

    def test_get_all_likes(self):
        all_links = Res_Links.get_all_links()
        self.assertEquals(len(all_links), 155)
        val = URLValidator()
        for link in all_links:
            try:
                link.url.decode('ascii')
            except UnicodeDecodeError:
                self.fail("%s url has non-ASCII text: %s" %
                          (link.title, link.url))

            try:
                val(link.url)
            except ValidationError:
                # allow relative references
                if not re.match('^/[\w/]+$', link.url):
                    self.fail("Invalid url:" + link.url)

            try:
                link.title.decode('ascii')
            except UnicodeDecodeError:
                self.fail("Link title has non-ASCII text:" + link.title)

    def test_undergrad_category(self):
        category_id = _get_category_id("Student & Campus Life")
        self.assertEquals(category_id, "studentcampuslife")
        affi = {"grad": False,
                "undergrad": True,
                "pce": False}
        links = _get_links_by_category_and_campus(category_id,
                                                  "seattle",
                                                  affi)
        self.assertEquals(len(links), 23)

        links = _get_links_by_category_and_campus(category_id,
                                                  "bothell",
                                                  affi)
        self.assertEquals(len(links), 22)

        links = _get_links_by_category_and_campus(category_id,
                                                  "tacoma",
                                                  affi)
        self.assertEquals(len(links), 22)

    def test_grad_category(self):
        category_id = _get_category_id("Student & Campus Life")
        self.assertEquals(category_id, "studentcampuslife")
        affi = {"grad": True,
                "undergrad": False,
                "pce": False}
        links = _get_links_by_category_and_campus(category_id,
                                                  "seattle",
                                                  affi)
        self.assertEquals(len(links), 24)

        links = _get_links_by_category_and_campus(category_id,
                                                  "bothell",
                                                  affi)
        self.assertEquals(len(links), 23)

        links = _get_links_by_category_and_campus(category_id,
                                                  "tacoma",
                                                  affi)
        self.assertEquals(len(links), 23)

    def test_pce_links(self):
        category_id = _get_category_id("Student & Campus Life")
        self.assertEquals(category_id, "studentcampuslife")
        affi = {"grad": False,
                "undergrad": False,
                "pce": True}
        links = _get_links_by_category_and_campus(category_id,
                                                  "",
                                                  affi)
        self.assertEquals(len(links), 6)
<|endoftext|>


<commit_before>from __future__ import print_function, unicode_literals

"""
An attempt to rewrite midi.py using named tuples (or something similar.

  - DRY (midi.py is notorious here)
  - MIDI messages should be immutable
  - the contructor must check if values are within range

Read and update (by cloning):

  - all data values

__getitem__() access to the raw bytes.

len() returns length in bytes?

Ole Martin Bjørndalen
ombdalen@gmail.com
http://nerdly.info/ole/
"""

#
# Messages can be defined something like this
# (I need to check these values.)
#

#
# Todo:
#   - support rich comparisons (easy with self.bytes and self.bin)
#

#
# Channel messages
#
channel_messages = [
    # Immutable                 Updatable (through cloning)
    # opcode  name
    (0x80, 'note_off',          'note velocity'),
    (0x90, 'note_on',           'note velocity'),
    (0xa0, 'poly_pressure',     'note  pressure'),
    (0xb0, 'program_change',    'program'),
    (0xc0, 'control_change',    'control value'),
    (0xd0, 'channel_pressure',  'pressure'),
    (0xe0, 'pitch_wheel',       'value:14'),

    (),
]

# Field specifiers
# The number after ':' is the number of bits used
'channel:4'    # four bit value (available in all channel messages)
'note:7'
'note:'        # 7 is the default size
'value:14'
'data:bytes'   # a tuple of byte values (on integer per byte)

#
# This could be auto-generated somehow from the list above# to avoid writing all that boilerplate.
#
# Using shorter names 'chan' and 'vel' here. Don't know
# if I'll stick with that.
#
class MidiMsg:
    """
    Abstract base class for MIDI messages
    """
    pass

def legal_data_byte(value):
    """Check if data byte is and integer in the range [0, 127], and return """

    if not isinstance(value, int):
        return False
    elif not 0 <= value <= 127:
        return False

    return True

def assert_data_byte(value):
    """Check if data byte is and int in the range [0, 127], and raise ValueError if not"""
    if not legal_data_byte(value):
        raise ValueError('MIDI data byte must be an integer in the range [0, 127] (was %r)' % value)

def assert_time_value(time):
    """Check if time value is a number >= 0"""

    # Todo: there must be a better way to check this
    if not (isinstance(time, int) or (isinstance(time, float))) or time < 0:
        raise ValueError('MIDI time value must be a number >= 0')

class NoteOff(MidiMsg):
    """
    Class for NoteOn MIDI messages.

    Todo: Should use descriptors to ensure correct values
    and handle conversion of two-byte values to more meaningful types.

    The 'pitch_wheel' message could even have a descriptor which
    acesses the 'hi' and 'lo' bytes as a float in the range -1 .. 1
    """
    opcode = 0x80
    type = 'note_off'

    #
    # Todo: there are essentially two contructors here. Use a function
    # instead, and have just one message class, and set __call__() to
    # a function with a closure on the message object?
    #
    def __init__(self, time=0, chan=None, note=None, vel=None):
        """Create a new MIDI message"""

        assert_time_value(time)
        self.time = time

        if chan == None:
            self.chan = 0
        else:
            self.chan = chan

        if note == None:
            self.note = 0
        else:
            self.note = note

        if vel == None:
            self.vel = 127
        else:
            self.vel = vel

        # Serialize
        self.bytes = (self.opcode | self.chan, self.note, self.vel)

        # Todo: This will only work in Python >= 3
        self.bin = bytes(self.bytes)

        self._assert_values()

    def __call__(self, time=0, chan=None, note=None, vel=None):
        """Clone message, allowing caller to override selected values."""

        if time == None:
            time = self.time

        if chan == None:
            chan = self.chan

        if note == None:
            note = self.note

        if vel == None:
            vel = self.vel

        return NoteOn(time=time, chan=chan, note=note, vel=vel)

    def _assert_values(self):
        [assert_data_byte(b) for b in [self.chan, self.note, self.vel]]

    def __repr__(self):
        """Return a human-readable representation of the message"""
        return '%s(time=%s, chan=%s, note=%s, vel=%s)' % (self.type, self.time, self.chan, self.note, self.vel)

    def __str__(self):
        """Return message serialized to text"""
        return '%s %s  %s %s %s' % (self.time, self.type, self.chan, self.note, self.vel)

    def __len__(self):
        """Returns number of bytes in the message"""
        return len(self.bytes)

class NoteOn(NoteOff):
    opcode = 0x90
    type = 'note_on'

if __name__ == '__main__':
    on = NoteOn()
    off = NoteOff()

    print(on)
    print(on(note=20))
    print(on.bin)
<commit_msg>Setting sensible default values, removing a lot of boilerplate<commit_after>from __future__ import print_function, unicode_literals

"""
An attempt to rewrite midi.py using named tuples (or something similar.

  - DRY (midi.py is notorious here)
  - MIDI messages should be immutable
  - the contructor must check if values are within range

Read and update (by cloning):

  - all data values

__getitem__() access to the raw bytes.

len() returns length in bytes?

Ole Martin Bjørndalen
ombdalen@gmail.com
http://nerdly.info/ole/
"""

#
# Messages can be defined something like this
# (I need to check these values.)
#

#
# Todo:
#   - support rich comparisons (easy with self.bytes and self.bin)
#

#
# Channel messages
#
channel_messages = [
    # Immutable                 Updatable (through cloning)
    # opcode  name
    (0x80, 'note_off',          'note velocity'),
    (0x90, 'note_on',           'note velocity'),
    (0xa0, 'poly_pressure',     'note  pressure'),
    (0xb0, 'program_change',    'program'),
    (0xc0, 'control_change',    'control value'),
    (0xd0, 'channel_pressure',  'pressure'),
    (0xe0, 'pitch_wheel',       'value:14'),

    (),
]

# Field specifiers
# The number after ':' is the number of bits used
'channel:4'    # four bit value (available in all channel messages)
'note:7'
'note:'        # 7 is the default size
'value:14'
'data:bytes'   # a tuple of byte values (on integer per byte)

#
# This could be auto-generated somehow from the list above# to avoid writing all that boilerplate.
#
# Using shorter names 'chan' and 'vel' here. Don't know
# if I'll stick with that.
#
class MidiMsg:
    """
    Abstract base class for MIDI messages
    """
    pass

def legal_data_byte(value):
    """Check if data byte is and integer in the range [0, 127], and return """

    if not isinstance(value, int):
        return False
    elif not 0 <= value <= 127:
        return False

    return True

def assert_data_byte(value):
    """Check if data byte is and int in the range [0, 127], and raise ValueError if not"""
    if not legal_data_byte(value):
        raise ValueError('MIDI data byte must be an integer in the range [0, 127] (was %r)' % value)

def assert_time_value(time):
    """Check if time value is a number >= 0"""

    # Todo: there must be a better way to check this
    if not (isinstance(time, int) or (isinstance(time, float))) or time < 0:
        raise ValueError('MIDI time value must be a number >= 0')

class NoteOff(MidiMsg):
    """
    Class for NoteOn MIDI messages.

    Todo: Should use descriptors to ensure correct values
    and handle conversion of two-byte values to more meaningful types.

    The 'pitch_wheel' message could even have a descriptor which
    acesses the 'hi' and 'lo' bytes as a float in the range -1 .. 1
    """
    opcode = 0x80
    type = 'note_off'

    #
    # Todo: there are essentially two contructors here. Use a function
    # instead, and have just one message class, and set __call__() to
    # a function with a closure on the message object?
    #
    def __init__(self, time=0, chan=0, note=60, vel=127):
        """Create a new MIDI message"""

        assert_time_value(time)
        self.time = time

        self.chan = chan
        self.note = note
        self.vel = vel

        # Serialize
        self.bytes = (self.opcode | self.chan, self.note, self.vel)

        # Todo: This will only work in Python >= 3
        self.bin = bytes(self.bytes)

        self._assert_values()

    def __call__(self, time=0, chan=0, note=60, vel=127):
        """Clone message, allowing caller to override selected values."""
        return NoteOn(time=time, chan=chan, note=note, vel=vel)

    def _assert_values(self):
        [assert_data_byte(b) for b in [self.chan, self.note, self.vel]]

    def __repr__(self):
        """Return a human-readable representation of the message"""
        return '%s(time=%s, chan=%s, note=%s, vel=%s)' % (self.type, self.time, self.chan, self.note, self.vel)

    def __str__(self):
        """Return message serialized to text"""
        return '%s %s  %s %s %s' % (self.time, self.type, self.chan, self.note, self.vel)

    def __len__(self):
        """Returns number of bytes in the message"""
        return len(self.bytes)

class NoteOn(NoteOff):
    opcode = 0x90
    type = 'note_on'

if __name__ == '__main__':
    on = NoteOn()
    off = NoteOff()

    print(on)
    print(on(note=20))
    print(on.bin)
<|endoftext|>