<commit_before>#!/usr/bin/python

# SWD adapter for the Nucon NUC1XX series.

# TODO: make it work, flash can't be written directly through ISP as it seems

__author__ = 'Pascal Hahn <ph@lxd.bz>'

import binascii
import SWDCommon

class NotReadyForCommandException(Exception):
  pass

class UnlockFailedException(Exception):
  pass

class NUC1XX(object):
  CONFIG0_ADDR = 0x00300000

  ISPCON_ADDR = 0x5000C000
  ISPADR_ADDR = 0x5000C004
  ISPDAT_ADDR = 0x5000C008
  ISPCMD_ADDR = 0x5000C00C
  ISPTRG_ADDR = 0x5000C010

  IPRSTC_ADDR = 0x50000008

  DHCSR_ADDR = 0xE000EDF0

  DCRSR_ADDR = 0xE000EDF4
  DCRDR_ADDR = 0xE000EDF8

  AIRCR_ADDR = 0xE000ED0C

  REGRWPROT_ADDR = 0x50000100

  def __init__(self, debugport):
    self.ahb = SWDCommon.MEM_AP(debugport, 0)

  def halt (self):
    self.ahb.writeWord(NUC1XX.DHCSR_ADDR, 0xA05F0003)

  def unhalt(self):
    self.ahb.writeWord(NUC1XX.DHCSR_ADDR, 0xA05F0001)

  def step(self):
    self.ahb.writeWord(NUC1XX.DHCSR_ADDR, 0xA05F0005)

  def reset(self):
    self.ahb.writeWord(NUC1XX.IPRSTC_ADDR, 0x01)

  def flashUnlock(self):
    if self.ahb.readBlock(NUC1XX.REGRWPROT_ADDR, 0x01) != [0x00]:
      print 'flashUnlock: Already unlocked'
    else:
      # disable write protection
      self.ahb.writeWord(NUC1XX.REGRWPROT_ADDR, 0x59)
      self.ahb.writeWord(NUC1XX.REGRWPROT_ADDR, 0x16)
      self.ahb.writeWord(NUC1XX.REGRWPROT_ADDR, 0x88)

    if self.ahb.readBlock(NUC1XX.REGRWPROT_ADDR, 0x01) != [0x01]:
      raise UnlockFailedException('flachUnlock: Unlock didn\'t work')

    # enable ISP
    self.ahb.writeWord(NUC1XX.ISPCON_ADDR, 0x00000031)

  def changeBS(self):
    val = self.ahb.readWord(NUC1XX.ISPCON_ADDR)
    self.ahb.writeWord(NUC1XX.ISPCON_ADDR, val | 2)

  def issueISPCommand(self, adr, cmd, data):
    if self.ahb.readBlock(NUC1XX.ISPTRG_ADDR, 0x01) != [0x00]:
      raise NotReadyForCommandException('not yet ready for command')
    self.ahb.writeWord(NUC1XX.ISPADR_ADDR, adr)
    self.ahb.writeWord(NUC1XX.ISPCMD_ADDR, cmd)
    self.ahb.writeWord(NUC1XX.ISPDAT_ADDR, data)

    # trigger command execution
    self.ahb.writeWord(NUC1XX.ISPTRG_ADDR, 0x00000001)

    while True:
      # successful execution
      if self.ahb.readBlock(NUC1XX.ISPTRG_ADDR, 0x01) == [0x00]:
        break
      # TODO: for now we assume success, actually verify it
    ispcon = self.ahb.readWord(NUC1XX.ISPCON_ADDR)
    self.ahb.writeWord(NUC1XX.ISPCON_ADDR, ispcon)
    if ispcon & 0x40:
      print 'issueISPCOMMAND: ISP command failed: %08X' % ispcon

  def writeFlash(self, addr, data):
    print 'writing 0x%x to 0x%x' % (data, addr)
    self.issueISPCommand(addr, 0x21, data)

  def readFlash(self, addr, length):
    for counter in range(0, length, 8):
      self.issueISPCommand(addr + counter, 0x00, 0x00)
      print 'reading %s: %s' % (
          hex(addr + counter), hex(self.ahb.readWord(NUC1XX.ISPDAT_ADDR)))

  def eraseFlash(self):
    self.issueISPCommand(0x00100000, 0x22, 0x00)

  def readRegister(self, register):
    self.ahb.writeWord(NUC1XX.DCRSR_ADDR, register)
    return self.ahb.readWord(NUC1XX.DCRDR_ADDR)

  def writeRegister(self, register, data):
    self.ahb.writeWord(NUC1XX.DCRDR_ADDR, data)
    self.ahb.writeWord(NUC1XX.DCRSR_ADDR, register)

  def readAllRom(self):
    for addr in range(0xfffffffC, 0xffffffff, 0x04):
      print hex(self.ahb.readWord(addr))

  def writeToRam(self):
    for addr in range(0x20000000, 0x200000d0, 0x04):
      print 'writing %s' % hex(addr)
      self.ahb.writeWord(addr, 0xDEADBEEF)

  def readFromRam(self):
    for addr in range(0x20000000, 0x200000d0, 0x04):
      print hex(self.ahb.readWord(addr))

  def readConfig(self):
    self.issueISPCommand(NUC1XX.CONFIG0_ADDR, 0x00, 0x00)
    print 'readConfig: %s' % hex(
        self.ahb.readWord(NUC1XX.CONFIG0_ADDR))

  def writeBinToFlash(self, binstr):
    start_addr = 0x00100000
    print 'length: %i' % len(binstr)
    for counter in range(0, len(binstr), 512):
      self.writeFlash(
          start_addr + counter,
          int('0x%s' % binascii.b2a_hex(binstr[counter:counter + 512]), 16))

def readFlashFile(filename):
  flashfile = open(filename, 'rb')
  return flashfile.read()

<commit_msg>correct NUC1XX.readFlash + cleanup<commit_after>#!/usr/bin/python

# SWD adapter for the Nucon NUC1XX series.

# TODO: make it work, flash can't be written directly through ISP as it seems

__author__ = 'Pascal Hahn <ph@lxd.bz>'

import binascii
import SWDCommon

class NotReadyForCommandException(Exception):
  pass

class UnlockFailedException(Exception):
  pass

class NUC1XX(object):
  CONFIG0_ADDR = 0x00300000

  ISPCON_ADDR = 0x5000C000
  ISPADR_ADDR = 0x5000C004
  ISPDAT_ADDR = 0x5000C008
  ISPCMD_ADDR = 0x5000C00C
  ISPTRG_ADDR = 0x5000C010

  IPRSTC_ADDR = 0x50000008

  DHCSR_ADDR = 0xE000EDF0

  DCRSR_ADDR = 0xE000EDF4
  DCRDR_ADDR = 0xE000EDF8

  AIRCR_ADDR = 0xE000ED0C

  REGRWPROT_ADDR = 0x50000100

  def __init__(self, debugport):
    self.ahb = SWDCommon.MEM_AP(debugport, 0)

  def halt (self):
    self.ahb.writeWord(NUC1XX.DHCSR_ADDR, 0xA05F0003)

  def unhalt(self):
    self.ahb.writeWord(NUC1XX.DHCSR_ADDR, 0xA05F0001)

  def step(self):
    self.ahb.writeWord(NUC1XX.DHCSR_ADDR, 0xA05F0005)

  def reset(self):
    self.ahb.writeWord(NUC1XX.IPRSTC_ADDR, 0x01)

  def flashUnlock(self):
    if self.ahb.readBlock(NUC1XX.REGRWPROT_ADDR, 0x01) != [0x00]:
      print 'flashUnlock: Already unlocked'
    else:
      # disable write protection
      self.ahb.writeWord(NUC1XX.REGRWPROT_ADDR, 0x59)
      self.ahb.writeWord(NUC1XX.REGRWPROT_ADDR, 0x16)
      self.ahb.writeWord(NUC1XX.REGRWPROT_ADDR, 0x88)

    if self.ahb.readBlock(NUC1XX.REGRWPROT_ADDR, 0x01) != [0x01]:
      raise UnlockFailedException('flachUnlock: Unlock didn\'t work')

    # enable ISP
    self.ahb.writeWord(NUC1XX.ISPCON_ADDR, 0x00000031)

  def changeBS(self):
    val = self.ahb.readWord(NUC1XX.ISPCON_ADDR)
    self.ahb.writeWord(NUC1XX.ISPCON_ADDR, val | 2)

  def issueISPCommand(self, adr, cmd, data):
    if self.ahb.readBlock(NUC1XX.ISPTRG_ADDR, 0x01) != [0x00]:
      raise NotReadyForCommandException('not yet ready for command')
    self.ahb.writeWord(NUC1XX.ISPADR_ADDR, adr)
    self.ahb.writeWord(NUC1XX.ISPCMD_ADDR, cmd)
    self.ahb.writeWord(NUC1XX.ISPDAT_ADDR, data)

    # trigger command execution
    self.ahb.writeWord(NUC1XX.ISPTRG_ADDR, 0x00000001)

    while True:
      # successful execution
      if self.ahb.readBlock(NUC1XX.ISPTRG_ADDR, 0x01) == [0x00]:
        break
      # TODO: for now we assume success, actually verify it
    ispcon = self.ahb.readWord(NUC1XX.ISPCON_ADDR)
    self.ahb.writeWord(NUC1XX.ISPCON_ADDR, ispcon)
    if ispcon & 0x40:
      print 'issueISPCOMMAND: ISP command failed: %08X' % ispcon

  def writeFlash(self, addr, data):
    print 'writing 0x%x to 0x%x' % (data, addr)
    self.issueISPCommand(addr, 0x21, data)

  def readFlash(self, start_addr, length):
    # TODO: maybe implement more efficient reading with only 1 dummy cmd
    for counter in range(0, length, 4):
      addr = start_addr + counter
      self.issueISPCommand(addr, 0x00, 0x00)
      data = self.ahb.readWord(NUC1XX.ISPDAT_ADDR)
      print 'reading 0x%x: 0x%x' % (addr, data)

  def eraseFlash(self):
    self.issueISPCommand(0x00100000, 0x22, 0x00)

  def readRegister(self, register):
    self.ahb.writeWord(NUC1XX.DCRSR_ADDR, register)
    return self.ahb.readWord(NUC1XX.DCRDR_ADDR)

  def writeRegister(self, register, data):
    self.ahb.writeWord(NUC1XX.DCRDR_ADDR, data)
    self.ahb.writeWord(NUC1XX.DCRSR_ADDR, register)

  def readAllRom(self):
    for addr in range(0xfffffffC, 0xffffffff, 0x04):
      print hex(self.ahb.readWord(addr))

  def writeToRam(self):
    for addr in range(0x20000000, 0x200000d0, 0x04):
      print 'writing %s' % hex(addr)
      self.ahb.writeWord(addr, 0xDEADBEEF)

  def readFromRam(self):
    for addr in range(0x20000000, 0x200000d0, 0x04):
      print hex(self.ahb.readWord(addr))

  def readConfig(self):
    self.issueISPCommand(NUC1XX.CONFIG0_ADDR, 0x00, 0x00)
    print 'readConfig: %s' % hex(
        self.ahb.readWord(NUC1XX.CONFIG0_ADDR))

  def writeBinToFlash(self, binstr):
    start_addr = 0x00100000
    print 'length: %i' % len(binstr)
    for counter in range(0, len(binstr), 512):
      self.writeFlash(
          start_addr + counter,
          int('0x%s' % binascii.b2a_hex(binstr[counter:counter + 512]), 16))

def readFlashFile(filename):
  flashfile = open(filename, 'rb')
  return flashfile.read()

<|endoftext|>


<commit_before>import logging, json
from lr.lib.harvest import harvest
import lr.lib.helpers as helpers
import iso8601
from datetime import datetime
from pylons import request, response, session, tmpl_context as c, url
from pylons.controllers.util import abort, redirect
from pylons.decorators import rest
from lr.lib.base import BaseController, render
import lr.lib.helpers
log = logging.getLogger(__name__)
import ast
import string
class HarvestController(BaseController):
    """REST Controller styled on the Atom Publishing Protocol"""
    # To properly map this controller, ensure your config/routing.py
    # file has a resource setup:
    #     map.resource('harvest', 'harvest')
    def __parse_date(self,date):
        last_update_date = iso8601.parse_date(date)
        last_update = helpers.convertToISO8601UTC(last_update_date)
        return last_update
    def _check_bool_param(self,params,key):
        if params.has_key(key):
            raw_value = string.lower(str(params[key]))
            if len(raw_value) > 1:
                print string.capitalize(raw_value)
                return ast.literal_eval(string.capitalize(raw_value))
            elif len(raw_value) == 1:
                if raw_value == 't':
                    return True
                elif raw_value == 'f':
                    return False
        return False
    def harvest(self, params, body, verb):
        h = harvest()
        def getrecord():
          data = self.get_base_response(verb,body)
          by_doc_ID = self._check_bool_param(params,'by_doc_ID')
          by_resource_ID = self._check_bool_param(params,'by_resource_ID')
          if not params.has_key('request_id'):
            data['OK'] = False
            data['error'] = 'badArgument'
            return json.dumps(data)
          if by_doc_ID and by_resource_ID:
            data['OK'] = False
            data['error'] = 'badArgument'
            return json.dumps(data)

          request_id = params['request_id']
          if by_doc_ID:
            document = h.get_record(request_id)
            if document is not None:
                records = map(lambda doc: {'record':{"header":{'identifier':doc.id, 'datestamp':helpers.convertToISO8601Zformat(datetime.today()),'status':'active'}},'resource_data':doc},[document])
            else:
                records = []
          else:
            records = map(lambda doc: {'record':{"header":{'identifier':doc.id, 'datestamp':helpers.convertToISO8601Zformat(datetime.today()),'status':'active'}},'resource_data':doc},h.get_records_by_resource(request_id))
          data['getrecord'] ={
            'record': records
            }
          return json.dumps(data)
        def listidentifiers():
            return self.list_identifiers(h,body,params,verb)
        def listrecords():
            return self.list_records(h,body,params,verb)
        def identify():
            data = self.get_base_response(verb,body)
            data['identify']={
                                    'node_id':        'string',
                                    'repositoryName':    'string',
                                    'baseURL':        'string',
                                    'protocolVersion':    '2.0',
                                    'service_version':    'string',
                                    'earliestDatestamp':    'string',
                                    'deletedRecord':    'string',
                                    'granularity':        'string',
                                    'adminEmail':        'string'
                                 }
            return json.dumps(data)
        def listmetadataformats():
            data = self.get_base_response(verb,body)
            data['listmetadataformats']=h.list_metadata_formats()
            return json.dumps(data)
        def listsets():
            data = self.get_base_response(verb,body)
            data['OK']=False
            data['error']='noSetHierarchy'
            return json.dumps(data)
        switch = {
                    'getrecord':getrecord,
                    'listrecords':listrecords,
                    'listidentifiers':listidentifiers,
                    'identify': identify,
                    'listmetadataformats': listmetadataformats,
                    'listsets': listsets
                 }
        return switch[verb]()
    def _test_time_params(self, params):

        if not params.has_key('from'):
            from_date = self.__parse_date('1990-10-10 12:12:12.0Z')
        else:
            from_date = self.__parse_date(params['from'])
        if not params.has_key('until'):
            until_date = self.__parse_date(datetime.utcnow().isoformat()+ "Z")
        else:
            until_date = self.__parse_date(params['until'])
        return from_date,until_date
    def list_records(self, h , body , params, verb = 'GET' ):
        data = self.get_base_response(verb,body)
        if params.has_key('from'):
            data['request']['from'] = params['from']
        if params.has_key('until'):
            data['request']['until'] = params['until']
        from_date, until_date = self._test_time_params(params)
        data['listrecords'] =  []
        base_response =  json.dumps(data).split('[')
        yield base_response[0] +'['
        def debug_map(doc):
            data ={'record':{"header":{'identifier':doc.id, 'datestamp':helpers.convertToISO8601Zformat(datetime.today()),'status':'active'},'resource_data':doc}}
            return data
        if from_date > until_date:
          data['OK'] = False
          data['error'] = 'badArgument'
        else:
            first = True
            for doc in h.list_records(from_date,until_date):
                if not first:
                    yield ','
                first = False
                yield json.dumps(debug_map(doc))
        yield base_response[1]

    def list_identifiers(self,h,body ,params, verb = 'GET'):
        data = self.get_base_response(verb,body)
        if params.has_key('from'):
            data['request']['from'] = params['from']
        if params.has_key('until'):
            data['request']['until'] = params['until']
        from_date, until_date = self._test_time_params(params)
        data['listidentifiers'] =  []
        base_response =  json.dumps(data).split('[')
        yield base_response[0] +'['
        first = True;
        for id in h.list_identifiers(from_date,until_date):
            if not first:
                yield ','
            first = False
            return_value = {"header":{'identifier':id, 'datestamp':helpers.convertToISO8601Zformat(datetime.today()) ,'status':'active'}}
            yield json.dumps(return_value)
        yield base_response[1]

    def get_base_response(self, verb, body):
      return {
               'OK':True,
               'error':'',
               'responseDate':helpers.convertToISO8601Zformat(datetime.today()),
               'request':{
                 'verb':verb,
                 'HTTP_request': body
                 }
              }

    def index(self, format='html'):
        """GET /harvest: All items in the collection"""
        abort(405,'Method not allowed')

    def create(self):
        """POST /harvest: Create a new item"""
        abort(405,'Method not allowed')

    def new(self, format='html'):
        """GET /harvest/new: Form to create a new item"""
        abort(405,'Method not allowed')
        # url('new_harvest')

    def update(self, id):
        """PUT /harvest/id: Update an existing item"""
        abort(405,'Method not allowed')
    def delete(self, id):
        """DELETE /harvest/id: Delete an existing item"""
        abort(405,'Method not allowed')
    def show(self, id, format='html'):
        """GET /harvest/id: Show a specific item"""
        if request.params.has_key('callback'):
            def jsonp(callback,params,body):
                yield '{0}('.format(callback)
                for i in self.harvest(params,body,id):
                    yield i
                yield ')'
            return jsonp(request.params['callback'],request.params,request.body)
        else:
            return self.harvest(request.params,request.body,id)
    def edit(self, id, format='html'):
        """GET /harvest/id/edit: Form to edit an existing item"""
        abort(405,'Method not allowed')

    #code below is to allow posting to /harvest/VERB
    #as REST uses POST only for creating, posting to an existing doc isn't allowed
    @rest.dispatch_on(POST='create_getrecord')
    def getrecord(self):
       """getrecord"""
    def create_getrecord(self):
        post_data = json.loads(request.body)
        return self.harvest(post_data,request.body,'getrecord')

    @rest.dispatch_on(POST='create_listrecords')
    def listrecords(self):
       """listrecords"""
       log.debug('test')
    def create_listrecords(self):
       params = json.loads(request.body)
       return self.harvest(params,request.body,'listrecords')

    @rest.dispatch_on(POST='create_listidentifiers')
    def listidentifiers(self):
       """listidentifiers"""
       log.debug('test')
    def create_listidentifiers(self):
       params = json.loads(request.body)
       return self.harvest(params,request.body,'listidentifiers')

    @rest.dispatch_on(POST='create_identify')
    def identify(self):
       """identify"""
       log.debug('test')
    def create_identify(self):
       return self.harvest(None,request.body,'identify')

    @rest.dispatch_on(POST='create_listmetadataformats')
    def listmetadataformats(self):
       """listmetadataformats"""
    def create_listmetadataformats(self):
       return self.harvest(None,request.body,'listmetadataformats')

    @rest.dispatch_on(POST='create_listsets')
    def listsets(self):
        """listsets"""
    def create_listsets(self):
       return self.harvest(None,request.body,'listsets')

<commit_msg>fixed get records to use ['id']<commit_after>import logging, json
from lr.lib.harvest import harvest
import lr.lib.helpers as helpers
import iso8601
from datetime import datetime
from pylons import request, response, session, tmpl_context as c, url
from pylons.controllers.util import abort, redirect
from pylons.decorators import rest
from lr.lib.base import BaseController, render
import lr.lib.helpers
log = logging.getLogger(__name__)
import ast
import string
class HarvestController(BaseController):
    """REST Controller styled on the Atom Publishing Protocol"""
    # To properly map this controller, ensure your config/routing.py
    # file has a resource setup:
    #     map.resource('harvest', 'harvest')
    def __parse_date(self,date):
        last_update_date = iso8601.parse_date(date)
        last_update = helpers.convertToISO8601UTC(last_update_date)
        return last_update
    def _check_bool_param(self,params,key):
        if params.has_key(key):
            raw_value = string.lower(str(params[key]))
            if len(raw_value) > 1:
                print string.capitalize(raw_value)
                return ast.literal_eval(string.capitalize(raw_value))
            elif len(raw_value) == 1:
                if raw_value == 't':
                    return True
                elif raw_value == 'f':
                    return False
        return False
    def harvest(self, params, body, verb):
        h = harvest()
        def getrecord():
          data = self.get_base_response(verb,body)
          by_doc_ID = self._check_bool_param(params,'by_doc_ID')
          by_resource_ID = self._check_bool_param(params,'by_resource_ID')
          if not params.has_key('request_id'):
            data['OK'] = False
            data['error'] = 'badArgument'
            return json.dumps(data)
          if by_doc_ID and by_resource_ID:
            data['OK'] = False
            data['error'] = 'badArgument'
            return json.dumps(data)

          request_id = params['request_id']
          if by_doc_ID:
            document = h.get_record(request_id)
            if document is not None:
                records = map(lambda doc: {'record':{"header":{'identifier':doc.id, 'datestamp':helpers.convertToISO8601Zformat(datetime.today()),'status':'active'}},'resource_data':doc},[document])
            else:
                records = []
          else:
            records = map(lambda doc: {'record':{"header":{'identifier':doc['_id'], 'datestamp':helpers.convertToISO8601Zformat(datetime.today()),'status':'active'}},'resource_data':doc},h.get_records_by_resource(request_id))
          data['getrecord'] ={
            'record': records
            }
          return json.dumps(data)
        def listidentifiers():
            return self.list_identifiers(h,body,params,verb)
        def listrecords():
            return self.list_records(h,body,params,verb)
        def identify():
            data = self.get_base_response(verb,body)
            data['identify']={
                                    'node_id':        'string',
                                    'repositoryName':    'string',
                                    'baseURL':        'string',
                                    'protocolVersion':    '2.0',
                                    'service_version':    'string',
                                    'earliestDatestamp':    'string',
                                    'deletedRecord':    'string',
                                    'granularity':        'string',
                                    'adminEmail':        'string'
                                 }
            return json.dumps(data)
        def listmetadataformats():
            data = self.get_base_response(verb,body)
            data['listmetadataformats']=h.list_metadata_formats()
            return json.dumps(data)
        def listsets():
            data = self.get_base_response(verb,body)
            data['OK']=False
            data['error']='noSetHierarchy'
            return json.dumps(data)
        switch = {
                    'getrecord':getrecord,
                    'listrecords':listrecords,
                    'listidentifiers':listidentifiers,
                    'identify': identify,
                    'listmetadataformats': listmetadataformats,
                    'listsets': listsets
                 }
        return switch[verb]()
    def _test_time_params(self, params):

        if not params.has_key('from'):
            from_date = self.__parse_date('1990-10-10 12:12:12.0Z')
        else:
            from_date = self.__parse_date(params['from'])
        if not params.has_key('until'):
            until_date = self.__parse_date(datetime.utcnow().isoformat()+ "Z")
        else:
            until_date = self.__parse_date(params['until'])
        return from_date,until_date
    def list_records(self, h , body , params, verb = 'GET' ):
        data = self.get_base_response(verb,body)
        if params.has_key('from'):
            data['request']['from'] = params['from']
        if params.has_key('until'):
            data['request']['until'] = params['until']
        from_date, until_date = self._test_time_params(params)
        data['listrecords'] =  []
        base_response =  json.dumps(data).split('[')
        yield base_response[0] +'['
        def debug_map(doc):
            data ={'record':{"header":{'identifier':doc.id, 'datestamp':helpers.convertToISO8601Zformat(datetime.today()),'status':'active'},'resource_data':doc}}
            return data
        if from_date > until_date:
          data['OK'] = False
          data['error'] = 'badArgument'
        else:
            first = True
            for doc in h.list_records(from_date,until_date):
                if not first:
                    yield ','
                first = False
                yield json.dumps(debug_map(doc))
        yield base_response[1]

    def list_identifiers(self,h,body ,params, verb = 'GET'):
        data = self.get_base_response(verb,body)
        if params.has_key('from'):
            data['request']['from'] = params['from']
        if params.has_key('until'):
            data['request']['until'] = params['until']
        from_date, until_date = self._test_time_params(params)
        data['listidentifiers'] =  []
        base_response =  json.dumps(data).split('[')
        yield base_response[0] +'['
        first = True;
        for id in h.list_identifiers(from_date,until_date):
            if not first:
                yield ','
            first = False
            return_value = {"header":{'identifier':id, 'datestamp':helpers.convertToISO8601Zformat(datetime.today()) ,'status':'active'}}
            yield json.dumps(return_value)
        yield base_response[1]

    def get_base_response(self, verb, body):
      return {
               'OK':True,
               'error':'',
               'responseDate':helpers.convertToISO8601Zformat(datetime.today()),
               'request':{
                 'verb':verb,
                 'HTTP_request': body
                 }
              }

    def index(self, format='html'):
        """GET /harvest: All items in the collection"""
        abort(405,'Method not allowed')

    def create(self):
        """POST /harvest: Create a new item"""
        abort(405,'Method not allowed')

    def new(self, format='html'):
        """GET /harvest/new: Form to create a new item"""
        abort(405,'Method not allowed')
        # url('new_harvest')

    def update(self, id):
        """PUT /harvest/id: Update an existing item"""
        abort(405,'Method not allowed')
    def delete(self, id):
        """DELETE /harvest/id: Delete an existing item"""
        abort(405,'Method not allowed')
    def show(self, id, format='html'):
        """GET /harvest/id: Show a specific item"""
        if request.params.has_key('callback'):
            def jsonp(callback,params,body):
                yield '{0}('.format(callback)
                for i in self.harvest(params,body,id):
                    yield i
                yield ')'
            return jsonp(request.params['callback'],request.params,request.body)
        else:
            return self.harvest(request.params,request.body,id)
    def edit(self, id, format='html'):
        """GET /harvest/id/edit: Form to edit an existing item"""
        abort(405,'Method not allowed')

    #code below is to allow posting to /harvest/VERB
    #as REST uses POST only for creating, posting to an existing doc isn't allowed
    @rest.dispatch_on(POST='create_getrecord')
    def getrecord(self):
       """getrecord"""
    def create_getrecord(self):
        post_data = json.loads(request.body)
        return self.harvest(post_data,request.body,'getrecord')

    @rest.dispatch_on(POST='create_listrecords')
    def listrecords(self):
       """listrecords"""
       log.debug('test')
    def create_listrecords(self):
       params = json.loads(request.body)
       return self.harvest(params,request.body,'listrecords')

    @rest.dispatch_on(POST='create_listidentifiers')
    def listidentifiers(self):
       """listidentifiers"""
       log.debug('test')
    def create_listidentifiers(self):
       params = json.loads(request.body)
       return self.harvest(params,request.body,'listidentifiers')

    @rest.dispatch_on(POST='create_identify')
    def identify(self):
       """identify"""
       log.debug('test')
    def create_identify(self):
       return self.harvest(None,request.body,'identify')

    @rest.dispatch_on(POST='create_listmetadataformats')
    def listmetadataformats(self):
       """listmetadataformats"""
    def create_listmetadataformats(self):
       return self.harvest(None,request.body,'listmetadataformats')

    @rest.dispatch_on(POST='create_listsets')
    def listsets(self):
        """listsets"""
    def create_listsets(self):
       return self.harvest(None,request.body,'listsets')

<|endoftext|>


<commit_before>from faker import Factory
from gittip import wireup, MAX_TIP, MIN_TIP
from gittip.models.participant import Participant

import decimal
import random
import string

faker = Factory.create()

platforms = ['github', 'twitter', 'bitbucket']

def _fake_thing(db, tablename, **kw):
    column_names = []
    column_value_placeholders = []
    column_values = []

    for k,v in kw.items():
        column_names.append(k)
        column_value_placeholders.append("%s")
        column_values.append(v)

    column_names = ", ".join(column_names)
    column_value_placeholders = ", ".join(column_value_placeholders)

    db.run( "INSERT INTO {} ({}) VALUES ({})"
            .format(tablename, column_names, column_value_placeholders)
          , column_values
           )

def fake_text_id(size=6, chars=string.ascii_lowercase + string.digits):
    """Create a random text id.
    """
    return ''.join(random.choice(chars) for x in range(size))

def fake_balance(max_amount=100):
    """Return a random amount between 0 and max_amount.
    """
    return random.random() * max_amount

def fake_int_id(nmax=2 ** 31 -1):
    """Create a random int id.
    """
    return random.randint(0, nmax)

def fake_participant(db, number="singular", is_admin=False, anonymous=False):
    """Create a fake User.
    """
    username = faker.firstName() + fake_text_id(3)
    _fake_thing( db
               , "participants"
               , id=fake_int_id()
               , username=username
               , username_lower=username.lower()
               , statement=faker.sentence()
               , ctime=faker.dateTimeThisYear()
               , is_admin=is_admin
               , balance=fake_balance()
               , anonymous=anonymous
               , goal=fake_balance()
               , balanced_account_uri=faker.uri()
               , last_ach_result=''
               , is_suspicious=False
               , last_bill_result=''  # Needed to not be suspicious
               , claimed_time=faker.dateTimeThisYear()
               , number=number
                )
    return Participant.from_username(username)

def fake_tip_amount():
    amount = ((decimal.Decimal(random.random()) * (MAX_TIP - MIN_TIP))
            + MIN_TIP)

    decimal_amount = decimal.Decimal(amount).quantize(decimal.Decimal('.01'))

    return decimal_amount

def fake_tip(db, tipper, tippee):
    """Create a fake tip.
    """
    _fake_thing( db
               , "tips"
               , id=fake_int_id()
               , ctime=faker.dateTimeThisYear()
               , mtime=faker.dateTimeThisMonth()
               , tipper=tipper.username
               , tippee=tippee.username
               , amount=fake_tip_amount()
                )

def fake_elsewhere(db, participant, platform=None):
    """Create a fake elsewhere.
    """
    if platform is None:
        platform = random.choice(platforms)

    info_templates = {
        "github": {
            "name": participant.username,
            "html_url": "https://github.com/" + participant.username,
            "type": "User",
            "login": participant.username
        },
        "twitter": {
            "name": participant.username,
            "html_url": "https://twitter.com/" + participant.username,
            "screen_name": participant.username
        },
        "bitbucket": {
            "display_name": participant.username,
            "username": participant.username,
            "is_team": "False",
            "html_url": "https://bitbucket.org/" + participant.username,
        }
    }

    _fake_thing( db
               , "elsewhere"
               , id=fake_int_id()
               , platform=platform
               , user_id=fake_text_id()
               , is_locked=False
               , participant=participant.username
               , user_info=info_templates[platform]
                )

def populate_db(db, num_participants=100, num_tips=50, num_teams=5):
    """Populate DB with fake data.
    """
    #Make the participants
    participants = []
    for i in xrange(num_participants):
        p = fake_participant(db)
        participants.append(p)

    #Make the "Elsewhere's"
    for p in participants:
        #All participants get between 1 and 3 elsewheres
        num_elsewheres = random.randint(1, 3)
        for platform_name in platforms[:num_elsewheres]:
            fake_elsewhere(db, p, platform_name)

    #Make teams
    for i in xrange(num_teams):
        t = fake_participant(db, number="plural")
        #Add 1 to 3 members to the team
        members = random.sample(participants, random.randint(1, 3))
        for p in members:
            t.add_member(p)

    #Make the tips
    for i in xrange(num_tips):
        tipper, tippee = random.sample(participants, 2)
        fake_tip(db, tipper, tippee)

def main():
    populate_db(wireup.db())

if __name__ == '__main__':
    main()
<commit_msg>Add fake transfers and paydays to fake_data script.<commit_after>from faker import Factory
from gittip import wireup, MAX_TIP, MIN_TIP
from gittip.models.participant import Participant

import decimal
import random
import string
import datetime

faker = Factory.create()

platforms = ['github', 'twitter', 'bitbucket']

def _fake_thing(db, tablename, **kw):
    column_names = []
    column_value_placeholders = []
    column_values = []

    for k,v in kw.items():
        column_names.append(k)
        column_value_placeholders.append("%s")
        column_values.append(v)

    column_names = ", ".join(column_names)
    column_value_placeholders = ", ".join(column_value_placeholders)

    db.run( "INSERT INTO {} ({}) VALUES ({})"
            .format(tablename, column_names, column_value_placeholders)
          , column_values
           )
    return kw

def fake_text_id(size=6, chars=string.ascii_lowercase + string.digits):
    """Create a random text id.
    """
    return ''.join(random.choice(chars) for x in range(size))

def fake_balance(max_amount=100):
    """Return a random amount between 0 and max_amount.
    """
    return random.random() * max_amount

def fake_int_id(nmax=2 ** 31 -1):
    """Create a random int id.
    """
    return random.randint(0, nmax)

def fake_participant(db, number="singular", is_admin=False, anonymous=False):
    """Create a fake User.
    """
    username = faker.firstName() + fake_text_id(3)
    d = _fake_thing( db
               , "participants"
               , id=fake_int_id()
               , username=username
               , username_lower=username.lower()
               , statement=faker.sentence()
               , ctime=faker.dateTimeThisYear()
               , is_admin=is_admin
               , balance=fake_balance()
               , anonymous=anonymous
               , goal=fake_balance()
               , balanced_account_uri=faker.uri()
               , last_ach_result=''
               , is_suspicious=False
               , last_bill_result=''  # Needed to not be suspicious
               , claimed_time=faker.dateTimeThisYear()
               , number=number
                )
    #Call participant constructor to perform other DB initialization
    return Participant.from_username(username)

def fake_tip_amount():
    amount = ((decimal.Decimal(random.random()) * (MAX_TIP - MIN_TIP))
            + MIN_TIP)

    decimal_amount = decimal.Decimal(amount).quantize(decimal.Decimal('.01'))

    return decimal_amount

def fake_tip(db, tipper, tippee):
    """Create a fake tip.
    """
    return _fake_thing( db
               , "tips"
               , id=fake_int_id()
               , ctime=faker.dateTimeThisYear()
               , mtime=faker.dateTimeThisMonth()
               , tipper=tipper.username
               , tippee=tippee.username
               , amount=fake_tip_amount()
                )

def fake_elsewhere(db, participant, platform=None):
    """Create a fake elsewhere.
    """
    if platform is None:
        platform = random.choice(platforms)

    info_templates = {
        "github": {
            "name": participant.username,
            "html_url": "https://github.com/" + participant.username,
            "type": "User",
            "login": participant.username
        },
        "twitter": {
            "name": participant.username,
            "html_url": "https://twitter.com/" + participant.username,
            "screen_name": participant.username
        },
        "bitbucket": {
            "display_name": participant.username,
            "username": participant.username,
            "is_team": "False",
            "html_url": "https://bitbucket.org/" + participant.username,
        }
    }

    _fake_thing( db
               , "elsewhere"
               , id=fake_int_id()
               , platform=platform
               , user_id=fake_text_id()
               , is_locked=False
               , participant=participant.username
               , user_info=info_templates[platform]
                )

def fake_transfer(db, tipper, tippee):
        return _fake_thing( db
               , "transfers"
               , id=fake_int_id()
               , timestamp=faker.dateTimeThisYear()
               , tipper=tipper.username
               , tippee=tippee.username
               , amount=fake_tip_amount()
                )

def populate_db(db, num_participants=300, num_tips=200, num_teams=5, num_transfers=300):
    """Populate DB with fake data.
    """
    #Make the participants
    participants = []
    for i in xrange(num_participants):
        participants.append(fake_participant(db))

    #Make the "Elsewhere's"
    for p in participants:
        #All participants get between 1 and 3 elsewheres
        num_elsewheres = random.randint(1, 3)
        for platform_name in platforms[:num_elsewheres]:
            fake_elsewhere(db, p, platform_name)

    #Make teams
    for i in xrange(num_teams):
        t = fake_participant(db, number="plural")
        #Add 1 to 3 members to the team
        members = random.sample(participants, random.randint(1, 3))
        for p in members:
            t.add_member(p)

    #Make the tips
    tips = []
    for i in xrange(num_tips):
        tipper, tippee = random.sample(participants, 2)
        tips.append(fake_tip(db, tipper, tippee))

    #Make the transfers
    transfers = []
    for i in xrange(num_tips):
        tipper, tippee = random.sample(participants, 2)
        transfers.append(fake_transfer(db, tipper, tippee))

    #Make some paydays
    #First determine the boundaries - min and max date
    min_date = min(min(x['ctime'] for x in tips), \
                   min(x['timestamp'] for x in transfers))
    max_date = max(max(x['ctime'] for x in tips), \
                   max(x['timestamp'] for x in transfers))
    #iterate through min_date, max_date one week at a time
    date = min_date
    while date < max_date:
        end_date = date + datetime.timedelta(days=7)
        week_tips = filter(lambda x: date <= x['ctime'] <= end_date, tips)
        week_transfers = filter(lambda x: date <= x['timestamp'] <= end_date, transfers)
        week_participants = filter(lambda x: x.ctime.replace(tzinfo=None) <= end_date, participants)
        actives=set()
        tippers=set()
        for xfers in week_tips, week_transfers:
            actives.update(x['tipper'] for x in xfers)
            actives.update(x['tippee'] for x in xfers)
            tippers.update(x['tipper'] for x in xfers)
        payday = {
            'id': fake_int_id(),
            'ts_start': date,
            'ts_end': end_date,
            'ntips': len(week_tips),
            'ntransfers': len(week_transfers),
            'nparticipants': len(week_participants),
            'ntippers': len(tippers),
            'nactive': len(actives),
            'transfer_volume': sum(x['amount'] for x in week_transfers)
        }
        #Make ach_volume and charge_volume between 0 and 10% of transfer volume
        def rand_part():
            return decimal.Decimal(random.random()* 0.1)
        payday['ach_volume']   = -1 * payday['transfer_volume'] * rand_part()
        payday['charge_volume'] = payday['transfer_volume'] * rand_part()
        _fake_thing(db, "paydays", **payday)
        date = end_date

def main():
    populate_db(wireup.db())

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Constrained network server (CNS) test base."""

import logging
import os
import Queue
import subprocess
import sys
import threading
import urllib2

import pyauto
import pyauto_paths

WINDOWS = 'win32' in sys.platform

# List of commonly used network constraints settings.
# Each setting is a tuppe of the form:
#    ('TEST_NAME', [BANDWIDTH_Kbps, LATENCY_ms, PACKET_LOSS_%])
#
# Note: The test name should satisfy the regex [\w\.-]+ (check
# tools/perf_expectations/tests/perf_expectations_unittest.py for details). It
# is used to name the result graphs on the dashboards.
#
# The WiFi, DSL, and Cable settings were taken from webpagetest.org as
# approximations of their respective real world networks. The settings were
# based on 2011 FCC Broadband Data report (http://www.fcc.gov/document/
# measuring-broadband-america-report-consumer-broadband-performance-us).
DialUp = ('DialUp', [56, 120, 5])
Slow = ('Slow', [256, 105, 1])
Wifi = ('Wifi', [1024, 60, 0])
DSL = ('DSL', [1541, 50, 0])
Cable = ('Cable', [5120, 28, 0])
NoConstraints = ('NoConstraints', [0, 0, 0])

# Path to CNS executable relative to source root.
_CNS_PATH = os.path.join(
    'media', 'tools', 'constrained_network_server', 'cns.py')

# Port to start the CNS on.
_CNS_PORT = 9000

# Base CNS URL, only requires & separated parameter names appended.
if WINDOWS:
  CNS_BASE_URL = 'http://chromeperf34.chrome:%d/ServeConstrained?' % _CNS_PORT
else:
  CNS_BASE_URL = 'http://127.0.0.1:%d/ServeConstrained?' % _CNS_PORT

# Used for server sanity check.
_TEST_VIDEO = 'roller.webm'

# Directory root to serve files from.
_ROOT_PATH = os.path.join(pyauto.PyUITest.DataDir(), 'pyauto_private', 'media')

class CNSTestBase(pyauto.PyUITest):
  """CNS test base hadles startup and teardown of CNS server."""

  def __init__(self, *args, **kwargs):
    """Initialize CNSTestBase by setting the arguments for CNS server.

    Args:
        Check cns.py command line argument list for details.
    """
    self._port = kwargs.get('port', _CNS_PORT)
    self._interface = kwargs.get('interface', 'lo')
    self._www_root = kwargs.get('www_root', _ROOT_PATH)
    self._verbose = kwargs.get('verbose', True)
    self._expiry_time = kwargs.get('expiry_time', 0)
    self._socket_timeout = kwargs.get('socket_timeout')
    pyauto.PyUITest.__init__(self, *args, **kwargs)

  def setUp(self):
    """Ensures the Constrained Network Server (CNS) server is up and running."""
    if WINDOWS:
      self._SetUpWin()
    else:
      self._SetUpLinux()

  def _SetUpWin(self):
    """Ensures the test can connect to the external CNS server."""
    if self.WaitUntil(self._CanAccessServer, retry_sleep=3, timeout=30,
                      debug=False):
      pyauto.PyUITest.setUp(self)
    else:
      self.fail('Failed to connect to CNS.')

  def _SetUpLinux(self):
    """Starts the CNS server locally."""
    cmd = [sys.executable, os.path.join(pyauto_paths.GetSourceDir(), _CNS_PATH),
           '--port', str(self._port),
           '--interface', self._interface,
           '--www-root', self._www_root,
           '--expiry-time', str(self._expiry_time)]

    if self._socket_timeout:
      cmd.extend(['--socket-timeout', str(self._socket_timeout)])
    if self._verbose:
      cmd.append('-v')
    logging.debug('Starting CNS server: %s ', ' '.join(cmd))

    self._cns_process = subprocess.Popen(cmd, stderr=subprocess.PIPE)
    ProcessLogger(self._cns_process)

    if self.WaitUntil(self._CanAccessServer, retry_sleep=3, timeout=30,
                      debug=False):
      pyauto.PyUITest.setUp(self)
    else:
      self.tearDown()
      self.fail('Failed to start CNS.')

  def _CanAccessServer(self):
    """Checks if the CNS server can serve a file with no network constraints."""
    test_url = ''.join([CNS_BASE_URL, 'f=', _TEST_VIDEO])
    try:
      return urllib2.urlopen(test_url) is not None
    except Exception:
      return False

  def tearDown(self):
    """Stops the Constrained Network Server (CNS)."""
    pyauto.PyUITest.tearDown(self)
    if not WINDOWS:
      logging.debug('Stopping CNS server.')
      # Do not use process.kill(), it will not clean up cns.
      self.Kill(self._cns_process.pid)
      # Need to wait since the process logger has a lock on the process stderr.
      self._cns_process.wait()
      self.assertFalse(self._cns_process.returncode is None)
      logging.debug('CNS server stopped.')

class ProcessLogger(threading.Thread):
  """A thread to log a process's stderr output."""

  def __init__(self, process):
    """Starts the process logger thread.

    Args:
      process: The process to log.
    """
    threading.Thread.__init__(self)
    self._process = process
    self.start()

  def run(self):
    """Adds debug statements for the process's stderr output."""
    line = True
    while line:
      line = self._process.stderr.readline()
      logging.debug(line.strip())

def GetFileURL(file_name, bandwidth=0, latency=0, loss=0, new_port=False):
  """Returns CNS URL for the file with specified constraints.

  Args:
    Check cns.ServeConstrained() args for more details.
  """
  video_url = [CNS_BASE_URL, 'f=' + file_name]
  if bandwidth > 0:
    video_url.append('bandwidth=%d' % bandwidth)
  if latency > 0:
    video_url.append('latency=%d' % latency)
  if loss > 0:
    video_url.append('loss=%d' % loss)
  if new_port:
    video_url.append('new_port=%s' % new_port)
  return '&'.join(video_url)

def CreateCNSPerfTasks(network_constraints_settings, test_media_files):
  """Returns a queue of tasks combinining network constrains with media files.

  Args:
    network_constraints_settings: List of (setting_name, setting_values)
        tupples.
    test_media_files: List of media files to run the tests on.
  """
  # Convert relative test path into an absolute path.
  tasks = Queue.Queue()
  for file_name in test_media_files:
    for series_name, settings in network_constraints_settings:
      logging.debug('Add test: %s\tSettings: %s\tMedia: %s', series_name,
                    settings, file_name)
      tasks.put((series_name, settings, file_name))

  return tasks
<commit_msg>Fix windows tests URL to CNS chromeperf34 bot.<commit_after># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Constrained network server (CNS) test base."""

import logging
import os
import Queue
import subprocess
import sys
import threading
import urllib2

import pyauto
import pyauto_paths

WINDOWS = 'win32' in sys.platform

# List of commonly used network constraints settings.
# Each setting is a tuppe of the form:
#    ('TEST_NAME', [BANDWIDTH_Kbps, LATENCY_ms, PACKET_LOSS_%])
#
# Note: The test name should satisfy the regex [\w\.-]+ (check
# tools/perf_expectations/tests/perf_expectations_unittest.py for details). It
# is used to name the result graphs on the dashboards.
#
# The WiFi, DSL, and Cable settings were taken from webpagetest.org as
# approximations of their respective real world networks. The settings were
# based on 2011 FCC Broadband Data report (http://www.fcc.gov/document/
# measuring-broadband-america-report-consumer-broadband-performance-us).
DialUp = ('DialUp', [56, 120, 5])
Slow = ('Slow', [256, 105, 1])
Wifi = ('Wifi', [1024, 60, 0])
DSL = ('DSL', [1541, 50, 0])
Cable = ('Cable', [5120, 28, 0])
NoConstraints = ('NoConstraints', [0, 0, 0])

# Path to CNS executable relative to source root.
_CNS_PATH = os.path.join(
    'media', 'tools', 'constrained_network_server', 'cns.py')

# Port to start the CNS on.
_CNS_PORT = 9000

# Base CNS URL, only requires & separated parameter names appended.
if WINDOWS:
  CNS_BASE_URL = 'http://chromeperf34:%d/ServeConstrained?' % _CNS_PORT
else:
  CNS_BASE_URL = 'http://127.0.0.1:%d/ServeConstrained?' % _CNS_PORT

# Used for server sanity check.
_TEST_VIDEO = 'roller.webm'

# Directory root to serve files from.
_ROOT_PATH = os.path.join(pyauto.PyUITest.DataDir(), 'pyauto_private', 'media')

class CNSTestBase(pyauto.PyUITest):
  """CNS test base hadles startup and teardown of CNS server."""

  def __init__(self, *args, **kwargs):
    """Initialize CNSTestBase by setting the arguments for CNS server.

    Args:
        Check cns.py command line argument list for details.
    """
    self._port = kwargs.get('port', _CNS_PORT)
    self._interface = kwargs.get('interface', 'lo')
    self._www_root = kwargs.get('www_root', _ROOT_PATH)
    self._verbose = kwargs.get('verbose', True)
    self._expiry_time = kwargs.get('expiry_time', 0)
    self._socket_timeout = kwargs.get('socket_timeout')
    pyauto.PyUITest.__init__(self, *args, **kwargs)

  def setUp(self):
    """Ensures the Constrained Network Server (CNS) server is up and running."""
    if WINDOWS:
      self._SetUpWin()
    else:
      self._SetUpLinux()

  def _SetUpWin(self):
    """Ensures the test can connect to the external CNS server."""
    if self.WaitUntil(self._CanAccessServer, retry_sleep=3, timeout=30,
                      debug=False):
      pyauto.PyUITest.setUp(self)
    else:
      self.fail('Failed to connect to CNS.')

  def _SetUpLinux(self):
    """Starts the CNS server locally."""
    cmd = [sys.executable, os.path.join(pyauto_paths.GetSourceDir(), _CNS_PATH),
           '--port', str(self._port),
           '--interface', self._interface,
           '--www-root', self._www_root,
           '--expiry-time', str(self._expiry_time)]

    if self._socket_timeout:
      cmd.extend(['--socket-timeout', str(self._socket_timeout)])
    if self._verbose:
      cmd.append('-v')
    logging.debug('Starting CNS server: %s ', ' '.join(cmd))

    self._cns_process = subprocess.Popen(cmd, stderr=subprocess.PIPE)
    ProcessLogger(self._cns_process)

    if self.WaitUntil(self._CanAccessServer, retry_sleep=3, timeout=30,
                      debug=False):
      pyauto.PyUITest.setUp(self)
    else:
      self.tearDown()
      self.fail('Failed to start CNS.')

  def _CanAccessServer(self):
    """Checks if the CNS server can serve a file with no network constraints."""
    test_url = ''.join([CNS_BASE_URL, 'f=', _TEST_VIDEO])
    try:
      return urllib2.urlopen(test_url) is not None
    except Exception:
      return False

  def tearDown(self):
    """Stops the Constrained Network Server (CNS)."""
    pyauto.PyUITest.tearDown(self)
    if not WINDOWS:
      logging.debug('Stopping CNS server.')
      # Do not use process.kill(), it will not clean up cns.
      self.Kill(self._cns_process.pid)
      # Need to wait since the process logger has a lock on the process stderr.
      self._cns_process.wait()
      self.assertFalse(self._cns_process.returncode is None)
      logging.debug('CNS server stopped.')

class ProcessLogger(threading.Thread):
  """A thread to log a process's stderr output."""

  def __init__(self, process):
    """Starts the process logger thread.

    Args:
      process: The process to log.
    """
    threading.Thread.__init__(self)
    self._process = process
    self.start()

  def run(self):
    """Adds debug statements for the process's stderr output."""
    line = True
    while line:
      line = self._process.stderr.readline()
      logging.debug(line.strip())

def GetFileURL(file_name, bandwidth=0, latency=0, loss=0, new_port=False):
  """Returns CNS URL for the file with specified constraints.

  Args:
    Check cns.ServeConstrained() args for more details.
  """
  video_url = [CNS_BASE_URL, 'f=' + file_name]
  if bandwidth > 0:
    video_url.append('bandwidth=%d' % bandwidth)
  if latency > 0:
    video_url.append('latency=%d' % latency)
  if loss > 0:
    video_url.append('loss=%d' % loss)
  if new_port:
    video_url.append('new_port=%s' % new_port)
  return '&'.join(video_url)

def CreateCNSPerfTasks(network_constraints_settings, test_media_files):
  """Returns a queue of tasks combinining network constrains with media files.

  Args:
    network_constraints_settings: List of (setting_name, setting_values)
        tupples.
    test_media_files: List of media files to run the tests on.
  """
  # Convert relative test path into an absolute path.
  tasks = Queue.Queue()
  for file_name in test_media_files:
    for series_name, settings in network_constraints_settings:
      logging.debug('Add test: %s\tSettings: %s\tMedia: %s', series_name,
                    settings, file_name)
      tasks.put((series_name, settings, file_name))

  return tasks
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# file: x-ornata-chroma-rgb.pyw
# vim:fileencoding=utf-8:fdm=marker:ft=python
#
# Copyright © 2020 R.F. Smith <rsmith@xs4all.nl>.
# SPDX-License-Identifier: MIT
# Created: 2020-03-14T22:44:16+0100
# Last modified: 2020-03-17T23:03:27+0100
"""Set the LEDs on a Razer Ornata Chroma keyboard to a static RGB color."""

from tkinter import messagebox
from tkinter import ttk
from tkinter.font import nametofont
from types import SimpleNamespace
import os
import sys
import tkinter as tk
import usb.core

__version__ = '1.0'

def create_widgets(root):
    """Create the window and its widgets.

    Arguments:
        root: the root window.

    Returns:
        A SimpleNamespace of widgets

    """
    # Set the font.
    default_font = nametofont("TkDefaultFont")
    default_font.configure(size=12)
    root.option_add("*Font", default_font)
    # General commands and bindings
    root.bind_all('q', do_q)
    root.wm_title('Razer Ornata Chroma color v' + __version__)
    root.columnconfigure(1, weight=1)
    root.rowconfigure(0, weight=1)
    root.rowconfigure(1, weight=1)
    root.rowconfigure(2, weight=1)
    # SimpleNamespace to save widgets that need to be accessed later.
    w = SimpleNamespace()
    # First row
    ttk.Button(root, text='Red', command=set_red).grid(row=0, column=0, sticky="ew")
    red = tk.Scale(root, from_=0, to=255, orient=tk.HORIZONTAL, length=255,
                   command=do_red)
    red.grid(row=0, column=1, sticky="nsew")
    w.red = red
    show = tk.Frame(root, width=100, height=100, bg='#000000')
    show.grid(row=0, column=2, rowspan=3)
    w.show = show
    # Second row
    ttk.Button(root, text='Green', command=set_green).grid(row=1, column=0, sticky="ew")
    green = tk.Scale(root, from_=0, to=255, orient=tk.HORIZONTAL, length=255,
                     command=do_green)
    green.grid(row=1, column=1, sticky="ew")
    w.green = green
    # Third row
    ttk.Button(root, text='Blue', command=set_blue).grid(row=2, column=0, sticky="ew")
    blue = tk.Scale(root, from_=0, to=255, orient=tk.HORIZONTAL, length=255,
                    command=do_blue)
    blue.grid(row=2, column=1, sticky="ew")
    w.blue = blue
    # Last row
    b = ttk.Button(root, text="Quit", command=do_exit)
    b.grid(row=3, column=0, sticky='ew')
    setb = ttk.Button(root, text="Set", command=do_set)
    setb.grid(row=3, column=2, sticky='e')
    w.setb = setb
    # Return the widgets that need to be accessed.
    return w

# Callbacks
def do_q(arg):
    """Handle the q key."""
    root.destroy()

def do_exit():
    """Callback to handle the quit button."""
    root.destroy()

def update_color():
    """Helper function to update the color example."""
    value = f'#{state.red:02x}{state.green:02x}{state.blue:02x}'
    w.show['bg'] = value

def set_red():
    """Set the color to pure red."""
    w.red.set(255)
    w.green.set(0)
    w.blue.set(0)

def set_green():
    """Set the color to pure green."""
    w.red.set(0)
    w.green.set(255)
    w.blue.set(0)

def set_blue():
    """Set the color to pure blue."""
    w.red.set(0)
    w.green.set(0)
    w.blue.set(255)

def do_red(r):
    """Process movement of the red slider."""
    state.red = int(r)
    update_color()

def do_green(g):
    """Process movement of the green slider."""
    state.green = int(g)
    update_color()

def do_blue(b):
    """Process movement of the blue slider."""
    state.blue = int(b)
    update_color()

def do_set():
    """Callback to set the color on the keyboard."""
    w.setb['state'] = tk.DISABLED
    msg = static_color_msg(state.red, state.green, state.blue)
    # 0x21: request_type USB_TYPE_CLASS | USB_RECIP_INTERFACE | USB_DIR_OUT
    # 0x09: request HID_REQ_SET_REPORT
    # 0x300: value
    # 0x01: report index HID_REQ_GET_REPORT
    read = state.dev.ctrl_transfer(0x21, 0x09, 0x300, 0x01, msg)
    if read != 90:
        messagebox.showerror('Set color', 'Operation failed.')
    w.setb['state'] = tk.NORMAL

def static_color_msg(red, green, blue):
    """
    Create a message to set the Razer Ornata Chroma lights to a static color.
    All arguments should be an integer in the range 0-255.

    Returns an bytes object containing the message ready to feed into a ctrl_transfer.
    """
    # Meaning of the nonzero bytes, in sequence: 0x3f = transaction id, 0x09
    # = length of used arguments, 0x0f = command class, 0x02 = command id
    hdr = b'\x00\x3f\x00\x00\x00\x09\x0f\x02'
    # Meaning of the nonzero bytes, in sequence: 0x01 = VARSTORE, 0x05
    # = BACKLIGHT_LED, 0x01 = effect id, 0x01 = unknown
    arguments = b'\x01\x05\x01\x00\x00\x01'
    msg = hdr + arguments + bytes([red, green, blue])  # Add color.
    chksum = 0
    for j in msg[2:]:  # Calculate the checksum
        chksum ^= j
    msg += bytes(80 - 9)  # Add filler; the variable message buffer is 80 bytes.
    msg += bytes([chksum, 0])  # Add checksum in penultimate byte
    return msg

if __name__ == '__main__':
    # Detach from the command line on UNIX systems.
    if os.name == 'posix':
        if os.fork():
            sys.exit()
    # Global state
    state = SimpleNamespace()
    state.red, state.green, state.blue = 0, 0, 0
    state.dev = usb.core.find(idVendor=0x1532, idProduct=0x021e)
    # Create the GUI window.
    root = tk.Tk(None)
    if state.dev is not None:
        # w is a namespace of widgets that needs to be accessed by the callbacks.
        w = create_widgets(root)
    else:
        messagebox.showerror('Device detection', 'No Ornata Chroma found')
        root.destroy()
    root.mainloop()
<commit_msg>Add Id for Razer Blackwidow Elite.<commit_after>#!/usr/bin/env python3
# file: x-ornata-chroma-rgb.pyw
# vim:fileencoding=utf-8:fdm=marker:ft=python
#
# Copyright © 2020 R.F. Smith <rsmith@xs4all.nl>.
# SPDX-License-Identifier: MIT
# Created: 2020-03-14T22:44:16+0100
# Last modified: 2020-03-20T18:54:11+0100
"""Set the LEDs on a Razer Ornata Chroma keyboard to a static RGB color."""

from tkinter import messagebox
from tkinter import ttk
from tkinter.font import nametofont
from types import SimpleNamespace
import os
import sys
import tkinter as tk
import usb.core

__version__ = '1.1'

def create_widgets(root):
    """Create the window and its widgets.

    Arguments:
        root: the root window.

    Returns:
        A SimpleNamespace of widgets

    """
    # Set the font.
    default_font = nametofont("TkDefaultFont")
    default_font.configure(size=12)
    root.option_add("*Font", default_font)
    # General commands and bindings
    root.bind_all('q', do_q)
    root.wm_title('Razer keyboard color v' + __version__)
    root.columnconfigure(1, weight=1)
    root.rowconfigure(0, weight=1)
    root.rowconfigure(1, weight=1)
    root.rowconfigure(2, weight=1)
    # SimpleNamespace to save widgets that need to be accessed later.
    w = SimpleNamespace()
    # First row
    ttk.Button(root, text='Red', command=set_red).grid(row=0, column=0, sticky="ew")
    red = tk.Scale(
        root, from_=0, to=255, orient=tk.HORIZONTAL, length=255, command=do_red
    )
    red.grid(row=0, column=1, sticky="nsew")
    w.red = red
    show = tk.Frame(root, width=100, height=100, bg='#000000')
    show.grid(row=0, column=2, rowspan=3)
    w.show = show
    # Second row
    ttk.Button(root, text='Green', command=set_green).grid(row=1, column=0, sticky="ew")
    green = tk.Scale(
        root, from_=0, to=255, orient=tk.HORIZONTAL, length=255, command=do_green
    )
    green.grid(row=1, column=1, sticky="ew")
    w.green = green
    # Third row
    ttk.Button(root, text='Blue', command=set_blue).grid(row=2, column=0, sticky="ew")
    blue = tk.Scale(
        root, from_=0, to=255, orient=tk.HORIZONTAL, length=255, command=do_blue
    )
    blue.grid(row=2, column=1, sticky="ew")
    w.blue = blue
    # Last row
    b = ttk.Button(root, text="Quit", command=do_exit)
    b.grid(row=3, column=0, sticky='ew')
    setb = ttk.Button(root, text="Set", command=do_set)
    setb.grid(row=3, column=2, sticky='e')
    w.setb = setb
    # Return the widgets that need to be accessed.
    return w

# Callbacks
def do_q(arg):
    """Handle the q key."""
    root.destroy()

def do_exit():
    """Callback to handle the quit button."""
    root.destroy()

def update_color():
    """Helper function to update the color example."""
    value = f'#{state.red:02x}{state.green:02x}{state.blue:02x}'
    w.show['bg'] = value

def set_red():
    """Set the color to pure red."""
    w.red.set(255)
    w.green.set(0)
    w.blue.set(0)

def set_green():
    """Set the color to pure green."""
    w.red.set(0)
    w.green.set(255)
    w.blue.set(0)

def set_blue():
    """Set the color to pure blue."""
    w.red.set(0)
    w.green.set(0)
    w.blue.set(255)

def do_red(r):
    """Process movement of the red slider."""
    state.red = int(r)
    update_color()

def do_green(g):
    """Process movement of the green slider."""
    state.green = int(g)
    update_color()

def do_blue(b):
    """Process movement of the blue slider."""
    state.blue = int(b)
    update_color()

def do_set():
    """Callback to set the color on the keyboard."""
    w.setb['state'] = tk.DISABLED
    msg = static_color_msg(state.red, state.green, state.blue)
    # 0x21: request_type USB_TYPE_CLASS | USB_RECIP_INTERFACE | USB_DIR_OUT
    # 0x09: request HID_REQ_SET_REPORT
    # 0x300: value
    # 0x01: report index HID_REQ_GET_REPORT
    read = state.dev.ctrl_transfer(0x21, 0x09, 0x300, 0x01, msg)
    if read != 90:
        messagebox.showerror('Set color', 'Operation failed.')
    w.setb['state'] = tk.NORMAL

def static_color_msg(red, green, blue):
    """
    Create a message to set the Razer Ornata Chroma lights to a static color.
    All arguments should be an integer in the range 0-255.

    Returns an bytes object containing the message ready to feed into a ctrl_transfer.
    """
    # Meaning of the nonzero bytes, in sequence: 0x3f = transaction id, 0x09
    # = length of used arguments, 0x0f = command class, 0x02 = command id
    hdr = b'\x00\x3f\x00\x00\x00\x09\x0f\x02'
    # Meaning of the nonzero bytes, in sequence: 0x01 = VARSTORE, 0x05
    # = BACKLIGHT_LED, 0x01 = effect id, 0x01 = unknown
    arguments = b'\x01\x05\x01\x00\x00\x01'
    msg = hdr + arguments + bytes([red, green, blue])  # Add color.
    chksum = 0
    for j in msg[2:]:  # Calculate the checksum
        chksum ^= j
    msg += bytes(80 - 9)  # Add filler; the variable message buffer is 80 bytes.
    msg += bytes([chksum, 0])  # Add checksum in penultimate byte
    return msg

if __name__ == '__main__':
    # Detach from the command line on UNIX systems.
    if os.name == 'posix':
        if os.fork():
            sys.exit()
    # Find devices
    devs = [usb.core.find(idVendor=0x1532, idProduct=p) for p in (0x021e, 0x0228)]
    devs = [dev for dev in devs if dev]
    # Global state
    state = SimpleNamespace()
    state.red, state.green, state.blue = 0, 0, 0
    state.dev = None
    if len(devs) == 1:
        state.dev = devs[0]
    # Create the GUI window.
    root = tk.Tk(None)
    if state.dev is not None:
        # w is a namespace of widgets that needs to be accessed by the callbacks.
        w = create_widgets(root)
    else:
        messagebox.showerror('Device detection', 'No Ornata Chroma found')
        root.destroy()
    root.mainloop()
<|endoftext|>


<commit_before># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import math
import numpy
import random

from nupic.encoders.utils import bitsToString

# Default SDR dimensions for 2% sparsity:
DEFAULT_N = 16384
DEFAULT_W = 328

class LanguageEncoder(object):
  """
  An encoder converts a value to a sparse distributed representation (SDR).

  The Encoder superclass implements:
  - bitmapToSDR() returns binary SDR of a bitmap
  - bitmapFromSDR() returns the bitmap rep of an SDR
  - pprintHeader() prints a header describing the encoding to the terminal
  - pprint() prints an encoding to the terminal
  - decodedToStr() returns pretty print string of decoded SDR

  Methods/properties that must be implemented by subclasses:
  - encode() returns a numpy array encoding the input
  - decode() returns a list of strings representing a decoded SDR
  - getWidth() returns the output width, in bits
  - getDescription() returns a dict describing the encoded output
  """

  def __init__(self, n=DEFAULT_N, w=DEFAULT_W, unionSparsity=20.0):
    """The SDR dimensions are standard for Cortical.io fingerprints."""
    self.n = n
    self.w = w
    self.unionSparsity = unionSparsity
    self.targetSparsity = float(self.w) / self.n

  def encode(self, inputText):
    """
    Encodes inputText and puts the encoded value into the numpy output array,
    which is a 1-D array of length returned by getWidth().
    Note: The numpy output array is reused, so clear it before updating it.

    @param inputData      (str)     Data to encode. This should be validated by
                                    the encoder subclass.
    @param output         (numpy)   1-D array of same length returned by
                                    getWidth().
    """
    raise NotImplementedError

  def encodeIntoArray(self, inputText, output):
    """
    Encodes inputData and puts the encoded value into the numpy output array,
    which is a 1-D array of length returned by getWidth().

    Note: The numpy output array is reused, so clear it before updating it.

    @param inputData Data to encode. This should be validated by the encoder.
    @param output numpy 1-D array of same length returned by getWidth()
    """
    raise NotImplementedError

  def decode(self, encoded):
    """
    Decodes the SDR encoded. See subclass implementation for details; the
    decoding approaches and return objects vary depending on the encoder.

    To pretty print the return value from this method, use decodedToStr().

    @param encoded        (numpy)     Encoded 1-d array (an SDR).
    """
    raise NotImplementedError

  def getWidth(self):
    """
    Get an encoding's output width in bits. See subclass implementation for
    details.
    """
    raise NotImplementedError()

  def getDescription(self):
    """
    Returns a tuple, each containing (name, offset).
    The name is a string description of each sub-field, and offset is the bit
    offset of the sub-field for that encoder; should be 0.
    """
    raise NotImplementedError()

  def bitmapToSDR(self, bitmap):
    """Convert SDR encoding from bitmap to binary numpy array."""
    sdr = numpy.zeros(self.n)
    sdr[bitmap] = 1
    return sdr

  def bitmapFromSDR(self, sdr):
    """Convert SDR encoding from binary numpy array to bitmap."""
    return numpy.array([i for i in range(len(sdr)) if sdr[i]==1])

  def encodeRandomly(self, text):
    """Return a random bitmap representation of the sample."""
    random.seed(sample)
    return numpy.sort(random.sample(xrange(self.n), self.w))

  def compare(self, bitmap1, bitmap2):
    """
    Compare bitmaps, returning a dict of similarity measures.

    @param bitmap1     (list)        Indices of ON bits.
    @param bitmap2     (list)        Indices of ON bits.
    @return distances  (dict)        Key-values of distance metrics and values.

    Example return dict:
      {
        "cosineSimilarity": 0.6666666666666666,
        "euclideanDistance": 0.3333333333333333,
        "jaccardDistance": 0.5,
        "overlappingAll": 6,
        "overlappingLeftRight": 0.6666666666666666,
        "overlappingRightLeft": 0.6666666666666666,
        "sizeLeft": 9,
        "sizeRight": 9
      }
    """
    if not len(bitmap1) > 0 or not len(bitmap2) > 0:
      raise ValueError("Bitmaps must have ON bits to compare.")

    sdr1 = self.bitmapToSDR(bitmap1)
    sdr2 = self.bitmapToSDR(bitmap2)

    distances = {
      "sizeLeft": float(len(bitmap1)),
      "sizeRight": float(len(bitmap2)),
      "overlappingAll": float(len(numpy.intersect1d(bitmap1, bitmap2))),
      "euclideanDistance": numpy.linalg.norm(sdr1 - sdr2)
    }

    distances["overlappingLeftRight"] = (distances["overlappingAll"] /
                                         distances["sizeLeft"])
    distances["overlappingRightLeft"] = (distances["overlappingAll"] /
                                         distances["sizeRight"])
    distances["cosineSimilarity"] = (distances["overlappingAll"] /
        (math.sqrt(distances["sizeLeft"]) * math.sqrt(distances["sizeRight"])))
    distances["jaccardDistance"] = 1 - (distances["overlappingAll"] /
        len(numpy.union1d(bitmap1, bitmap2)))

    return distances

  def sparseUnion(self, counts):
    """
    Bits from the input patterns are unionized and then sparsified.

    @param counts     (Counter)   A count of the ON bits for the union bitmap.

    @return           (list)      A sparsified union bitmap.
    """
    max_sparsity = int((self.unionSparsity / 100) * self.n)
    w = min(len(counts), max_sparsity)
    return [c[0] for c in counts.most_common(w)]  # TODO: how does this break ties?

  @staticmethod
  def scaleEncoding(encoding, scaleFactor):
    """
    Scale down the size of the encoding by a factor between 0 and 1.

    @param encoding     (array)   Bitmap encoding.
    @param scaleFactor  (float)   Factor between 0 and 1 to scale down the SDR's
                                  size (n) by.
    @return             (list)    Scaled down bitmap of the encoding.
    """
    scaledBitmap = [int(i * float(scaleFactor)) for i in encoding]
    return sorted(set(scaledBitmap), key=lambda x: scaledBitmap.index(x))

  def pprintHeader(self, prefix=""):
    """
    Pretty-print a header that labels the sub-fields of the encoded output.
    This can be used in conjuction with pprint().
    @param prefix printed before the header if specified
    """
    print prefix,
    description = self.getDescription() + [("end", self.getWidth())]
    for i in xrange(len(description) - 1):
      name = description[i][0]
      width = description[i+1][1] - description[i][1]
      formatStr = "%%-%ds |" % width
      if len(name) > width:
        pname = name[0:width]
      else:
        pname = name
      print formatStr % pname,
    print
    print prefix, "-" * (self.getWidth() + (len(description) - 1)*3 - 1)

  def pprint(self, output, prefix=""):
    """
    Pretty-print the encoded output using ascii art.
    @param output to print
    @param prefix printed before the header if specified
    """
    print prefix,
    description = self.getDescription() + [("end", self.getWidth())]
    for i in xrange(len(description) - 1):
      offset = description[i][1]
      nextoffset = description[i+1][1]
      print "%s |" % bitsToString(output[offset:nextoffset]),
    print

  def decodedToStr(self, decodeResults):
    """
    Return a pretty print string representing the return value from decode().
    """

    (fieldsDict, fieldsOrder) = decodeResults

    desc = ''
    for fieldName in fieldsOrder:
      (ranges, rangesStr) = fieldsDict[fieldName]
      if len(desc) > 0:
        desc += ", %s:" % (fieldName)
      else:
        desc += "%s:" % (fieldName)

      desc += "[%s]" % (rangesStr)

    return desc
<commit_msg>seed random encodings properly<commit_after># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import math
import numpy
import random

from nupic.encoders.utils import bitsToString

# Default SDR dimensions for 2% sparsity:
DEFAULT_N = 16384
DEFAULT_W = 328

class LanguageEncoder(object):
  """
  An encoder converts a value to a sparse distributed representation (SDR).

  The Encoder superclass implements:
  - bitmapToSDR() returns binary SDR of a bitmap
  - bitmapFromSDR() returns the bitmap rep of an SDR
  - pprintHeader() prints a header describing the encoding to the terminal
  - pprint() prints an encoding to the terminal
  - decodedToStr() returns pretty print string of decoded SDR

  Methods/properties that must be implemented by subclasses:
  - encode() returns a numpy array encoding the input
  - decode() returns a list of strings representing a decoded SDR
  - getWidth() returns the output width, in bits
  - getDescription() returns a dict describing the encoded output
  """

  def __init__(self, n=DEFAULT_N, w=DEFAULT_W, unionSparsity=20.0):
    """The SDR dimensions are standard for Cortical.io fingerprints."""
    self.n = n
    self.w = w
    self.unionSparsity = unionSparsity
    self.targetSparsity = float(self.w) / self.n

  def encode(self, inputText):
    """
    Encodes inputText and puts the encoded value into the numpy output array,
    which is a 1-D array of length returned by getWidth().
    Note: The numpy output array is reused, so clear it before updating it.

    @param inputData      (str)     Data to encode. This should be validated by
                                    the encoder subclass.
    @param output         (numpy)   1-D array of same length returned by
                                    getWidth().
    """
    raise NotImplementedError

  def encodeIntoArray(self, inputText, output):
    """
    Encodes inputData and puts the encoded value into the numpy output array,
    which is a 1-D array of length returned by getWidth().

    Note: The numpy output array is reused, so clear it before updating it.

    @param inputData Data to encode. This should be validated by the encoder.
    @param output numpy 1-D array of same length returned by getWidth()
    """
    raise NotImplementedError

  def decode(self, encoded):
    """
    Decodes the SDR encoded. See subclass implementation for details; the
    decoding approaches and return objects vary depending on the encoder.

    To pretty print the return value from this method, use decodedToStr().

    @param encoded        (numpy)     Encoded 1-d array (an SDR).
    """
    raise NotImplementedError

  def getWidth(self):
    """
    Get an encoding's output width in bits. See subclass implementation for
    details.
    """
    raise NotImplementedError()

  def getDescription(self):
    """
    Returns a tuple, each containing (name, offset).
    The name is a string description of each sub-field, and offset is the bit
    offset of the sub-field for that encoder; should be 0.
    """
    raise NotImplementedError()

  def bitmapToSDR(self, bitmap):
    """Convert SDR encoding from bitmap to binary numpy array."""
    sdr = numpy.zeros(self.n)
    sdr[bitmap] = 1
    return sdr

  def bitmapFromSDR(self, sdr):
    """Convert SDR encoding from binary numpy array to bitmap."""
    return numpy.array([i for i in range(len(sdr)) if sdr[i]==1])

  def encodeRandomly(self, text, w, n):
    """Return a random bitmap representation of the text."""
    random.seed(text)
    return numpy.sort(random.sample(xrange(n), w))

  def compare(self, bitmap1, bitmap2):
    """
    Compare bitmaps, returning a dict of similarity measures.

    @param bitmap1     (list)        Indices of ON bits.
    @param bitmap2     (list)        Indices of ON bits.
    @return distances  (dict)        Key-values of distance metrics and values.

    Example return dict:
      {
        "cosineSimilarity": 0.6666666666666666,
        "euclideanDistance": 0.3333333333333333,
        "jaccardDistance": 0.5,
        "overlappingAll": 6,
        "overlappingLeftRight": 0.6666666666666666,
        "overlappingRightLeft": 0.6666666666666666,
        "sizeLeft": 9,
        "sizeRight": 9
      }
    """
    if not len(bitmap1) > 0 or not len(bitmap2) > 0:
      raise ValueError("Bitmaps must have ON bits to compare.")

    sdr1 = self.bitmapToSDR(bitmap1)
    sdr2 = self.bitmapToSDR(bitmap2)

    distances = {
      "sizeLeft": float(len(bitmap1)),
      "sizeRight": float(len(bitmap2)),
      "overlappingAll": float(len(numpy.intersect1d(bitmap1, bitmap2))),
      "euclideanDistance": numpy.linalg.norm(sdr1 - sdr2)
    }

    distances["overlappingLeftRight"] = (distances["overlappingAll"] /
                                         distances["sizeLeft"])
    distances["overlappingRightLeft"] = (distances["overlappingAll"] /
                                         distances["sizeRight"])
    distances["cosineSimilarity"] = (distances["overlappingAll"] /
        (math.sqrt(distances["sizeLeft"]) * math.sqrt(distances["sizeRight"])))
    distances["jaccardDistance"] = 1 - (distances["overlappingAll"] /
        len(numpy.union1d(bitmap1, bitmap2)))

    return distances

  def sparseUnion(self, counts):
    """
    Bits from the input patterns are unionized and then sparsified.

    @param counts     (Counter)   A count of the ON bits for the union bitmap.

    @return           (list)      A sparsified union bitmap.
    """
    max_sparsity = int((self.unionSparsity / 100) * self.n)
    w = min(len(counts), max_sparsity)
    return [c[0] for c in counts.most_common(w)]  # TODO: how does this break ties?

  @staticmethod
  def scaleEncoding(encoding, scaleFactor):
    """
    Scale down the size of the encoding by a factor between 0 and 1.

    @param encoding     (array)   Bitmap encoding.
    @param scaleFactor  (float)   Factor between 0 and 1 to scale down the SDR's
                                  size (n) by.
    @return             (list)    Scaled down bitmap of the encoding.
    """
    scaledBitmap = [int(i * float(scaleFactor)) for i in encoding]
    return sorted(set(scaledBitmap), key=lambda x: scaledBitmap.index(x))

  def pprintHeader(self, prefix=""):
    """
    Pretty-print a header that labels the sub-fields of the encoded output.
    This can be used in conjuction with pprint().
    @param prefix printed before the header if specified
    """
    print prefix,
    description = self.getDescription() + [("end", self.getWidth())]
    for i in xrange(len(description) - 1):
      name = description[i][0]
      width = description[i+1][1] - description[i][1]
      formatStr = "%%-%ds |" % width
      if len(name) > width:
        pname = name[0:width]
      else:
        pname = name
      print formatStr % pname,
    print
    print prefix, "-" * (self.getWidth() + (len(description) - 1)*3 - 1)

  def pprint(self, output, prefix=""):
    """
    Pretty-print the encoded output using ascii art.
    @param output to print
    @param prefix printed before the header if specified
    """
    print prefix,
    description = self.getDescription() + [("end", self.getWidth())]
    for i in xrange(len(description) - 1):
      offset = description[i][1]
      nextoffset = description[i+1][1]
      print "%s |" % bitsToString(output[offset:nextoffset]),
    print

  def decodedToStr(self, decodeResults):
    """
    Return a pretty print string representing the return value from decode().
    """

    (fieldsDict, fieldsOrder) = decodeResults

    desc = ''
    for fieldName in fieldsOrder:
      (ranges, rangesStr) = fieldsDict[fieldName]
      if len(desc) > 0:
        desc += ", %s:" % (fieldName)
      else:
        desc += "%s:" % (fieldName)

      desc += "[%s]" % (rangesStr)

    return desc
<|endoftext|>


<commit_before>from splitwise import Splitwise
from splitwise.comment import Comment
import unittest
try:
    from unittest.mock import patch
except ImportError:  # Python 2
    from mock import patch

@patch('splitwise.Splitwise._Splitwise__makeRequest')
class CreateCommentTestCase(unittest.TestCase):

    def setUp(self):
        self.sObj = Splitwise('consumerkey', 'consumersecret')

    def test_createComment_success(self, mockMakeRequest):
        mockMakeRequest.return_value = '{"comment": {"id": 211110000, "content": "Test for create comment", "comment_type": "User", "relation_type": "ExpenseComment", "relation_id": 982430660, "created_at": "2020-08-09T16:14:52Z", "deleted_at": null, "user": {"id": 784241, "first_name": "ruks", "last_name": null, "picture": {"medium": "https://s3.amazonaws.com/splitwise/uploads/user/default_avatars/avatar-ruby47-100px.png"}}}, "errors":{}}'  # noqa: E501
        # data = {}
        # data["expense_id"] = 982430660
        # data["content"] = "Test for create comment"
        data = '{"expense_id": 982430660, "content": "Test for create comment"}'
        comment, errors = self.sObj.createComment(data)
        mockMakeRequest.assert_called_with("https://secure.splitwise.com/api/v3.0/create_comment",
                                           "POST",
                                           data)
        self.assertEqual(comment.getId(), 211110000)
        self.assertEqual(comment.getContent(), 'Test for create comment')
        self.assertEqual(comment.getCommentType(), 'User')
        self.assertEqual(comment.getRelationType(), 'ExpenseComment')
        self.assertEqual(comment.getRelationId(), 982430660)
        self.assertEqual(comment.getCreatedAt(), '2020-08-09T16:14:52Z')
        self.assertEqual(comment.getDeletedAt(), None)
        user = comment.getCommentedUser()
        self.assertEqual(user.getFirstName(), 'ruks')
        self.assertEqual(user.getLastName(), None)

    def test_createComment_error(self, mockMakeRequest):
        mockMakeRequest.return_value = '"comment":{},"errors":{"base":["An unknown error occurred. Please try again or contact support@splitwise.com if you experience repeated issues. Sorry for the trouble!"]}'  # noqa: E501
        data = Comment()
        comment, errors = self.sObj.createComment(data)
        mockMakeRequest.assert_called_with("https://secure.splitwise.com/api/v3.0/create_comment",
                                           "POST",
                                           data)
        self.assertEqual(errors.getErrors(), {
            'base': [
                'An unknown error occurred. Please try again or contact support@splitwise.com if you experience repeated issues. \
Sorry for the trouble!'
            ]})

    def test_createComment_exception(self, mockMakeRequest):
        mockMakeRequest.side_effect = Exception(
            "Invalid response %s. Please check your consumer key and secret." % 404)
        data = Comment()
        comment, errors = self.sObj.createComment(data)
        with self.assertRaises(Exception):
            mockMakeRequest.assert_called_with("https://secure.splitwise.com/api/v3.0/create_comment",
                                               "POST",
                                               data)
<commit_msg>Rectified lint errors createComment<commit_after>from splitwise import Splitwise
from splitwise.comment import Comment
import unittest
try:
    from unittest.mock import patch
except ImportError:  # Python 2
    from mock import patch

@patch('splitwise.Splitwise._Splitwise__makeRequest')
class CreateCommentTestCase(unittest.TestCase):

    def setUp(self):
        self.sObj = Splitwise('consumerkey', 'consumersecret')

    def test_createComment_success(self, mockMakeRequest):
        mockMakeRequest.return_value = '{"comment": {"id": 211110000, "content": "Test for create comment", "comment_type": "User", "relation_type": "ExpenseComment", "relation_id": 982430660, "created_at": "2020-08-09T16:14:52Z", "deleted_at": null, "user": {"id": 784241, "first_name": "ruks", "last_name": null, "picture": {"medium": "https://s3.amazonaws.com/splitwise/uploads/user/default_avatars/avatar-ruby47-100px.png"}}}, "errors":{}}'  # noqa: E501
        data = '{"expense_id": 982430660, "content": "Test for create comment"}'
        comment, errors = self.sObj.createComment(data)
        mockMakeRequest.assert_called_with("https://secure.splitwise.com/api/v3.0/create_comment",
                                           "POST",
                                           data)
        self.assertEqual(comment.getId(), 211110000)
        self.assertEqual(comment.getContent(), 'Test for create comment')
        self.assertEqual(comment.getCommentType(), 'User')
        self.assertEqual(comment.getRelationType(), 'ExpenseComment')
        self.assertEqual(comment.getRelationId(), 982430660)
        self.assertEqual(comment.getCreatedAt(), '2020-08-09T16:14:52Z')
        self.assertEqual(comment.getDeletedAt(), None)
        user = comment.getCommentedUser()
        self.assertEqual(user.getFirstName(), 'ruks')
        self.assertEqual(user.getLastName(), None)

    def test_createComment_error(self, mockMakeRequest):
        mockMakeRequest.return_value = '{"comment":{},"errors":{"base":["An unknown error occurred. Please try again or contact support@splitwise.com if you experience repeated issues. Sorry for the trouble!"]}}'  # noqa: E501
        data = '{"expense_id": 982430660, "content": "Test for create comment"}'

        comment, errors = self.sObj.createComment(data)
        mockMakeRequest.assert_called_with("https://secure.splitwise.com/api/v3.0/create_comment",
                                           "POST",
                                           data)
        self.assertEqual(errors.getErrors(), {
            'base': [
                'An unknown error occurred. Please try again or contact support@splitwise.com if you experience repeated issues. \
Sorry for the trouble!'
            ]})

    def test_createComment_exception(self, mockMakeRequest):
        mockMakeRequest.side_effect = Exception(
            "Invalid response %s. Please check your consumer key and secret." % 404)
        data = '{"expense_id": 982430660, "content": "Test for create comment"}'
        with self.assertRaises(Exception):
            comment, errors = self.sObj.createComment(data)
            mockMakeRequest.assert_called_with("https://secure.splitwise.com/api/v3.0/create_comment",
                                               "POST",
                                               data)
<|endoftext|>


<commit_before>from __future__ import print_function
from svgpathtools import parse_path, Path, Line, CubicBezier

def formatPoint(point):
    return '{0:10.3f}, {1:10.3f}'.format(point.real, point.imag)

class Part(dict):
    """implemented as a fluent interface"""

    # -------------------------------------------------------------------------
    # Customization Methods

    def __init__(self, d):
        self['d'] = d
        self['x'] = 0
        self['y'] = 0
        self['z'] = 0
        self['other'] = ''

    def __repr__(self):
        return 'Part(\'%s\')' % self['d']

    def __str__(self):
        return ', '.join([
            '{}={}'.format(k, self[k]) for k in ['x', 'y', 'z', 'other']
        ])

    # -------------------------------------------------------------------------
    # Properties

    @property
    def ID(self):
        s = '_'.join([
            '{}{}'.format(k, self[k]) for k in ['x', 'y', 'z'] if self[k]
        ])

        if self['other']:
            s += '_' + self['other'] if s else self['other']

        return '_' + s if s else ''

    @ID.setter
    def ID(self, tokens):
        for token in tokens:
            for dim in ['x', 'y', 'z']:
                if dim in token:
                    try:
                        self[dim] = int(token[1:])
                    except:
                        self['other'] = token
        return self

    # -------------------------------------------------------------------------
    # Actions

    def quantize(self):
        path = parse_path(self['d'])
        newPath = Path()
        for seg in path:
            if isinstance(seg, Line):
                newSeg = Line(
                    complex(round(seg.start.real), round(seg.start.imag)),
                    complex(round(seg.end.real), round(seg.end.imag))
                )
                newPath.append(newSeg)

            elif isinstance(seg, CubicBezier):
                newSeg = CubicBezier(
                    complex(round(seg.start.real), round(seg.start.imag)),
                    complex(
                        round(seg.control1.real), round(seg.control1.imag)
                    ),
                    complex(
                        round(seg.control2.real), round(seg.control2.imag)
                    ),
                    complex(round(seg.end.real), round(seg.end.imag))
                )
                newPath.append(newSeg)

        self['d'] = newPath.d()
        return self

    def toConsole(self):
        print('\t', self)
        path = parse_path(self['d'])
        for seg in path:
            if isinstance(seg, Line):
                print('\t\tLine',
                      formatPoint(seg.start),
                      ' ' * 22,
                      ' ' * 22,
                      formatPoint(seg.end))
            elif isinstance(seg, CubicBezier):
                print('\t\tCubB',
                      formatPoint(seg.start),
                      formatPoint(seg.control1),
                      formatPoint(seg.control2),
                      formatPoint(seg.end))
            else:
                raise TypeError("seg must be a path segment.")

        return self

    def translate(self, x, y):
        path = parse_path(self['d'])
        newPath = Path()
        for seg in path:
            if isinstance(seg, Line):
                newSeg = Line(
                    complex(seg.start.real + x, seg.start.imag + y),
                    complex(seg.end.real + x, seg.end.imag + y)
                )
                newPath.append(newSeg)

            elif isinstance(seg, CubicBezier):
                newSeg = CubicBezier(
                    complex(seg.start.real + x, seg.start.imag + y),
                    complex(seg.control1.real + x, seg.control1.imag + y),
                    complex(seg.control2.real + x, seg.control2.imag + y),
                    complex(seg.end.real + x, seg.end.imag + y)
                )
                newPath.append(newSeg)

        self['d'] = newPath.d()
        return self
<commit_msg>Use the translate function from the library<commit_after>from __future__ import print_function
from svgpathtools import parse_path, Path, Line, CubicBezier

def formatPoint(point):
    return '{0:10.3f}, {1:10.3f}'.format(point.real, point.imag)

class Part(dict):
    """implemented as a fluent interface"""

    # -------------------------------------------------------------------------
    # Customization Methods

    def __init__(self, d):
        self['d'] = d
        self['x'] = 0
        self['y'] = 0
        self['z'] = 0
        self['other'] = ''

    def __repr__(self):
        return 'Part(\'%s\')' % self['d']

    def __str__(self):
        return ', '.join([
            '{}={}'.format(k, self[k]) for k in ['x', 'y', 'z', 'other']
        ])

    # -------------------------------------------------------------------------
    # Properties

    @property
    def ID(self):
        s = '_'.join([
            '{}{}'.format(k, self[k]) for k in ['x', 'y', 'z'] if self[k]
        ])

        if self['other']:
            s += '_' + self['other'] if s else self['other']

        return '_' + s if s else ''

    @ID.setter
    def ID(self, tokens):
        for token in tokens:
            for dim in ['x', 'y', 'z']:
                if dim in token:
                    try:
                        self[dim] = int(token[1:])
                    except:
                        self['other'] = token
        return self

    # -------------------------------------------------------------------------
    # Actions

    def quantize(self):
        path = parse_path(self['d'])
        newPath = Path()
        for seg in path:
            if isinstance(seg, Line):
                newSeg = Line(
                    complex(round(seg.start.real), round(seg.start.imag)),
                    complex(round(seg.end.real), round(seg.end.imag))
                )
                newPath.append(newSeg)

            elif isinstance(seg, CubicBezier):
                newSeg = CubicBezier(
                    complex(round(seg.start.real), round(seg.start.imag)),
                    complex(
                        round(seg.control1.real), round(seg.control1.imag)
                    ),
                    complex(
                        round(seg.control2.real), round(seg.control2.imag)
                    ),
                    complex(round(seg.end.real), round(seg.end.imag))
                )
                newPath.append(newSeg)

        self['d'] = newPath.d()
        return self

    def toConsole(self):
        print('\t', self)
        path = parse_path(self['d'])
        for seg in path:
            if isinstance(seg, Line):
                print('\t\tLine',
                      formatPoint(seg.start),
                      ' ' * 22,
                      ' ' * 22,
                      formatPoint(seg.end))
            elif isinstance(seg, CubicBezier):
                print('\t\tCubB',
                      formatPoint(seg.start),
                      formatPoint(seg.control1),
                      formatPoint(seg.control2),
                      formatPoint(seg.end))
            else:
                raise TypeError("seg must be a path segment.")

        return self

    def translate(self, x, y):
        path = parse_path(self['d'])
        newPath = path.translated(complex(x, y))
        self['d'] = newPath.d()
        return self
<|endoftext|>