<commit_before># Copyright 2004-2005 Grant T. Olson.
# See license.txt for terms.

"""
x86tokenizer.py
---------------

This provides the framework to tokenize x86 instructions from strings.  There
are two major functions:

tokenizeInstDef()
    This tokenizes an instruction definition so we can build the appropriate
    tables to lookup at a later date.

tokenizeInst()
    This tokenizes an actual instruction, in code, to help build an instruction
    instance.

Much of the stuff going on here is the same, but an actual instruction may (and
will) contain info that is invalid for a definition.  i.e.::

    # an instance can have concrete values, but a def can't
    tokenizeInst("MOV EAX,0xF000")
    tokenizeInst("MOV [EBP+8],0x10")

Of course I still need to implement the tokenizeInst function.
"""

import re

#
class tokenizeError(Exception):pass

#token IDs
REGISTER,OPCODE,COMMA,OPERAND,LBRACKET,RBRACKET,NUMBER,SYMBOL,STRING = range(1,10)

tokLookup = {'REGISTER':REGISTER,
             'OPCODE':OPCODE,
             'COMMA':COMMA,
             'OPERAND':OPERAND,
             'LBRACKET':LBRACKET,
             'RBRACKET':RBRACKET,
             'NUMBER':NUMBER,
             'SYMBOL':SYMBOL,
             }
#Re's

# Re's used by both tokeinizers
commaRe = '(?P<COMMA>,)'

# Instruction defs can have hard-coded indirect r/m's in them.
# An actual instruction will extract the LBRACKET and RBRACKET
# and include any appropriate offsets: 'MOV [EDP + 4], AX'
# This will never happen in an instruction def so we need to parse differently
basicRegisterRe = 'AL|CL|DL|BL|AH|CH|DH|BH|AX|CX|DX|BX|SP|BP|SI|DI|'\
        'EAX|ECX|EDX|EBX|ESP|EBP|ESI|EDI'
indirectRegisterRe = '\[AL\]|\[CL\]|\[DL\]|\[BL\]|\[AH\]|\[CH\]|\[DH\]|\[BH\]|'\
        '\[AX\]|\[CX\]|\[DX\]|\[BX\]|\[SP\]|\[BP\]|\[SI\]|\[DI\]|'\
        '\[EAX\]|\[ECX\]|\[EDX\]|\[EBX\]|\[ESP\]|\[EBP\]|\[ESI\]|\[EDI\]'
defRegRe = '(?P<REGISTER>%s|%s)' % (basicRegisterRe,indirectRegisterRe)
instRegRe= '(?P<REGISTER>%s)' % (basicRegisterRe)

# def specific stuff
#TODO: fix this better
ptrRe = 'ptr16\:16|ptr32\:32'
memRe = 'm16\:16|m32\:32'
sregRe = 'Sreg'
moffsRe = 'moffs8|moffs16|moffs32'
immediateRe = 'imm8|imm16|imm32'
relativeRe = 'rel8|rel16|rel32'
rRe = 'r8|r16|r32'
mmRe = 'mm|xmm'
rmRe = 'r/m8|r/m16|r/m32|m8|m16|m32|m'
otherRe = '/digit|REG'
operandRe = '(?P<OPERAND>%s|%s|%s|%s|%s|%s|%s|%s|%s|%s)' % (ptrRe, memRe, sregRe,
                                                            moffsRe, immediateRe,
                                                            relativeRe, rRe, mmRe,
                                                            rmRe, otherRe)

whitespaceRe = '[\s\+]'

#opcode is a symbol that is all caps, no lowercase or underscore
opcodeRe = re.compile('^[A-Z]+$')

#instructionRe specific stuff
stringRe = "((?P<q>'|\")(?P<STRING>.*)(?P=q))"
lbracketRe = '(?P<LBRACKET>\[)'
rbracketRe = '(?P<RBRACKET>\])'
numberRe = '(?P<NUMBER>[\+\-]?(0x[0-9A-Fa-f]+|[0-9]+))'
symbolRe = '(?P<SYMBOL>[A-Za-z_@]+)'

#define final re's
instructionDefRe = re.compile("(?:%s*(?:%s|%s|%s|%s|%s|%s|%s)(?P<rest>.*))" % \
                           (whitespaceRe,defRegRe,operandRe,symbolRe,commaRe,numberRe,lbracketRe,rbracketRe))

instructionRe = re.compile("(?:%s*(?:%s|%s|%s|%s|%s|%s|%s)(?P<rest>.*))" % \
                           (whitespaceRe,lbracketRe,rbracketRe,instRegRe,
                            commaRe,numberRe,symbolRe,stringRe))

def tokenizeString(s,reToProcess):
    lst = []
    rest = s
    while rest:
        instMatch = reToProcess.match(rest)
        if not instMatch:
            raise tokenizeError("Couldn't find match for string '%s' from '%s'" % (rest,s))

        instDict = instMatch.groupdict()
        if instDict['REGISTER']: lst.append((REGISTER,instDict['REGISTER']))
        elif instDict['SYMBOL']:
            if opcodeRe.match(instDict['SYMBOL']):
                lst.append((OPCODE,instDict['SYMBOL']))
            else:
                lst.append((SYMBOL,instDict['SYMBOL']))
        elif instDict['COMMA']: lst.append((COMMA,instDict['COMMA']))
        elif instDict.has_key('OPERAND') and instDict['OPERAND']:
            # only defs have operands.
            #only instructions have anything below here, but if it's a def
            #we've already (hopefully) found a match so we don't need to check
            #for key existance.

            opText = instDict['OPERAND']
            # Hack for 'm' codes.  These are stored in the RM field,
            # But register values are technically invalid.  Should this be
            # verified while compiling?
            if opText == 'm32':
                opText = 'r/m32'
            elif opText == 'm16':
                opText = 'r/m16'
            elif opText == 'm8':
                opText = 'r/m8'
            lst.append((OPERAND,opText))
        elif instDict['LBRACKET']: lst.append((LBRACKET,instDict['LBRACKET']))
        elif instDict['RBRACKET']: lst.append((RBRACKET,instDict['RBRACKET']))
        elif instDict['NUMBER']: lst.append((NUMBER,instDict['NUMBER']))
        elif instDict['STRING']: lst.append((STRING,instDict['STRING']))
        else:
            raise tokenizeError("Tokenization failed on string %s, match %s" \
                                  % (s,rest))
        rest = instDict['rest']
    return tuple(lst)

def tokenizeInstDef(s):
    toks = tokenizeString(s, instructionDefRe)
    index,length = 0,len(toks)
    if length == 0:
        raise tokenizeError("Invalid Instruction.  Cannot be blank")
    if toks[index][0] != OPCODE:
        raise tokenizeError("Invalid Instruction: '%s' " \
                            "Must start with an OPCODE" % s)

    while index < length and toks[index][0] == OPCODE:
        index += 1

    while index < length:
        if toks[index][0] not in (REGISTER,OPERAND,NUMBER):
            raise tokenizeError("Invalid Instruction Definition: '%s' " \
                                "Expected a REGISTER OR OPERAND ENTRY" % s)
        index += 1
        if index < length:
            if toks[index][0] != COMMA:
                raise tokenizeError("Invalid Instruction Def: '%s' Expected " \
                                    "a COMMA" % s)
            index += 1
    return toks

def tokenizeInst(s):
    toks = tokenizeString(s, instructionRe)
    index,length = 0,len(toks)
    if length == 0:
        raise tokenizeError("Invalid Instruction.  Cannot be blank")
    if toks[index][0] != OPCODE:
        raise tokenizeError("Invalid Instruction: '%s' " \
                            "Must start with an OPCODE" % s)

    while index < length and toks[index][0] == OPCODE:
        index += 1

    while index < length:
        if toks[index][0] in (REGISTER,NUMBER,SYMBOL,STRING):
            index += 1
        elif toks[index][0] == LBRACKET:
            index += 1
            if toks[index][0] in (NUMBER,SYMBOL):
                index += 1
            elif toks[index][0] == REGISTER:
                index += 1
                if toks[index][0] in (NUMBER,SYMBOL):
                    index += 1
            else:
                raise tokenizeError("Invalid Instruction: '%s'  Expected a " \
                                    "REGISTER inside the [brackets]" %s)
            if toks[index][0] != RBRACKET:
                raise tokenizeError("Invalid Instruction: '%s' Expected an " \
                                    "ending BRACKET here." % s)
            else:
                index += 1
        else:
            raise tokenizeError("Invalid Instruction: '%s' " \
                                "Expected a REGISTER,LBRACKET,NUMBER,SYMBOL, or STRING" % s)

        if index < length:
            if toks[index][0] != COMMA:
                raise tokenizeError("Invalid Instruction: '%s' Expected " \
                                    "a COMMA" % s)
            index += 1
    return toks<commit_msg>Symbols can be one letter long; can't start with a digit.<commit_after># Copyright 2004-2005 Grant T. Olson.
# See license.txt for terms.

"""
x86tokenizer.py
---------------

This provides the framework to tokenize x86 instructions from strings.  There
are two major functions:

tokenizeInstDef()
    This tokenizes an instruction definition so we can build the appropriate
    tables to lookup at a later date.

tokenizeInst()
    This tokenizes an actual instruction, in code, to help build an instruction
    instance.

Much of the stuff going on here is the same, but an actual instruction may (and
will) contain info that is invalid for a definition.  i.e.::

    # an instance can have concrete values, but a def can't
    tokenizeInst("MOV EAX,0xF000")
    tokenizeInst("MOV [EBP+8],0x10")

Of course I still need to implement the tokenizeInst function.
"""

import re

#
class tokenizeError(Exception):pass

#token IDs
REGISTER,OPCODE,COMMA,OPERAND,LBRACKET,RBRACKET,NUMBER,SYMBOL,STRING = range(1,10)

tokLookup = {'REGISTER':REGISTER,
             'OPCODE':OPCODE,
             'COMMA':COMMA,
             'OPERAND':OPERAND,
             'LBRACKET':LBRACKET,
             'RBRACKET':RBRACKET,
             'NUMBER':NUMBER,
             'SYMBOL':SYMBOL,
             }
#Re's

# Re's used by both tokeinizers
commaRe = '(?P<COMMA>,)'

# Instruction defs can have hard-coded indirect r/m's in them.
# An actual instruction will extract the LBRACKET and RBRACKET
# and include any appropriate offsets: 'MOV [EDP + 4], AX'
# This will never happen in an instruction def so we need to parse differently
basicRegisterRe = 'AL|CL|DL|BL|AH|CH|DH|BH|AX|CX|DX|BX|SP|BP|SI|DI|'\
        'EAX|ECX|EDX|EBX|ESP|EBP|ESI|EDI'
indirectRegisterRe = '\[AL\]|\[CL\]|\[DL\]|\[BL\]|\[AH\]|\[CH\]|\[DH\]|\[BH\]|'\
        '\[AX\]|\[CX\]|\[DX\]|\[BX\]|\[SP\]|\[BP\]|\[SI\]|\[DI\]|'\
        '\[EAX\]|\[ECX\]|\[EDX\]|\[EBX\]|\[ESP\]|\[EBP\]|\[ESI\]|\[EDI\]'
defRegRe = '(?P<REGISTER>%s|%s)' % (basicRegisterRe,indirectRegisterRe)
instRegRe= '(?P<REGISTER>%s)' % (basicRegisterRe)

# def specific stuff
#TODO: fix this better
ptrRe = 'ptr16\:16|ptr32\:32'
memRe = 'm16\:16|m32\:32'
sregRe = 'Sreg'
moffsRe = 'moffs8|moffs16|moffs32'
immediateRe = 'imm8|imm16|imm32'
relativeRe = 'rel8|rel16|rel32'
rRe = 'r8|r16|r32'
mmRe = 'mm|xmm'
rmRe = 'r/m8|r/m16|r/m32|m8|m16|m32|m'
otherRe = '/digit|REG'
operandRe = '(?P<OPERAND>%s|%s|%s|%s|%s|%s|%s|%s|%s|%s)' % (ptrRe, memRe, sregRe,
                                                            moffsRe, immediateRe,
                                                            relativeRe, rRe, mmRe,
                                                            rmRe, otherRe)

whitespaceRe = '[\s\+]'

#opcode is a symbol that is all caps, no lowercase or underscore
opcodeRe = re.compile('^[A-Z]+$')

#instructionRe specific stuff
stringRe = "((?P<q>'|\")(?P<STRING>.*)(?P=q))"
lbracketRe = '(?P<LBRACKET>\[)'
rbracketRe = '(?P<RBRACKET>\])'
numberRe = '(?P<NUMBER>[\+\-]?(0x[0-9A-Fa-f]+|[0-9]+))'
symbolRe = '(?P<SYMBOL>[A-Za-z_@][A-Za-z_@0-9]*)'

#define final re's
instructionDefRe = re.compile("(?:%s*(?:%s|%s|%s|%s|%s|%s|%s)(?P<rest>.*))" % \
                           (whitespaceRe,defRegRe,operandRe,symbolRe,commaRe,numberRe,lbracketRe,rbracketRe))

instructionRe = re.compile("(?:%s*(?:%s|%s|%s|%s|%s|%s|%s)(?P<rest>.*))" % \
                           (whitespaceRe,lbracketRe,rbracketRe,instRegRe,
                            commaRe,numberRe,symbolRe,stringRe))

def tokenizeString(s,reToProcess):
    lst = []
    rest = s
    while rest:
        instMatch = reToProcess.match(rest)
        if not instMatch:
            raise tokenizeError("Couldn't find match for string '%s' from '%s'" % (rest,s))

        instDict = instMatch.groupdict()
        if instDict['REGISTER']: lst.append((REGISTER,instDict['REGISTER']))
        elif instDict['SYMBOL']:
            if opcodeRe.match(instDict['SYMBOL']):
                lst.append((OPCODE,instDict['SYMBOL']))
            else:
                lst.append((SYMBOL,instDict['SYMBOL']))
        elif instDict['COMMA']: lst.append((COMMA,instDict['COMMA']))
        elif instDict.has_key('OPERAND') and instDict['OPERAND']:
            # only defs have operands.
            #only instructions have anything below here, but if it's a def
            #we've already (hopefully) found a match so we don't need to check
            #for key existance.

            opText = instDict['OPERAND']
            # Hack for 'm' codes.  These are stored in the RM field,
            # But register values are technically invalid.  Should this be
            # verified while compiling?
            if opText == 'm32':
                opText = 'r/m32'
            elif opText == 'm16':
                opText = 'r/m16'
            elif opText == 'm8':
                opText = 'r/m8'
            lst.append((OPERAND,opText))
        elif instDict['LBRACKET']: lst.append((LBRACKET,instDict['LBRACKET']))
        elif instDict['RBRACKET']: lst.append((RBRACKET,instDict['RBRACKET']))
        elif instDict['NUMBER']: lst.append((NUMBER,instDict['NUMBER']))
        elif instDict['STRING']: lst.append((STRING,instDict['STRING']))
        else:
            raise tokenizeError("Tokenization failed on string %s, match %s" \
                                  % (s,rest))
        rest = instDict['rest']
    return tuple(lst)

def tokenizeInstDef(s):
    toks = tokenizeString(s, instructionDefRe)
    index,length = 0,len(toks)
    if length == 0:
        raise tokenizeError("Invalid Instruction.  Cannot be blank")
    if toks[index][0] != OPCODE:
        raise tokenizeError("Invalid Instruction: '%s' " \
                            "Must start with an OPCODE" % s)

    while index < length and toks[index][0] == OPCODE:
        index += 1

    while index < length:
        if toks[index][0] not in (REGISTER,OPERAND,NUMBER):
            raise tokenizeError("Invalid Instruction Definition: '%s' " \
                                "Expected a REGISTER OR OPERAND ENTRY" % s)
        index += 1
        if index < length:
            if toks[index][0] != COMMA:
                raise tokenizeError("Invalid Instruction Def: '%s' Expected " \
                                    "a COMMA" % s)
            index += 1
    return toks

def tokenizeInst(s):
    toks = tokenizeString(s, instructionRe)
    index,length = 0,len(toks)
    if length == 0:
        raise tokenizeError("Invalid Instruction.  Cannot be blank")
    if toks[index][0] != OPCODE:
        raise tokenizeError("Invalid Instruction: '%s' " \
                            "Must start with an OPCODE" % s)

    while index < length and toks[index][0] == OPCODE:
        index += 1

    while index < length:
        if toks[index][0] in (REGISTER,NUMBER,SYMBOL,STRING):
            index += 1
        elif toks[index][0] == LBRACKET:
            index += 1
            if toks[index][0] in (NUMBER,SYMBOL):
                index += 1
            elif toks[index][0] == REGISTER:
                index += 1
                if toks[index][0] in (NUMBER,SYMBOL):
                    index += 1
            else:
                raise tokenizeError("Invalid Instruction: '%s'  Expected a " \
                                    "REGISTER inside the [brackets]" %s)
            if toks[index][0] != RBRACKET:
                raise tokenizeError("Invalid Instruction: '%s' Expected an " \
                                    "ending BRACKET here." % s)
            else:
                index += 1
        else:
            raise tokenizeError("Invalid Instruction: '%s' " \
                                "Expected a REGISTER,LBRACKET,NUMBER,SYMBOL, or STRING" % s)

        if index < length:
            if toks[index][0] != COMMA:
                raise tokenizeError("Invalid Instruction: '%s' Expected " \
                                    "a COMMA" % s)
            index += 1
    return toks
<|endoftext|>


<commit_before>import xam.clustering
import xam.nlp
import xam.preprocessing
import xam.stacking
import xam.tsa
import xam.util
<commit_msg>Fix imports<commit_after>from . import clustering
from . import nlp
from . import preprocessing
from . import stacking
from . import tsa
from . import util
<|endoftext|>


<commit_before>from xml.dom.minidom import parseString
from xml.etree.ElementTree import tostring, SubElement, Element
from datetime import datetime
from dateutil.parser import parse
import urllib
import requests
from urlparse import parse_qs

from .constants import XERO_API_URL
from .exceptions import *

class Manager(object):
    DECORATED_METHODS = ('get', 'save', 'filter', 'all', 'put')

    DATETIME_FIELDS = (u'UpdatedDateUTC', u'Updated', u'FullyPaidOnDate')
    DATE_FIELDS = (u'DueDate', u'Date')
    BOOLEAN_FIELDS = (u'IsSupplier', u'IsCustomer')

    MULTI_LINES = (u'LineItem', u'Phone', u'Address', 'TaxRate')
    PLURAL_EXCEPTIONS = {'Addresse': 'Address'}

    NO_SEND_FIELDS = (u'UpdatedDateUTC',)

    def __init__(self, name, oauth):
        self.oauth = oauth
        self.name = name

        # setup our singular variants of the name
        # only if the name ends in 0
        if name[-1] == "s":
            self.singular = name[:len(name)-1]
        else:
            self.singular = name

        for method_name in self.DECORATED_METHODS:
            method = getattr(self, method_name)
            setattr(self, method_name, self._get_data(method))

    def walk_dom(self, dom):
        tree_list = tuple()
        for node in dom.childNodes:
            tagName = getattr(node, 'tagName', None)
            if tagName:
                tree_list += (tagName, self.walk_dom(node),)
            else:
                data = node.data.strip()
                if data:
                    tree_list += (node.data.strip(),)
        return tree_list

    def convert_to_dict(self, deep_list):
        out = {}
        if len(deep_list) > 2:
            lists = [l for l in deep_list if isinstance(l, tuple)]
            keys = [l for l in deep_list if isinstance(l, unicode)]
            for key, data in zip(keys, lists):

                if len(data) == 1:
                    # we're setting a value
                    # check to see if we need to apply any special
                    # formatting to the value
                    val = data[0]
                    if key in self.BOOLEAN_FIELDS:
                        val = True if val.lower() == 'true' else False
                    if key in self.DATETIME_FIELDS:
                        val = parse(val)
                    if key in self.DATE_FIELDS:
                        val = parse(val).date()

                    out[key] = val

                elif len(data) > 1 and ((key in self.MULTI_LINES) or (key == self.singular)):
                    # our data is a collection and needs to be handled as such
                    if out:
                        out.append(self.convert_to_dict(data))
                    else:
                        out = [self.convert_to_dict(data)]

                elif len(data) > 1:
                    out[key] = self.convert_to_dict(data)

        elif len(deep_list) == 2:
            key = deep_list[0]
            data = deep_list[1]
            out[key] = self.convert_to_dict(data)
        else:
            out = deep_list[0]
        return out

    def dict_to_xml(self, root_elm, data):
        for key in data.keys():
            # Xero will complain if we send back these fields.
            if key in self.NO_SEND_FIELDS:
                continue

            sub_data = data[key]
            elm = SubElement(root_elm, key)

            is_list = isinstance(sub_data, list) or isinstance(sub_data, tuple)
            is_plural = key[len(key)-1] == "s"
            plural_name = key[:len(key)-1]

            # Key references a dict. Unroll the dict
            # as it's own XML node with subnodes
            if isinstance(sub_data, dict):
                self.dict_to_xml(elm, sub_data)

            # Key references a list/tuple
            elif is_list:
                # key name is a plural. This means each item
                # in the list needs to be wrapped in an XML
                # node that is a singular version of the list name.
                if is_plural:
                    for d in sub_data:
                        plural_name = self.PLURAL_EXCEPTIONS.get(plural_name, plural_name)
                        self.dict_to_xml(SubElement(elm, plural_name), d)

                # key name isn't a plural. Just insert the content
                # as an XML node with subnodes
                else:
                    for d in sub_data:
                        self.dict_to_xml(elm, d)

            # Normal element - just inser the data.
            else:
                elm.text = str(sub_data)

        return root_elm

    def _prepare_data_for_save(self, data):
        if isinstance(data, list) or isinstance(data, tuple):
            root_elm = Element(self.name)
            for d in data:
                sub_elm = SubElement(root_elm, self.singular)
                self.dict_to_xml(sub_elm, d)
        else:
            root_elm = self.dict_to_xml(Element(self.singular), data)

        return tostring(root_elm)

    def _get_results(self, data):
        response = data[u'Response']
        result = response.get(self.name, {})

        if isinstance(result, tuple) or isinstance(result, list):
            return result

        if isinstance(result, dict) and self.singular in result:
            return result[self.singular]

    def _get_data(self, func):
        def wrapper(*args, **kwargs):
            uri, method, body, headers = func(*args, **kwargs)
            response = getattr(requests, method)(uri, data=body, headers=headers, auth=self.oauth)

            if response.status_code == 200:
                if response.headers['content-type'] == 'application/pdf':
                    return response.text
                # parseString takes byte content, not unicode.
                dom = parseString(response.text.encode(response.encoding))
                data = self.convert_to_dict(self.walk_dom(dom))
                return self._get_results(data)

            elif response.status_code == 400:
                raise XeroBadRequest(response)

            elif response.status_code == 401:
                raise XeroUnauthorized(response)

            elif response.status_code == 403:
                raise XeroForbidden(response)

            elif response.status_code == 404:
                raise XeroNotFound(response)

            elif response.status_code == 500:
                raise XeroInternalError(response)

            elif response.status_code == 501:
                raise XeroNotImplemented(response)

            elif response.status_code == 503:
                # Two 503 responses are possible. Rate limit errors
                # return encoded content; offline errors don't.
                # If you parse the response text and there's nothing
                # encoded, it must be a not-available error.
                payload = parse_qs(response.text)
                if payload:
                    raise XeroRateLimitExceeded(response, payload)
                else:
                    raise XeroNotAvailable(response)
            else:
                raise XeroExceptionUnknown(response)

        return wrapper

    def get(self, id, headers=None):
        uri = '/'.join([XERO_API_URL, self.name, id])
        return uri, 'get', None, headers

    def save_or_put(self, data, method='post', headers=None):
        uri = '/'.join([XERO_API_URL, self.name])
        body = {'xml': self._prepare_data_for_save(data)}
        return uri, method, body, headers

    def save(self, data):
        return self.save_or_put(data, method='post')

    def put(self, data):
        return self.save_or_put(data, method='put')

    def prepare_filtering_date(self, val):
        if isinstance(val, datetime):
            val = val.strftime('%a, %d %b %Y %H:%M:%S GMT')
        else:
            val = '"%s"' % val
        return {'If-Modified-Since': val}

    def filter(self, **kwargs):
        headers = None
        uri = '/'.join([XERO_API_URL, self.name])
        if kwargs:
            if 'since' in kwargs:
                val = kwargs['since']
                headers = self.prepare_filtering_date(val)
                del kwargs['since']

            def get_filter_params():
                if key in self.BOOLEAN_FIELDS:
                    return 'true' if kwargs[key] else 'false'
                elif key in self.DATETIME_FIELDS:
                    return kwargs[key].isoformat()
                else:
                    return '"%s"' % str(kwargs[key])

            def generate_param(key):
                parts = key.split("__")
                field = key.replace('_', '.')
                fmt = '%s==%s'
                if len(parts) == 2:
                    # support filters:
                    # Name__Contains=John becomes Name.Contains("John")
                    if parts[1] in ["contains", "startswith", "endswith"]:
                        field = parts[0]
                        fmt = ''.join(['%s.', parts[1], '(%s)'])

                return fmt % (
                    field,
                    get_filter_params()
                )

            params = [generate_param(key) for key in kwargs.keys()]

            if params:
                uri += '?where=' + urllib.quote('&&'.join(params))

        return uri, 'get', None, headers

    def all(self):
        uri = '/'.join([XERO_API_URL, self.name])
        return uri, 'get', None, None
<commit_msg>Support Xero's Guid datatypes.<commit_after>from xml.dom.minidom import parseString
from xml.etree.ElementTree import tostring, SubElement, Element
from datetime import datetime
from dateutil.parser import parse
import urllib
import requests
from urlparse import parse_qs

from .constants import XERO_API_URL
from .exceptions import *

class Manager(object):
    DECORATED_METHODS = ('get', 'save', 'filter', 'all', 'put')

    DATETIME_FIELDS = (u'UpdatedDateUTC', u'Updated', u'FullyPaidOnDate')
    DATE_FIELDS = (u'DueDate', u'Date')
    BOOLEAN_FIELDS = (u'IsSupplier', u'IsCustomer')
    GUID_FIELDS = (u'ContactID', u'InvoiceID')

    MULTI_LINES = (u'LineItem', u'Phone', u'Address', 'TaxRate')
    PLURAL_EXCEPTIONS = {'Addresse': 'Address'}

    NO_SEND_FIELDS = (u'UpdatedDateUTC',)

    def __init__(self, name, oauth):
        self.oauth = oauth
        self.name = name

        # setup our singular variants of the name
        # only if the name ends in 0
        if name[-1] == "s":
            self.singular = name[:len(name)-1]
        else:
            self.singular = name

        for method_name in self.DECORATED_METHODS:
            method = getattr(self, method_name)
            setattr(self, method_name, self._get_data(method))

    def walk_dom(self, dom):
        tree_list = tuple()
        for node in dom.childNodes:
            tagName = getattr(node, 'tagName', None)
            if tagName:
                tree_list += (tagName, self.walk_dom(node),)
            else:
                data = node.data.strip()
                if data:
                    tree_list += (node.data.strip(),)
        return tree_list

    def convert_to_dict(self, deep_list):
        out = {}
        if len(deep_list) > 2:
            lists = [l for l in deep_list if isinstance(l, tuple)]
            keys = [l for l in deep_list if isinstance(l, unicode)]
            for key, data in zip(keys, lists):

                if len(data) == 1:
                    # we're setting a value
                    # check to see if we need to apply any special
                    # formatting to the value
                    val = data[0]
                    if key in self.BOOLEAN_FIELDS:
                        val = True if val.lower() == 'true' else False
                    if key in self.DATETIME_FIELDS:
                        val = parse(val)
                    if key in self.DATE_FIELDS:
                        val = parse(val).date()

                    out[key] = val

                elif len(data) > 1 and ((key in self.MULTI_LINES) or (key == self.singular)):
                    # our data is a collection and needs to be handled as such
                    if out:
                        out.append(self.convert_to_dict(data))
                    else:
                        out = [self.convert_to_dict(data)]

                elif len(data) > 1:
                    out[key] = self.convert_to_dict(data)

        elif len(deep_list) == 2:
            key = deep_list[0]
            data = deep_list[1]
            out[key] = self.convert_to_dict(data)
        else:
            out = deep_list[0]
        return out

    def dict_to_xml(self, root_elm, data):
        for key in data.keys():
            # Xero will complain if we send back these fields.
            if key in self.NO_SEND_FIELDS:
                continue

            sub_data = data[key]
            elm = SubElement(root_elm, key)

            is_list = isinstance(sub_data, list) or isinstance(sub_data, tuple)
            is_plural = key[len(key)-1] == "s"
            plural_name = key[:len(key)-1]

            # Key references a dict. Unroll the dict
            # as it's own XML node with subnodes
            if isinstance(sub_data, dict):
                self.dict_to_xml(elm, sub_data)

            # Key references a list/tuple
            elif is_list:
                # key name is a plural. This means each item
                # in the list needs to be wrapped in an XML
                # node that is a singular version of the list name.
                if is_plural:
                    for d in sub_data:
                        plural_name = self.PLURAL_EXCEPTIONS.get(plural_name, plural_name)
                        self.dict_to_xml(SubElement(elm, plural_name), d)

                # key name isn't a plural. Just insert the content
                # as an XML node with subnodes
                else:
                    for d in sub_data:
                        self.dict_to_xml(elm, d)

            # Normal element - just inser the data.
            else:
                elm.text = str(sub_data)

        return root_elm

    def _prepare_data_for_save(self, data):
        if isinstance(data, list) or isinstance(data, tuple):
            root_elm = Element(self.name)
            for d in data:
                sub_elm = SubElement(root_elm, self.singular)
                self.dict_to_xml(sub_elm, d)
        else:
            root_elm = self.dict_to_xml(Element(self.singular), data)

        return tostring(root_elm)

    def _get_results(self, data):
        response = data[u'Response']
        result = response.get(self.name, {})

        if isinstance(result, tuple) or isinstance(result, list):
            return result

        if isinstance(result, dict) and self.singular in result:
            return result[self.singular]

    def _get_data(self, func):
        def wrapper(*args, **kwargs):
            uri, method, body, headers = func(*args, **kwargs)
            response = getattr(requests, method)(uri, data=body, headers=headers, auth=self.oauth)

            if response.status_code == 200:
                if response.headers['content-type'] == 'application/pdf':
                    return response.text
                # parseString takes byte content, not unicode.
                dom = parseString(response.text.encode(response.encoding))
                data = self.convert_to_dict(self.walk_dom(dom))
                return self._get_results(data)

            elif response.status_code == 400:
                raise XeroBadRequest(response)

            elif response.status_code == 401:
                raise XeroUnauthorized(response)

            elif response.status_code == 403:
                raise XeroForbidden(response)

            elif response.status_code == 404:
                raise XeroNotFound(response)

            elif response.status_code == 500:
                raise XeroInternalError(response)

            elif response.status_code == 501:
                raise XeroNotImplemented(response)

            elif response.status_code == 503:
                # Two 503 responses are possible. Rate limit errors
                # return encoded content; offline errors don't.
                # If you parse the response text and there's nothing
                # encoded, it must be a not-available error.
                payload = parse_qs(response.text)
                if payload:
                    raise XeroRateLimitExceeded(response, payload)
                else:
                    raise XeroNotAvailable(response)
            else:
                raise XeroExceptionUnknown(response)

        return wrapper

    def get(self, id, headers=None):
        uri = '/'.join([XERO_API_URL, self.name, id])
        return uri, 'get', None, headers

    def save_or_put(self, data, method='post', headers=None):
        uri = '/'.join([XERO_API_URL, self.name])
        body = {'xml': self._prepare_data_for_save(data)}
        return uri, method, body, headers

    def save(self, data):
        return self.save_or_put(data, method='post')

    def put(self, data):
        return self.save_or_put(data, method='put')

    def prepare_filtering_date(self, val):
        if isinstance(val, datetime):
            val = val.strftime('%a, %d %b %Y %H:%M:%S GMT')
        else:
            val = '"%s"' % val
        return {'If-Modified-Since': val}

    def filter(self, **kwargs):
        headers = None
        uri = '/'.join([XERO_API_URL, self.name])
        if kwargs:
            if 'since' in kwargs:
                val = kwargs['since']
                headers = self.prepare_filtering_date(val)
                del kwargs['since']

            def get_filter_params():
                last_key = key.split('_')[-1]
                if last_key in self.GUID_FIELDS:
                    return '%s("Guid")' % str(last_key)

                if key in self.BOOLEAN_FIELDS:
                    return 'true' if kwargs[key] else 'false'
                elif key in self.DATETIME_FIELDS:
                    return kwargs[key].isoformat()
                else:
                    return '"%s"' % str(kwargs[key])

            def generate_param(key):
                parts = key.split("__")
                field = key.replace('_', '.')
                fmt = '%s==%s'
                if len(parts) == 2:
                    # support filters:
                    # Name__Contains=John becomes Name.Contains("John")
                    if parts[1] in ["contains", "startswith", "endswith"]:
                        field = parts[0]
                        fmt = ''.join(['%s.', parts[1], '(%s)'])

                return fmt % (
                    field,
                    get_filter_params()
                )

            params = [generate_param(key) for key in kwargs.keys()]

            if params:
                uri += '?where=' + urllib.quote('&&'.join(params))

        return uri, 'get', None, headers

    def all(self):
        uri = '/'.join([XERO_API_URL, self.name])
        return uri, 'get', None, None
<|endoftext|>


<commit_before>"""Implementation of Basic KNX datatypes."""
from enum import Enum

from xknx.exceptions import ConversionError

class DPTBase:
    """
    Base class for KNX data types.

    KNX communicates using Group-addresses, and every Group Object represents a data point of some type.
    To have a standardized interpretation of the data there are a number of Data Point types (DPT).
    The DPT's is written like: xx.yyy, for example 14.056 for a 4-octet float, with Power info in Watts.
    The Major number (xx) describes the data type (format and encoding) - while the the minor (YYY) number
    describes the measurement with value range and unit.

    More DTP's are added as new needs come up, but this a list of some of the commonly used ones:
    1.yyy  boolean, like switching, on/off, open/close, move up/down, step
    2.yyy  2 x boolean, e.g. switching + priority control
    3.yyy  boolean + 3-bit unsigned value, e.g. dimming up/down
    4.yyy  character (8-bit)
    5.yyy  8-bit unsigned value, like dim value (0..100%), blinds position (0..100%)
    6.yyy  8-bit signed (2's complement), e.g. +/- %
    7.yyy  2-byte unsigned value, i.e. pulse counter
    8.yyy  2-byte signed (2's complement), e.g. +/- %
    9.yyy  2-byte float, e.g. temperature
    10.yyy time (3 bytes)
    11.yyy date (3 bytes)
    12.yyy 4-byte unsigned value, i.e. pulse counter
    13.yyy 4-byte signed (2's complement), i.e. flow, energy
    14.yyy 4-byte float, IEEE 754, i.e. Electrical measurements: current, power
    15.yyy access control
    16.yyy string -> 14 characters (14 x 8-bit)
    17.yyy scene number
    18.yyy scene control
    19.yyy date / time
    20.yyy 8-bit enumeration, e.g. HVAC mode ('auto', 'comfort', 'standby', 'economy', 'protection')
    28.yyy UTF-8
    29.yyy V64, 64-bit signed value
    232.yyy RGB [0,0,0]...[255,255,255]

    """

    payload_length = None

    @classmethod
    def test_bytesarray(cls, raw):
        """Test if array of raw bytes has the correct length and values of correct type."""
        if cls.payload_length is None:
            raise NotImplementedError(f"payload_length has to be defined for: {cls}")
        if not isinstance(raw, (tuple, list)) \
                or len(raw) != cls.payload_length \
                or any(not isinstance(byte, int) for byte in raw) \
                or any(byte < 0 for byte in raw) \
                or any(byte > 255 for byte in raw):
            raise ConversionError("Invalid raw bytes", raw=raw)

    @classmethod
    def __recursive_subclasses__(cls):
        """Yield all subclasses and their subclasses."""
        for subclass in cls.__subclasses__():
            yield from subclass.__recursive_subclasses__()
            yield subclass

    @classmethod
    def has_distinct_dpt_numbers(cls):
        """Return True if dpt numbers are defined (not inherited)."""
        return 'dpt_main_number' in cls.__dict__ and 'dpt_sub_number' in cls.__dict__

    @classmethod
    def has_distinct_value_type(cls):
        """Return True if value_type is defined (not inherited)."""
        return 'value_type' in cls.__dict__

    @staticmethod
    def transcoder_by_dpt(dpt_main, dpt_sub=None):
        """Return Class reference of DPTBase subclass with matching DPT number."""
        for dpt in DPTBase.__recursive_subclasses__():
            if dpt.has_distinct_dpt_numbers():
                if dpt_main == dpt.dpt_main_number and dpt_sub == dpt.dpt_sub_number:
                    return dpt
        return None

    @staticmethod
    def transcoder_by_value_type(value_type):
        """Return Class reference of DPTBase subclass with matching value_type."""
        for dpt in DPTBase.__recursive_subclasses__():
            if dpt.has_distinct_value_type:
                if value_type == dpt.value_type:
                    return dpt
        return None

class DPTBinary():
    """The DPTBinary is a base class for all datatypes encoded directly into the first Byte of the payload (mostly integer)."""

    # pylint: disable=too-few-public-methods

    # APCI (application layer control information)
    APCI_BITMASK = 0x3F
    APCI_MAX_VALUE = APCI_BITMASK

    def __init__(self, value):
        """Initialize DPTBinary class."""
        if not isinstance(value, int):
            raise TypeError()
        if value > DPTBinary.APCI_BITMASK:
            raise ConversionError("Cant init DPTBinary", value=value)

        self.value = value

    def __eq__(self, other):
        """Equal operator."""
        return DPTComparator.compare(self, other)

    def __str__(self):
        """Return object as readable string."""
        return '<DPTBinary value="{0}" />'.format(self.value)

class DPTArray():
    """The DPTArray is a base class for all datatypes appended to the KNX telegram."""

    # pylint: disable=too-few-public-methods
    def __init__(self, value):
        """Initialize DPTArray class."""
        if isinstance(value, int):
            self.value = (value,)
        elif isinstance(value, (list, bytes)):
            self.value = tuple(value,)
        elif isinstance(value, tuple):
            self.value = value
        else:
            raise TypeError()

    def __eq__(self, other):
        """Equal operator."""
        return DPTComparator.compare(self, other)

    def __str__(self):
        """Return object as readable string."""
        return '<DPTArray value="[{0}]" />'.format(
            ','.join(hex(b) for b in self.value))

class DPTComparator():
    """Helper class to compare different types of DPT objects."""

    # pylint: disable=too-few-public-methods

    @staticmethod
    def compare(a, b):
        """Test if 'a' and 'b' are the same."""
        # pylint: disable=invalid-name,too-many-return-statements,len-as-condition
        if a is None and b is None:
            return True

        if a is None:
            if isinstance(b, DPTBinary):
                return b.value == 0
            if isinstance(b, DPTArray):
                return len(b.value) == 0

        if b is None:
            if isinstance(a, DPTBinary):
                return a.value == 0
            if isinstance(a, DPTArray):
                return len(a.value) == 0

        if isinstance(a, DPTArray) and isinstance(b, DPTArray):
            return a.value == b.value

        if isinstance(a, DPTBinary) and isinstance(b, DPTBinary):
            return a.value == b.value

        if isinstance(a, DPTBinary) and isinstance(b, DPTArray):
            return a.value == 0 and len(b.value) == 0

        if isinstance(a, DPTArray) and isinstance(b, DPTBinary):
            return len(a.value) == 0 and b.value == 0

        raise TypeError()

class DPTWeekday(Enum):
    """Enum class for week days."""

    MONDAY = 1
    TUESDAY = 2
    WEDNESDAY = 3
    THURSDAY = 4
    FRIDAY = 5
    SATURDAY = 6
    SUNDAY = 7
    NONE = 0
<commit_msg>add missing parenthesis<commit_after>"""Implementation of Basic KNX datatypes."""
from enum import Enum

from xknx.exceptions import ConversionError

class DPTBase:
    """
    Base class for KNX data types.

    KNX communicates using Group-addresses, and every Group Object represents a data point of some type.
    To have a standardized interpretation of the data there are a number of Data Point types (DPT).
    The DPT's is written like: xx.yyy, for example 14.056 for a 4-octet float, with Power info in Watts.
    The Major number (xx) describes the data type (format and encoding) - while the the minor (YYY) number
    describes the measurement with value range and unit.

    More DTP's are added as new needs come up, but this a list of some of the commonly used ones:
    1.yyy  boolean, like switching, on/off, open/close, move up/down, step
    2.yyy  2 x boolean, e.g. switching + priority control
    3.yyy  boolean + 3-bit unsigned value, e.g. dimming up/down
    4.yyy  character (8-bit)
    5.yyy  8-bit unsigned value, like dim value (0..100%), blinds position (0..100%)
    6.yyy  8-bit signed (2's complement), e.g. +/- %
    7.yyy  2-byte unsigned value, i.e. pulse counter
    8.yyy  2-byte signed (2's complement), e.g. +/- %
    9.yyy  2-byte float, e.g. temperature
    10.yyy time (3 bytes)
    11.yyy date (3 bytes)
    12.yyy 4-byte unsigned value, i.e. pulse counter
    13.yyy 4-byte signed (2's complement), i.e. flow, energy
    14.yyy 4-byte float, IEEE 754, i.e. Electrical measurements: current, power
    15.yyy access control
    16.yyy string -> 14 characters (14 x 8-bit)
    17.yyy scene number
    18.yyy scene control
    19.yyy date / time
    20.yyy 8-bit enumeration, e.g. HVAC mode ('auto', 'comfort', 'standby', 'economy', 'protection')
    28.yyy UTF-8
    29.yyy V64, 64-bit signed value
    232.yyy RGB [0,0,0]...[255,255,255]

    """

    payload_length = None

    @classmethod
    def test_bytesarray(cls, raw):
        """Test if array of raw bytes has the correct length and values of correct type."""
        if cls.payload_length is None:
            raise NotImplementedError(f"payload_length has to be defined for: {cls}")
        if not isinstance(raw, (tuple, list)) \
                or len(raw) != cls.payload_length \
                or any(not isinstance(byte, int) for byte in raw) \
                or any(byte < 0 for byte in raw) \
                or any(byte > 255 for byte in raw):
            raise ConversionError("Invalid raw bytes", raw=raw)

    @classmethod
    def __recursive_subclasses__(cls):
        """Yield all subclasses and their subclasses."""
        for subclass in cls.__subclasses__():
            yield from subclass.__recursive_subclasses__()
            yield subclass

    @classmethod
    def has_distinct_dpt_numbers(cls):
        """Return True if dpt numbers are defined (not inherited)."""
        return 'dpt_main_number' in cls.__dict__ and 'dpt_sub_number' in cls.__dict__

    @classmethod
    def has_distinct_value_type(cls):
        """Return True if value_type is defined (not inherited)."""
        return 'value_type' in cls.__dict__

    @staticmethod
    def transcoder_by_dpt(dpt_main, dpt_sub=None):
        """Return Class reference of DPTBase subclass with matching DPT number."""
        for dpt in DPTBase.__recursive_subclasses__():
            if dpt.has_distinct_dpt_numbers():
                if dpt_main == dpt.dpt_main_number and dpt_sub == dpt.dpt_sub_number:
                    return dpt
        return None

    @staticmethod
    def transcoder_by_value_type(value_type):
        """Return Class reference of DPTBase subclass with matching value_type."""
        for dpt in DPTBase.__recursive_subclasses__():
            if dpt.has_distinct_value_type():
                if value_type == dpt.value_type:
                    return dpt
        return None

class DPTBinary():
    """The DPTBinary is a base class for all datatypes encoded directly into the first Byte of the payload (mostly integer)."""

    # pylint: disable=too-few-public-methods

    # APCI (application layer control information)
    APCI_BITMASK = 0x3F
    APCI_MAX_VALUE = APCI_BITMASK

    def __init__(self, value):
        """Initialize DPTBinary class."""
        if not isinstance(value, int):
            raise TypeError()
        if value > DPTBinary.APCI_BITMASK:
            raise ConversionError("Cant init DPTBinary", value=value)

        self.value = value

    def __eq__(self, other):
        """Equal operator."""
        return DPTComparator.compare(self, other)

    def __str__(self):
        """Return object as readable string."""
        return '<DPTBinary value="{0}" />'.format(self.value)

class DPTArray():
    """The DPTArray is a base class for all datatypes appended to the KNX telegram."""

    # pylint: disable=too-few-public-methods
    def __init__(self, value):
        """Initialize DPTArray class."""
        if isinstance(value, int):
            self.value = (value,)
        elif isinstance(value, (list, bytes)):
            self.value = tuple(value,)
        elif isinstance(value, tuple):
            self.value = value
        else:
            raise TypeError()

    def __eq__(self, other):
        """Equal operator."""
        return DPTComparator.compare(self, other)

    def __str__(self):
        """Return object as readable string."""
        return '<DPTArray value="[{0}]" />'.format(
            ','.join(hex(b) for b in self.value))

class DPTComparator():
    """Helper class to compare different types of DPT objects."""

    # pylint: disable=too-few-public-methods

    @staticmethod
    def compare(a, b):
        """Test if 'a' and 'b' are the same."""
        # pylint: disable=invalid-name,too-many-return-statements,len-as-condition
        if a is None and b is None:
            return True

        if a is None:
            if isinstance(b, DPTBinary):
                return b.value == 0
            if isinstance(b, DPTArray):
                return len(b.value) == 0

        if b is None:
            if isinstance(a, DPTBinary):
                return a.value == 0
            if isinstance(a, DPTArray):
                return len(a.value) == 0

        if isinstance(a, DPTArray) and isinstance(b, DPTArray):
            return a.value == b.value

        if isinstance(a, DPTBinary) and isinstance(b, DPTBinary):
            return a.value == b.value

        if isinstance(a, DPTBinary) and isinstance(b, DPTArray):
            return a.value == 0 and len(b.value) == 0

        if isinstance(a, DPTArray) and isinstance(b, DPTBinary):
            return len(a.value) == 0 and b.value == 0

        raise TypeError()

class DPTWeekday(Enum):
    """Enum class for week days."""

    MONDAY = 1
    TUESDAY = 2
    WEDNESDAY = 3
    THURSDAY = 4
    FRIDAY = 5
    SATURDAY = 6
    SUNDAY = 7
    NONE = 0
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

#
# Copyright © 2012-2013 by its contributors. See AUTHORS for details.
#
# Distributed under the MIT/X11 software license, see the accompanying
# file LICENSE or http://www.opensource.org/licenses/mit-license.php.
#

# Python standard library, unit-testing
import unittest2

# Python patterns, scenario unit-testing
from python_patterns.unittest.scenario import ScenarioMeta, ScenarioTest

from bitcoin.base58 import *

# ===----------------------------------------------------------------------===

BASE58_CODEC = [
    dict(data='', result=''),
    dict(data=chr(0), result='1'),
    dict(data=chr(0)+('1f8b1340c286881bcc449c37569ae320b013785d'+'48427760').decode('hex'),
         result='13snZ4ZyCzaL7358SmgvHGC9AxskqumNxP'),
    # src/test/data/base58_encode_decode.json:
    dict(data='61'.decode('hex'), result='2g'),
    dict(data='626262'.decode('hex'), result='a3gV'),
    dict(data='636363'.decode('hex'), result='aPEr'),
    dict(data='516b6fcd0f'.decode('hex'), result='ABnLTmg'),
    dict(data='bf4f89001e670274dd'.decode('hex'), result='3SEo3LWLoPntC'),
    dict(data='572e4794'.decode('hex'), result='3EFU7m'),
    dict(data='ecac89cad93923c02321'.decode('hex'), result='EJDM8drfXA6uyA'),
    dict(data='10c8511e'.decode('hex'), result='Rt5zm'),
    dict(data='00000000000000000000'.decode('hex'), result='1111111111'),
    dict(data='73696d706c792061206c6f6e6720737472696e67'.decode('hex'),
         result='2cFupjhnEsSn59qHXstmK2ffpLv2'),
    dict(data='00eb15231dfceb60925886b67d065299925915aeb172c06647'.decode('hex'),
         result='1NS17iag9jJgTHD1VXjvLCEnZuQ3rJDE9L'),
]

class TestCodecBase58(unittest2.TestCase):
    "Test encoding and decoding of base58 values."
    __metaclass__ = ScenarioMeta
    class test_serialize(ScenarioTest):
        scenarios = BASE58_CODEC
        def __test__(self, data, result):
            self.assertEqual(data.encode('base58'), result)
    class test_deserialize(ScenarioTest):
        scenarios = BASE58_CODEC
        def __test__(self, data, result):
            self.assertEqual(result.decode('base58'), data)

# ===----------------------------------------------------------------------===

from bitcoin.errors import HashChecksumError

HASH_CHECKED_DATA = [
    dict(data     = b'\x00' + '1f8b1340c286881bcc449c37569ae320b013785d'.decode('hex'),
         checksum = '48427760'.decode('hex'),
         base58   = u"13snZ4ZyCzaL7358SmgvHGC9AxskqumNxP"),
    dict(data     = b'\x05' + '5ece0cadddc415b1980f001785947120acdb36fc'.decode('hex'),
         checksum = 'b43c48af'.decode('hex'),
         base58   = u"3ALJH9Y951VCGcVZYAdpA3KchoP9McEj1G"),
]

class TestHashCheckedData(unittest2.TestCase):
    "Test checksummed base58 values"
    __metaclass__ = ScenarioMeta
    class test_init(ScenarioTest):
        scenarios = HASH_CHECKED_DATA
        def __test__(self, data, checksum, base58):
            checked = HashCheckedData(base58.decode('base58'))
            self.assertEqual(checked, b''.join([data, checksum]))
            self.assertEqual(checked.data, data)
            self.assertEqual(checked.checksum, checksum)
            self.assertEqual(checked.encode('base58'), base58)
            checked = HashCheckedData(data, add_hash=True)
            self.assertEqual(checked, b''.join([data, checksum]))
            self.assertEqual(checked.data, data)
            self.assertEqual(checked.checksum, checksum)
            self.assertEqual(checked.encode('base58'), base58)

BAD_HASH_DATA = [
    # Too short:
    dict(data=b''),
    dict(data=b'\x00'),
    dict(data=b'\x00'*2),
    dict(data=b'\x00'*3),
    # Bad checksum:
    dict(data=b'\x00'*4),
]

class TestBadHashChecksum(unittest2.TestCase):
    "Test bad checksums for HashCheckedData"
    __metaclass__ = ScenarioMeta
    class test_bad_checksum(ScenarioTest):
        scenarios = BAD_HASH_DATA
        def __test__(self, data):
            with self.assertRaises(HashChecksumError):
                checked = HashCheckedData(data, add_hash=False)

# ===----------------------------------------------------------------------===

VERSIONED_PAYLOAD = [
    dict(version  = 0,
         payload  = '1f8b1340c286881bcc449c37569ae320b013785d'.decode('hex'),
         checksum = '48427760'.decode('hex'),
         base58   = u"13snZ4ZyCzaL7358SmgvHGC9AxskqumNxP"),
    dict(version  = 5,
         payload  = '5ece0cadddc415b1980f001785947120acdb36fc'.decode('hex'),
         checksum = 'b43c48af'.decode('hex'),
         base58   = u"3ALJH9Y951VCGcVZYAdpA3KchoP9McEj1G"),
]

class TestVersionedPayload(unittest2.TestCase):
    "Test versioned base58 payloads"
    __metaclass__ = ScenarioMeta
    class test_init(ScenarioTest):
        scenarios = VERSIONED_PAYLOAD
        def __test__(self, version, payload, checksum, base58):
            data = b''.join([six.int2byte(version), payload])
            checked = VersionedPayload(base58.decode('base58'))
            self.assertEqual(checked, b''.join([data, checksum]))
            self.assertEqual(checked.version, version)
            self.assertEqual(checked.payload, payload)
            self.assertEqual(checked.data, data)
            self.assertEqual(checked.checksum, checksum)
            self.assertEqual(checked.encode('base58'), base58)
    class test_init_with_version(ScenarioTest):
        scenarios = VERSIONED_PAYLOAD
        def __test__(self, version, payload, checksum, base58):
            data = b''.join([six.int2byte(version), payload])
            checked = VersionedPayload(payload, version=version)
            self.assertEqual(checked, b''.join([data, checksum]))
            self.assertEqual(checked.version, version)
            self.assertEqual(checked.payload, payload)
            self.assertEqual(checked.data, data)
            self.assertEqual(checked.checksum, checksum)
            self.assertEqual(checked.encode('base58'), base58)
    class test_init_with_version_explicit_hash(ScenarioTest):
        scenarios = VERSIONED_PAYLOAD
        def __test__(self, version, payload, checksum, base58):
            data = b''.join([six.int2byte(version), payload])
            checked = VersionedPayload(payload + checksum,
                version = version, add_hash = False)
            self.assertEqual(checked, b''.join([data, checksum]))
            self.assertEqual(checked.version, version)
            self.assertEqual(checked.payload, payload)
            self.assertEqual(checked.data, data)
            self.assertEqual(checked.checksum, checksum)
            self.assertEqual(checked.encode('base58'), base58)

#
# End of File
#
<commit_msg>Clean up unit tests.<commit_after># -*- coding: utf-8 -*-

#
# Copyright © 2012-2013 by its contributors. See AUTHORS for details.
#
# Distributed under the MIT/X11 software license, see the accompanying
# file LICENSE or http://www.opensource.org/licenses/mit-license.php.
#

# Python standard library, unit-testing
import unittest2

# Python patterns, scenario unit-testing
from python_patterns.unittest.scenario import ScenarioMeta, ScenarioTest

from bitcoin.base58 import *

# ===----------------------------------------------------------------------===

BASE58_CODEC = [
    dict(data='', result=u""),
    dict(data=chr(0), result=u"1"),
    dict(data     = chr(0)+('1f8b1340c286881bcc449c37569ae320b013785d'+'48427760').decode('hex'),
         result   = u"13snZ4ZyCzaL7358SmgvHGC9AxskqumNxP"),
    # src/test/data/base58_encode_decode.json:
    dict(data='61'.decode('hex'), result=u"2g"),
    dict(data='626262'.decode('hex'), result=u"a3gV"),
    dict(data='636363'.decode('hex'), result=u"aPEr"),
    dict(data='516b6fcd0f'.decode('hex'), result=u"ABnLTmg"),
    dict(data='bf4f89001e670274dd'.decode('hex'), result=u"3SEo3LWLoPntC"),
    dict(data='572e4794'.decode('hex'), result=u"3EFU7m"),
    dict(data='ecac89cad93923c02321'.decode('hex'), result=u"EJDM8drfXA6uyA"),
    dict(data='10c8511e'.decode('hex'), result=u"Rt5zm"),
    dict(data='00000000000000000000'.decode('hex'), result=u"1111111111"),
    dict(data   = '73696d706c792061206c6f6e6720737472696e67'.decode('hex'),
         result = u"2cFupjhnEsSn59qHXstmK2ffpLv2"),
    dict(data   = '00eb15231dfceb60925886b67d065299925915aeb172c06647'.decode('hex'),
         result = u"1NS17iag9jJgTHD1VXjvLCEnZuQ3rJDE9L"),
]

class TestCodecBase58(unittest2.TestCase):
    "Test encoding and decoding of base58 values."
    __metaclass__ = ScenarioMeta
    class test_serialize(ScenarioTest):
        scenarios = BASE58_CODEC
        def __test__(self, data, result):
            self.assertEqual(data.encode('base58'), result)
    class test_deserialize(ScenarioTest):
        scenarios = BASE58_CODEC
        def __test__(self, data, result):
            self.assertEqual(result.decode('base58'), data)

# ===----------------------------------------------------------------------===

from bitcoin.errors import HashChecksumError

HASH_CHECKED_DATA = [
    dict(data     = b'\x00' + '1f8b1340c286881bcc449c37569ae320b013785d'.decode('hex'),
         checksum = '48427760'.decode('hex'),
         base58   = u"13snZ4ZyCzaL7358SmgvHGC9AxskqumNxP"),
    dict(data     = b'\x05' + '5ece0cadddc415b1980f001785947120acdb36fc'.decode('hex'),
         checksum = 'b43c48af'.decode('hex'),
         base58   = u"3ALJH9Y951VCGcVZYAdpA3KchoP9McEj1G"),
]

class TestHashCheckedData(unittest2.TestCase):
    "Test checksummed base58 values"
    __metaclass__ = ScenarioMeta
    class test_init(ScenarioTest):
        scenarios = HASH_CHECKED_DATA
        def __test__(self, data, checksum, base58):
            checked = HashCheckedData(base58.decode('base58'))
            self.assertEqual(checked, b''.join([data, checksum]))
            self.assertEqual(checked.data, data)
            self.assertEqual(checked.checksum, checksum)
            self.assertEqual(checked.encode('base58'), base58)
            checked = HashCheckedData(data, add_hash=True)
            self.assertEqual(checked, b''.join([data, checksum]))
            self.assertEqual(checked.data, data)
            self.assertEqual(checked.checksum, checksum)
            self.assertEqual(checked.encode('base58'), base58)

BAD_HASH_DATA = [
    # Too short:
    dict(data=b''),
    dict(data=b'\x00'),
    dict(data=b'\x00'*2),
    dict(data=b'\x00'*3),
    # Bad checksum:
    dict(data=b'\x00'*4),
]

class TestBadHashChecksum(unittest2.TestCase):
    "Test bad checksums for HashCheckedData"
    __metaclass__ = ScenarioMeta
    class test_bad_checksum(ScenarioTest):
        scenarios = BAD_HASH_DATA
        def __test__(self, data):
            with self.assertRaises(HashChecksumError):
                checked = HashCheckedData(data, add_hash=False)

# ===----------------------------------------------------------------------===

VERSIONED_PAYLOAD = [
    dict(version  = 0,
         payload  = '1f8b1340c286881bcc449c37569ae320b013785d'.decode('hex'),
         checksum = '48427760'.decode('hex'),
         base58   = u"13snZ4ZyCzaL7358SmgvHGC9AxskqumNxP"),
    dict(version  = 5,
         payload  = '5ece0cadddc415b1980f001785947120acdb36fc'.decode('hex'),
         checksum = 'b43c48af'.decode('hex'),
         base58   = u"3ALJH9Y951VCGcVZYAdpA3KchoP9McEj1G"),
]

class TestVersionedPayload(unittest2.TestCase):
    "Test versioned base58 payloads"
    __metaclass__ = ScenarioMeta
    class test_init(ScenarioTest):
        scenarios = VERSIONED_PAYLOAD
        def __test__(self, version, payload, checksum, base58):
            data = b''.join([six.int2byte(version), payload])
            checked = VersionedPayload(base58.decode('base58'))
            self.assertEqual(checked, b''.join([data, checksum]))
            self.assertEqual(checked.version, version)
            self.assertEqual(checked.payload, payload)
            self.assertEqual(checked.data, data)
            self.assertEqual(checked.checksum, checksum)
            self.assertEqual(checked.encode('base58'), base58)
    class test_init_with_version(ScenarioTest):
        scenarios = VERSIONED_PAYLOAD
        def __test__(self, version, payload, checksum, base58):
            data = b''.join([six.int2byte(version), payload])
            checked = VersionedPayload(payload, version=version)
            self.assertEqual(checked, b''.join([data, checksum]))
            self.assertEqual(checked.version, version)
            self.assertEqual(checked.payload, payload)
            self.assertEqual(checked.data, data)
            self.assertEqual(checked.checksum, checksum)
            self.assertEqual(checked.encode('base58'), base58)
    class test_init_with_version_explicit_hash(ScenarioTest):
        scenarios = VERSIONED_PAYLOAD
        def __test__(self, version, payload, checksum, base58):
            data = b''.join([six.int2byte(version), payload])
            checked = VersionedPayload(payload + checksum,
                version = version, add_hash = False)
            self.assertEqual(checked, b''.join([data, checksum]))
            self.assertEqual(checked.version, version)
            self.assertEqual(checked.payload, payload)
            self.assertEqual(checked.data, data)
            self.assertEqual(checked.checksum, checksum)
            self.assertEqual(checked.encode('base58'), base58)

#
# End of File
#
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2022 The Trax Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Copyright 2020 The Trax Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3
"""Tests for trax.layers.relattention."""

from absl.testing import absltest
import numpy as np

import trax.layers as tl
import trax.layers.research.rel_attention as ra

class RelAttentionTest(absltest.TestCase):

  def test_fast_shift_matrix(self):
    layer = ra._fast_matrix_shift
    x = np.array([[[[-3., -2., -1., 0.], [-3., -2., -1.,
                                          0.], [-3., -2., -1., 0.],
                    [-3., -2., -1., 0.]]]]).astype(np.float32)

    y = layer(x)
    self.assertEqual(y.dtype, np.float32)
    self.assertEqual(
        tl.to_list(y), [[[[0., 0., -3., -2.], [-1., 0., 0., -3.],
                          [-2., -1., 0., 0.], [-3., -2., -1., 0.]]]])

if __name__ == '__main__':
  absltest.main()
<commit_msg>Remove unused comments related to Python 2 compatibility.<commit_after># coding=utf-8
# Copyright 2022 The Trax Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Copyright 2020 The Trax Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for trax.layers.relattention."""

from absl.testing import absltest
import numpy as np

import trax.layers as tl
import trax.layers.research.rel_attention as ra

class RelAttentionTest(absltest.TestCase):

  def test_fast_shift_matrix(self):
    layer = ra._fast_matrix_shift
    x = np.array([[[[-3., -2., -1., 0.], [-3., -2., -1.,
                                          0.], [-3., -2., -1., 0.],
                    [-3., -2., -1., 0.]]]]).astype(np.float32)

    y = layer(x)
    self.assertEqual(y.dtype, np.float32)
    self.assertEqual(
        tl.to_list(y), [[[[0., 0., -3., -2.], [-1., 0., 0., -3.],
                          [-2., -1., 0., 0.], [-3., -2., -1., 0.]]]])

if __name__ == '__main__':
  absltest.main()
<|endoftext|>


<commit_before>"""Create a Graph class data structure."""
from stack import Stack

class Graph(object):
    """Graph object."""

    def __init__(self):
        """Graph things on initialization."""
        self._nodes = {}
        self._weights = {}

    def nodes(self):
        """List all nodes in the Graph."""
        return [key for key in self._nodes]

    def edges(self):
        """List all edges in the Graph."""
        return [[self._nodes], [self._weights]]

    def weights(self):
        """List all the edges weights in the Graph."""
        return self._weights

    def add_node(self, *args):
        """Add a new node to the Graph."""
        for arg in args:
            self._nodes.setdefault(arg, [])

    def add_edge(self, val1, val2):
        """Add a new edge to the Graph and connects the values.

        If either value doesn't exist they will be created.
        """
        self.add_node(val1, val2)
        if val2 in self._nodes[val1]:  # pragma: no cover
            pass
        else:
            weight = self.add_weight(val1, val2)
            self._nodes[val1].append(val2)
            self._weights['{}-{}'.format(val1, val2)] = weight

    def del_node(self, val):
        """Delete the node containing the given value."""
        try:
            del self._nodes[val]
            for key in self._nodes:
                if val in self._nodes[key]:
                    self._nodes[key].remove(val)
        except KeyError:
            raise KeyError("No such Node exists")

    def del_edge(self, val1, val2):
        """Delete the edge connecting the two values."""
        try:
            for node in self._nodes[val1]:
                if val2 == node:
                    self._nodes[val1].remove(node)
                    del self._weights['{}-{}'.format(val1, val2)]
                    return
            raise IndexError("No connection between those two Nodes.")
        except KeyError:
            raise KeyError("No Node with given first value.")

    def has_node(self, val):
        """Return True is node containing value exists in Graph."""
        return True if val in self._nodes else False

    def neighbors(self, val):
        """Return list of all nodes connected to node containing the value."""
        return self._nodes[val]

    def adjacent(self, val1, val2):
        """Return True if there is an edge connecting the two values."""
        return val2 in self._nodes[val1]

    def depth_first_traversal(self, start):
        """Perform a depth-first traversal and return full visited path."""
        if start not in self._nodes:
            raise KeyError("No such value")
        path = []
        visit = Stack()
        curr = start
        while True:
            try:
                neighbors = self.neighbors(curr)
                for n in neighbors:
                    visit.push(n)
                path.append(curr)
                curr = visit.pop()
            except IndexError:
                break
        return path

    def breadth_first_traversal(self, start):
        """Perform a breadth-first traversal and return full visited path."""
        if start not in self._nodes:
            raise KeyError("No such value")
        path, visit = [], []
        curr = start
        while True:
            try:
                path.append(curr)
                visit.extend(self.neighbors(curr))
                curr = visit[0]
                visit = visit[1:]
            except IndexError:
                break
        return path

    def add_weight(self, val1, val2):
        """Add weights for all edges."""
        if isinstance(val1, str):
            val1 = ord(val1)
        if isinstance(val2, str):
            val2 = ord(val2)
        return abs(val1 - val2)
<commit_msg>workin on graph<commit_after>"""Create a Graph class data structure."""
from stack import Stack

class Graph(object):
    """Graph object."""

    def __init__(self):
        """Graph things on initialization."""
        self._nodes = {}

    def nodes(self):
        """List all nodes in the Graph."""
        return [key for key in self._nodes]

    def edges(self):
        """List all edges in the Graph."""
        return self._nodes

    def add_node(self, *args):
        """Add a new node to the Graph."""
        for arg in args:
            self._nodes.setdefault(arg, {})

    def add_edge(self, val1, val2, weight):
        """Add a new edge to the Graph and connects the values.

        If either value doesn't exist they will be created.
        """
        self.add_node(val1, val2)
        if val2 in self._nodes[val1]:  # pragma: no cover
            pass
        else:
            self._nodes[val1].update({val2: weight})

    def del_node(self, val):
        """Delete the node containing the given value."""
        try:
            if val in self._nodes:
                del self._nodes[val]
            for key in self._nodes:
                if val in list(self._nodes[key]):
                    self._nodes[key].pop(val)
        except KeyError:
            raise KeyError("No such Node exists")

    def del_edge(self, val1, val2):
        """Delete the edge connecting the two values."""
        try:
            for node in self._nodes[val1]:
                if val2 == node:
                    self._nodes[val1].pop(node)
                    return
            raise IndexError("No connection between those two Nodes.")
        except KeyError:
            raise KeyError("No Node with given first value.")

    def has_node(self, val):
        """Return True is node containing value exists in Graph."""
        return True if val in self._nodes else False

    def neighbors(self, val):
        """Return list of all nodes connected to node containing the value."""
        return list(self._nodes[val].keys())

    def adjacent(self, val1, val2):
        """Return True if there is an edge connecting the two values."""
        return val2 in self._nodes[val1]

    def depth_first_traversal(self, start):
        """Perform a depth-first traversal and return full visited path."""
        if start not in self._nodes:
            raise KeyError("No such value")
        path = []
        visit = Stack()
        curr = start
        while True:
            try:
                neighbors = self.neighbors(curr)
                for n in neighbors:
                    visit.push(n)
                path.append(curr)
                curr = visit.pop()
            except IndexError:
                break
        return path

    def breadth_first_traversal(self, start):
        """Perform a breadth-first traversal and return full visited path."""
        if start not in self._nodes:
            raise KeyError("No such value")
        path, visit = [], []
        curr = start
        while True:
            try:
                path.append(curr)
                visit.extend(self.neighbors(curr))
                curr = visit[0]
                visit = visit[1:]
            except IndexError:
                break
        return path
<|endoftext|>


<commit_before>'''
Created on June 1, 2011

@note:
@author: John Cornwell
@contact: JohnWCornwellV@gmail.com
@summary: Used to extract Compustat data from a csv dump and convert into individual pickle files
'''

import numpy as np
import datetime as dt
import pickle as pkl
import qstkutil.utils as utils
import qstkutil.DataAccess as da
import os
import dircache
import time
import csv
import sys

''' dump files from dictionary dData, set of paths lSets '''
def _dumpFiles( dData, lSets, lsOutPaths ):
    lKeys = dData.keys()

    for key in lKeys:
        for i, symSet in enumerate( lSets ):
            if key in symSet:
                sFilename = lsOutPaths[i] + key + '.pkl'
                break

        fOut = open( sFilename, 'wb' )
        pkl.dump( dData[key], fOut, -1)
        fOut.close()

''' Analyzes the csv data to determine which columns are float values '''
def _analyze():
    try:
        rootdir = os.environ['QSDATA']
    except KeyError:
        print "Please be sure to set the value for QSDATA in config.sh or local.sh\n"

    ''' Create lists of input and output paths '''
    fFile = ( rootdir + "\Raw\Compustat\Compustat.csv")

    spamReader = csv.reader(open(fFile, 'rb'), delimiter=',')

    ''' Take first row as the labels '''
    for row in spamReader:
        lsLabels = row
        break

    used = [0] * len(lsLabels)
    badSet = set()

    ''' find non-float valued entities '''
    for i, row in enumerate(spamReader):
        for j, elem in enumerate(row):
            if used[j] == 1:
                continue

            if elem == '':
                elem = 0.0

            try:
                float(elem)
            except:
                badSet.add( lsLabels[j] )
                used[j] = 1

        if( i % 10000 == 0 ):
            print (i / 1378625.0)*100, '%'
    print badSet

def convert ():

    print "Starting..."+ str(time.strftime("%H:%M:%S"))

    ''' Write every so often to save memory, 20k lines is usually < .5GB '''
    lSaveMem = 20000

    try:
        rootdir = os.environ['QSDATA']
    except KeyError:
        print "Please be sure to set the value for QSDATA in config.sh or local.sh\n"

    ''' Create lists of input and output paths '''
    fFile = ( rootdir + "\Raw\Compustat\Compustat.csv")

    listOfOutputPaths= []
    listOfOutputPaths.append(rootdir + "/Processed/Compustat/US/AMEX/")
    listOfOutputPaths.append(rootdir + "/Processed/Compustat/US/NASDAQ/")
    listOfOutputPaths.append(rootdir + "/Processed/Compustat/US/NYSE/")

    #If the output paths don't exist, then create them...
    for path in listOfOutputPaths:
        if not (os.access(path, os.F_OK)):
            #Path does not exist, so create it
            os.makedirs(path) #Makes paths recursively
    #done making all output paths!

    #In case there are already some files there- remove them. This will remove all the pkl fils from the previous run
    utils.clean_paths (listOfOutputPaths)

    spamReader = csv.reader(open(fFile, 'rb'), delimiter=',')
    ''' 1378625 rows typical '''

    ''' Take first row as the labels '''
    for row in spamReader:
        lsLabels = row
        break

    ''' Generated from _Analyze() '''
    lsBadLabels = set(['LOC', 'ADD4', 'ADD3', 'ADD2', 'ADD1', 'ACCTCHGQ', 'WEBURL', 'IDBFLAG', 'popsrc', 'DATACQTR', 'conm', 'COSTAT', 'FINALQ', 'fdateq', 'FAX', 'RP', 'PRIROW', 'dldte', 'indfmt', 'SPCSRC', 'BUSDESC', 'ipodate', 'PHONE', 'CURCDQ', 'pdateq', 'DATAFQTR', 'PRICAN', 'EIN', 'datadate', 'tic', 'ADDZIP', 'CONML', 'consol', 'datafmt', 'cusip', 'BSPRQ', 'OGMQ', 'COMPSTQ', 'COUNTY', 'STATE', 'CURNCDQ', 'CITY', 'rdq', 'apdedateq', 'STALTQ', 'INCORP'])

    ''' get list of stocks in 3 US indexes '''
    Access = da.DataAccess( 'Norgate' )
    setNyse = set( Access.get_symbols_in_sublist("/US/NYSE") )
    setNasdaq = set( Access.get_symbols_in_sublist("/US/NASDAQ") )
    setAmex = set( Access.get_symbols_in_sublist("/US/AMEX") )

    ''' If stock appears in more than one index, remove to avoid ambiguity '''
    print 'Ignoring duplicate stocks:',
    dup1 =  setNyse.intersection( setNasdaq.union(setAmex))
    dup2 =  setNasdaq.intersection( setAmex )
    print dup1.union(dup2)

    setNyse   = setNyse - dup1.union(dup2)
    setAmex   = setAmex - dup1.union(dup2)
    setNasdaq = setNasdaq - dup1.union(dup2)

    ''' Note the two lists below must be in the same order '''
    lsOutPaths = []
    lsOutPaths.append(rootdir + "/Processed/Compustat/US/AMEX/")
    lsOutPaths.append(rootdir + "/Processed/Compustat/US/NASDAQ/")
    lsOutPaths.append(rootdir + "/Processed/Compustat/US/NYSE/")

    lSets = [setAmex, setNasdaq, setNyse]

    #If the output paths don't exist, then create them...
    for path in lsOutPaths:
        if not (os.access(path, os.F_OK)):
            #Path does not exist, so create it
            os.makedirs(path) #Makes paths recursively
    #done making all output paths!

    #In case there are already some files there- remove them. This will remove all the pkl fils from the previous run
    utils.clean_paths (lsOutPaths)

    lDateCol = 0
    llUseCols = []

    ''' We have first row (the labels), loop through saving good label indicies '''
    for j, sElem in enumerate(lsLabels):
        if( sElem not in lsBadLabels ):
            llUseCols.append(j)

        ''' Keep track of ticker column and date, specially handled later '''
        if( sElem == 'datadate' ):
            lDateCol = j
        if( sElem == 'tic' ):
            lTicCol = j

    ''' Dict of ticker->numpy array mapping '''
    dData = dict()
    print ''

    ''' Main loop, iterate over the rows in the csv file '''
    for j, row in enumerate(spamReader):
        lsLabels = row
        sTic = row[lTicCol]

        ''' Find out what index this belongs to '''
        lIndex = -1
        for i, symSet in enumerate( lSets ):
            if sTic in symSet:
                lIndex = i
                break
        if lIndex == -1:
            continue

        sFilename = lsOutPaths[lIndex] + sTic + '.pkl'

        ''' If the file exists (temporary memory saving measure), read it in and delete file from disk '''
        if( os.path.isfile(sFilename)):
            if dData.has_key(sTic):
               print 'File should not be both on disk and in dict'
               sys.exit("FAILURE")

            fIn = open( sFilename, 'rb' )
            dData[sTic] = pkl.load( fIn )
            fIn.close()
            os.remove( sFilename )

        fDate = float( dt.datetime.strptime( row[lDateCol], "%m/%d/%Y").strftime("%Y%m%d") )

        ''' convert blanks to nans '''
        for i in llUseCols:
            if row[i] == '':
                row[i] = 'nan'

        ''' Add row if data exists, if not, create new array '''
        if dData.has_key(sTic):
            dData[sTic] = np.vstack( (dData[sTic], np.array([fDate] + [row[i] for i in llUseCols], dtype=np.float)) )
        else:
            dData[sTic]= np.array( [fDate] + [row[i] for i in llUseCols], dtype=np.float )

        if( (j+1) % 1000 == 0):
            fDone = (j / 1378625.0) * 100
            print '\rApprox %.2lf%%'%((j / 1378625.0) * 100),

        if( (j+1) % lSaveMem == 0):
            ''' Write all the pickle files we currently have '''

            print '\nWriting %i lines to pickle files to save memory\n'%(lSaveMem)
            _dumpFiles( dData, lSets, lsOutPaths)
            ''' Remember to delete! '''
            del dData
            dData = dict()

        # Done writing files
    # Done with main loop

    print ''
    print 'Writing final pickle files\n'
    _dumpFiles( dData, lSets, lsOutPaths)
    del dData

    print "Finished..."+ str(time.strftime("%H:%M:%S"))

    return

if __name__ == '__main__':

    #_analyze()

    convert()

    <commit_msg>added print for good labels<commit_after>'''
Created on June 1, 2011

@note:
@author: John Cornwell
@contact: JohnWCornwellV@gmail.com
@summary: Used to extract Compustat data from a csv dump and convert into individual pickle files
'''

import numpy as np
import datetime as dt
import pickle as pkl
import qstkutil.utils as utils
import qstkutil.DataAccess as da
import os
import dircache
import time
import csv
import sys

''' dump files from dictionary dData, set of paths lSets '''
def _dumpFiles( dData, lSets, lsOutPaths ):
    lKeys = dData.keys()

    for key in lKeys:
        for i, symSet in enumerate( lSets ):
            if key in symSet:
                sFilename = lsOutPaths[i] + key + '.pkl'
                break

        fOut = open( sFilename, 'wb' )
        pkl.dump( dData[key], fOut, -1)
        fOut.close()

''' Analyzes the csv data to determine which columns are float values '''
def _analyze():
    try:
        rootdir = os.environ['QSDATA']
    except KeyError:
        print "Please be sure to set the value for QSDATA in config.sh or local.sh\n"

    ''' Create lists of input and output paths '''
    fFile = ( rootdir + "\Raw\Compustat\Compustat.csv")

    spamReader = csv.reader(open(fFile, 'rb'), delimiter=',')

    ''' Take first row as the labels '''
    for row in spamReader:
        lsLabels = row
        break

    used = [0] * len(lsLabels)
    badSet = set()

    ''' find non-float valued entities '''
    for i, row in enumerate(spamReader):
        for j, elem in enumerate(row):
            if used[j] == 1:
                continue

            if elem == '':
                elem = 0.0

            try:
                float(elem)
            except:
                badSet.add( lsLabels[j] )
                used[j] = 1

        if( i % 10000 == 0 ):
            print (i / 1378625.0)*100, '%'

    print 'Bad (non float) labels:'
    print badSet

    for i,label in enumerate(lsLabels):
        if label in lsBadLabels:
            del lsLabels[i]

    print '\n\nGood (float) labels:'
    print lsLabels

    return

def convert ():

    print "Starting..."+ str(time.strftime("%H:%M:%S"))

    ''' Write every so often to save memory, 20k lines is usually < .5GB '''
    lSaveMem = 20000

    try:
        rootdir = os.environ['QSDATA']
    except KeyError:
        print "Please be sure to set the value for QSDATA in config.sh or local.sh\n"

    ''' Create lists of input and output paths '''
    fFile = ( rootdir + "\Raw\Compustat\Compustat.csv")

    listOfOutputPaths= []
    listOfOutputPaths.append(rootdir + "/Processed/Compustat/US/AMEX/")
    listOfOutputPaths.append(rootdir + "/Processed/Compustat/US/NASDAQ/")
    listOfOutputPaths.append(rootdir + "/Processed/Compustat/US/NYSE/")

    #If the output paths don't exist, then create them...
    for path in listOfOutputPaths:
        if not (os.access(path, os.F_OK)):
            #Path does not exist, so create it
            os.makedirs(path) #Makes paths recursively
    #done making all output paths!

    #In case there are already some files there- remove them. This will remove all the pkl fils from the previous run
    utils.clean_paths (listOfOutputPaths)

    spamReader = csv.reader(open(fFile, 'rb'), delimiter=',')
    ''' 1378625 rows typical '''

    ''' Take first row as the labels '''
    for row in spamReader:
        lsLabels = row
        break

    ''' Generated from _Analyze() '''
    lsBadLabels = set(['LOC', 'ADD4', 'ADD3', 'ADD2', 'ADD1', 'ACCTCHGQ', 'WEBURL', 'IDBFLAG', 'popsrc', 'DATACQTR', 'conm', 'COSTAT', 'FINALQ', 'fdateq', 'FAX', 'RP', 'PRIROW', 'dldte', 'indfmt', 'SPCSRC', 'BUSDESC', 'ipodate', 'PHONE', 'CURCDQ', 'pdateq', 'DATAFQTR', 'PRICAN', 'EIN', 'datadate', 'tic', 'ADDZIP', 'CONML', 'consol', 'datafmt', 'cusip', 'BSPRQ', 'OGMQ', 'COMPSTQ', 'COUNTY', 'STATE', 'CURNCDQ', 'CITY', 'rdq', 'apdedateq', 'STALTQ', 'INCORP'])

    ''' get list of stocks in 3 US indexes '''
    Access = da.DataAccess( 'Norgate' )
    setNyse = set( Access.get_symbols_in_sublist("/US/NYSE") )
    setNasdaq = set( Access.get_symbols_in_sublist("/US/NASDAQ") )
    setAmex = set( Access.get_symbols_in_sublist("/US/AMEX") )

    ''' If stock appears in more than one index, remove to avoid ambiguity '''
    print 'Ignoring duplicate stocks:',
    dup1 =  setNyse.intersection( setNasdaq.union(setAmex))
    dup2 =  setNasdaq.intersection( setAmex )
    print dup1.union(dup2)

    setNyse   = setNyse - dup1.union(dup2)
    setAmex   = setAmex - dup1.union(dup2)
    setNasdaq = setNasdaq - dup1.union(dup2)

    ''' Note the two lists below must be in the same order '''
    lsOutPaths = []
    lsOutPaths.append(rootdir + "/Processed/Compustat/US/AMEX/")
    lsOutPaths.append(rootdir + "/Processed/Compustat/US/NASDAQ/")
    lsOutPaths.append(rootdir + "/Processed/Compustat/US/NYSE/")

    lSets = [setAmex, setNasdaq, setNyse]

    #If the output paths don't exist, then create them...
    for path in lsOutPaths:
        if not (os.access(path, os.F_OK)):
            #Path does not exist, so create it
            os.makedirs(path) #Makes paths recursively
    #done making all output paths!

    #In case there are already some files there- remove them. This will remove all the pkl fils from the previous run
    utils.clean_paths (lsOutPaths)

    lDateCol = 0
    llUseCols = []

    ''' We have first row (the labels), loop through saving good label indicies '''
    for j, sElem in enumerate(lsLabels):
        if( sElem not in lsBadLabels ):
            llUseCols.append(j)

        ''' Keep track of ticker column and date, specially handled later '''
        if( sElem == 'datadate' ):
            lDateCol = j
        if( sElem == 'tic' ):
            lTicCol = j

    ''' Dict of ticker->numpy array mapping '''
    dData = dict()
    print ''

    ''' Main loop, iterate over the rows in the csv file '''
    for j, row in enumerate(spamReader):
        lsLabels = row
        sTic = row[lTicCol]

        ''' Find out what index this belongs to '''
        lIndex = -1
        for i, symSet in enumerate( lSets ):
            if sTic in symSet:
                lIndex = i
                break
        if lIndex == -1:
            continue

        sFilename = lsOutPaths[lIndex] + sTic + '.pkl'

        ''' If the file exists (temporary memory saving measure), read it in and delete file from disk '''
        if( os.path.isfile(sFilename)):
            if dData.has_key(sTic):
               print 'File should not be both on disk and in dict'
               sys.exit("FAILURE")

            fIn = open( sFilename, 'rb' )
            dData[sTic] = pkl.load( fIn )
            fIn.close()
            os.remove( sFilename )

        fDate = float( dt.datetime.strptime( row[lDateCol], "%m/%d/%Y").strftime("%Y%m%d") )

        ''' convert blanks to nans '''
        for i in llUseCols:
            if row[i] == '':
                row[i] = 'nan'

        ''' Add row if data exists, if not, create new array '''
        if dData.has_key(sTic):
            dData[sTic] = np.vstack( (dData[sTic], np.array([fDate] + [row[i] for i in llUseCols], dtype=np.float)) )
        else:
            dData[sTic]= np.array( [fDate] + [row[i] for i in llUseCols], dtype=np.float )

        if( (j+1) % 1000 == 0):
            fDone = (j / 1378625.0) * 100
            print '\rApprox %.2lf%%'%((j / 1378625.0) * 100),

        if( (j+1) % lSaveMem == 0):
            ''' Write all the pickle files we currently have '''

            print '\nWriting %i lines to pickle files to save memory\n'%(lSaveMem)
            _dumpFiles( dData, lSets, lsOutPaths)
            ''' Remember to delete! '''
            del dData
            dData = dict()

        # Done writing files
    # Done with main loop

    print ''
    print 'Writing final pickle files\n'
    _dumpFiles( dData, lSets, lsOutPaths)
    del dData

    print "Finished..."+ str(time.strftime("%H:%M:%S"))

    return

if __name__ == '__main__':

    #_analyze()
    execfile( "C:\John's Folder\Code\Eclipse64\Python\QuantSoftware\src\openquantsoftware\config.py" )
    convert()

    <|endoftext|>


<commit_before>"""
基本コンテナオブジェクトの読み取り専用モードオブジェクトについてのサンプルです。
"""

from trypython.common.commoncls import SampleBase

class Sample(SampleBase):
    def exec(self):
        ###################################################################
        #
        # 基本コンテナオブジェクトと読み取り専用オブジェクト
        #
        # Python の基本コンテナとして
        #   * リスト (list)
        #   * ディクショナリ (dict)
        #   * セット (set)
        # が存在するが、それぞれ読み取り専用として扱えるオブジェクトが
        # 対で存在する。対応としては以下のようになる。
        #
        # リスト
        #    tuple に変更すると読み取り専用のリストとして使える
        # ディクショナリ
        #    types.MappingProxyType オブジェクトが読み取り専用として使える
        # セット
        #    frozenset が読み取り専用として使える
        #
        ##################################################################
        pass

def go():
    obj = Sample()
    obj.exec()

if __name__ == '__main__':
    go()
<commit_msg>Add some code<commit_after>"""
基本コンテナオブジェクトの読み取り専用モードオブジェクトについてのサンプルです。
"""
import types

from trypython.common.commoncls import SampleBase
from trypython.common.commonfunc import pr, hr

class Sample(SampleBase):
    def exec(self):
        ###################################################################
        #
        # 基本コンテナオブジェクトと読み取り専用オブジェクト
        #
        # Python の基本コンテナとして
        #   * リスト (list)
        #   * ディクショナリ (dict)
        #   * セット (set)
        # が存在するが、それぞれ読み取り専用として扱えるオブジェクトが
        # 対で存在する。対応としては以下のようになる。
        #
        # リスト
        #    tuple に変更すると読み取り専用のリストとして使える
        # ディクショナリ
        #    types.MappingProxyType オブジェクトが読み取り専用として使える
        # セット
        #    frozenset が読み取り専用として使える
        #
        ##################################################################
        #
        # listとtuple
        #
        a_list = list(range(10))
        pr('list', a_list)

        a_list.append(100)
        a_list.insert(0, 101)
        del a_list[1]

        pr('list is mutable', a_list)

        a_tuple = tuple(a_list)
        pr('tuple', a_tuple)

        try:
            # noinspection PyUnresolvedReferences
            a_tuple.append(102)
        except AttributeError as attr_ex:
            pr('tuple is immutable', attr_ex)

        try:
            # noinspection PyUnresolvedReferences
            del a_tuple[0]
        except TypeError as type_ex:
            pr('tuple is immutable2', type_ex)

        # tupleは生成時に元データをコピーして生成されるので
        # 元のデータに変更があっても影響はない
        a_list.clear()
        a_list.append(999)
        pr('tuple', a_tuple)

        hr()

        #
        # dictとtypes.MappingProxyType
        #
        a_dict = dict(hello=1, world=2)
        pr('dict', a_dict)

        a_dict['spam'] = 100
        del a_dict['world']

        pr('dict is mutable', a_dict)

        a_proxy = types.MappingProxyType(a_dict)
        try:
            # noinspection PyUnresolvedReferences
            a_proxy['hello'] = 200
        except TypeError as type_ex:
            pr('MappingProxyType is immutable', type_ex)

        try:
            # noinspection PyUnresolvedReferences
            del a_proxy['hello']
        except TypeError as type_ex:
            pr('MappingProxyType is immutable2', type_ex)

        # MappingProxyTypeのみの特殊動作。
        # このオブジェクト自体が動的ビューなので
        # 元オブジェクトの変更は即座に反映される。
        # (tupleやfrozensetの場合は元の値がコピーされているため元データ
        #  変更しても影響はない。)
        a_dict['world'] = 999
        pr('proxy', a_proxy['world'])

        hr()

        #
        # setとfrozenset
        #
        a_set = set(a_tuple)
        pr('set', a_set)

        a_set.add(10)
        a_set.add(3)
        a_set.remove(2)

        pr('set is mutable', a_set)

        a_frozenset = frozenset(a_set)

        try:
            # noinspection PyUnresolvedReferences
            a_frozenset.add(4)
        except AttributeError as attr_ex:
            pr('frozenset is immutable', attr_ex)

        try:
            # noinspection PyUnresolvedReferences
            a_frozenset.remove(10)
        except AttributeError as attr_ex:
            pr('frozenset is immutable2', attr_ex)

        # frozensetは生成時に元データをコピーして生成されるので
        # 元のデータに変更があっても影響はない
        a_set.clear()
        a_set.add(999)
        pr('frozenset', a_frozenset)

        hr()

def go():
    obj = Sample()
    obj.exec()

if __name__ == '__main__':
    go()
<|endoftext|>


<commit_before>##
# Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# DRI: Wilfredo Sanchez, wsanchez@apple.com
##

import os

from twisted.python.filepath import FilePath

import twistedcaldav.directory.test.util
import twistedcaldav.directory.test.test_xmlfile
from twistedcaldav.directory.sqldb import SQLDirectoryService

xmlFile = FilePath(os.path.join(os.path.dirname(__file__), "accounts.xml"))

# FIXME: Add tests for GUID hooey, once we figure out what that means here

class SQLDB (
    twistedcaldav.directory.test.test_xmlfile.XMLFileBase,
    twistedcaldav.directory.test.util.BasicTestCase,
    twistedcaldav.directory.test.util.DigestTestCase
):
    """
    Test SQL directory implementation.
    """
    def service(self):
        return SQLDirectoryService(os.getcwd(), self.xmlFile())

    def test_verifyCredentials_digest(self):
        super(SQLDB, self).test_verifyCredentials_digest()
    test_verifyCredentials_digest.todo = ""

    def test_verifyRealmFromDB(self):

        def _service():
            return SQLDirectoryService(os.getcwd(), None)

        self.assertEquals(_service().realmName, "Test")
<commit_msg>Clean up test_verifyRealmFromDB.  - Don't assume that the DB already exists initialized from XML; order of tests isn't guaranteed.  - Not sure why were were defining a function that is only ever called once.<commit_after>##
# Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# DRI: Wilfredo Sanchez, wsanchez@apple.com
##

import os

from twisted.python.filepath import FilePath

import twistedcaldav.directory.test.util
import twistedcaldav.directory.test.test_xmlfile
from twistedcaldav.directory.sqldb import SQLDirectoryService

xmlFile = FilePath(os.path.join(os.path.dirname(__file__), "accounts.xml"))

# FIXME: Add tests for GUID hooey, once we figure out what that means here

class SQLDB (
    twistedcaldav.directory.test.test_xmlfile.XMLFileBase,
    twistedcaldav.directory.test.util.BasicTestCase,
    twistedcaldav.directory.test.util.DigestTestCase
):
    """
    Test SQL directory implementation.
    """
    def service(self):
        return SQLDirectoryService(os.getcwd(), self.xmlFile())

    def test_verifyCredentials_digest(self):
        super(SQLDB, self).test_verifyCredentials_digest()
    test_verifyCredentials_digest.todo = ""

    def test_verifyRealmFromDB(self):
        # Make sure the database has been initialized with the XML file
        self.service()

        # Then get an instance without using the XML file
        service = SQLDirectoryService(os.getcwd(), None)

        self.assertEquals(service.realmName, "Test")
<|endoftext|>