<commit_before>from ggame import App, Sprite, ImageAsset, Frame
import math

class Stars(Sprite):

    asset = ImageAsset("starfield.jpg")
    width = 512
    height = 512

    def __init__(self, position):
        super().__init__(Stars.asset, position)

class Sun(Sprite):

    asset = ImageAsset("sun.png")
    width = 80
    height = 76

    def __init__(self, position):
        super().__init__(Sun.asset, position)
        self.mass = 30*1000
        self.fxcenter = 0.5
        self.fycenter = 0.5

class Vector(object):

    def __init__(self, x, y):
        self.x = x
        self.y = y

    def mag(self):
        return math.sqrt(self.x*self.x + self.y*self.y)

    def unit(self):
        r = self.mag()
        if r == 0:
            return Vector(0,0)
        else:
            return Vector(self.x/r, self.y/r)

class GravitySprite(Sprite):

    G = 1.0
    T = 3.0

    def __init__(self, asset, position, velocity, sun):
        super().__init__(asset, position)
        self.vx = velocity[0]
        self.vy = velocity[1]
        self.sun = sun
        self.fxcenter = 0.5
        self.fycenter = 0.5
        self.rrate = 0.0
        self.thrust = 0.0

    def step(self):
        dt = 0.033
        R = Vector(self.sun.x-self.x, self.sun.y-self.y)
        Ur = R.unit()
        ag = GravitySprite.G*self.sun.mass/R.mag()**2
        Ag = Vector(Ur.x*ag, Ur.y*ag)
        vx = self.vx
        vy = self.vy
        self.vx += (Ag.x + self.thrust*GravitySprite.T*math.sin(self.rotation))* dt
        self.vy += (Ag.y - self.thrust*GravitySprite.T*math.cos(self.rotation))* dt
        self.x += self.vx + 0.5*Ag.x*dt*dt
        self.y += self.vy + 0.5*Ag.y*dt*dt

class Ship(GravitySprite):

    R = 2.0
    bullets = 4

    def __init__(self, asset, app, position, velocity, sun):
        super().__init__(asset, position, velocity, sun)
        self.app = app
        bullets = []
        for i in range(Ship.bullets):
            bullets.append(Bullet(app, sun))

    def registerKeys(self, keys):
        commands = ["left", "right", "forward", "fire"]
        self.keymap = dict(zip(keys, commands))
        [self.app.listenKeyEvent("keydown", k, self.controldown) for k in keys]
        [self.app.listenKeyEvent("keyup", k, self.controlup) for k in keys]

    def shootvector(self):
        vel = 2
        xv = vel*math.sin(self.rotation)
        yv = vel*(-math.cos(self.rotation))
        return xv + self.xv, yv + self.yv

    def controldown(self, event):
        command = self.keymap[event.key]
        if command == "left":
            self.rrate = -0.01*Ship.R
        elif command == "right":
            self.rrate = Ship.R*0.01
        elif command == "forward":
            self.thrust = 0.1
        elif command == "fire":
            for bullet in self.bullets:
                if bullet.time == 0:
                    bullet.shoot(self.position, self.shootvector(), 15)
                    break

    def controlup(self, event):
        command = self.keymap[event.key]
        if command in ["left", "right"]:
            self.rrate = 0.0
        elif command == "forward":
            self.thrust = 0.0

    def step(self):
        super().step()
        self.rotation += self.rrate

class Bullet(GravitySprite):

    asset = ImageAsset("blast.png", Frame(0,0,8,8), 8)

    def __init__(self, app, sun):
        super().__init__(Bullet.asset, (0,0), (0,0), sun)
        self.visible = False
        self.time = 0

    def shoot(self, position, velocity, time):
        self.position = position
        self.vx = velocity[0]
        self.vy = velocity[1]
        self.time = time*30
        self.visible = True
        print("shoot")

    def step(self):
        if self.time > 0:
            self.time -= 1
            super().step()
        elif self.visible:
            self.visible = False

class Ship1(Ship):

    asset = ImageAsset("four_spaceship_by_albertov.png",
        Frame(227,0,292-227,92), 1)

    def __init__(self, app, position, velocity, sun):
        super().__init__(Ship1.asset, app, position, velocity, sun)
        self.registerKeys(["left arrow", "right arrow", "up arrow", "enter"])

class Ship2(Ship):

    asset = ImageAsset("four_spaceship_by_albertov.png",
        Frame(0,0,86,92), 1)

    def __init__(self, app, position, velocity, sun):
        super().__init__(Ship2.asset, app, position, velocity, sun)
        self.registerKeys(["a", "d", "w", "space"])

class Spacewar(App):

    def __init__(self, width, height):
        super().__init__(width, height)
        for x in range(self.width//Stars.width + 1):
            for y in range(self.height//Stars.height + 1):
                Stars((x*Stars.width, y*Stars.height))
        self.sun = Sun((self.width/2, self.height/2))
        self.ship1 = Ship1(self, (self.width/2+100,self.height/2), (0,-4), self.sun)
        self.ship2 = Ship2(self, (self.width/2-100,self.height/2), (0,4), self.sun)

    def step(self):
        #for ship in self.getSpritesbyClass(Ship1
        self.ship1.step()
        self.ship2.step()

app = Spacewar(0,0)
app.run()

<commit_msg>Updated from Brython Server: 8/7/2015 5:43:13 PM<commit_after>from ggame import App, Sprite, ImageAsset, Frame
import math

class Stars(Sprite):

    asset = ImageAsset("starfield.jpg")
    width = 512
    height = 512

    def __init__(self, position):
        super().__init__(Stars.asset, position)

class Sun(Sprite):

    asset = ImageAsset("sun.png")
    width = 80
    height = 76

    def __init__(self, position):
        super().__init__(Sun.asset, position)
        self.mass = 30*1000
        self.fxcenter = 0.5
        self.fycenter = 0.5

class Vector(object):

    def __init__(self, x, y):
        self.x = x
        self.y = y

    def mag(self):
        return math.sqrt(self.x*self.x + self.y*self.y)

    def unit(self):
        r = self.mag()
        if r == 0:
            return Vector(0,0)
        else:
            return Vector(self.x/r, self.y/r)

class GravitySprite(Sprite):

    G = 1.0
    T = 3.0

    def __init__(self, asset, position, velocity, sun):
        super().__init__(asset, position)
        self.vx = velocity[0]
        self.vy = velocity[1]
        self.sun = sun
        self.fxcenter = 0.5
        self.fycenter = 0.5
        self.rrate = 0.0
        self.thrust = 0.0

    def step(self):
        dt = 0.033
        R = Vector(self.sun.x-self.x, self.sun.y-self.y)
        Ur = R.unit()
        ag = GravitySprite.G*self.sun.mass/R.mag()**2
        Ag = Vector(Ur.x*ag, Ur.y*ag)
        vx = self.vx
        vy = self.vy
        self.vx += (Ag.x + self.thrust*GravitySprite.T*math.sin(self.rotation))* dt
        self.vy += (Ag.y - self.thrust*GravitySprite.T*math.cos(self.rotation))* dt
        self.x += self.vx + 0.5*Ag.x*dt*dt
        self.y += self.vy + 0.5*Ag.y*dt*dt

class Ship(GravitySprite):

    R = 2.0
    bullets = 4

    def __init__(self, asset, app, position, velocity, sun):
        super().__init__(asset, position, velocity, sun)
        self.app = app
        self.bullets = []
        for i in range(Ship.bullets):
            bullets.append(Bullet(app, sun))

    def registerKeys(self, keys):
        commands = ["left", "right", "forward", "fire"]
        self.keymap = dict(zip(keys, commands))
        [self.app.listenKeyEvent("keydown", k, self.controldown) for k in keys]
        [self.app.listenKeyEvent("keyup", k, self.controlup) for k in keys]

    def shootvector(self):
        vel = 2
        xv = vel*math.sin(self.rotation)
        yv = vel*(-math.cos(self.rotation))
        return xv + self.xv, yv + self.yv

    def controldown(self, event):
        command = self.keymap[event.key]
        if command == "left":
            self.rrate = -0.01*Ship.R
        elif command == "right":
            self.rrate = Ship.R*0.01
        elif command == "forward":
            self.thrust = 0.1
        elif command == "fire":
            for bullet in self.bullets:
                if bullet.time == 0:
                    bullet.shoot(self.position, self.shootvector(), 15)
                    break

    def controlup(self, event):
        command = self.keymap[event.key]
        if command in ["left", "right"]:
            self.rrate = 0.0
        elif command == "forward":
            self.thrust = 0.0

    def step(self):
        super().step()
        self.rotation += self.rrate

class Bullet(GravitySprite):

    asset = ImageAsset("blast.png", Frame(0,0,8,8), 8)

    def __init__(self, app, sun):
        super().__init__(Bullet.asset, (0,0), (0,0), sun)
        self.visible = False
        self.time = 0

    def shoot(self, position, velocity, time):
        self.position = position
        self.vx = velocity[0]
        self.vy = velocity[1]
        self.time = time*30
        self.visible = True

    def step(self):
        if self.time > 0:
            self.time -= 1
            super().step()
        elif self.visible:
            self.visible = False

class Ship1(Ship):

    asset = ImageAsset("four_spaceship_by_albertov.png",
        Frame(227,0,292-227,92), 1)

    def __init__(self, app, position, velocity, sun):
        super().__init__(Ship1.asset, app, position, velocity, sun)
        self.registerKeys(["left arrow", "right arrow", "up arrow", "enter"])

class Ship2(Ship):

    asset = ImageAsset("four_spaceship_by_albertov.png",
        Frame(0,0,86,92), 1)

    def __init__(self, app, position, velocity, sun):
        super().__init__(Ship2.asset, app, position, velocity, sun)
        self.registerKeys(["a", "d", "w", "space"])

class Spacewar(App):

    def __init__(self, width, height):
        super().__init__(width, height)
        for x in range(self.width//Stars.width + 1):
            for y in range(self.height//Stars.height + 1):
                Stars((x*Stars.width, y*Stars.height))
        self.sun = Sun((self.width/2, self.height/2))
        self.ship1 = Ship1(self, (self.width/2+100,self.height/2), (0,-4), self.sun)
        self.ship2 = Ship2(self, (self.width/2-100,self.height/2), (0,4), self.sun)

    def step(self):
        #for ship in self.getSpritesbyClass(Ship1
        self.ship1.step()
        self.ship2.step()

app = Spacewar(0,0)
app.run()

<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2014 Corgan Labs
# See LICENSE.txt for distribution terms
#

import os
import hmac
import hashlib
import ecdsa
import struct
import Base58

from hashlib import sha256
from ecdsa.curves import SECP256k1
from ecdsa.ecdsa import int_to_string, string_to_int
from ecdsa.numbertheory import square_root_mod_prime as sqrt_mod

MIN_ENTROPY_LEN = 128        # bits
BIP32_HARDEN    = 0x80000000 # choose from hardened set of child keys
CURVE_GEN       = ecdsa.ecdsa.generator_secp256k1
CURVE_ORDER     = CURVE_GEN.order()
FIELD_ORDER     = SECP256k1.curve.p()
INFINITY        = ecdsa.ellipticcurve.INFINITY

class BIP32Key(object):

    # Static initializers to create from entropy or external formats
    #
    @staticmethod
    def fromEntropy(entropy, public=False):
        "Create a BIP32Key using supplied entropy >= MIN_ENTROPY_LEN"
        if entropy == None:
            entropy = os.urandom(MIN_ENTROPY_LEN/8) # Python doesn't have os.random()
        if not len(entropy) >= MIN_ENTROPY_LEN/8:
            raise ValueError("Initial entropy %i must be at least %i bits" %
                                (len(entropy), MIN_ENTROPY_LEN))
        I = hmac.new("Bitcoin seed", entropy, hashlib.sha512).digest()
        Il, Ir = I[:32], I[32:]
        # FIXME test Il for 0 or less than SECP256k1 prime field order
        key = BIP32Key(secret=Il, chain=Ir, depth=0, index=0, fpr='\0\0\0\0', public=False)
        if public:
            key.SetPublic()
        return key

    @staticmethod
    def fromExtendedKey(xkey, public=False):
        """
        Create a BIP32Key by importing from extended private or public key string

        If public is True, return a public-only key regardless of input type.
        """
        rawchk = Base58.decode(xkey)
        if len(rawchk) != 82:
            raise ValueError("extended key format wrong length")

        # Verify encoding checksum
        raw, chk = rawchk[:-4], rawchk[-4:]
        if (chk != sha256(sha256(raw).digest()).digest()[:4]):
            raise ValueError("base58 decoding checksum failure")

        # Verify address version/type
        version = raw[:4]
        if version.encode('hex') == '0488ade4':
            keytype = 'xprv'
        elif version.encode('hex') == '0488b21e':
            keytype = 'xpub'
        else:
            raise ValueError("unknown extended key version")

        # Extract remaining fields
        depth = ord(raw[4])
        fpr = raw[5:9]
        child = struct.unpack(">L", raw[9:13])[0]
        chain = raw[13:45]
        secret = raw[45:78]

        # Extract private key or public key point
        if keytype == 'xprv':
            secret = secret[1:]
        else:
            # Recover public curve point from compressed key
            lsb = ord(secret[0]) & 1
            x = string_to_int(secret[1:])
            ys = (x**3+7) % FIELD_ORDER # y^2 = x^3 + 7 mod p
            y = sqrt_mod(ys, FIELD_ORDER)
            if y & 1 != lsb:
                y = FIELD_ORDER-y
            point = ecdsa.ellipticcurve.Point(SECP256k1.curve, x, y)
            secret = ecdsa.VerifyingKey.from_public_point(point, curve=SECP256k1)

        is_pubkey = (keytype == 'xpub')
        key = BIP32Key(secret=secret, chain=chain, depth=depth, index=child, fpr=fpr, public=is_pubkey)
        if not is_pubkey and public:
            key = key.SetPublic()
        return key

    # Normal class initializer
    def __init__(self, secret, chain, depth, index, fpr, public=False):
        """
        Create a public or private BIP32Key using key material and chain code.

        secret   This is the source material to generate the keypair, either a
                 32-byte string representation of a private key, or the ECDSA
                 library object representing a public key.

        chain    This is a 32-byte string representation of the chain code

        depth    Child depth; parent increments its own by one when assigning this

        index    Child index

        fpr      Parent fingerprint

        public   If true, this keypair will only contain a public key and can only create
                 a public key chain.
        """

        self.public = public
        if public is False:
            self.k = ecdsa.SigningKey.from_string(secret, curve=SECP256k1)
            self.K = self.k.get_verifying_key()
        else:
            self.k = None
            self.K = secret

        self.C = chain
        self.depth = depth
        self.index = index
        self.parent_fpr = fpr

    # Internal methods not intended to be called externally
    #
    def hmac(self, data):
        """
        Calculate the HMAC-SHA512 of input data using the chain code as key.

        Returns a tuple of the HMAC and its left and right halves.
        """
        I = hmac.new(self.C, data, hashlib.sha512).digest()
        return (I, I[:32], I[32:])

    def CKDpriv(self, i):
        """
        Create a child key of index 'i'.

        If the most significant bit of 'i' is set, then select from the
        hardened key set, otherwise, select a regular child key.

        Returns a BIP32Key constructed with the child key parameters,
        or None if i index would result in an invalid key.
        """
        # Index as bytes, BE
        i_str = struct.pack(">L", i)

        # Data to HMAC
        if i & BIP32_HARDEN:
            data = b'\0' + self.k.to_string() + i_str
        else:
            data = self.PublicKey() + i_str

        # Get HMAC of data
        (I, Il, Ir) = self.hmac(data)

        # Construct new key material from Il and current private key
        Il_int = string_to_int(Il)
        if Il_int > CURVE_ORDER:
            return None
        pvt_int = string_to_int(self.k.to_string())
        k_int = (Il_int + pvt_int) % CURVE_ORDER
        if (k_int == 0):
            return None
        secret = int_to_string(k_int)

        # Construct and return a new BIP32Key
        return BIP32Key(secret=secret, chain=Ir, depth=self.depth+1, index=i, fpr=self.Fingerprint(), public=False)

    def CKDpub(self, i):
        """
        Create a publicly derived child key of index 'i'.

        If the most significant bit of 'i' is set, this is
        an error.

        Returns a BIP32Key constructed with the child key parameters,
        or None if index would result in invalid key.
        """

        if i & BIP32_HARDEN:
            raise Exception("Cannot create a hardened child key using public child derivation")

        # Data to HMAC.  Same as CKDpriv() for public child key.
        data = self.PublicKey() + struct.pack(">L", i)

        # Get HMAC of data
        (I, Il, Ir) = self.hmac(data)

        # Construct curve point Il*G+K
        Il_int = string_to_int(Il)
        if Il_int >= CURVE_ORDER:
            return None
        point = Il_int*CURVE_GEN + self.K.pubkey.point
        if point == INFINITY:
            return None

        # Retrieve public key based on curve point
        K_i = ecdsa.VerifyingKey.from_public_point(point, curve=SECP256k1)

        # Construct and return a new BIP32Key
        return BIP32Key(secret=K_i, chain=Ir, depth=self.depth, index=i, fpr=self.Fingerprint(), public=True)

    # Public methods
    #
    def ChildKey(self, i):
        """
        Create and return a child key of this one at index 'i'.

        The index 'i' should be summed with BIP32_HARDEN to indicate
        to use the private derivation algorithm.
        """
        if self.public is False:
            return self.CKDpriv(i)
        else:
            return self.CKDpub(i)

    def SetPublic(self):
        "Convert a private BIP32Key into a public one"
        self.k = None
        self.public = True

    def PrivateKey(self):
        "Return private key as string"
        if self.public:
            raise Exception("Publicly derived deterministic keys have no private half")
        else:
            return self.k.to_string()

    def PublicKey(self):
        "Return compressed public key encoding"
        if self.K.pubkey.point.y() & 1:
            ck = b'\3'+int_to_string(self.K.pubkey.point.x())
        else:
            ck = b'\2'+int_to_string(self.K.pubkey.point.x())
        return ck

    def ChainCode(self):
        "Return chain code as string"
        return self.C

    def Identifier(self):
        "Return key identifier as string"
        cK = self.PublicKey()
        return hashlib.new('ripemd160', sha256(cK).digest()).digest()

    def Fingerprint(self):
        "Return key fingerprint as string"
        return self.Identifier()[:4]

    def Address(self):
        "Return compressed public key address"
        vh160 = '\x00'+self.Identifier()
        raw = vh160+sha256(sha256(vh160).digest()).digest()[:4]
        return Base58.encode(raw)

    def WalletImportFormat(self):
        "Returns private key encoded for wallet import"
        if self.public:
            raise Exception("Publicly derived deterministic keys have no private half")
        raw = '\x80' + self.k.to_string() + '\x01' # Always compressed
        chk = sha256(sha256(raw).digest()).digest()[:4]
        return Base58.encode(raw+chk)

    def ExtendedKey(self, private=True, encoded=True):
        "Return extended private or public key as string, optionally Base58 encoded"
        if self.public is True and private is True:
            raise Exception("Cannot export an extended private key from a public-only deterministic key")
        version = '\x04\x88\xB2\x1E' if private is False else '\x04\x88\xAD\xE4'
        depth = chr(self.depth)
        fpr = self.parent_fpr
        child = struct.pack('>L', self.index)
        chain = self.C
        if self.public is True or private is False:
            data = self.PublicKey()
        else:
            data = '\x00' + self.PrivateKey()
        raw = version+depth+fpr+child+chain+data
        if not encoded:
            return raw
        else:
            chk = sha256(sha256(raw).digest()).digest()[:4]
            return Base58.encode(raw+chk)

    # Debugging methods
    #
    def dump(self):
        "Dump key fields mimicking the BIP0032 test vector format"
        print "   * Identifier"
        print "     * (hex):      ", self.Identifier().encode('hex')
        print "     * (fpr):      ", self.Fingerprint().encode('hex')
        print "     * (main addr):", self.Address()
        if self.public is False:
            print "   * Secret key"
            print "     * (hex):      ", self.PrivateKey().encode('hex')
            print "     * (wif):      ", self.WalletImportFormat()
        print "   * Public key"
        print "     * (hex):      ", self.PublicKey().encode('hex')
        print "   * Chain code"
        print "     * (hex):      ", self.C.encode('hex')
        print "   * Serialized"
        print "     * (pub hex):  ", self.ExtendedKey(private=False, encoded=False).encode('hex')
        print "     * (prv hex):  ", self.ExtendedKey(private=True, encoded=False).encode('hex')
        print "     * (pub b58):  ", self.ExtendedKey(private=False, encoded=True)
        print "     * (prv b58):  ", self.ExtendedKey(private=True, encoded=True)

if __name__ == "__main__":
    import sys

    # BIP0032 Test vector 1
    entropy='000102030405060708090A0B0C0D0E0F'.decode('hex')
    m = BIP32Key.fromEntropy(entropy)
    print "Test vector 1:"
    print "Master (hex):", entropy.encode('hex')
    print "* [Chain m]"
    m.dump()

    print "* [Chain m/0h]"
    m = m.ChildKey(0+BIP32_HARDEN)
    m.dump()

    print "* [Chain m/0h/1]"
    m = m.ChildKey(1)
    m.dump()

    print "* [Chain m/0h/1/2h]"
    m = m.ChildKey(2+BIP32_HARDEN)
    m.dump()

    print "* [Chain m/0h/1/2h/2]"
    m = m.ChildKey(2)
    m.dump()

    print "* [Chain m/0h/1/2h/2/1000000000]"
    m = m.ChildKey(1000000000)
    m.dump()

    # BIP0032 Test vector 2
    entropy = 'fffcf9f6f3f0edeae7e4e1dedbd8d5d2cfccc9c6c3c0bdbab7b4b1aeaba8a5a29f9c999693908d8a8784817e7b7875726f6c696663605d5a5754514e4b484542'.decode('hex')
    m = BIP32Key.fromEntropy(entropy)
    print "Test vector 2:"
    print "Master (hex):", entropy.encode('hex')
    print "* [Chain m]"
    m.dump()

    print "* [Chain m/0]"
    m = m.ChildKey(0)
    m.dump()

    print "* [Chain m/0/2147483647h]"
    m = m.ChildKey(2147483647+BIP32_HARDEN)
    m.dump()

    print "* [Chain m/0/2147483647h/1]"
    m = m.ChildKey(1)
    m.dump()

    print "* [Chain m/0/2147483647h/1/2147483646h]"
    m = m.ChildKey(2147483646+BIP32_HARDEN)
    m.dump()

    print "* [Chain m/0/2147483647h/1/2147483646h/2]"
    m = m.ChildKey(2)
    m.dump()
<commit_msg>BIP32Key: make hard-coded constants into module constants<commit_after>#!/usr/bin/env python
#
# Copyright 2014 Corgan Labs
# See LICENSE.txt for distribution terms
#

import os
import hmac
import hashlib
import ecdsa
import struct
import Base58

from hashlib import sha256
from ecdsa.curves import SECP256k1
from ecdsa.ecdsa import int_to_string, string_to_int
from ecdsa.numbertheory import square_root_mod_prime as sqrt_mod

MIN_ENTROPY_LEN = 128        # bits
BIP32_HARDEN    = 0x80000000 # choose from hardened set of child keys
CURVE_GEN       = ecdsa.ecdsa.generator_secp256k1
CURVE_ORDER     = CURVE_GEN.order()
FIELD_ORDER     = SECP256k1.curve.p()
INFINITY        = ecdsa.ellipticcurve.INFINITY
EX_MAIN_PRIVATE = '0488ade4'.decode('hex') # Version string for mainnet extended private keys
EX_MAIN_PUBLIC  = '0488b21e'.decode('hex') # Version string for mainnet extended public keys

class BIP32Key(object):

    # Static initializers to create from entropy or external formats
    #
    @staticmethod
    def fromEntropy(entropy, public=False):
        "Create a BIP32Key using supplied entropy >= MIN_ENTROPY_LEN"
        if entropy == None:
            entropy = os.urandom(MIN_ENTROPY_LEN/8) # Python doesn't have os.random()
        if not len(entropy) >= MIN_ENTROPY_LEN/8:
            raise ValueError("Initial entropy %i must be at least %i bits" %
                                (len(entropy), MIN_ENTROPY_LEN))
        I = hmac.new("Bitcoin seed", entropy, hashlib.sha512).digest()
        Il, Ir = I[:32], I[32:]
        # FIXME test Il for 0 or less than SECP256k1 prime field order
        key = BIP32Key(secret=Il, chain=Ir, depth=0, index=0, fpr='\0\0\0\0', public=False)
        if public:
            key.SetPublic()
        return key

    @staticmethod
    def fromExtendedKey(xkey, public=False):
        """
        Create a BIP32Key by importing from extended private or public key string

        If public is True, return a public-only key regardless of input type.
        """
        rawchk = Base58.decode(xkey)
        if len(rawchk) != 82:
            raise ValueError("extended key format wrong length")

        # Verify encoding checksum
        raw, chk = rawchk[:-4], rawchk[-4:]
        if (chk != sha256(sha256(raw).digest()).digest()[:4]):
            raise ValueError("base58 decoding checksum failure")

        # Verify address version/type
        version = raw[:4]
        if version == EX_MAIN_PRIVATE:
            keytype = 'xprv'
        elif version == EX_MAIN_PUBLIC:
            keytype = 'xpub'
        else:
            raise ValueError("unknown extended key version")

        # Extract remaining fields
        depth = ord(raw[4])
        fpr = raw[5:9]
        child = struct.unpack(">L", raw[9:13])[0]
        chain = raw[13:45]
        secret = raw[45:78]

        # Extract private key or public key point
        if keytype == 'xprv':
            secret = secret[1:]
        else:
            # Recover public curve point from compressed key
            lsb = ord(secret[0]) & 1
            x = string_to_int(secret[1:])
            ys = (x**3+7) % FIELD_ORDER # y^2 = x^3 + 7 mod p
            y = sqrt_mod(ys, FIELD_ORDER)
            if y & 1 != lsb:
                y = FIELD_ORDER-y
            point = ecdsa.ellipticcurve.Point(SECP256k1.curve, x, y)
            secret = ecdsa.VerifyingKey.from_public_point(point, curve=SECP256k1)

        is_pubkey = (keytype == 'xpub')
        key = BIP32Key(secret=secret, chain=chain, depth=depth, index=child, fpr=fpr, public=is_pubkey)
        if not is_pubkey and public:
            key = key.SetPublic()
        return key

    # Normal class initializer
    def __init__(self, secret, chain, depth, index, fpr, public=False):
        """
        Create a public or private BIP32Key using key material and chain code.

        secret   This is the source material to generate the keypair, either a
                 32-byte string representation of a private key, or the ECDSA
                 library object representing a public key.

        chain    This is a 32-byte string representation of the chain code

        depth    Child depth; parent increments its own by one when assigning this

        index    Child index

        fpr      Parent fingerprint

        public   If true, this keypair will only contain a public key and can only create
                 a public key chain.
        """

        self.public = public
        if public is False:
            self.k = ecdsa.SigningKey.from_string(secret, curve=SECP256k1)
            self.K = self.k.get_verifying_key()
        else:
            self.k = None
            self.K = secret

        self.C = chain
        self.depth = depth
        self.index = index
        self.parent_fpr = fpr

    # Internal methods not intended to be called externally
    #
    def hmac(self, data):
        """
        Calculate the HMAC-SHA512 of input data using the chain code as key.

        Returns a tuple of the HMAC and its left and right halves.
        """
        I = hmac.new(self.C, data, hashlib.sha512).digest()
        return (I, I[:32], I[32:])

    def CKDpriv(self, i):
        """
        Create a child key of index 'i'.

        If the most significant bit of 'i' is set, then select from the
        hardened key set, otherwise, select a regular child key.

        Returns a BIP32Key constructed with the child key parameters,
        or None if i index would result in an invalid key.
        """
        # Index as bytes, BE
        i_str = struct.pack(">L", i)

        # Data to HMAC
        if i & BIP32_HARDEN:
            data = b'\0' + self.k.to_string() + i_str
        else:
            data = self.PublicKey() + i_str

        # Get HMAC of data
        (I, Il, Ir) = self.hmac(data)

        # Construct new key material from Il and current private key
        Il_int = string_to_int(Il)
        if Il_int > CURVE_ORDER:
            return None
        pvt_int = string_to_int(self.k.to_string())
        k_int = (Il_int + pvt_int) % CURVE_ORDER
        if (k_int == 0):
            return None
        secret = int_to_string(k_int)

        # Construct and return a new BIP32Key
        return BIP32Key(secret=secret, chain=Ir, depth=self.depth+1, index=i, fpr=self.Fingerprint(), public=False)

    def CKDpub(self, i):
        """
        Create a publicly derived child key of index 'i'.

        If the most significant bit of 'i' is set, this is
        an error.

        Returns a BIP32Key constructed with the child key parameters,
        or None if index would result in invalid key.
        """

        if i & BIP32_HARDEN:
            raise Exception("Cannot create a hardened child key using public child derivation")

        # Data to HMAC.  Same as CKDpriv() for public child key.
        data = self.PublicKey() + struct.pack(">L", i)

        # Get HMAC of data
        (I, Il, Ir) = self.hmac(data)

        # Construct curve point Il*G+K
        Il_int = string_to_int(Il)
        if Il_int >= CURVE_ORDER:
            return None
        point = Il_int*CURVE_GEN + self.K.pubkey.point
        if point == INFINITY:
            return None

        # Retrieve public key based on curve point
        K_i = ecdsa.VerifyingKey.from_public_point(point, curve=SECP256k1)

        # Construct and return a new BIP32Key
        return BIP32Key(secret=K_i, chain=Ir, depth=self.depth, index=i, fpr=self.Fingerprint(), public=True)

    # Public methods
    #
    def ChildKey(self, i):
        """
        Create and return a child key of this one at index 'i'.

        The index 'i' should be summed with BIP32_HARDEN to indicate
        to use the private derivation algorithm.
        """
        if self.public is False:
            return self.CKDpriv(i)
        else:
            return self.CKDpub(i)

    def SetPublic(self):
        "Convert a private BIP32Key into a public one"
        self.k = None
        self.public = True

    def PrivateKey(self):
        "Return private key as string"
        if self.public:
            raise Exception("Publicly derived deterministic keys have no private half")
        else:
            return self.k.to_string()

    def PublicKey(self):
        "Return compressed public key encoding"
        if self.K.pubkey.point.y() & 1:
            ck = b'\3'+int_to_string(self.K.pubkey.point.x())
        else:
            ck = b'\2'+int_to_string(self.K.pubkey.point.x())
        return ck

    def ChainCode(self):
        "Return chain code as string"
        return self.C

    def Identifier(self):
        "Return key identifier as string"
        cK = self.PublicKey()
        return hashlib.new('ripemd160', sha256(cK).digest()).digest()

    def Fingerprint(self):
        "Return key fingerprint as string"
        return self.Identifier()[:4]

    def Address(self):
        "Return compressed public key address"
        vh160 = '\x00'+self.Identifier()
        raw = vh160+sha256(sha256(vh160).digest()).digest()[:4]
        return Base58.encode(raw)

    def WalletImportFormat(self):
        "Returns private key encoded for wallet import"
        if self.public:
            raise Exception("Publicly derived deterministic keys have no private half")
        raw = '\x80' + self.k.to_string() + '\x01' # Always compressed
        chk = sha256(sha256(raw).digest()).digest()[:4]
        return Base58.encode(raw+chk)

    def ExtendedKey(self, private=True, encoded=True):
        "Return extended private or public key as string, optionally Base58 encoded"
        if self.public is True and private is True:
            raise Exception("Cannot export an extended private key from a public-only deterministic key")
        version = EX_MAIN_PRIVATE if private else EX_MAIN_PUBLIC
        depth = chr(self.depth)
        fpr = self.parent_fpr
        child = struct.pack('>L', self.index)
        chain = self.C
        if self.public is True or private is False:
            data = self.PublicKey()
        else:
            data = '\x00' + self.PrivateKey()
        raw = version+depth+fpr+child+chain+data
        if not encoded:
            return raw
        else:
            chk = sha256(sha256(raw).digest()).digest()[:4]
            return Base58.encode(raw+chk)

    # Debugging methods
    #
    def dump(self):
        "Dump key fields mimicking the BIP0032 test vector format"
        print "   * Identifier"
        print "     * (hex):      ", self.Identifier().encode('hex')
        print "     * (fpr):      ", self.Fingerprint().encode('hex')
        print "     * (main addr):", self.Address()
        if self.public is False:
            print "   * Secret key"
            print "     * (hex):      ", self.PrivateKey().encode('hex')
            print "     * (wif):      ", self.WalletImportFormat()
        print "   * Public key"
        print "     * (hex):      ", self.PublicKey().encode('hex')
        print "   * Chain code"
        print "     * (hex):      ", self.C.encode('hex')
        print "   * Serialized"
        print "     * (pub hex):  ", self.ExtendedKey(private=False, encoded=False).encode('hex')
        print "     * (prv hex):  ", self.ExtendedKey(private=True, encoded=False).encode('hex')
        print "     * (pub b58):  ", self.ExtendedKey(private=False, encoded=True)
        print "     * (prv b58):  ", self.ExtendedKey(private=True, encoded=True)

if __name__ == "__main__":
    import sys

    # BIP0032 Test vector 1
    entropy='000102030405060708090A0B0C0D0E0F'.decode('hex')
    m = BIP32Key.fromEntropy(entropy)
    print "Test vector 1:"
    print "Master (hex):", entropy.encode('hex')
    print "* [Chain m]"
    m.dump()

    print "* [Chain m/0h]"
    m = m.ChildKey(0+BIP32_HARDEN)
    m.dump()

    print "* [Chain m/0h/1]"
    m = m.ChildKey(1)
    m.dump()

    print "* [Chain m/0h/1/2h]"
    m = m.ChildKey(2+BIP32_HARDEN)
    m.dump()

    print "* [Chain m/0h/1/2h/2]"
    m = m.ChildKey(2)
    m.dump()

    print "* [Chain m/0h/1/2h/2/1000000000]"
    m = m.ChildKey(1000000000)
    m.dump()

    # BIP0032 Test vector 2
    entropy = 'fffcf9f6f3f0edeae7e4e1dedbd8d5d2cfccc9c6c3c0bdbab7b4b1aeaba8a5a29f9c999693908d8a8784817e7b7875726f6c696663605d5a5754514e4b484542'.decode('hex')
    m = BIP32Key.fromEntropy(entropy)
    print "Test vector 2:"
    print "Master (hex):", entropy.encode('hex')
    print "* [Chain m]"
    m.dump()

    print "* [Chain m/0]"
    m = m.ChildKey(0)
    m.dump()

    print "* [Chain m/0/2147483647h]"
    m = m.ChildKey(2147483647+BIP32_HARDEN)
    m.dump()

    print "* [Chain m/0/2147483647h/1]"
    m = m.ChildKey(1)
    m.dump()

    print "* [Chain m/0/2147483647h/1/2147483646h]"
    m = m.ChildKey(2147483646+BIP32_HARDEN)
    m.dump()

    print "* [Chain m/0/2147483647h/1/2147483646h/2]"
    m = m.ChildKey(2)
    m.dump()
<|endoftext|>


<commit_before>from bcbio.ngsalign import tophat
import os
from bcbio.utils import file_exists, safe_makedir
from bipy.utils import remove_suffix, is_pair, get_in
from bipy.toolbox import fastqc
from bipy.pipeline.stages import AbstractStage
from bcbio.distributed.transaction import file_transaction
import sh
#from bipy.log import logger
from bcbio.log import logger
import sys

FASTQ_FORMAT_TO_BCBIO = {"sanger": None,
                         "illumina_1.3+": "illumina",
                         "illumina_1.5+": "illumina",
                         "illumina_1.8+": None,
                         "solexa": "solexa"}

def run_with_config(fastq_file, pair_file, ref_file,
                    stage_name, config):
    if not any_bowtie_reference_exists(ref_file):
        bowtie_reference_not_found_error()

    out_file = _bcbio_tophat_wrapper(fastq_file, pair_file,
                                     ref_file, stage_name,
                                     config)
    return out_file

def _bcbio_tophat_wrapper(fastq_file, pair_file, ref_file,
                          stage_name, config):
    bcbio_config = {}
    stage_config = config["stage"][stage_name]
    cores = stage_config.get("cores", 1)
    # use the listed quality format, if there isn't one, try to figure
    # out what format it is
    quality_format = stage_config.get("quality_format", None)
    if quality_format is None:
        fastq_format = fastqc.detect_fastq_format(fastq_file)
        quality_format = FASTQ_FORMAT_TO_BCBIO[fastq_format]

    max_errors = stage_config.get("max_errors", None)
    options = stage_config.get("options", {})
    tophat_loc = config["program"].get("tophat", "tophat")
    bowtie_loc = config["program"].get("bowtie", "bowtie")
    out_base = remove_suffix(os.path.basename(fastq_file))
    align_dir = os.path.join(config["dir"]["results"], stage_name)

    bcbio_config["resources"] = {"tophat": {"cores": cores,
                                            "options": options}}
    bcbio_config["algorithm"] = {}
    bcbio_config["program"] = {}
    bcbio_config["algorithm"]["quality_format"] = quality_format
    bcbio_config["algorithm"]["max_errors"] = max_errors
    bcbio_config["gtf"] = config.get("gtf", None)
    if bcbio_config["gtf"]:
        if not file_exists(bcbio_config["gtf"]):
            raise ValueError("GTF file does not exist. Please check to make sure that "
                             "the value of gtf is set corretly in the configuration file.")
            sys.exit(1)
    bcbio_config["program"]["tophat"] = tophat_loc
    bcbio_config["program"]["bowtie"] = bowtie_loc
    bcbio_config["program"]["picard"] = config["program"]["picard"]
    bcbio_config["program"]["gatk"] = {"dir": ""}

    out_file = tophat.align(fastq_file, pair_file, ref_file, out_base,
                            align_dir, bcbio_config)
    return out_file

class Tophat(AbstractStage):

    stage = "tophat"

    def __init__(self, config):
        self.config = config
        self.ref = config.get("ref", None)

    def __call__(self, in_file):
        self._start_message(in_file)
        if isinstance(in_file, basestring):
            logger.info("Detected %s as non-paired." % in_file)
            out_file = run_with_config(in_file, None, self.ref,
                                       self.stage, self.config)
        elif is_pair(in_file):
            logger.info("Detected %s as a pair." % in_file)
            out_file = run_with_config(in_file[0], in_file[1],
                                       self.ref, self.stage, self.config)
        else:
            logger.info("Detected %s as non-paired." % in_file)
            out_file = run_with_config(in_file[0], None, self.ref,
                                       self.stage, self.config)
        self._end_message(in_file)
        return out_file

class Bowtie(AbstractStage):

    stage = "bowtie"

    def __init__(self, config):
        self.config = config
        self.stage_config = config["stage"][self.stage]
        # defaults = {"q": True, "n": 2, "k": 1,
        #            "X": 2000, "best": True,
        #            "sam": True,
        #            "phred64-quals": True}
        #self.options = dict(defaults.items() +
        #                    self.stage_config.get("options", {}).items())
        self.options = self.stage_config.get("options", {})
        self.bowtie = sh.Command(self.stage_config.get("program", "bowtie"))
        self.out_prefix = os.path.join(get_in(self.config,
                                              ("dir", "results"), "results"),
                                              self.stage)
        self.ref_file = self.config["ref"]
        if not any_bowtie_reference_exists(self.ref_file):
            bowtie_reference_not_found_error()

    def _bowtie_se(self, in_file, out_file):
        self.bowtie(self.options, self.ref_file, in_file, out_file)

    def _bowtie_pe(self, in_file, out_file):
        self.bowtie(self.options, self.ref_file,
                    "-1", in_file[0], "-2", in_file[1], out_file)

    def _get_out_file(self, in_file):
        base, _ = os.path.splitext(os.path.basename(in_file))
        out_prefix = os.path.join(get_in(self.config,
                                         ("dir", "results"), "results"),
                                         self.stage)
        out_dir = os.path.join(out_prefix, base)
        out_file = os.path.join(out_dir, base + ".sam")
        return out_file

    def out_file(self, in_file):
        if is_pair(in_file):
            return self._get_out_file(in_file[0])
        else:
            return self._get_out_file(in_file)

    def __call__(self, in_file):
        self._start_message(in_file)
        out_file = self.out_file(in_file)

        if file_exists(out_file):
            return out_file

        with file_transaction(out_file) as tmp_out_file:
            if is_pair(in_file):
                self._bowtie_pe(in_file, tmp_out_file)
            else:
                self._bowtie_se(in_file, tmp_out_file)
        self._end_message(in_file)

        return out_file

def any_bowtie_reference_exists(prefix):
    return bowtie_1_reference_exists(prefix) or bowtie_2_reference_exists(prefix)

def bowtie_1_reference_exists(prefix):
    return file_exists(prefix + ".1.ebwt")
def bowtie_2_reference_exists(prefix):
    return file_exists(prefix + ".1.bt2")

def bowtie_reference_not_found_error():
    raise ValueError("Bowtie 1 or Bowtie 2 reference not found. Please make "
                     "sure that ref: is set correctly in the config file.")
    sys.exit(1)
<commit_msg>Logging edit in Tophat.<commit_after>from bcbio.ngsalign import tophat
import os
from bcbio.utils import file_exists, safe_makedir
from bipy.utils import remove_suffix, is_pair, get_in
from bipy.toolbox import fastqc
from bipy.pipeline.stages import AbstractStage
from bcbio.distributed.transaction import file_transaction
import sh
#from bipy.log import logger
from bcbio.log import logger
import sys

FASTQ_FORMAT_TO_BCBIO = {"sanger": None,
                         "illumina_1.3+": "illumina",
                         "illumina_1.5+": "illumina",
                         "illumina_1.8+": None,
                         "solexa": "solexa"}

def run_with_config(fastq_file, pair_file, ref_file,
                    stage_name, config):
    if not any_bowtie_reference_exists(ref_file):
        bowtie_reference_not_found_error()

    out_file = _bcbio_tophat_wrapper(fastq_file, pair_file,
                                     ref_file, stage_name,
                                     config)
    return out_file

def _bcbio_tophat_wrapper(fastq_file, pair_file, ref_file,
                          stage_name, config):
    bcbio_config = {}
    stage_config = config["stage"][stage_name]
    cores = stage_config.get("cores", 1)
    # use the listed quality format, if there isn't one, try to figure
    # out what format it is
    quality_format = stage_config.get("quality_format", None)
    if quality_format is None:
        fastq_format = fastqc.detect_fastq_format(fastq_file)
        quality_format = FASTQ_FORMAT_TO_BCBIO[fastq_format]

    max_errors = stage_config.get("max_errors", None)
    options = stage_config.get("options", {})
    tophat_loc = config["program"].get("tophat", "tophat")
    bowtie_loc = config["program"].get("bowtie", "bowtie")
    out_base = remove_suffix(os.path.basename(fastq_file))
    align_dir = os.path.join(config["dir"]["results"], stage_name)

    bcbio_config["resources"] = {"tophat": {"cores": cores,
                                            "options": options}}
    bcbio_config["algorithm"] = {}
    bcbio_config["program"] = {}
    bcbio_config["algorithm"]["quality_format"] = quality_format
    bcbio_config["algorithm"]["max_errors"] = max_errors
    bcbio_config["gtf"] = config.get("gtf", None)
    if bcbio_config["gtf"]:
        if not file_exists(bcbio_config["gtf"]):
            raise ValueError("GTF file does not exist. Please check to make sure that "
                             "the value of gtf is set corretly in the configuration file.")
            sys.exit(1)
    bcbio_config["program"]["tophat"] = tophat_loc
    bcbio_config["program"]["bowtie"] = bowtie_loc
    bcbio_config["program"]["picard"] = config["program"]["picard"]
    bcbio_config["program"]["gatk"] = {"dir": ""}

    out_file = tophat.align(fastq_file, pair_file, ref_file, out_base,
                            align_dir, bcbio_config)
    return out_file

class Tophat(AbstractStage):

    stage = "tophat"

    def __init__(self, config):
        self.config = config
        self.ref = config.get("ref", None)

    def __call__(self, in_file):
        if isinstance(in_file, basestring):
            logger.info("Detected %s as non-paired." % in_file)
            out_file = run_with_config(in_file, None, self.ref,
                                       self.stage, self.config)
        elif is_pair(in_file):
            logger.info("Detected %s as a pair." % in_file)
            out_file = run_with_config(in_file[0], in_file[1],
                                       self.ref, self.stage, self.config)
        else:
            logger.info("Detected %s as non-paired." % in_file)
            out_file = run_with_config(in_file[0], None, self.ref,
                                       self.stage, self.config)
        return out_file

class Bowtie(AbstractStage):

    stage = "bowtie"

    def __init__(self, config):
        self.config = config
        self.stage_config = config["stage"][self.stage]
        # defaults = {"q": True, "n": 2, "k": 1,
        #            "X": 2000, "best": True,
        #            "sam": True,
        #            "phred64-quals": True}
        #self.options = dict(defaults.items() +
        #                    self.stage_config.get("options", {}).items())
        self.options = self.stage_config.get("options", {})
        self.bowtie = sh.Command(self.stage_config.get("program", "bowtie"))
        self.out_prefix = os.path.join(get_in(self.config,
                                              ("dir", "results"), "results"),
                                              self.stage)
        self.ref_file = self.config["ref"]
        if not any_bowtie_reference_exists(self.ref_file):
            bowtie_reference_not_found_error()

    def _bowtie_se(self, in_file, out_file):
        self.bowtie(self.options, self.ref_file, in_file, out_file)

    def _bowtie_pe(self, in_file, out_file):
        self.bowtie(self.options, self.ref_file,
                    "-1", in_file[0], "-2", in_file[1], out_file)

    def _get_out_file(self, in_file):
        base, _ = os.path.splitext(os.path.basename(in_file))
        out_prefix = os.path.join(get_in(self.config,
                                         ("dir", "results"), "results"),
                                         self.stage)
        out_dir = os.path.join(out_prefix, base)
        out_file = os.path.join(out_dir, base + ".sam")
        return out_file

    def out_file(self, in_file):
        if is_pair(in_file):
            return self._get_out_file(in_file[0])
        else:
            return self._get_out_file(in_file)

    def __call__(self, in_file):
        self._start_message(in_file)
        out_file = self.out_file(in_file)

        if file_exists(out_file):
            return out_file

        with file_transaction(out_file) as tmp_out_file:
            if is_pair(in_file):
                self._bowtie_pe(in_file, tmp_out_file)
            else:
                self._bowtie_se(in_file, tmp_out_file)
        self._end_message(in_file)

        return out_file

def any_bowtie_reference_exists(prefix):
    return bowtie_1_reference_exists(prefix) or bowtie_2_reference_exists(prefix)

def bowtie_1_reference_exists(prefix):
    return file_exists(prefix + ".1.ebwt")
def bowtie_2_reference_exists(prefix):
    return file_exists(prefix + ".1.bt2")

def bowtie_reference_not_found_error():
    raise ValueError("Bowtie 1 or Bowtie 2 reference not found. Please make "
                     "sure that ref: is set correctly in the config file.")
    sys.exit(1)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
Copyright 2015 Zalando SE

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 language governing permissions and limitations under the License.
"""

import logging

import apscheduler.schedulers.background as scheduler_background
import apscheduler.triggers.interval as scheduler_interval
import connexion

import birdwatch.configuration

logger = logging.getLogger('lizzy')
logging.basicConfig(level=logging.DEBUG, style='{', format="{asctime} | {levelname: <7} | {name: <20.20} | {message}")

def setup_scheduler(config):
    # configure scheduler
    scheduler = scheduler_background.BackgroundScheduler()
    interval = scheduler_interval.IntervalTrigger(seconds=config.job_interval)
    scheduler.add_job(JOB_FUNCTION_PLEASE_REPLACE, interval, max_instances=10)
    return scheduler

def setup_webapp(config):
    app = connexion.App(__name__, config.port, specification_dir='swagger/')
    app.add_api('YAML_FILE')
    return app

def main():
    config = configuration.Configuration()

    logger.info('Starting Scheduler')
    scheduler = setup_scheduler(config)
    scheduler.start()
    logger.info('Scheduler running')

    logger.info('Starting web app')
    app = setup_webapp(config)
    app.run()

if __name__ == '__main__':
    main()
<commit_msg>Requirements<commit_after>#!/usr/bin/env python3

"""
Copyright 2015 Zalando SE

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
 language governing permissions and limitations under the License.
"""

import logging

import apscheduler.schedulers.background as scheduler_background
import apscheduler.triggers.interval as scheduler_interval
import connexion

import birdwatch.configuration as configuration

logger = logging.getLogger('birdwatch')
logging.basicConfig(level=logging.DEBUG, style='{', format="{asctime} | {levelname: <7} | {name: <20.20} | {message}")

def setup_scheduler(config):
    # configure scheduler
    scheduler = scheduler_background.BackgroundScheduler()
    interval = scheduler_interval.IntervalTrigger(seconds=config.job_interval)
    scheduler.add_job(JOB_FUNCTION_PLEASE_REPLACE, interval, max_instances=10)
    return scheduler

def setup_webapp(config):
    app = connexion.App(__name__, config.port, specification_dir='swagger/')
    app.add_api('YAML_FILE')
    return app

def main():
    config = configuration.Configuration()

    logger.info('Starting Scheduler')
    scheduler = setup_scheduler(config)
    scheduler.start()
    logger.info('Scheduler running')

    logger.info('Starting web app')
    app = setup_webapp(config)
    app.run()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import io, struct, os, sys
from binascii import crc32, unhexlify, hexlify
from bitcoin.main import *
from bitcoin.bci import unspent, blockr_unspent, blockr_pushtx, blockr_fetchtx, eligius_pushtx, helloblock_unspent, bci_unspent
from bitcoin.transaction import mk_multisig_script, select, multiaccess, access, sum, mktx, mksend, mk_pubkey_script, sign, multisign
from bitcoin.pyspecials import safe_hexlify, safe_unhexlify, st, by

def mk_multisig_scriptpubkey(fo):
    # takes file_object fo and returns scriptpubkey as hex
    data = fo.read(65*3)

    if not data:
        return None

    script_pubkeys = []
    while data:
        chunk = data[:65]; data = data[65:]
        # pad right side with null bytes
        if len(chunk) < 33:   chunk += b'\x00' * (33-len(chunk))
        elif len(chunk) < 65: chunk += b'\x00' * (65-len(chunk))
        script_pubkeys.append(chunk)

    pubz = list(map(safe_hexlify, script_pubkeys))
    return mk_multisig_script(pubz, 1)

def mk_txouts(fo, value=None, jsonfmt=1):
    """Make Tx Outputs as json (or hex)"""
    value = 547 if not value else int(value)
    txouts = []
    while True:
        scriptPubKey = mk_multisig_scriptpubkey(fo)
        if scriptPubKey is None: break
        txouts.append( {'script': scriptPubKey, 'value': value} )
    if jsonfmt:
        return txouts
    return ''.join([(safe_hexlify(struct.pack('<Q', value)) +
                     str(wrap_script(x['script']))) for x in txouts])

def file_insert(filename, value=None, jsonfmt=1):
    """Encode filename into the blockchain using multisig addresses"""
    # TODO: sum (outsputs*547) + (10000*kBytes)
    try:
        fileobj = open(filename, 'rb').read()
    except:
        raise Exception("can't find file!")

    data = struct.pack('<L', len(fileobj)) + \
           struct.pack('<L', crc32(fileobj)) + fileobj
    fd = io.BytesIO(data)
    TXOUTS = mk_txouts(fd, value, jsonfmt)
    if jsonfmt:
        return list(TXOUTS)
    return wrap_varint(TXOUTS)

def encode_file(filename, privkey, value=None, input_address=None, network=None):
    network = 'testnet' if network is None else network
    if input_address is not None:
        u = blockr_unspent(input_address, 'testnet')
    if not value: value = 547
    OUTS = file_insert(filename, value)
    TXFEE = int(1.1 * (10000*os.path.getsize(filename)/1000))
    TOTALFEE = TXFEE + int(value)*len(OUTS)
    INS = select(u, TXFEE)
    rawtx = mksend(INS, OUTS, input_address, TXFEE)
    signedtx = sign(rawtx, 0, privkey, 1)
    return signedtx

# if __name__ == '__main__':
#     import sys, os
#     if len(sys.argv) < 2:
#         print("file_insert.py FILENAME OUTPUT_VALUE [INPUT_ADDRESS]")
#     elif len(sys.argv) == 3:
#         filename = sys.argv[-1]     # TODO: check file exists
#     elif len(sys.argv) == 4:
<commit_msg>Update file_insert.py<commit_after>import io, struct, os, sys, math
from binascii import crc32, unhexlify, hexlify
from bitcoin.main import *
from bitcoin.bci import unspent, blockr_unspent, blockr_pushtx, blockr_fetchtx, eligius_pushtx, helloblock_unspent, bci_unspent
from bitcoin.transaction import mk_multisig_script, select, multiaccess, access, sum, mktx, mksend, mk_pubkey_script, sign, multisign
from bitcoin.pyspecials import safe_hexlify, safe_unhexlify, st, by

def mk_multisig_scriptpubkey(fo):
    # takes file_object fo and returns scriptpubkey as hex
    data = fo.read(65*3)

    if not data:
        return None

    script_pubkeys = []
    while data:
        chunk = data[:65]; data = data[65:]
        # pad right side with null bytes
        if len(chunk) < 33:   chunk += b'\x00' * (33-len(chunk))
        elif len(chunk) < 65: chunk += b'\x00' * (65-len(chunk))
        script_pubkeys.append(chunk)

    pubz = list(map(safe_hexlify, script_pubkeys))
    return mk_multisig_script(pubz, 1)

def mk_txouts(fo, value=None, jsonfmt=1):
    """Make Tx Outputs as json (or hex)"""
    value = 547 if not value else int(value)
    txouts = []
    while True:
        scriptPubKey = mk_multisig_scriptpubkey(fo)
        if scriptPubKey is None: break
        txouts.append( {'script': scriptPubKey, 'value': value} )
    if jsonfmt:
        return txouts
    return ''.join([(safe_hexlify(struct.pack('<Q', value)) +
                     str(wrap_script(x['script']))) for x in txouts])

def mk_binary_txouts(filename, value=None, jsonfmt=1):
    """Encode file into the blockchain (with prepended file length, crc32) using multisig addresses"""
    try: fileobj = open(filename, 'rb').read()
    except: raise Exception("can't find file!")

    data = struct.pack('<I', len(fileobj)) + \
           struct.pack('<I', crc32(fileobj) & 0xffffffff) + fileobj
    fd = io.BytesIO(data)
    TXOUTS = mk_txouts(fd, value, jsonfmt)
    if jsonfmt:
        return list(TXOUTS)
    return wrap_varint(TXOUTS)

def encode_file(filename, privkey, value=None, input_address=None, network=None):
    """Takes binary file, returns signed Tx"""
    if not network:
        network = 'testnet'

    if input_address is None:
        input_address = privtoaddr(privkey, 111) if network == 'testnet' else privtoaddr(privkey)

    u = blockr_unspent(input_address, 'testnet') if network == 'testnet' else unspent(input_address)
    value = 547 if value is None else int(value)

    TXFEE = math.ceil(1.1 * (10000*os.path.getsize(filename)/1000))
    OUTS = mk_binary_txouts(filename, value)
    TOTALFEE = TXFEE + int(value)*len(OUTS)
    INS = select(u, TOTALFEE)

    rawtx = mksend(INS, OUTS, input_address, TXFEE)
    signedtx = sign(rawtx, 0, privkey, 1)
    return signedtx

# if __name__ == '__main__':
#     import sys, os
#     if len(sys.argv) < 2:
#         print("mk_binary_txouts.py FILENAME OUTPUT_VALUE [INPUT_ADDRESS]")
#     elif len(sys.argv) == 3:
#         filename = sys.argv[-1]     # TODO: check file exists
#     elif len(sys.argv) == 4:
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding:utf-8 -*-
from random import shuffle

class Blackjack(object):
    def __init__(self):
        self.money = 2000.00

    def creat_deck(self, deck=1):
        '''
        Create card function, it is possible to create a deck with more cards.
        Increases the difficulty for the player. Maximum of 8 decks.
        '''
        suits = ["♣", "♦", "♥", "♠"]
        numbers = ["A", "2", "3", "4", "5", "6", "7", "8", "9", "10", "Q",
                        "J", "K"]
        self.decks = []
        count = 0
        while count < deck:
            count += 1
            for suit in suits:
                for number in numbers:
                    self.decks.append('{}{}'.format(number, suit))
        return shuffle(self.decks)

    def bet(self, coin, quantity):
        '''
        Betting values are defined as casino chips. That is why exist the
        exception error.
        '''
        if coin not in (1, 5, 10, 25, 50, 100):
            raise Exception('Invalid coin for bet.')
        elif self.money < (coin * quantity):
            raise Exception('Value of bet larger what your money.')
        else:
            self.money -= (coin * quantity)
            self.money += 0.01
            return self.money

    def play(self):
        '''
        To start the function it is necessary to have a bet, that is why the
        variable self.money receives 0.01 in the betting function, ensuring
        that a bet was made. Remember that function remove 4 cards of decks.
        '''
        self.hand = []
        self.house = []
        if self.money == 2000.00:
            raise Exception('Bet is necessary for player.')
        else:
            while len(self.hand) < 2:
                self.hand.append(self.decks.pop(0))
                self.house.append(self.decks.pop(0))
            return self.hand, self.house

    def show_hand(self):
        '''
        This function only shows the cards to the player. His and those in the
        house.
        '''
        msg = ('Your card: {}')
        cards = ', '.join(self.hand)
        return(msg.format(cards))
        return('House: {}, X'.format(self.house[0]))

    def show_points(self, count):
        '''
        Function counting the value of the cards. According to the rule if you
        have an Ace and a J, complete the value 21 or Blackjack.
        '''
        self.points = 0
        for card in count:
            value = card[:-1]
            if value in ('A' and 'J'):
                self.points += 21
                self.points -= 1
            elif value == 'A':
                self.points += 1
            elif value in ('J', 'Q', 'K'):
                self.points += 10
            else:
                self.points += int(value)
        return self.points
<commit_msg>Add hit function<commit_after>#!/usr/bin/env python
# -*- coding:utf-8 -*-
from random import shuffle

class Blackjack(object):
    def __init__(self):
        self.money = 2000.00

    def creat_deck(self, deck=1):
        '''
        Create card function, it is possible to create a deck with more cards.
        Increases the difficulty for the player. Maximum of 8 decks.
        '''
        suits = ["♣", "♦", "♥", "♠"]
        numbers = ["A", "2", "3", "4", "5", "6", "7", "8", "9", "10", "Q",
                        "J", "K"]
        self.decks = []
        count = 0
        while count < deck:
            count += 1
            for suit in suits:
                for number in numbers:
                    self.decks.append('{}{}'.format(number, suit))
        return shuffle(self.decks)

    def bet(self, coin, quantity):
        '''
        Betting values are defined as casino chips. That is why exist the
        exception error.
        '''
        if coin not in (1, 5, 10, 25, 50, 100):
            raise Exception('Invalid coin for bet.')
        elif self.money < (coin * quantity):
            raise Exception('Value of bet larger what your money.')
        else:
            self.money -= (coin * quantity)
            self.money += 0.01
            return self.money

    def play(self):
        '''
        To start the function it is necessary to have a bet, that is why the
        variable self.money receives 0.01 in the betting function, ensuring
        that a bet was made. Remember that function remove 4 cards of decks.
        '''
        self.hand = []
        self.house = []
        if self.money == 2000.00:
            raise Exception('Bet is necessary for player.')
        else:
            while len(self.hand) < 2:
                self.hand.append(self.decks.pop(0))
                self.house.append(self.decks.pop(0))
            return self.hand, self.house

    def show_hand(self):
        '''
        This function only shows the cards to the player. His and those in the
        house.
        '''
        msg = ('Your card: {}')
        cards = ', '.join(self.hand)
        return(msg.format(cards))
        return('House: {}, X'.format(self.house[0]))

    def show_points(self, count):
        '''
        Function counting the value of the cards. According to the rule if you
        have an Ace and a J, complete the value 21 or Blackjack.
        '''
        self.points = 0
        for card in count:
            value = card[:-1]
            if value in ('A' and 'J', 'Q', 'K', 10):
                self.points += 21
                self.points -= 1
            elif value == 'A':
                self.points += 1
            elif value in ('J', 'Q', 'K'):
                self.points += 10
            else:
                self.points += int(value)
        return self.points

    def hit(self):
        '''
        Purchase function of cards. Allows purchase while value of sum of cards
        is lower than 21.
        '''
        if self.points < 21:
            self.hand.append(self.decks.pop(0))
        else:
            raise Exception(None)
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

import warnings

from datashape import dshape
import flask
from flask.testing import FlaskClient
from odo import resource
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
from toolz import assoc

from ..expr import Expr
from ..dispatch import dispatch
from .server import DEFAULT_PORT
from .serialization import json

# These are a hack for testing
# It's convenient to use requests for live production but use
# flask for testing.  Sadly they have different Response objects,
# hence the dispatched functions

__all__ = 'Client',

def content(response):
    if isinstance(response, flask.Response):
        return response.data
    if isinstance(response, requests.Response):
        return response.content

def ok(response):
    if isinstance(response, flask.Response):
        return 200 <= response.status_code <= 299
    if isinstance(response, requests.Response):
        return response.ok

def reason(response):
    if isinstance(response, flask.Response):
        return response.status
    if isinstance(response, requests.Response):
        return response.text

def _request(method, client, url, params=None, auth=None, **kwargs):
    if not isinstance(requests, FlaskClient):
        kwargs['verify'] = client.verify_ssl
        kwargs['params'] = params
        kwargs['auth'] = auth

    with warnings.catch_warnings():
        warnings.simplefilter('ignore', InsecureRequestWarning)
        return method(
            '{base}{url}'.format(
                base=client.url,
                url=url,
            ),
            **kwargs
        )

def get(*args, **kwargs):
    return _request(requests.get, *args, **kwargs)

def post(*args, **kwargs):
    return _request(requests.post, *args, **kwargs)

class Client(object):

    """ Client for Blaze Server

    Provides programmatic access to datasets living on Blaze Server

    Parameters
    ----------

    url : str
        URL of a Blaze server
    serial : SerializationFormat, optional
        The serialization format object to use. Defaults to JSON.
        A serialization format is an object that supports:
        name, loads, and dumps.
    verify_ssl : bool, optional
        Verify the ssl certificate from the server.
        This is enabled by default.
    auth : tuple, optional
        The username and password to use when connecting to the server.
        If not provided, no auth header will be sent.

    Examples
    --------

    >>> # This example matches with the docstring of ``Server``
    >>> from blaze import data
    >>> c = Client('localhost:6363')
    >>> t = data(c) # doctest: +SKIP

    See Also
    --------

    blaze.server.server.Server
    """
    __slots__ = 'url', 'serial', 'verify_ssl', 'auth'

    def __init__(self, url, serial=json, verify_ssl=True, auth=None, **kwargs):
        url = url.strip('/')
        if not url.startswith('http'):
            url = 'http://' + url
        self.url = url
        self.serial = serial
        self.verify_ssl = verify_ssl
        self.auth = auth

    @property
    def dshape(self):
        """The datashape of the client"""
        response = get(self, '/datashape', auth=self.auth)
        if not ok(response):
            raise ValueError("Bad Response: %s" % reason(response))

        return dshape(content(response).decode('utf-8'))

    def add(self, name, resource_uri, *args, **kwargs):
        """Add the given resource URI to the Blaze server.

        Parameters
        ----------
        name : str
            The name to give the resource
        resource_uri : str
            The URI string describing the resource to add to the server, e.g
            'sqlite:///path/to/file.db::table'
        imports : list
            A list of string names for any modules that must be imported on
            the Blaze server before the resource can be added. This is identical
            to the `imports` field in a Blaze server YAML file.
        args : any, optional
            Any additional positional arguments that can be passed to the
            ``blaze.resource`` constructor for this resource type
        kwargs : any, optional
            Any additional keyword arguments that can be passed to the
            ``blaze.resource`` constructor for this resource type
        """
        payload = {name: {'source': resource_uri}}
        imports = kwargs.pop('imports', None)
        if imports is not None:
            payload[name]['imports'] = imports
        if args:
            payload[name]['args'] = args
        if kwargs:
            payload[name]['kwargs'] = kwargs

        response = post(self, '/add', auth=self.auth,
                        data=self.serial.dumps(payload),
                        headers={'Content-Type': 'application/vnd.blaze+' + self.serial.name})
        # A special case for the "Not Found" error, since that means that this
        # server doesn't support adding datasets, and the user should see a more
        # helpful response
        if response.status_code == 404:
            raise ValueError("Server does not support dynamically adding datasets")

        if not ok(response):
            raise ValueError("Bad Response: %s" % reason(response))

@dispatch(Client)
def discover(c):
    return c.dshape

def mimetype(serial):
    """Function to generate a blaze serialization format mimetype put into a
    dictionary of headers for consumption by requests.

    Examples
    --------
    >>> from blaze.server.serialization import msgpack
    >>> mimetype(msgpack)
    {'Content-Type': 'application/vnd.blaze+msgpack'}
    """
    return {'Content-Type': 'application/vnd.blaze+%s' % serial.name}

@dispatch(Expr, Client)
def compute_down(expr, ec, profiler_output=None, **kwargs):
    """Compute down for blaze clients.

    Parameters
    ----------
    expr : Expr
        The expression to send to the server.
    ec : Client
        The blaze client to compute against.
    namespace : dict[Symbol -> any], optional
        The namespace to compute the expression in. This will be amended to
        include that data for the server. By default this will just be the
        client mapping to the server's data.
    compute_kwargs : dict, optional
        Extra kwargs to pass to compute on the server.
    odo_kwargs : dict, optional
        Extra kwargs to pass to odo on the server.
    profile : bool, optional
        Should blaze server run cProfile over the computation of the expression
        and the serialization of the response.
    profiler_output : file-like object, optional
        A file like object to hold the profiling output from the server.
        If this is not passed then the server will write the data to the
        server's filesystem
    """
    from .server import to_tree

    tree = to_tree(expr)
    serial = ec.serial
    if profiler_output is not None:
        kwargs['profiler_output'] = ':response'
    r = post(
        ec,
        '/compute',
        data=serial.dumps(assoc(kwargs, 'expr', tree)),
        auth=ec.auth,
        headers=mimetype(serial),
    )

    if not ok(r):
        raise ValueError("Bad response: %s" % reason(r))
    response = serial.loads(content(r))
    if profiler_output is not None:
        profiler_output.write(response['profiler_output'])
    return serial.data_loads(response['data'])

@resource.register('blaze://.+')
def resource_blaze(uri, leaf=None, **kwargs):
    if leaf is not None:
        raise ValueError('The syntax blaze://...::{leaf} is no longer '
                         'supported as of version 0.8.1.\n'
                         'You can access {leaf!r} using this syntax:\n'
                         'data({uri})[{leaf!r}]'
                         .format(leaf=leaf, uri=uri))
    uri = uri[len('blaze://'):]
    sp = uri.split('/')
    tld, rest = sp[0], sp[1:]
    if ':' not in tld:
        tld += ':%d' % DEFAULT_PORT
    uri = '/'.join([tld] + list(rest))
    return Client(uri)
<commit_msg>BUG: if profiler_output is not None pass profile=True<commit_after>from __future__ import absolute_import, division, print_function

import warnings

from datashape import dshape
import flask
from flask.testing import FlaskClient
from odo import resource
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
from toolz import assoc

from ..expr import Expr
from ..dispatch import dispatch
from .server import DEFAULT_PORT
from .serialization import json

# These are a hack for testing
# It's convenient to use requests for live production but use
# flask for testing.  Sadly they have different Response objects,
# hence the dispatched functions

__all__ = 'Client',

def content(response):
    if isinstance(response, flask.Response):
        return response.data
    if isinstance(response, requests.Response):
        return response.content

def ok(response):
    if isinstance(response, flask.Response):
        return 200 <= response.status_code <= 299
    if isinstance(response, requests.Response):
        return response.ok

def reason(response):
    if isinstance(response, flask.Response):
        return response.status
    if isinstance(response, requests.Response):
        return response.text

def _request(method, client, url, params=None, auth=None, **kwargs):
    if not isinstance(requests, FlaskClient):
        kwargs['verify'] = client.verify_ssl
        kwargs['params'] = params
        kwargs['auth'] = auth

    with warnings.catch_warnings():
        warnings.simplefilter('ignore', InsecureRequestWarning)
        return method(
            '{base}{url}'.format(
                base=client.url,
                url=url,
            ),
            **kwargs
        )

def get(*args, **kwargs):
    return _request(requests.get, *args, **kwargs)

def post(*args, **kwargs):
    return _request(requests.post, *args, **kwargs)

class Client(object):

    """ Client for Blaze Server

    Provides programmatic access to datasets living on Blaze Server

    Parameters
    ----------

    url : str
        URL of a Blaze server
    serial : SerializationFormat, optional
        The serialization format object to use. Defaults to JSON.
        A serialization format is an object that supports:
        name, loads, and dumps.
    verify_ssl : bool, optional
        Verify the ssl certificate from the server.
        This is enabled by default.
    auth : tuple, optional
        The username and password to use when connecting to the server.
        If not provided, no auth header will be sent.

    Examples
    --------

    >>> # This example matches with the docstring of ``Server``
    >>> from blaze import data
    >>> c = Client('localhost:6363')
    >>> t = data(c) # doctest: +SKIP

    See Also
    --------

    blaze.server.server.Server
    """
    __slots__ = 'url', 'serial', 'verify_ssl', 'auth'

    def __init__(self, url, serial=json, verify_ssl=True, auth=None, **kwargs):
        url = url.strip('/')
        if not url.startswith('http'):
            url = 'http://' + url
        self.url = url
        self.serial = serial
        self.verify_ssl = verify_ssl
        self.auth = auth

    @property
    def dshape(self):
        """The datashape of the client"""
        response = get(self, '/datashape', auth=self.auth)
        if not ok(response):
            raise ValueError("Bad Response: %s" % reason(response))

        return dshape(content(response).decode('utf-8'))

    def add(self, name, resource_uri, *args, **kwargs):
        """Add the given resource URI to the Blaze server.

        Parameters
        ----------
        name : str
            The name to give the resource
        resource_uri : str
            The URI string describing the resource to add to the server, e.g
            'sqlite:///path/to/file.db::table'
        imports : list
            A list of string names for any modules that must be imported on
            the Blaze server before the resource can be added. This is identical
            to the `imports` field in a Blaze server YAML file.
        args : any, optional
            Any additional positional arguments that can be passed to the
            ``blaze.resource`` constructor for this resource type
        kwargs : any, optional
            Any additional keyword arguments that can be passed to the
            ``blaze.resource`` constructor for this resource type
        """
        payload = {name: {'source': resource_uri}}
        imports = kwargs.pop('imports', None)
        if imports is not None:
            payload[name]['imports'] = imports
        if args:
            payload[name]['args'] = args
        if kwargs:
            payload[name]['kwargs'] = kwargs

        response = post(self, '/add', auth=self.auth,
                        data=self.serial.dumps(payload),
                        headers={'Content-Type': 'application/vnd.blaze+' + self.serial.name})
        # A special case for the "Not Found" error, since that means that this
        # server doesn't support adding datasets, and the user should see a more
        # helpful response
        if response.status_code == 404:
            raise ValueError("Server does not support dynamically adding datasets")

        if not ok(response):
            raise ValueError("Bad Response: %s" % reason(response))

@dispatch(Client)
def discover(c):
    return c.dshape

def mimetype(serial):
    """Function to generate a blaze serialization format mimetype put into a
    dictionary of headers for consumption by requests.

    Examples
    --------
    >>> from blaze.server.serialization import msgpack
    >>> mimetype(msgpack)
    {'Content-Type': 'application/vnd.blaze+msgpack'}
    """
    return {'Content-Type': 'application/vnd.blaze+%s' % serial.name}

@dispatch(Expr, Client)
def compute_down(expr, ec, profiler_output=None, **kwargs):
    """Compute down for blaze clients.

    Parameters
    ----------
    expr : Expr
        The expression to send to the server.
    ec : Client
        The blaze client to compute against.
    namespace : dict[Symbol -> any], optional
        The namespace to compute the expression in. This will be amended to
        include that data for the server. By default this will just be the
        client mapping to the server's data.
    compute_kwargs : dict, optional
        Extra kwargs to pass to compute on the server.
    odo_kwargs : dict, optional
        Extra kwargs to pass to odo on the server.
    profile : bool, optional
        Should blaze server run cProfile over the computation of the expression
        and the serialization of the response.
    profiler_output : file-like object, optional
        A file like object to hold the profiling output from the server.
        If this is not passed then the server will write the data to the
        server's filesystem
    """
    from .server import to_tree

    tree = to_tree(expr)
    serial = ec.serial
    if profiler_output is not None:
        kwargs['profile'] = True
        kwargs['profiler_output'] = ':response'
    r = post(
        ec,
        '/compute',
        data=serial.dumps(assoc(kwargs, 'expr', tree)),
        auth=ec.auth,
        headers=mimetype(serial),
    )

    if not ok(r):
        raise ValueError("Bad response: %s" % reason(r))
    response = serial.loads(content(r))
    if profiler_output is not None:
        profiler_output.write(response['profiler_output'])
    return serial.data_loads(response['data'])

@resource.register('blaze://.+')
def resource_blaze(uri, leaf=None, **kwargs):
    if leaf is not None:
        raise ValueError('The syntax blaze://...::{leaf} is no longer '
                         'supported as of version 0.8.1.\n'
                         'You can access {leaf!r} using this syntax:\n'
                         'data({uri})[{leaf!r}]'
                         .format(leaf=leaf, uri=uri))
    uri = uri[len('blaze://'):]
    sp = uri.split('/')
    tld, rest = sp[0], sp[1:]
    if ':' not in tld:
        tld += ':%d' % DEFAULT_PORT
    uri = '/'.join([tld] + list(rest))
    return Client(uri)
<|endoftext|>


<commit_before># Reads tweets, stored in json format in compressed text files (one tweet per line) and creates a
# CSV file from the output. Data to be extracted can be chosen using the command-line, but if there
# are no arguments then default commonly-used fields will be extracted. THese are:
#
# Tweet_ID, User_ID, Screen_Name, Lat, Lon, Loc_Txt, Time, Text
#
# Note: for convenience some time columns are also added: Year, Month, Day, Hour, Minute, Seconds

import argparse # For parsing command line arguments
import gzip # For compressing files
import os
import json
import sys

import datetime as dt
# Can't use dt.strptime() to convert time string to date because %z isn't valid! Instead use this:
import email.utils
# NOTE ABOUT TIME ZONES: I'm using local time, not UTC. So there will be two kinks in the data when
# UK changes time zone. Advantage is that work patterns are constant (e.g. 9am-5pm) so don't want to
# change these when the time zone changes.

def fix(text):
    """Encodes a string as unicode and replaces difficult characters. Also checks for and removes
    things that will break the csv file like newline characters.
    Also adds quote characters around strings.
    If the given 'text' doesn't have an encode method (e.g. it is a number) then it is just returned
    as a string (using str)."""
    try:
        return '"'+text.encode(encoding='UTF-8', errors='xmlcharrefreplace').replace("\n"," ")+'"'
    except AttributeError:
        return str(text)

# A list to store the fields to retrieve from the json files. Start with some commonly used fields.
# Each dimension needs to be separated with commas
fields = [
        "id",                   # ID of the tweet
        "user,id",              # ID of the user
        "user,screen_name",     # Name of the user
        "geo,coordinates,0",    # Latitude (y)
        "geo,coordinates,1",    # Longitude (x)
        "place,full_name",      # Name of the location
        "created_at",           # Time created
        "text"                  # Message text
        ]

# ****** Create the command-line parser ******
parser = argparse.ArgumentParser(description='Convert JSON tweets to CSV.')

# Filenames
parser.add_argument('files', nargs="+", metavar="FILE",
        help="List of input files (at least one must be specified")
parser.add_argument('-o', '--outfile', default="tweets.csv",
        help="Output file name (default tweets.csv")

# Specify fields to extract
parser.add_argument('-f', '--field', nargs=1,
        help="Add a field to the output, with each dimension separated by commas. \
                E.g. to get the ID of the user who posted the message, use 'user,id'.")

# Whether or not to clear all the defaults or add time columns
parser.add_argument('-nd', '--no_defaults', action="store_true", default=False,
        help="Don't add any of the default fields to the CSV output")
parser.add_argument('-ntc', '--no_time_columns', action="store_true", default=False,
        help="Don't add the extra time columns to the CSV output")

# Parse command-line arguments
args = parser.parse_args()

if args.no_defaults:
    print "-nd specified: not extracting the default fields"
    fields = {}

# XXXX add any specified columns to the fields that we're interested in

if len(fields)==0:
    print "No fields have been specified, you have probably set the '--no_defaults' flag "+\
            "but not specified any of your own fields.\n"+\
            "This is probably a mistake so I wont continue.\n"+\
            "If you want to add fields yourself use --field"
    sys.exit(1)

print "Will write output to: ",args.outfile

print "Will extract data in the following json fields: ",fields

print "Will {s} time columns".format(s="not add" if args.no_time_columns else "add")

# ****** Start interrogating the tweet files (one tweet per line, possibly compressed) ******

with open(args.outfile, 'w') as of:

    for i, fname in enumerate(args.files):
        # work out if it is a compressed file or plain text and then open as a file object called
        # 'f'. (gzip and normal files can be treated the same)
        with GzipFile(fname, mode="rb") if fname[:-3] == ".gz" else open(fname, "r") as f:

        #f = None
        #if fname[:-3] == ".gz":
        #    f = GzipFile(fname, mode="rb")
        #else
        # There is no data for this field in this tweet:
        #    f = open(fname, "r")

            print "Interrogating file {i}/{num}: {f}".format(i=(i+1), num=len(args.files), f=f.name)

            # If this is the first file, prepare the output file
            if i==0:
                for field in fields:
                    of.write(field.replace(",","-") +", ") # (note: no commas in fieldnames)
                # Also add the extra time columns (added for convenience)
                if not args.no_time_columns:
                    of.write("Year, Month, Day, Hour, Minute, Seconds")
                of.write("\n")

            for line in f:
                tweet = json.loads(line) # Create a dictionary from the tweet on the current line

                # For each field that we're interested in, extract the data and write to the file
                for field in fields:
                    nodata = False # Set to true if there is no data for this field
                    # Need to do a multi-dimensional dictionary lookup from the string field (Sam P
                    # helped me with this). I.e. convert 's = "user,name,firstname" to
                    # tweet["user"]["name"]["firstname"]
                    value = tweet
                    #print "\n* FIELD: {f}\n".format(f=field)
                    for key in field.split(","):
                        #print "KEY:",key, "\nVALUE",value
                        if value == None:# or key not in value:
                            # There is no data for this field in this tweet
                            print "No data in tweet for field: {f}".format(f=field)
                            print "Value: '{v}'. Key: '{k}'".format(v=value, k=key)
                            nodata = True
                            continue # Move onto the next field
                        try:
                            value = value[key]
                        except TypeError:
                            # This happens if the key should be an integer. E.g. with coordinates,
                            # instead of being a dictionary they are a two-element list
                            value = value[int(key)]

                    if nodata:
                        of.write(" , ") # Nothing to write for this field
                    else:
                        of.write(fix(value)+", ")

                # Have added all the required fields, now add time columns for convenience.This is
                # much more complicated than it should be because the %z symbol in strftime()
                # doesn't work on my mac, might work on unix. Instead use email.util library (!)

                time_str = tweet['created_at']
                # Make a tuple from the time (e.g. (year,month,day,hour,minute,second)
                time_tpl = email.utils.parsedate_tz(time_str)
                # Now use those indivdual components to make a datetime object
                time = dt.datetime(*[time_tpl[i] for i in range(7)])

                print time_str, time
                XXXX HERE ADD Separat columns for month etc...

                of.write("\n") # Finished this message, on to next one

<commit_msg>Finished parser, mostly works ok now.<commit_after># Reads tweets, stored in json format in compressed text files (one tweet per line) and creates a
# CSV file from the output. Data to be extracted can be chosen using the command-line, but if there
# are no arguments then default commonly-used fields will be extracted. THese are:
#
# Tweet_ID, User_ID, Screen_Name, Lat, Lon, Loc_Txt, Time, Text
#
# Note: for convenience some time columns are also added: Year, Month, Day, Hour, Minute, Seconds

import argparse # For parsing command line arguments
import gzip # For compressing files
import os
import json
import sys

import pprint # For pretty-printing errors
pp = pprint.PrettyPrinter(depth=1)

import datetime as dt
# Can't use dt.strptime() to convert time string to date because %z isn't valid! Instead use this:
import email.utils
# NOTE ABOUT TIME ZONES: I'm using local time, not UTC. So there will be two kinks in the data when
# UK changes time zone. Advantage is that work patterns are constant (e.g. 9am-5pm) so don't want to
# change these when the time zone changes.

def fix(text):
    """Encodes a string as unicode and replaces difficult characters. Also checks for and removes
    things that will break the csv file like newline characters.
    Also adds quote characters around strings.
    If the given 'text' doesn't have an encode method (e.g. it is a number) then it is just returned
    as a string (using str)."""
    try:
        return '"'+text.encode(encoding='UTF-8', errors='xmlcharrefreplace').replace("\n"," ")+'"'
    except AttributeError:
        return str(text)

# A list to store the fields to retrieve from the json files. Start with some commonly used fields.
# Each dimension needs to be separated with commas
fields = [
        "id",                   # ID of the tweet
        "user,id",              # ID of the user
        "user,screen_name",     # Name of the user
        "geo,coordinates,0",    # Latitude (y)
        "geo,coordinates,1",    # Longitude (x)
        "place,full_name",      # Name of the location
        "created_at",           # Time created
        "text"                  # Message text
        ]

# Remember number of errors (e.g. unrecognisable fields)
error_fields = dict()
error_count = 0

# ****** Create the command-line parser ******
parser = argparse.ArgumentParser(description='Convert JSON tweets to CSV.')

# Filenames
parser.add_argument('files', nargs="+", metavar="FILE",
        help="List of input files (at least one must be specified")
parser.add_argument('-o', '--outfile', default="tweets.csv",
        help="Output file name (default tweets.csv")

# Specify fields to extract
parser.add_argument('-f', '--field', nargs=1,
        help="Add a field to the output, with each dimension separated by commas. \
                E.g. to get the ID of the user who posted the message, use 'user,id'.")

# Whether or not to clear all the defaults or add time columns
parser.add_argument('-nd', '--no_defaults', action="store_true", default=False,
        help="Don't add any of the default fields to the CSV output")
parser.add_argument('-ntc', '--no_time_columns', action="store_true", default=False,
        help="Don't add the extra time columns to the CSV output")

# Whether to run multi-threaded

# Parse command-line arguments
args = parser.parse_args()

if args.no_defaults:
    print "-nd specified: not extracting the default fields"
    fields = {}

# XXXX add any specified columns to the fields that we're interested in

if len(fields)==0:
    print "No fields have been specified, you have probably set the '--no_defaults' flag "+\
            "but not specified any of your own fields.\n"+\
            "This is probably a mistake so I wont continue.\n"+\
            "If you want to add fields yourself use --field"
    sys.exit(1)

print "Will write output to: ",args.outfile

print "Will extract data in the following json fields: ",fields

print "Will {s} time columns".format(s="not add" if args.no_time_columns else "add")

# ****** Start interrogating the tweet files (one tweet per line, possibly compressed) ******

with open(args.outfile, 'w') as of:

    for i, fname in enumerate(args.files):
        # work out if it is a compressed file or plain text and then open as a file object called
        # 'f'. (gzip and normal files can be treated the same)
        with GzipFile(fname, mode="rb") if fname[:-3] == ".gz" else open(fname, "r") as f:

        #f = None
        #if fname[:-3] == ".gz":
        #    f = GzipFile(fname, mode="rb")
        #else
        # There is no data for this field in this tweet:
        #    f = open(fname, "r")

            print "Interrogating file {i}/{num}: {f}".format(i=(i+1), num=len(args.files), f=f.name)

            # If this is the first file, prepare the output file
            if i==0:
                for field in fields:
                    of.write(field.replace(",","-") +", ") # (note: no commas in fieldnames)
                # Also add the extra time columns (added for convenience)
                if not args.no_time_columns:
                    of.write("Year, Month, Day, Hour, Minute, Second")
                of.write("\n")

            for line in f:
                tweet = json.loads(line) # Create a dictionary from the tweet on the current line

                # For each field that we're interested in, extract the data and write to the file
                for field in fields:
                    nodata = False # Set to true if there is no data for this field
                    # Need to do a multi-dimensional dictionary lookup from the string field (Sam P
                    # helped me with this). I.e. convert 's = "user,name,firstname" to
                    # tweet["user"]["name"]["firstname"]
                    value = tweet
                    #print "\n* FIELD: {f}\n".format(f=field)
                    for key in field.split(","):
                        #print "KEY:",key, "\nVALUE",value
                        if value == None:# or key not in value:
                            # There is no data for this field in this tweet
                            #print "No data in tweet for field: {f}".format(f=field)
                            #print "Value: '{v}'. Key: '{k}'".format(v=value, k=key)
                            nodata = True
                            error_count += error_count
                            if field in error_fields:
                                error_fields[field] += 1
                            else:
                                error_fields[field] = 0
                            continue # Move onto the next field
                        try:
                            value = value[key]
                        except TypeError:
                            # This happens if the key should be an integer. E.g. with coordinates,
                            # instead of being a dictionary they are a two-element list
                            value = value[int(key)]

                    if nodata:
                        of.write(" , ") # Nothing to write for this field
                    else:
                        of.write(fix(value)+", ")

                if not args.no_time_columns:

                    # Have added all the required fields, now add time columns for convenience.This is
                    # much more complicated than it should be because the %z symbol in strftime()
                    # doesn't work on my mac, might work on unix. Instead use email.util library (!)

                    time_str = tweet['created_at']
                    # Make a tuple from the time (e.g. (year,month,day,hour,minute,second)
                    time_tpl = email.utils.parsedate_tz(time_str)
                    # Now use those indivdual components to make a datetime object
                    t = dt.datetime(*[time_tpl[i] for i in range(7)])

                    of.write("{Yr}, {Mo}, {D}, {H}, {M}, {S}".format(
                        Yr=t.year, Mo=t.month, D=t.day, M=t.minute, H=t.hour, S=t.second )
                    )

                of.write("\n") # Finished this message, on to next one

print "Finished. There were a total of {e} fields that could not be extracted from the"+\
        "json file. These are: ".format(e=error_count)
pp.pprint(error_fields)

<|endoftext|>