<commit_before>#!/usr/bin/env python

import unittest
from unittest import TestCase
from plasTeX.DOM import *

class Construction(TestCase):

    def _checkPositions(self, node):
        """ Check the postions of all contained nodes """

        maxidx = len(node) - 1

        # Check firstChild and lastChild
        if node:
            assert node.firstChild is node[0], 'firstChild is incorrect'
            assert node.lastChild is node[maxidx], 'lastChild is incorrect'

        # Check nextSibling
        for i, item in enumerate(node):
            if i == maxidx:
                assert item.nextSibling is None, \
                       'nextSibling in position %s should be None' % i
            else:
                assert item.nextSibling is node[i+1], \
                       'nextSibling in position %s is incorrect (%s)' % \
                       (i, item.nextSibling)

        # Check previousSibling
        for i, item in enumerate(node):
            if i == 0:
                assert item.previousSibling is None, \
                       'previousSibling in position %s should be None' % i
            else:
                assert item.previousSibling is node[i-1], \
                       'previousSibling in position %s is incorrect (%s)' % \
                       (i, item.previousSibling)

        # Check parentNode
        for i, item in enumerate(node):
            assert item.parentNode is node, \
                   'parentNode in position %s is incorrect' % i

        # Check ownerDocument
        for i, item in enumerate(node):
            assert item.ownerDocument is node.ownerDocument, \
                   'ownerDocument in position %s is incorrect' % i

        # Check attributes
        if node.attributes:
            for key, value in node.attributes.items():
                if isinstance(value, Node):
                    assert value.parentNode is node, \
                           'parentNode is incorrect (%s)' % value.parentNode
                    self._checkPositions(value)

                elif isinstance(value, list):
                    for item in value:
                        assert item.parentNode is node, \
                               'parentNode is incorrect (%s)' % item.parentNode
                        self._checkPositions(item)

                elif isinstance(value, dict):
                    for item in value.values():
                        assert item.parentNode is node, \
                               'parentNode is incorrect (%s)' % item.parentNode
                        self._checkPositions(item)

    def testConstructor(self):
        """ Passing list of nodes to constructor """
        doc = Document()
        one = doc.createElement('one')
        two = doc.createElement('two')
        three = doc.createElement('three')
        node = Node([one,two,three])
        expected = [one,two,three]
        for i, item in enumerate(node):
            assert item is expected[i], '"%s" != "%s"' % (item, expected[i])
        self._checkPositions(node)

    def testInsert(self):
        """ Insert into empty node """
        doc = Document()
        one = doc.createElement('one')
        two = doc.createElement('two')
        three = doc.createElement('three')
        node = Node()
        node.insert(0, one)
        node.insert(1, two)
        node.insert(2, three)
        expected = [one,two,three]
        for i, item in enumerate(node):
            assert item is expected[i], '"%s" != "%s"' % (item, expected[i])
        self._checkPositions(node)

    def testInsert2(self):
        """ Insert into populated node """
        doc = Document()
        one = doc.createElement('one')
        two = doc.createElement('two')
        three = doc.createElement('three')
        node = Node([one,two,three])
        i0 = doc.createElement('i0')
        i3 = doc.createTextNode('i3')
        node.insert(0, i0)
        node.insert(3, i3)
        expected = [i0,one,two,i3,three]
        for i, item in enumerate(node):
            assert item is expected[i], '"%s" != "%s"' % (item, expected[i])
        self._checkPositions(node)

    def testAttributes(self):
        doc = Document()
        one = doc.createElement('one')
        two = doc.createElement('two')
        three = doc.createTextNode('three')
        node = Node()
        node.attributes['one'] = one
        one.attributes['two'] = two
        two.attributes['three'] = three
        self._checkPositions(node)

if __name__ == '__main__':
    unittest.main()

<commit_msg>Updated DOM tests<commit_after>#!/usr/bin/env python

import unittest
from unittest import TestCase
from plasTeX.DOM import *

class Construction(TestCase):

    def _checkPositions(self, node):
        """ Check the postions of all contained nodes """
        if not(isinstance(node, Node)):
            return

        maxidx = len(node) - 1

        # Check firstChild and lastChild
        if node:
            assert node.firstChild is node[0], 'firstChild is incorrect'
            assert node.lastChild is node[maxidx], 'lastChild is incorrect'

        # Check nextSibling
        for i, item in enumerate(node):
            if i == maxidx:
                assert item.nextSibling is None, \
                       'nextSibling in position %s should be None' % i
            else:
                assert item.nextSibling is node[i+1], \
                       'nextSibling in position %s is incorrect (%s)' % \
                       (i, item.nextSibling)

        # Check previousSibling
        for i, item in enumerate(node):
            if i == 0:
                assert item.previousSibling is None, \
                       'previousSibling in position %s should be None' % i
            else:
                assert item.previousSibling is node[i-1], \
                       'previousSibling in position %s is incorrect (%s)' % \
                       (i, item.previousSibling)

        # Check parentNode
        for i, item in enumerate(node):
            assert item.parentNode is node, \
                   'parentNode in position %s is incorrect' % i

        # Check ownerDocument
        for i, item in enumerate(node):
            assert item.ownerDocument is node.ownerDocument, \
                   'ownerDocument in position %s is incorrect' % i

        # Check attributes
        if node.attributes:
            for key, value in node.attributes.items():
                if isinstance(value, Node):
                    assert value.parentNode is node, \
                           'parentNode is incorrect (%s)' % value.parentNode
                    self._checkPositions(value)

                elif isinstance(value, list):
                    for item in value:
                        assert getattr(item, 'parentNode', node) is node, \
                               'parentNode is incorrect (%s)' % item.parentNode
                        self._checkPositions(item)

                elif isinstance(value, dict):
                    for item in value.values():
                        assert getattr(item, 'parentNode', node) is node, \
                               'parentNode is incorrect (%s)' % item.parentNode
                        self._checkPositions(item)

    def testConstructor(self):
        """ Passing list of nodes to constructor """
        doc = Document()
        one = doc.createElement('one')
        two = doc.createElement('two')
        three = doc.createElement('three')
        node = Node([one,two,three])
        expected = [one,two,three]
        for i, item in enumerate(node):
            assert item is expected[i], '"%s" != "%s"' % (item, expected[i])
        self._checkPositions(node)

    def testAttributes(self):
        """ Set attributes on a node """
        doc = Document()
        one = doc.createElement('one')
        two = doc.createElement('two')
        three = doc.createTextNode('three')
        four = ['hi','bye',Text('text node')]
        node = Node()
        node.attributes['one'] = one
        one.attributes['two'] = two
        two.attributes['three'] = three
        two.attributes['four'] = four
        self._checkPositions(node)

    def testFirstChild(self):
        doc = Document()
        node = doc.createElement('node')
        one = doc.createElement('one')
        two = doc.createElement('two')
        assert node.firstChild is None, '"%s" != None' % node.firstChild
        node.append(one)
        assert node.firstChild is one, '"%s" != "%s"' % (node.firstChild, one)
        node.insert(0, two)
        assert node.firstChild is two, '"%s" != "%s"' % (node.firstChild, two)
        self._checkPositions(node)

    def testLastChild(self):
        doc = Document()
        node = doc.createElement('node')
        one = doc.createElement('one')
        two = doc.createElement('two')
        assert node.lastChild is None, '"%s" != None' % node.lastChild
        node.append(one)
        assert node.lastChild is one, '"%s" != "%s"' % (node.lastChild, one)
        node.append(two)
        assert node.lastChild is two, '"%s" != "%s"' % (node.lastChild, two)
        self._checkPositions(node)

    def testChildNodes(self):
        doc = Document()
        node = doc.createElement('node')
        one = doc.createElement('one')
        two = doc.createTextNode('two')
        three = doc.createElement('three')
        node.append(one)
        node.append(two)
        node.append(three)
        assert node[0] is one, '"%s" != "%s"' % (node[0], one)
        assert node[1] is two, '"%s" != "%s"' % (node[1], two)
        assert node[2] is three, '"%s" != "%s"' % (node[2], three)
        assert node.childNodes[0] is one, '"%s" != "%s"' % (node.childNodes[0], one)
        assert node.childNodes[1] is two, '"%s" != "%s"' % (node.childNodes[1], two)
        assert node.childNodes[2] is three, '"%s" != "%s"' % (node.childNodes[2], three)
        self._checkPositions(node)

    def testPreviousSibling(self):
        doc = Document()
        node = doc.createElement('node')
        one = doc.createElement('one')
        two = doc.createTextNode('two')
        three = doc.createElement('three')
        node.append(one)
        node.append(two)
        node.append(three)
        assert None is one.previousSibling, 'None != "%s"' % one.previousSibling
        assert one is two.previousSibling, '"%s" != "%s"' % (one, two.previousSibling)
        assert two is three.previousSibling, '"%s" != "%s"' % (two, three.previousSibling)

    def testNextSibling(self):
        doc = Document()
        node = doc.createElement('node')
        one = doc.createElement('one')
        two = doc.createTextNode('two')
        three = doc.createElement('three')
        node.append(one)
        node.append(two)
        node.append(three)
        assert two is one.nextSibling, '"%s" != "%s"' % (two, one.nextSibling)
        assert three is two.nextSibling, '"%s" != "%s"' % (three, two.nextSibling)
        assert None is three.nextSibling, 'None != "%s"' % three.nextSibling

    def testOwnerDocument(self):
        doc = Document()
        node = doc.createElement('node')
        one = doc.createElement('one')
        two = doc.createTextNode('two')
        three = doc.createElement('three')
        node.append(one)
        node.append(two)
        node.attributes['three'] = three
        a = Text('a')
        b = Text('b')
        node.attributes['four'] = {'a':a, 'b':1}
        node.attributes['five'] = [b, 1, 'c']
        assert node.ownerDocument is doc, '"%s" != "%s"' % (node.ownerDocument, doc)
        assert one.ownerDocument is doc, '"%s" != "%s"' % (one.ownerDocument, doc)
        assert two.ownerDocument is doc, '"%s" != "%s"' % (two.ownerDocument, doc)
        assert three.ownerDocument is doc, '"%s" != "%s"' % (three.ownerDocument, doc)
        assert a.ownerDocument is doc, '"%s" != "%s"' % (a.ownerDocument, doc)
        assert b.ownerDocument is doc, '"%s" != "%s"' % (b.ownerDocument, doc)
        self._checkPositions(node)

    def testCompareDocumentPosition(self):
        doc = Document()
        node = doc.createElement('node')
        one = doc.createElement('one')
        two = doc.createTextNode('two')
        three = doc.createElement('three')
        four = doc.createElement('four')
        node.append(one)
        node.append(two)
        node.append(three)
        three.append(four)
        five = doc.createElement('five')

        expected = Node.DOCUMENT_POSITION_FOLLOWING
        rc = one.compareDocumentPosition(four)
        assert rc == expected, '"%s" != "%s"' % (rc, expected)

        expected = Node.DOCUMENT_POSITION_PRECEDING
        rc = four.compareDocumentPosition(one)
        assert rc == expected, '"%s" != "%s"' % (rc, expected)

        expected = Node.DOCUMENT_POSITION_CONTAINED_BY
        rc = node.compareDocumentPosition(four)
        assert rc == expected, '"%s" != "%s"' % (rc, expected)

        expected = Node.DOCUMENT_POSITION_CONTAINS
        rc = four.compareDocumentPosition(node)
        assert rc == expected, '"%s" != "%s"' % (rc, expected)

        expected = Node.DOCUMENT_POSITION_DISCONNECTED
        rc = five.compareDocumentPosition(node)
        assert rc == expected, '"%s" != "%s"' % (rc, expected)

    def testInsertBefore(self):
        doc = Document()
        node = doc.createElement('node')
        one = doc.createElement('one')
        two = doc.createTextNode('two')
        three = doc.createElement('three')
        node.append(one)
        node.append(two)
        node.insertBefore(three, two)
        assert node[1] is three, '"%s" != "%s"' % (node[1], three)
        node.insertBefore(three, one)
        assert node[0] is three, '"%s" != "%s"' % (node[0], three)
        assert node[1] is one, '"%s" != "%s"' % (node[1], one)
        assert node[2] is two, '"%s" != "%s"' % (node[2], two)
        self._checkPositions(node)

    def testReplaceChild(self):
        doc = Document()
        node = doc.createElement('node')
        one = doc.createElement('one')
        two = doc.createTextNode('two')
        three = doc.createElement('three')
        node.append(one)
        node.append(two)
        node.replaceChild(three, two)
        assert node[0] is one, '"%s" != "%s"' % (node[0], one)
        assert node[1] is three, '"%s" != "%s"' % (node[1], three)
        assert len(node) == 2, '%s != %s' % (len(node), 2)
        self._checkPositions(node)

    def testRemoveChild(self):
        doc = Document()
        node = doc.createElement('node')
        one = doc.createElement('one')
        two = doc.createTextNode('two')
        three = doc.createElement('three')
        node.append(one)
        node.append(two)

        res = node.removeChild(one)
        assert res is one, '"%s" != "%s"' % (res, one)
        assert len(node) == 1, '%s != %s' % (len(node), 1)
        assert node[0] is two, '"%s" != "%s"' % (node[0], two)
        self._checkPositions(node)

        res = node.removeChild(two)
        assert res is two, '"%s" != "%s"' % (res, two)
        assert len(node) == 0, '%s != %s' % (len(node), 0)

    def testPop(self):
        doc = Document()
        node = doc.createElement('node')
        one = doc.createElement('one')
        two = doc.createTextNode('two')
        three = doc.createElement('three')
        node.extend([one, two, three])

        res = node.pop()
        assert res is three, '"%s" != "%s"' % (res, three)
        assert len(node) == 2, '%s != %s' % (len(node), 2)
        self._checkPositions(node)

        res = node.pop(0)
        assert res is one, '"%s" != "%s"' % (res, one)
        assert len(node) == 1, '%s != %s' % (len(node), 1)
        self._checkPositions(node)

    def testAppendChild(self):
        doc = Document()
        node = doc.createElement('node')
        one = doc.createElement('one')
        two = doc.createTextNode('two')
        three = doc.createElement('three')
        node.appendChild(one)
        frag = DocumentFragment()
        frag.appendChild(two)
        frag.appendChild(three)
        node.appendChild(frag)

        assert node[0] is one, '"%s" != "%s"' % (node[0], one)
        assert node[1] is two, '"%s" != "%s"' % (node[1], two)
        assert node[2] is three, '"%s" != "%s"' % (node[2], three)
        self._checkPositions(node)

    def testInsert(self):
        """ Insert into empty node """
        doc = Document()
        one = doc.createElement('one')
        two = doc.createElement('two')
        three = doc.createElement('three')
        node = Node()
        node.insert(0, one)
        node.insert(1, two)
        node.insert(2, three)
        expected = [one,two,three]
        for i, item in enumerate(node):
            assert item is expected[i], '"%s" != "%s"' % (item, expected[i])
        self._checkPositions(node)

    def testInsert2(self):
        """ Insert into populated node """
        doc = Document()
        one = doc.createElement('one')
        two = doc.createElement('two')
        three = doc.createElement('three')
        node = Node([one,two,three])
        i0 = doc.createElement('i0')
        i3 = doc.createTextNode('i3')
        node.insert(0, i0)
        node.insert(3, i3)
        expected = [i0,one,two,i3,three]
        for i, item in enumerate(node):
            assert item is expected[i], '"%s" != "%s"' % (item, expected[i])
        self._checkPositions(node)

    def testInsert3(self):
        """ Insert document fragment """
        doc = Document()
        node = doc.createElement('node')
        one = doc.createElement('one')
        two = doc.createTextNode('two')
        three = doc.createElement('three')
        four = doc.createElement('four')
        node.appendChild(one)
        node.appendChild(two)
        frag = DocumentFragment()
        frag.appendChild(three)
        frag.appendChild(four)
        node.insert(1,frag)

        assert node[0] is one, '"%s" != "%s"' % (node[0], one)
        assert node[1] is three, '"%s" != "%s"' % (node[1], three)
        assert node[2] is four, '"%s" != "%s"' % (node[2], four)
        assert node[3] is two, '"%s" != "%s"' % (node[3], two)
        self._checkPositions(node)

    def testSetItem(self):
        doc = Document()
        node = doc.createElement('node')
        one = doc.createElement('one')
        two = doc.createTextNode('two')
        three = doc.createElement('three')
        four = doc.createElement('four')
        five = doc.createElement('five')
        node.appendChild(one)
        node.appendChild(two)
        node.appendChild(three)

        node[1] = four
        assert node[0] is one, '"%s" != "%s"' % (node[0], one)
        assert node[1] is four, '"%s" != "%s"' % (node[1], four)
        assert node[2] is three, '"%s" != "%s"' % (node[2], three)
        assert len(node) == 3, '%s != %s' % (len(node), 3)
        self._checkPositions(node)

        node[2] = five
        assert node[0] is one, '"%s" != "%s"' % (node[0], one)
        assert node[1] is four, '"%s" != "%s"' % (node[1], four)
        assert node[2] is five, '"%s" != "%s"' % (node[2], five)
        assert len(node) == 3, '%s != %s' % (len(node), 3)
        self._checkPositions(node)

    def testExtend(self):
        doc = Document()
        node = doc.createElement('node')
        one = doc.createElement('one')
        two = doc.createTextNode('two')
        three = doc.createElement('three')
        four = doc.createElement('four')
        five = doc.createElement('five')
        node.appendChild(one)

        node.extend([two, three])
        assert node[0] is one, '"%s" != "%s"' % (node[0], one)
        assert node[1] is two, '"%s" != "%s"' % (node[1], two)
        assert node[2] is three, '"%s" != "%s"' % (node[2], three)
        assert len(node) == 3, '%s != %s' % (len(node), 3)
        self._checkPositions(node)

        node += [four, five]
        assert node[3] is four, '"%s" != "%s"' % (node[3], four)
        assert node[4] is five, '"%s" != "%s"' % (node[4], five)
        assert len(node) == 5, '%s != %s' % (len(node), 5)
        self._checkPositions(node)

    def testAdd(self):
        doc = Document()
        node = doc.createElement('node')
        one = doc.createElement('one')
        two = doc.createTextNode('two')
        node.appendChild(one)
        node.appendChild(two)

        node2 = doc.createElement('node2')
        three = doc.createElement('three')
        four = doc.createElement('four')
        node.appendChild(three)
        node.appendChild(four)

        res = node + node2
        assert isinstance(res, Node), '%s is not a Node' % res
        assert res[0] is one, '"%s" != "%s"' % (res[0], one)
        assert res[1] is two, '"%s" != "%s"' % (res[1], two)
        assert res[2] is three, '"%s" != "%s"' % (res[2], three)
        assert res[3] is four, '"%s" != "%s"' % (res[3], four)
        self._checkPositions(res)

    def testHasChildNodes(self):
        doc = Document()
        node = doc.createElement('node')
        one = doc.createElement('one')
        two = doc.createTextNode('two')

        assert not node.hasChildNodes()

        node.appendChild(one)
        node.appendChild(two)

        assert node.hasChildNodes()

    def testCloneNode(self):
        doc = Document()
        one = doc.createElement('one')
        two = doc.createElement('two')
        three = doc.createTextNode('three')
        two.append(three)
        one.append(two)

        res = one.cloneNode(1)
        assert type(res) is type(one), '"%s" != "%s"' % (type(res), type(one))
        assert type(res[0]) is type(one[0])
        assert one == res
        assert one is not res
        assert one[0] is not res[0]

    def testNormalize(self):
        pass

    def testIsSupported(self):
        pass

    def testHasAttributes(self):
        pass

    def testTextContent(self):
        pass

    def testIsSameNode(self):
        pass

    def testLookupPrefix(self):
        pass

    def testIsDefaultNamespace(self):
        pass

    def testLookupNamespaceURI(self):
        pass

    def testIsEqualNode(self):
        pass

    def testGetFeature(self):
        pass

    def testSetUserData(self):
        pass

    def testGetUserData(self):
        pass

if __name__ == '__main__':
    unittest.main()

<|endoftext|>


<commit_before>import itertools
import time
import numpy
import inverse_kinematics
import direct_kinematics
import math
from contextlib import closing
import pypot.robot

import pygame				#Init de Pygame
from pygame.locals import *	#Import de Pygame locals en direct

##********** VARIABLES **********##
T_Fenetre = 600						#Hauteur Fenetre
Fenetre_2 = T_Fenetre/2				#Fenetre/2 souvent utilise
T_Reticule = 50						#Taille Reticule
Reticule_2 = T_Reticule/2			#Reticule/2 souvent utilise

Initial_Altitude = -90
Alt_Var = 20
Alt_Axis = 0
Limit_Dist=50						#Distance Limite
#Ce n'est pas une limite reelle c'est utile pour avoir
#un ratio coherent en fonction de la position du curseur
Coeff_Dist = Fenetre_2/Limit_Dist	#Ratio en question
Pos_Center = [100, 0, Initial_Altitude]			#Position par defaut du robot
Is_Mouse = True						#Gestion de la souris
Is_Joystick = False					#ou du Joystick (par ergonomie l'un desactive l'autre)
Joy_X_Axis = 0						#Position du Joystick sur X
Joy_Y_Axis = 0						#Position du Joystick sur Y

##********** PYGAME **********##
#Init
pygame.init()
H_Fenetre = 600
L_Fenetre = 600
H_Reticule = 50
L_Reticule = 50

limit_distance=50
coeff_distance=(L_Fenetre/2)/limit_distance

#Ouverture de la fenetre Pygame
fenetre = pygame.display.set_mode((T_Fenetre, T_Fenetre))
#Chargement, redimensionnement et collage de la grille
fond = pygame.image.load("../Ressources/Grille.jpg").convert()
fond = pygame.transform.scale(fond, (T_Fenetre, T_Fenetre))
fenetre.blit(fond, (0,0))
#Chargement, redimensionnement, positionnement et collage du Reticule
Reticule = pygame.image.load("../Ressources/Reticule.png").convert_alpha()
Reticule = pygame.transform.scale(Reticule, (T_Reticule, T_Reticule))
Ret_x = Fenetre_2-Reticule_2
Ret_y = Fenetre_2-Reticule_2
fenetre.blit(Reticule, (Ret_x, Ret_y))
#Rafraichissement de l'ecran
pygame.display.flip()

##********** MOUVEMENTS **********##
#Recupere les angles
def get_angles (pos):
	angles = inverse_kinematics.leg_ik(pos[0], pos[1], pos[2])
	return [ angles.posx, angles.posy, angles.posz ]
#Ordonne a la jambe d'atteindre une position
def leg_goto(leg, pos):
	angles = get_angles(pos)
	i = 0
	for m in leg :
		m.goal_position = angles[i]
		i += 1

##********** CLASSES **********##
#Positionnement "correct" d'une patte
class corrected_leg:
	leg = None
	delta_angle = 0
	leg_buffer = []

	def __init__(self, leg, delta_angle):
		self.leg = leg
		self.delta_angle = delta_angle
		self.leg_buffer = Pos_Center

	def get_corrected_angle(self, initial_angle):
		return initial_angle - self.delta_angle

	def goto(self, pos, absolute = False, ref = Pos_Center):
		Angle_Rad = math.radians(-self.delta_angle)
		if not(absolute) :
			dy = math.cos(Angle_Rad) * pos[0] +  math.sin(Angle_Rad) * pos[1]
			dx = (-1 * math.sin(Angle_Rad) * pos[0]) +  math.cos(Angle_Rad) * pos[1]
		else:
			dx = pos[0]
			dy = pos[1]
		Goal_Pos = [ref[0]+dx, ref[1]+dy, ref[2] + pos[2]]

		leg_goto(self.leg, Goal_Pos)

#Positionnement "correct" du Robot
class corrected_robot:
	legs = []
	rigth_legs = []
	left_legs = []
	state = -1
	start_time = 0
	actual_time = 0
	ast_period_time = 0

	def __init__(self, legs, rl, ll):
		self.legs = legs
		self.rigth_legs = rl
		self.left_legs = ll
		self.state = -1
		self.start_time = time.clock() * 1000
		self.actual_time = self.start_time
		self.last_period_time = self.start_time;

	def goto(self, pos):
		for l in self.legs:
			l.goto(pos)

	def moveto(self, pos, speed, amp, absolute = False):
		t = ((self.actual_time-self.last_period_time)/(speed))
		z = amp * math.sin(math.pi * t)
		old_pos = [pos[0], pos[1], pos[2]]
		pos[0] = pos[0] * t
		pos[1] = pos[1] * t
		alt_pos = [old_pos[0] - pos[0], old_pos[1] - pos[1], pos[2]]
		pos[2] = z
		R_pos = pos		# RIGTH LEGS
		L_pos = alt_pos	# LEFT LEGS

		if self.state == 1 :
			R_pos = alt_pos
			L_pos = pos

		for l in self.left_legs:
			l.goto(L_pos, absolute)
		for l in self.rigth_legs:
			l.goto(R_pos, absolute)

		self.actual_time = time.clock() * 1000
		if ((self.actual_time - speed) >= self.last_period_time):
			self.last_period_time = self.actual_time
			self.state = -1 *  self.state

##********** TESTS **********##
#Simplification Between
def between ( x,  val1, val2) : # return val1 <= x <= val2
	return (val1 <= x and x <= val2)
#Simplification outof
def outof ( x,  val1, val2) : # return val1 > x  or  x > val2
	return (not(between(x, val1, val2)))

def update_center_pos_z(z):
	Pos_Center[2] = z
#Simplification speed
def check_Speed_Axis (Speed_Axis, gaz) :
	return (outof(Speed_Axis, 0, 0.1) and not(gaz)) or ( between(Speed_Axis, 0, 0.1) and gaz )

##********** MAIN **********##
if __name__ == '__main__':

	#Test du joystick present
	try:
		joystick = pygame.joystick.Joystick(0)
		joystick.init()
		Is_Joystick = True
		Is_Mouse = False
	except:
		Is_Joystick = False
		Is_Mouse = True

	#On va maintenant utiliser un fichier de configuration
	#Il nous permettra de limiter nos demandes aux angles valides
	with closing(pypot.robot.from_json('config.json')) as megabot:
		for m in megabot.motors :
			m.compliant = False;
		#Positionnement "correct" des pattes
		leg1 = corrected_leg(megabot.leg1, 0)
		leg2 = corrected_leg(megabot.leg2, -65)
		leg3 = corrected_leg(megabot.leg3, -115)
		leg4 = corrected_leg(megabot.leg4, -180)
		leg5 = corrected_leg(megabot.leg5, 115)
		leg6 = corrected_leg(megabot.leg6, 65)
		#Positionnement "correct" du Robot
		robot = corrected_robot([leg1, leg2, leg3, leg4, leg5, leg6], [leg1, leg3, leg5], [leg2, leg4, leg6])
		#Position Initiale du Robot
		robot.goto([0, 0, 0])

		continuer = True
		Gaz=False
		circle = False
		Joy_X_Axis = 0
		Joy_Y_Axis = 0
		Speed_Axis = 0.0
		Alt_Axis = 0
		circle_rigth = False
		circle_left = False

		while continuer:

			if Is_Joystick :
				Joy_X_Axis = joystick.get_axis(0)
				Joy_Y_Axis = joystick.get_axis(1)
				Abs_Speed_Axis = joystick.get_axis(2)
				Speed_Axis = math.fabs((Abs_Speed_Axis - 1 ) / 2 )
				Alt_Axis = joystick.get_axis(6)

			for event in pygame.event.get():	#Attente des evenements
				if event.type == QUIT or (event.type == KEYDOWN and event.key == K_ESCAPE):
					Continue = False
				#Tab pour switch entre Joystick et Souris
				if event.type == KEYDOWN and event.key == K_TAB:
							Is_Joystick = not(Is_Joystick)
							Is_Mouse = not(Is_Mouse)
				#Mouvement de la souris si elle est utilisee
				if Is_Mouse and event.type == MOUSEMOTION :
					#On change les coordonnees du Reticule
					Ret_x = event.pos[0]-Reticule_2
					Ret_y = event.pos[1]-Reticule_2
				#Activation des Gaz Si clic souris, espace ou activation des Gaz
				if (event.type == MOUSEBUTTONDOWN) or (event.type == KEYDOWN and event.key == K_SPACE) or (check_Speed_Axis(Speed_Axis, Gaz) and not(Is_Mouse)) :
					Gaz = not(Gaz)
					#On change la couleur du Reticule
					if Gaz :
						Reticule = pygame.image.load("../Ressources/Red_Reticule.png").convert_alpha()
						Reticule = pygame.transform.scale(Reticule, (T_Reticule, T_Reticule))
					else :
						Reticule = pygame.image.load("../Ressources/Reticule.png").convert_alpha()
						Reticule = pygame.transform.scale(Reticule, (T_Reticule, T_Reticule))
				if (event.type == KEYDOWN and event.key == K_LCTRL and not(Gaz)):
					circle = not (circle)
				if (event.type == KEYDOWN and event.key == K_LEFT and circle and not(circle_rigth)):
					circle_left = not(circle_left)
				if (event.type == KEYDOWN and event.key == K_RIGHT and circle and not(circle_left)):
					circle_rigth = not(circle_rigth)

			#Fin de l'attente des evenements

			#GESTION DES DEPLACEMENTS (OU PAS)
			#Definition de la periode en fonction des Gaz
			period = (1600 - (Speed_Axis*1500))
			#Position a atteindre
			print Speed_Axis
			if Is_Joystick :
				Ret_x = Fenetre_2 + (Fenetre_2 * Joy_X_Axis) - Reticule_2
				Ret_y = Fenetre_2 + (Fenetre_2 * Joy_Y_Axis) - Reticule_2

			Pos_Final = [(Ret_x - Fenetre_2)/Coeff_Dist, (Ret_y - Fenetre_2)/Coeff_Dist, 0]

			new_altitude = Initial_Altitude - ( Alt_Axis * Alt_Var)

			update_center_pos_z(new_altitude)

			if Gaz  and (outof(Joy_X_Axis, -0.1, 0.1) or outof(Joy_Y_Axis, -0.1, 0.1)):
				robot.moveto(Pos_Final, period, 35, False)
			elif circle :
				if circle_rigth :
					robot.moveto([0, 50, 0], period, 35, True)
				elif circle_left :
					robot.moveto([0, -50, 0], period, 35, True)
				else :
					robot.goto([0, 0, 0])
			else :
				robot.goto(Pos_Final)

			#PYGAME Re-collage
			fenetre.blit(fond, (0,0))
			fenetre.blit(Reticule, (Ret_x, Ret_y))
			#PYGAME Rafraichissement
			pygame.display.flip()<commit_msg>blablabl<commit_after>import itertools
import time
import numpy
import inverse_kinematics
import direct_kinematics
import math
from contextlib import closing
import pypot.robot

import pygame				#Init de Pygame
from pygame.locals import *	#Import de Pygame locals en direct

##********** VARIABLES **********##
T_Fenetre = 600						#Hauteur Fenetre
Fenetre_2 = T_Fenetre/2				#Fenetre/2 souvent utilise
T_Reticule = 50						#Taille Reticule
Reticule_2 = T_Reticule/2			#Reticule/2 souvent utilise

Initial_Altitude = -90
Alt_Var = 20
Alt_Axis = 0
Limit_Dist=50						#Distance Limite
#Ce n'est pas une limite reelle c'est utile pour avoir
#un ratio coherent en fonction de la position du curseur
Coeff_Dist = Fenetre_2/Limit_Dist	#Ratio en question
Pos_Center = [100, 0, Initial_Altitude]			#Position par defaut du robot
Is_Mouse = True						#Gestion de la souris
Is_Joystick = False					#ou du Joystick (par ergonomie l'un desactive l'autre)
Joy_X_Axis = 0						#Position du Joystick sur X
Joy_Y_Axis = 0						#Position du Joystick sur Y

Left_Right_Axis = 0

##********** PYGAME **********##
#Init
pygame.init()
H_Fenetre = 600
L_Fenetre = 600
H_Reticule = 50
L_Reticule = 50

limit_distance=50
coeff_distance=(L_Fenetre/2)/limit_distance

#Ouverture de la fenetre Pygame
fenetre = pygame.display.set_mode((T_Fenetre, T_Fenetre))
#Chargement, redimensionnement et collage de la grille
fond = pygame.image.load("../Ressources/Grille.jpg").convert()
fond = pygame.transform.scale(fond, (T_Fenetre, T_Fenetre))
fenetre.blit(fond, (0,0))
#Chargement, redimensionnement, positionnement et collage du Reticule
Reticule = pygame.image.load("../Ressources/Reticule.png").convert_alpha()
Reticule = pygame.transform.scale(Reticule, (T_Reticule, T_Reticule))
Ret_x = Fenetre_2-Reticule_2
Ret_y = Fenetre_2-Reticule_2
fenetre.blit(Reticule, (Ret_x, Ret_y))
#Rafraichissement de l'ecran
pygame.display.flip()

##********** MOUVEMENTS **********##
#Recupere les angles
def get_angles (pos):
	angles = inverse_kinematics.leg_ik(pos[0], pos[1], pos[2])
	return [ angles.posx, angles.posy, angles.posz ]
#Ordonne a la jambe d'atteindre une position
def leg_goto(leg, pos):
	angles = get_angles(pos)
	i = 0
	for m in leg :
		m.goal_position = angles[i]
		i += 1

##********** CLASSES **********##
#Positionnement "correct" d'une patte
class corrected_leg:
	leg = None
	delta_angle = 0
	leg_buffer = []

	def __init__(self, leg, delta_angle):
		self.leg = leg
		self.delta_angle = delta_angle
		self.leg_buffer = Pos_Center

	def get_corrected_angle(self, initial_angle):
		return initial_angle - self.delta_angle

	def goto(self, pos, absolute = False, ref = Pos_Center):
		Angle_Rad = math.radians(-self.delta_angle)
		if not(absolute) :
			dy = math.cos(Angle_Rad) * pos[0] +  math.sin(Angle_Rad) * pos[1]
			dx = (-1 * math.sin(Angle_Rad) * pos[0]) +  math.cos(Angle_Rad) * pos[1]
		else:
			dx = pos[0]
			dy = pos[1]
		Goal_Pos = [ref[0]+dx, ref[1]+dy, ref[2] + pos[2]]

		leg_goto(self.leg, Goal_Pos)

#Positionnement "correct" du Robot
class corrected_robot:
	legs = []
	rigth_legs = []
	left_legs = []
	state = -1
	start_time = 0
	actual_time = 0
	ast_period_time = 0

	def __init__(self, legs, rl, ll):
		self.legs = legs
		self.rigth_legs = rl
		self.left_legs = ll
		self.state = -1
		self.start_time = time.clock() * 1000
		self.actual_time = self.start_time
		self.last_period_time = self.start_time;

	def goto(self, pos):
		for l in self.legs:
			l.goto(pos)

	def moveto(self, pos, speed, amp, absolute = False):
		t = ((self.actual_time-self.last_period_time)/(speed))
		z = amp * math.sin(math.pi * t)
		old_pos = [pos[0], pos[1], pos[2]]
		pos[0] = pos[0] * t
		pos[1] = pos[1] * t
		alt_pos = [old_pos[0] - pos[0], old_pos[1] - pos[1], pos[2]]
		pos[2] = z
		R_pos = pos		# RIGTH LEGS
		L_pos = alt_pos	# LEFT LEGS

		if self.state == 1 :
			R_pos = alt_pos
			L_pos = pos

		for l in self.left_legs:
			l.goto(L_pos, absolute)
		for l in self.rigth_legs:
			l.goto(R_pos, absolute)

		self.actual_time = time.clock() * 1000
		if ((self.actual_time - speed) >= self.last_period_time):
			self.last_period_time = self.actual_time
			self.state = -1 *  self.state

##********** TESTS **********##
#Simplification Between
def between ( x,  val1, val2) : # return val1 <= x <= val2
	return (val1 <= x and x <= val2)
#Simplification outof
def outof ( x,  val1, val2) : # return val1 > x  or  x > val2
	return (not(between(x, val1, val2)))

def update_center_pos_z(z):
	Pos_Center[2] = z
#Simplification speed
def check_Speed_Axis (Speed_Axis, gaz) :
	return (outof(Speed_Axis, 0, 0.1) and not(gaz)) or ( between(Speed_Axis, 0, 0.1) and gaz )

##********** MAIN **********##
if __name__ == '__main__':

	#Test du joystick present
	try:
		joystick = pygame.joystick.Joystick(0)
		joystick.init()
		Is_Joystick = True
		Is_Mouse = False
	except:
		Is_Joystick = False
		Is_Mouse = True

	#On va maintenant utiliser un fichier de configuration
	#Il nous permettra de limiter nos demandes aux angles valides
	with closing(pypot.robot.from_json('config.json')) as megabot:
		for m in megabot.motors :
			m.compliant = False;
		#Positionnement "correct" des pattes
		leg1 = corrected_leg(megabot.leg1, 0)
		leg2 = corrected_leg(megabot.leg2, -65)
		leg3 = corrected_leg(megabot.leg3, -115)
		leg4 = corrected_leg(megabot.leg4, -180)
		leg5 = corrected_leg(megabot.leg5, 115)
		leg6 = corrected_leg(megabot.leg6, 65)
		#Positionnement "correct" du Robot
		robot = corrected_robot([leg1, leg2, leg3, leg4, leg5, leg6], [leg1, leg3, leg5], [leg2, leg4, leg6])
		#Position Initiale du Robot
		robot.goto([0, 0, 0])

		Move = False
		Continue = True
		Gaz=False
		circle = False
		Joy_X_Axis = 0
		Joy_Y_Axis = 0
		Speed_Axis = 0.0
		Alt_Axis = 0

		while Continue:

			if Is_Joystick :
				Joy_X_Axis = joystick.get_axis(0)
				Joy_Y_Axis = joystick.get_axis(1)
				Abs_Speed_Axis = joystick.get_axis(2)
				Speed_Axis = math.fabs((Abs_Speed_Axis - 1 ) / 2 )
				Alt_Axis = joystick.get_axis(6)
				Left_Right_Axis = joystick.get_axis(5)

			for event in pygame.event.get():	#Attente des evenements
				if event.type == QUIT or (event.type == KEYDOWN and event.key == K_ESCAPE):
					Continue = False
				#Tab pour switch entre Joystick et Souris
				if event.type == KEYDOWN and event.key == K_TAB:
							Is_Joystick = not(Is_Joystick)
							Is_Mouse = not(Is_Mouse)
				#Mouvement de la souris si elle est utilisee
				if Is_Mouse and event.type == MOUSEMOTION :
					#On change les coordonnees du Reticule
					Ret_x = event.pos[0]-Reticule_2
					Ret_y = event.pos[1]-Reticule_2
				#Activation des Gaz Si clic souris, espace ou activation des Gaz
				if (event.type == MOUSEBUTTONDOWN) or (event.type == KEYDOWN and event.key == K_SPACE) or (check_Speed_Axis(Speed_Axis, Gaz) and not(Is_Mouse)) :
					Gaz = not(Gaz)
					#On change la couleur du Reticule
					if Gaz :
						Reticule = pygame.image.load("../Ressources/Red_Reticule.png").convert_alpha()
						Reticule = pygame.transform.scale(Reticule, (T_Reticule, T_Reticule))
					else :
						Move = False
						Reticule = pygame.image.load("../Ressources/Reticule.png").convert_alpha()
						Reticule = pygame.transform.scale(Reticule, (T_Reticule, T_Reticule))
				if (Gaz and (outof(Joy_X_Axis, -0.1, 0.1) or outof(Joy_Y_Axis, -0.1, 0.1))) :
					Move = True
				else :
					Move = False
				if (not(Move) and Gaz and outof(Left_Right_Axis, -0.1, 0.1)):
					circle = True
				else :
					circle = False
			#Fin de l'attente des evenements

			#GESTION DES DEPLACEMENTS (OU PAS)
			#Definition de la periode en fonction des Gaz
			period = (1500*(1-Speed_Axis)) + 100
			#Position a atteindre
			if Is_Joystick :
				Ret_x = Fenetre_2 + (Fenetre_2 * Joy_X_Axis) - Reticule_2
				Ret_y = Fenetre_2 + (Fenetre_2 * Joy_Y_Axis) - Reticule_2

			Pos_Final = [(Ret_x - Fenetre_2)/Coeff_Dist, (Ret_y - Fenetre_2)/Coeff_Dist, 0]

			new_altitude = Initial_Altitude - ( Alt_Axis * Alt_Var)

			update_center_pos_z(new_altitude)

			new_rotation_pos = [0, -50 * Left_Right_Axis, 0]

			if Move :
				robot.moveto(Pos_Final, period, 35, False)
			elif circle :
				robot.moveto(new_rotation_pos, period, 35, True)
			else :
				robot.goto(Pos_Final)

			#PYGAME Re-collage
			fenetre.blit(fond, (0,0))
			fenetre.blit(Reticule, (Ret_x, Ret_y))
			#PYGAME Rafraichissement
			pygame.display.flip()<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import re
import phonenumbers

class FindSpam:
    bad_keywords = ["baba(ji)?", "fifa.*coins?", "fifabay", "Long Path Tool",
                    "brianfo", "nike", "tosterone", "bajotz",
                    "vashi?k[ae]r[ae]n", "sumer", "kolcak"
                    "porn", "molvi", "judi bola", "ituBola.com", "lost lover",
                    "acai", "skin care", "rejuvenated skin",
                    "LifeForce", "swtor2credits", "me2.do", "black magic",
                    "bam2u", "Neuro(3X|flexyn)", "Nutra", "TesteroneXL",
                    "Bowtrol", "Slim ?Genix", "Cleanse EFX", "Babyliss ?Pro",
                    "Forskolin", "Blackline Elite", "TestCore Pro",
                    "Xtreme Antler", "Maxx Test 3000", "orvigomax",
                    "Cheap Wigs?", "jivam", "(Improve )?Brain Power", "Maximum ?Shred",
                    "aging skin", "acne( prone)? skin", "(skin )?eye serum",
                    "skin (serum|eye)", "bagprada", "6611165613" "Apowersoft",
                    "Service Solahart",
                    "(fake|original) (passports?|driver'?s? licen[cs]e|ID cards?)",
                    "(support|service|helpline)( phone)? number|1[ -]?[ -]?[ -]?866[ -]?978[ -]?(6819|6762)",
                    "(hotmail|gmail|outlook|yahoo|lexmark (printer)?) ?(password( recovery)?|tech)? ?((customer|technical) (support|service))? (support|contact|telephone|help(line)?|phone) number"]
    blacklisted_websites = ["online ?kelas", "careyourhealths", "wowtoes",
                            "ipubsoft", "orabank", "powerigfaustralia",
                            "cfpchampionship2015playofflive", "optimalstackfacts",
                            "maletestosteronebooster", "x4facts",
                            "tripleeffectseyeserum", "healthcaresup",
                            "garciniacambogiaprofacts", "filerepairforum",
                            "lxwpro-t", "casque-beatsbydre", "tenderpublish",
                            "elliskinantiaging", "funmac", "lovebiscuits",
                            "Eglobalfitness", "musclezx90site", "fifapal",
                            "hits4slim", "screenshot\\.net", "downloadavideo\\.net",
                            "strongmenmuscle", "sh\\.st/"]
    rules = [
        {'regex': u"(?i)\\b(%s)\\b|ಌ(>>>>|===>|==>>>)(?s).*http" % "|".join(bad_keywords), 'all': True,
         'sites': [], 'reason': "Bad keyword in {}", 'title': True, 'body': True, 'username': True, 'stripcodeblocks': False},
        {'regex': u"(?i)\\b(weight (body ?builder|loo?s[es]|reduction)|muscles? build(ing)?|muscles?( (grow(th)?|diets?))?|anti aging|SkinCentric|loo?s[es] weight|wrinkles?)\\b", 'all': True,
         'sites': ["fitness.stackexchange.com"], 'reason': "Bad keyword in {}", 'title': True, 'body': False, 'username': True, 'stripcodeblocks': False},
        {'regex': u"(?i)^(?:(?=.*?\\b(?:online|hd)\\b)(?=.*?(?:free|full|unlimited)).*?movies?\\b|(?=.*?\\b(?:acai|kisn)\\b)(?=.*?care).*products?\\b|(?=.*?packer).*mover)", 'all': True,
         'sites': [], 'reason': "Bad keywords in {}", 'title': True, 'body': False, 'username': True, 'stripcodeblocks': False},
        {'regex': u"\\d(?:_*\\d){9}|\\+?\\d_*\\d[\\s\\-]?(?:_*\\d){8,10}|\\d[ -]?\\d{3}[ -]?\\d{3}[ -]?\\d{4}", 'all': True,
         'sites': ["patents.stackexchange.com"], 'reason': "Phone number detected", 'validation_method': 'check_phone_numbers', 'title': True, 'body': False, 'username': False, 'stripcodeblocks': False},
        {'regex': u"(?i)\\b(nigg(a|er)|asshole|fag|fuck(ing?)?|shit|whore)s?\\b", 'all': True,
         'sites': [], 'reason': "Offensive {} detected", 'insensitive':True, 'title': True, 'body': True, 'username': False, 'stripcodeblocks': False},
        {'regex': u"(?i)\\b(crap)\\b", 'all': True, 'sites': [], 'reason': "Offensive {} detected", 'insensitive': True, 'title': True, 'body': False, 'username': False, 'stripcodeblocks': False},
        {'regex': u"^(?=.*[A-Z])[^a-z]*$", 'all': True, 'sites': [], 'reason': "All-caps title", 'title': True, 'body': False, 'username': False, 'stripcodeblocks': False},
        {'regex': u"^(?=.*[0-9])[^a-zA-Z]*$", 'all': True, 'sites': [], 'reason': "Numbers-only title", 'title': True, 'body': False, 'username': False, 'stripcodeblocks': False},
        {'regex': u"https?://[a-zA-Z0-9_.-]+\\.[a-zA-Z]{2,4}(/[a-zA-Z0-9_/?=.-])?", 'all': True,
         'sites': ["stackoverflow.com", "superuser.com", "askubuntu.com"], 'reason': "URL in title", 'title': True, 'body': False, 'username': False, 'stripcodeblocks': False},
        {'regex': u"(?i)(%s)" % "|".join(blacklisted_websites), 'all': True,
         'sites': [], 'reason': "Blacklisted website", 'title': True, 'body': True, 'username': True, 'stripcodeblocks': False},
        {'regex': u"([^\\s_.?!=-])\\1{10,}", 'all': True, 'sites': [], 'reason': "Repeating characters in {}", 'title': True, 'body': True, 'username': False, 'stripcodeblocks': True},
        {'regex': u"(?i)(?P<word>\w+).*((\\b| )+(?P=word)){5,}", 'all': True, 'sites': [], 'reason': "Repeating words in {}", 'title': True, 'body': True, 'username': False, 'stripcodeblocks': True},
        {'regex': u"(?i)(tejveer ?iq|ser?vice pemanas?)", 'all': True, 'sites': [], 'reason': "Blacklisted username", 'title': False, 'body': False, 'username': True, 'stripcodeblocks': False}
    ]

    @staticmethod
    def test_post(title, body, user_name, site, is_answer):
        result = []
        for rule in FindSpam.rules:
            body_to_check = body
            if rule['stripcodeblocks']:
                body_to_check = re.sub("
.*?
", "", body, flags=re.DOTALL)
                body_to_check = re.sub(".*?", "", body_to_check, flags=re.DOTALL)
            if rule['all'] != (site in rule['sites']):
                matched_title = re.compile(rule['regex'], re.UNICODE).findall(title)
                matched_username = re.compile(rule['regex'], re.UNICODE).findall(user_name)
                matched_body = re.compile(rule['regex'], re.UNICODE).findall(body_to_check)
                if matched_title and rule['title']:
                    try:
                        if getattr(FindSpam, "%s" % rule['validation_method'])(matched_title):
                            result.append(rule['reason'])
                    except KeyError:  # There is no special logic for this rule
                        result.append(rule['reason'].replace("{}", "title"))
                if matched_username and rule['username']:
                    try:
                        if getattr(FindSpam, "%s" % rule['validation_method'])(matched_username):
                            result.append(rule['reason'])
                    except KeyError:  # There is no special logic for this rule
                        result.append(rule['reason'].replace("{}", "username"))
                if matched_body and rule['body']:
                    type_of_post = "answer" if is_answer else "body"
                    try:
                        if getattr(FindSpam, "%s" % rule['validation_method'])(matched_body):
                            result.append(rule['reason'].replace("{}", type_of_post))
                    except KeyError:  # There is no special logic for this rule
                        result.append(rule['reason'].replace("{}", type_of_post))
        return result

    @staticmethod
    def check_phone_numbers(matched):
        test_formats = ["IN", "US", None]
        for phone_number in matched:
            for testf in test_formats:
                try:
                    z = phonenumbers.parse(phone_number, testf)
                    if phonenumbers.is_possible_number(z) and phonenumbers.is_valid_number(z):
                        print "Possible %s, Valid %s, Explain: %s" % (phonenumbers.is_possible_number(z), phonenumbers.is_valid_number(z), z)
                        return True
                except phonenumbers.phonenumberutil.NumberParseException:
                    pass
        return False
<commit_msg>+musclehealthfitness<commit_after># -*- coding: utf-8 -*-
import re
import phonenumbers

class FindSpam:
    bad_keywords = ["baba(ji)?", "fifa.*coins?", "fifabay", "Long Path Tool",
                    "brianfo", "nike", "tosterone", "bajotz",
                    "vashi?k[ae]r[ae]n", "sumer", "kolcak"
                    "porn", "molvi", "judi bola", "ituBola.com", "lost lover",
                    "acai", "skin care", "rejuvenated skin",
                    "LifeForce", "swtor2credits", "me2.do", "black magic",
                    "bam2u", "Neuro(3X|flexyn)", "Nutra", "TesteroneXL",
                    "Bowtrol", "Slim ?Genix", "Cleanse EFX", "Babyliss ?Pro",
                    "Forskolin", "Blackline Elite", "TestCore Pro",
                    "Xtreme Antler", "Maxx Test 3000", "orvigomax",
                    "Cheap Wigs?", "jivam", "(Improve )?Brain Power", "Maximum ?Shred",
                    "aging skin", "acne( prone)? skin", "(skin )?eye serum",
                    "skin (serum|eye)", "bagprada", "6611165613" "Apowersoft",
                    "Service Solahart",
                    "(fake|original) (passports?|driver'?s? licen[cs]e|ID cards?)",
                    "(support|service|helpline)( phone)? number|1[ -]?[ -]?[ -]?866[ -]?978[ -]?(6819|6762)",
                    "(hotmail|gmail|outlook|yahoo|lexmark (printer)?) ?(password( recovery)?|tech)? ?((customer|technical) (support|service))? (support|contact|telephone|help(line)?|phone) number"]
    blacklisted_websites = ["online ?kelas", "careyourhealths", "wowtoes",
                            "ipubsoft", "orabank", "powerigfaustralia",
                            "cfpchampionship2015playofflive", "optimalstackfacts",
                            "maletestosteronebooster", "x4facts",
                            "tripleeffectseyeserum", "healthcaresup",
                            "garciniacambogiaprofacts", "filerepairforum",
                            "lxwpro-t", "casque-beatsbydre", "tenderpublish",
                            "elliskinantiaging", "funmac", "lovebiscuits",
                            "Eglobalfitness", "musclezx90site", "fifapal",
                            "hits4slim", "screenshot\\.net", "downloadavideo\\.net",
                            "strongmenmuscle", "sh\\.st/", "musclehealthfitness"]
    rules = [
        {'regex': u"(?i)\\b(%s)\\b|ಌ(>>>>|===>|==>>>)(?s).*http" % "|".join(bad_keywords), 'all': True,
         'sites': [], 'reason': "Bad keyword in {}", 'title': True, 'body': True, 'username': True, 'stripcodeblocks': False},
        {'regex': u"(?i)\\b(weight (body ?builder|loo?s[es]|reduction)|muscles? build(ing)?|muscles?( (grow(th)?|diets?))?|anti aging|SkinCentric|loo?s[es] weight|wrinkles?)\\b", 'all': True,
         'sites': ["fitness.stackexchange.com"], 'reason': "Bad keyword in {}", 'title': True, 'body': False, 'username': True, 'stripcodeblocks': False},
        {'regex': u"(?i)^(?:(?=.*?\\b(?:online|hd)\\b)(?=.*?(?:free|full|unlimited)).*?movies?\\b|(?=.*?\\b(?:acai|kisn)\\b)(?=.*?care).*products?\\b|(?=.*?packer).*mover)", 'all': True,
         'sites': [], 'reason': "Bad keywords in {}", 'title': True, 'body': False, 'username': True, 'stripcodeblocks': False},
        {'regex': u"\\d(?:_*\\d){9}|\\+?\\d_*\\d[\\s\\-]?(?:_*\\d){8,10}|\\d[ -]?\\d{3}[ -]?\\d{3}[ -]?\\d{4}", 'all': True,
         'sites': ["patents.stackexchange.com"], 'reason': "Phone number detected", 'validation_method': 'check_phone_numbers', 'title': True, 'body': False, 'username': False, 'stripcodeblocks': False},
        {'regex': u"(?i)\\b(nigg(a|er)|asshole|fag|fuck(ing?)?|shit|whore)s?\\b", 'all': True,
         'sites': [], 'reason': "Offensive {} detected", 'insensitive':True, 'title': True, 'body': True, 'username': False, 'stripcodeblocks': False},
        {'regex': u"(?i)\\b(crap)\\b", 'all': True, 'sites': [], 'reason': "Offensive {} detected", 'insensitive': True, 'title': True, 'body': False, 'username': False, 'stripcodeblocks': False},
        {'regex': u"^(?=.*[A-Z])[^a-z]*$", 'all': True, 'sites': [], 'reason': "All-caps title", 'title': True, 'body': False, 'username': False, 'stripcodeblocks': False},
        {'regex': u"^(?=.*[0-9])[^a-zA-Z]*$", 'all': True, 'sites': [], 'reason': "Numbers-only title", 'title': True, 'body': False, 'username': False, 'stripcodeblocks': False},
        {'regex': u"https?://[a-zA-Z0-9_.-]+\\.[a-zA-Z]{2,4}(/[a-zA-Z0-9_/?=.-])?", 'all': True,
         'sites': ["stackoverflow.com", "superuser.com", "askubuntu.com"], 'reason': "URL in title", 'title': True, 'body': False, 'username': False, 'stripcodeblocks': False},
        {'regex': u"(?i)(%s)" % "|".join(blacklisted_websites), 'all': True,
         'sites': [], 'reason': "Blacklisted website", 'title': True, 'body': True, 'username': True, 'stripcodeblocks': False},
        {'regex': u"([^\\s_.?!=-])\\1{10,}", 'all': True, 'sites': [], 'reason': "Repeating characters in {}", 'title': True, 'body': True, 'username': False, 'stripcodeblocks': True},
        {'regex': u"(?i)(?P<word>\w+).*((\\b| )+(?P=word)){5,}", 'all': True, 'sites': [], 'reason': "Repeating words in {}", 'title': True, 'body': True, 'username': False, 'stripcodeblocks': True},
        {'regex': u"(?i)(tejveer ?iq|ser?vice pemanas?)", 'all': True, 'sites': [], 'reason': "Blacklisted username", 'title': False, 'body': False, 'username': True, 'stripcodeblocks': False}
    ]

    @staticmethod
    def test_post(title, body, user_name, site, is_answer):
        result = []
        for rule in FindSpam.rules:
            body_to_check = body
            if rule['stripcodeblocks']:
                body_to_check = re.sub("
.*?
", "", body, flags=re.DOTALL)
                body_to_check = re.sub(".*?", "", body_to_check, flags=re.DOTALL)
            if rule['all'] != (site in rule['sites']):
                matched_title = re.compile(rule['regex'], re.UNICODE).findall(title)
                matched_username = re.compile(rule['regex'], re.UNICODE).findall(user_name)
                matched_body = re.compile(rule['regex'], re.UNICODE).findall(body_to_check)
                if matched_title and rule['title']:
                    try:
                        if getattr(FindSpam, "%s" % rule['validation_method'])(matched_title):
                            result.append(rule['reason'])
                    except KeyError:  # There is no special logic for this rule
                        result.append(rule['reason'].replace("{}", "title"))
                if matched_username and rule['username']:
                    try:
                        if getattr(FindSpam, "%s" % rule['validation_method'])(matched_username):
                            result.append(rule['reason'])
                    except KeyError:  # There is no special logic for this rule
                        result.append(rule['reason'].replace("{}", "username"))
                if matched_body and rule['body']:
                    type_of_post = "answer" if is_answer else "body"
                    try:
                        if getattr(FindSpam, "%s" % rule['validation_method'])(matched_body):
                            result.append(rule['reason'].replace("{}", type_of_post))
                    except KeyError:  # There is no special logic for this rule
                        result.append(rule['reason'].replace("{}", type_of_post))
        return result

    @staticmethod
    def check_phone_numbers(matched):
        test_formats = ["IN", "US", None]
        for phone_number in matched:
            for testf in test_formats:
                try:
                    z = phonenumbers.parse(phone_number, testf)
                    if phonenumbers.is_possible_number(z) and phonenumbers.is_valid_number(z):
                        print "Possible %s, Valid %s, Explain: %s" % (phonenumbers.is_possible_number(z), phonenumbers.is_valid_number(z), z)
                        return True
                except phonenumbers.phonenumberutil.NumberParseException:
                    pass
        return False
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
""" OAuth support, facebook and google.

Session variables:

user.biv_id: User's id, survives after log out
user.oauth_type, User's oauth_type, survives after log out
user.is_logged_in, User's logged in status
user.display_name, User's full name

oauth.next_uri, redirect uri after successful login,
    cleared after authorize_complete()
oauth.state, oauth state var, cleared after authorize_complete()
oauth.<oauth_type>.token, user's oauth token, cleared during log out

    :copyright: Copyright (c) 2014 Bivio Software, Inc.  All Rights Reserved.
    :license: Apache, see LICENSE for more details.
"""

import flask
import flask_oauthlib.client
import publicprize.controller as ppc
import publicprize.auth.model as ppam
import werkzeug

def _oauth_provider(name, **kwargs):
    """Creates a OAuth client for the named provider."""
    c = ppc.app().config['PUBLICPRIZE']
    return flask_oauthlib.client.OAuth(ppc.app()).remote_app(
        name,
        consumer_key=c[name.upper() + '_APP_ID'],
        consumer_secret=c[name.upper() + '_APP_SECRET'],
        request_token_url=None,
        **kwargs
    )

_OAUTH_PROVIDER = {
    'facebook': _oauth_provider(
        'facebook',
        request_token_params={'scope': 'email'},
        base_url='https://graph.facebook.com',
        access_token_url='/oauth/access_token',
        authorize_url='https://www.facebook.com/dialog/oauth'
    ),
    'google': _oauth_provider(
        'google',
        request_token_params={
            'scope': 'https://www.googleapis.com/auth/userinfo.email'
        },
        base_url='https://www.googleapis.com/oauth2/v1/',
        access_token_method='POST',
        access_token_url='https://accounts.google.com/o/oauth2/token',
        authorize_url='https://accounts.google.com/o/oauth2/auth',
    )
}

_OAUTH_PROVIDER_DATA_PATH = {
    'facebook': 'me',
    'google': 'userinfo',
}

def add_user_to_session(user):
    """Store user info on session"""
    ppc.db.session.add(user)
    ppc.db.session.flush()
    flask.session['user.biv_id'] = user.biv_id
    flask.session['user.oauth_type'] = user.oauth_type
    flask.session['user.is_logged_in'] = True
    flask.session['user.display_name'] = user.display_name

def authorize(oauth_type, callback):
    """Call the OAuth provider's server which returns to the callback
    on completion or failer.
    Store the "next" or referrer page to return to after callback."""
    flask.session['oauth.next_uri'] = flask.request.args.get('next') \
        or flask.request.referrer or None
    # state variable is uri randomizer, verfied in authorize_complete()
    state = werkzeug.security.gen_salt(64)
    flask.session['oauth.state'] = state
    return _OAUTH_PROVIDER[oauth_type].authorize(
        callback=callback,
        state=state
    )

def authorize_complete(oauth_type):
    """Handle oauth callback, validate the response and update the user
    model."""
    resp = _OAUTH_PROVIDER[oauth_type].authorized_response()
    next_uri = None
    if 'oauth.next_uri' in flask.session:
        next_uri = flask.session['oauth.next_uri']
        del flask.session['oauth.next_uri']
    if _validate_auth(resp, oauth_type):
        flask.session['oauth.{}.token'.format(oauth_type)] = (
            resp['access_token'], '')
        _user_from_info(
            oauth_type,
            _OAUTH_PROVIDER[oauth_type].get(
                _OAUTH_PROVIDER_DATA_PATH[oauth_type]).data
        )
    return flask.redirect(next_uri or '/')

def logout():
    """Clear the login state from the session and redirect to root."""
    _clear_session()
    flask.flash('You have successfully logged out.')
    return flask.redirect('/')

def _clear_session():
    """Clear the login state from the session"""
    flask.session['user.is_logged_in'] = False
    if 'oauth.state' in flask.session:
        del flask.session['oauth.state']
    oauth_type = flask.session.get('user.oauth_type')
    if oauth_type:
        key = 'oauth.{}.token'.format(oauth_type)
        if flask.session.get(key):
            del flask.session[key]

def _client_error(oauth_type, message=None):
    _clear_session()
    flask.flash(message \
        or '{} has denied access to this App.'.format(oauth_type))

def _user_from_info(oauth_type, info):
    """Saves oauth provider user info to user model.
    info arg contains email, id, name."""
    ppc.app().logger.info(info)
    if not info.get('email'):
        _client_error(
            oauth, 'Your email must be provided to this App to login.')
        return
    user = ppam.User.query.filter_by(
        oauth_type=oauth_type,
        oauth_id=info['id']
    ).first()
    if user:
        user.display_name = info['name']
        user.user_email = info['email']
    else:
        user = ppam.User(
            display_name=info['name'],
            user_email=info['email'],
            oauth_type=oauth_type,
            oauth_id=info['id']
        )
    # TODO(pjm): download user avatar
    # facebook: https://graph.facebook.com/{id}/picture?type=square
    # google: image url provided in info.picture field
    add_user_to_session(user)

def _validate_auth(resp, oauth_type):
    """Validates oauth providers's auth response"""
    app = ppc.app()

    if resp is None or isinstance(resp, flask_oauthlib.client.OAuthException):
        _client_error(oauth_type)
        app.logger.warn(
            'Access denied: {}, resp: {}'.format(flask.request.args, resp))
        return False
    state = flask.session['oauth.state']
    del flask.session['oauth.state']
    if state != flask.request.args.get('state'):
        _client_error(oauth_type)
        app.logger.warn(
            'Invalid oauth state, expected: {} response: {}'.format(
                state,
                flask.request.args.get('state'))
        )
        return False
    return True

_FACEBOOK = _OAUTH_PROVIDER['facebook']
_GOOGLE = _OAUTH_PROVIDER['google']

@_FACEBOOK.tokengetter
def _get_facebook_oauth_token():
    """Callback for facebook auth"""
    return flask.session.get('oauth.facebook.token')

@_GOOGLE.tokengetter
def _get_google_access_token():
    """Callback for google auth"""
    return flask.session.get('oauth.google.token')
<commit_msg>clear user.oauth_type session value if oauth fails<commit_after># -*- coding: utf-8 -*-
""" OAuth support, facebook and google.

Session variables:

user.biv_id: User's id, survives after log out
user.oauth_type, User's oauth_type, survives after log out
user.is_logged_in, User's logged in status
user.display_name, User's full name

oauth.next_uri, redirect uri after successful login,
    cleared after authorize_complete()
oauth.state, oauth state var, cleared after authorize_complete()
oauth.<oauth_type>.token, user's oauth token, cleared during log out

    :copyright: Copyright (c) 2014 Bivio Software, Inc.  All Rights Reserved.
    :license: Apache, see LICENSE for more details.
"""

import flask
import flask_oauthlib.client
import publicprize.controller as ppc
import publicprize.auth.model as ppam
import werkzeug

def _oauth_provider(name, **kwargs):
    """Creates a OAuth client for the named provider."""
    c = ppc.app().config['PUBLICPRIZE']
    return flask_oauthlib.client.OAuth(ppc.app()).remote_app(
        name,
        consumer_key=c[name.upper() + '_APP_ID'],
        consumer_secret=c[name.upper() + '_APP_SECRET'],
        request_token_url=None,
        **kwargs
    )

_OAUTH_PROVIDER = {
    'facebook': _oauth_provider(
        'facebook',
        request_token_params={'scope': 'email'},
        base_url='https://graph.facebook.com',
        access_token_url='/oauth/access_token',
        authorize_url='https://www.facebook.com/dialog/oauth'
    ),
    'google': _oauth_provider(
        'google',
        request_token_params={
            'scope': 'https://www.googleapis.com/auth/userinfo.email'
        },
        base_url='https://www.googleapis.com/oauth2/v1/',
        access_token_method='POST',
        access_token_url='https://accounts.google.com/o/oauth2/token',
        authorize_url='https://accounts.google.com/o/oauth2/auth',
    )
}

_OAUTH_PROVIDER_DATA_PATH = {
    'facebook': 'me',
    'google': 'userinfo',
}

def add_user_to_session(user):
    """Store user info on session"""
    ppc.db.session.add(user)
    ppc.db.session.flush()
    flask.session['user.biv_id'] = user.biv_id
    flask.session['user.oauth_type'] = user.oauth_type
    flask.session['user.is_logged_in'] = True
    flask.session['user.display_name'] = user.display_name

def authorize(oauth_type, callback):
    """Call the OAuth provider's server which returns to the callback
    on completion or failer.
    Store the "next" or referrer page to return to after callback."""
    flask.session['oauth.next_uri'] = flask.request.args.get('next') \
        or flask.request.referrer or None
    # state variable is uri randomizer, verfied in authorize_complete()
    state = werkzeug.security.gen_salt(64)
    flask.session['oauth.state'] = state
    return _OAUTH_PROVIDER[oauth_type].authorize(
        callback=callback,
        state=state
    )

def authorize_complete(oauth_type):
    """Handle oauth callback, validate the response and update the user
    model."""
    resp = _OAUTH_PROVIDER[oauth_type].authorized_response()
    next_uri = None
    if 'oauth.next_uri' in flask.session:
        next_uri = flask.session['oauth.next_uri']
        del flask.session['oauth.next_uri']
    if _validate_auth(resp, oauth_type):
        flask.session['oauth.{}.token'.format(oauth_type)] = (
            resp['access_token'], '')
        _user_from_info(
            oauth_type,
            _OAUTH_PROVIDER[oauth_type].get(
                _OAUTH_PROVIDER_DATA_PATH[oauth_type]).data
        )
    return flask.redirect(next_uri or '/')

def logout():
    """Clear the login state from the session and redirect to root."""
    _clear_session()
    flask.flash('You have successfully logged out.')
    return flask.redirect('/')

def _clear_session(clear_oauth_type=False):
    """Clear the login state from the session"""
    flask.session['user.is_logged_in'] = False
    if 'oauth.state' in flask.session:
        del flask.session['oauth.state']
    oauth_type = flask.session.get('user.oauth_type')
    if oauth_type:
        key = 'oauth.{}.token'.format(oauth_type)
        if flask.session.get(key):
            del flask.session[key]
        if clear_oauth_type:
            del flask.session['user.oauth_type']

def _client_error(oauth_type, message=None):
    _clear_session(True)
    flask.flash(message \
        or '{} has denied access to this App.'.format(oauth_type))

def _user_from_info(oauth_type, info):
    """Saves oauth provider user info to user model.
    info arg contains email, id, name."""
    ppc.app().logger.info(info)
    if not info.get('email'):
        _client_error(
            oauth, 'Your email must be provided to this App to login.')
        return
    user = ppam.User.query.filter_by(
        oauth_type=oauth_type,
        oauth_id=info['id']
    ).first()
    if user:
        user.display_name = info['name']
        user.user_email = info['email']
    else:
        user = ppam.User(
            display_name=info['name'],
            user_email=info['email'],
            oauth_type=oauth_type,
            oauth_id=info['id']
        )
    # TODO(pjm): download user avatar
    # facebook: https://graph.facebook.com/{id}/picture?type=square
    # google: image url provided in info.picture field
    add_user_to_session(user)

def _validate_auth(resp, oauth_type):
    """Validates oauth providers's auth response"""
    app = ppc.app()

    if resp is None or isinstance(resp, flask_oauthlib.client.OAuthException):
        _client_error(oauth_type)
        app.logger.warn(
            'Access denied: {}, resp: {}'.format(flask.request.args, resp))
        return False
    state = flask.session['oauth.state']
    del flask.session['oauth.state']
    if state != flask.request.args.get('state'):
        _client_error(oauth_type)
        app.logger.warn(
            'Invalid oauth state, expected: {} response: {}'.format(
                state,
                flask.request.args.get('state'))
        )
        return False
    return True

_FACEBOOK = _OAUTH_PROVIDER['facebook']
_GOOGLE = _OAUTH_PROVIDER['google']

@_FACEBOOK.tokengetter
def _get_facebook_oauth_token():
    """Callback for facebook auth"""
    return flask.session.get('oauth.facebook.token')

@_GOOGLE.tokengetter
def _get_google_access_token():
    """Callback for google auth"""
    return flask.session.get('oauth.google.token')
<|endoftext|>


<commit_before>import os
import json
from io import BytesIO
from .utils import BinaryReader

SIGNATURE_WEB = "UnityWeb"
SIGNATURE_RAW = "UnityRaw"

with open(os.path.join(os.path.dirname(__file__), "strings.dat"), "rb") as f:
	STRINGS_DAT = f.read()

with open(os.path.join(os.path.dirname(__file__), "classes.json"), "r") as f:
	UNITY_CLASSES = json.load(f)

def UnityClass(i):
	return UNITY_CLASSES[str(i)]

class TypeTree:
	NULL = "(null)"

	def __init__(self):
		self.children = []

	def __repr__(self):
		return "<%s %s (size=%r, index=%r, is_array=%r, flags=%r)>" % (
			self.type, self.name, self.size, self.index, self.is_array, self.flags
		)

	@property
	def post_align(self):
		return self.flags & 0x4000

	def load_blob(self, buf):
		self.num_nodes = buf.read_uint()
		self.buffer_bytes = buf.read_uint()
		node_data = BytesIO(buf.read(24 * self.num_nodes))
		self.data = buf.read(self.buffer_bytes)

		parents = [self]

		buf = BinaryReader(node_data)

		for i in range(self.num_nodes):
			version = buf.read_int16()
			depth = buf.read_byte()
			is_array = buf.read_byte()
			type_offset = buf.read_int()
			name_offset = buf.read_int()
			size = buf.read_int()
			index = buf.read_uint()
			flags = buf.read_int()
			type = self.NULL
			name = self.NULL

			type = self.get_string(type_offset)
			name = self.get_string(name_offset)

			if depth == 0:
				curr = self
			else:
				while len(parents) > depth:
					parents.pop()
				curr = TypeTree()
				parents[-1].children.append(curr)
				parents.append(curr)

			curr.type = type
			curr.name = name
			curr.size = size
			curr.index = index
			curr.is_array = is_array
			curr.version = version
			curr.flags = flags

	def get_string(self, offset):
		if offset < 0:
			offset &= 0x7fffffff
			data = STRINGS_DAT
		elif offset < self.buffer_bytes:
			data = self.data
		else:
			return self.NULL
		return data[offset:].partition(b"\0")[0].decode("utf-8")

class TypeMetadata:
	def __init__(self):
		self.type_trees = {}
		self.hashes = {}

	def load(self, buf):
		offset = buf.tell()
		self.generator_version = buf.read_string()
		self.target_platform = buf.read_uint()

		assert self.target_platform == 5  # Windows. RuntimePlatform?

		# if format >= 13
		self.has_type_trees = bool(buf.read_byte())
		self.num_types = buf.read_int()

		for i in range(self.num_types):
			class_id = buf.read_int()  # TODO get unity class
			if class_id < 0:
				hash = buf.read(0x20)
			else:
				hash = buf.read(0x10)

			self.hashes[class_id] = hash

			if self.has_type_trees:
				tree = TypeTree()
				tree.load_blob(buf)
				self.type_trees[UnityClass(class_id)] = tree
				self.type_trees

class ObjectInfo:
	def __init__(self, asset):
		self.asset = asset

	def __repr__(self):
		return "<%s %i>" % (self.type, self.class_id)

	def bytes(self):
		self.asset.data.seek(self.asset.data_offset + self.data_offset)
		return self.asset.data.read(self.size)

	def load(self, buf):
		self.data_offset = buf.read_uint() + self.asset.data_offset
		self.size = buf.read_uint()
		self.type = UnityClass(buf.read_uint())
		self.class_id = buf.read_int16()

		if self.asset.format <= 10:
			self.is_destroyed = bool(buf.read_int16())
		elif self.asset.format >= 11:
			self.unk0 = buf.read_int16()

			if self.asset.format >= 15:
				self.unk1 = buf.read_byte()

	def read(self):
		type = self.asset.types[self.type]
		buf = BinaryReader(self.asset.data)
		buf.seek(self.data_offset)
		return self.read_value(type, buf)

	def read_value(self, type, buf):
		align = False
		t = type.type
		first_child = type.children[0] if type.children else TypeTree()
		if t == "string":
			size = buf.read_uint()
			result = buf.read_string(size)
			align = type.children[0].post_align
		else:
			if type.is_array:
				first_child = type

			if first_child and first_child.is_array:
				align = first_child.post_align
				size = buf.read_uint()
				array_type = first_child.children[1]
				if array_type.type in ("char", "UInt8"):
					data = buf.read(size)
					result = repr(data)
				else:
					result = []
					for i in range(size):
						result.append(self.read_value(array_type, buf))

			else:
				assert t == "TextAsset", t
				result = {}
				for child in type.children:
					result[child.name] = self.read_value(child, buf)

		if align or type.post_align:
			buf.align()

		return result

class Asset:
	def __init__(self):
		self.objects = {}
		self.adds = []
		self.asset_refs = []
		self.types = {}

	def __repr__(self):
		return "<%s %s>" % (self.__class__.__name__, self.name)

	def load(self, buf):
		offset = buf.tell()
		self.name = buf.read_string()
		self.header_size = buf.read_uint()
		self.size = buf.read_uint()

		buf.seek(offset + self.header_size - 4)
		self.data = BytesIO(buf.read(self.size))
		self.prepare()

	def prepare(self):
		buf = BinaryReader(self.data, endian=">")

		self.metadata_size = buf.read_uint()
		self.file_size = buf.read_uint()
		self.format = buf.read_uint()
		self.data_offset = buf.read_uint()
		self.endianness = buf.read_uint()

		if self.endianness == 0:
			buf.endian = "<"

		assert self.format >= 9

		self.tree = TypeMetadata()
		self.tree.load(buf)

		self.num_objects = buf.read_uint()
		for i in range(self.num_objects):
			if self.format >= 14:
				buf.align()
				path_id = buf.read_int64()
			else:
				path_id = buf.read_int()

			obj = ObjectInfo(self)
			obj.load(buf)

			if obj.type in self.tree.type_trees:
				self.types[obj.type] = self.tree.type_trees[obj.type]
			elif obj.type not in self.types:
				self.types[obj.type] = TypeMetadata.default().type_trees[obj.type]

			if path_id in self.objects:
				raise ValueError("Duplicate asset object: %r" % (obj))

			self.objects[path_id] = obj

		if self.format >= 11:
			num_adds = buf.read_uint()
			for i in range(num_adds):
				if self.format >= 14:
					buf.align()
					id = buf.read_int64()
				else:
					id = buf.read_int()
				self.adds.append((id, buf.read_int()))

		self.num_refs = buf.read_uint()
		for i in range(self.num_refs):
			ref = AssetRef()  # TODO
			ref.load(buf)
			self.asset_refs.append(ref)

		unk_string = buf.read_string()
		assert not unk_string, unk_string

class AssetBundle:
	def __init__(self):
		self.assets = []

	def load(self, file):
		buf = BinaryReader(file, endian=">")

		self.signature = buf.read_string()
		self.format_version = buf.read_int()
		self.unity_version = buf.read_string()
		self.generator_version = buf.read_string()
		self.file_size = buf.read_uint()
		self.header_size = buf.read_int()

		self.file_count = buf.read_int()
		self.bundle_count = buf.read_int()

		if self.format_version >= 2:
			self.complete_file_size = buf.read_uint()

			if self.format_version >= 3:
				self.data_header_size = buf.read_uint()

		if self.header_size >= 60:
			self.uncompressed_file_size = buf.read_uint()
			self.bundle_header_size = buf.read_uint()

		assert self.signature == SIGNATURE_RAW

		# Preload assets
		buf.seek(self.header_size)
		self.num_assets = buf.read_int()
		for i in range(self.num_assets):
			asset = Asset()
			asset.load(buf)
			self.assets.append(asset)

def load(file):
	ret = AssetBundle()
	ret.load(file)
	return ret
<commit_msg>Implement AssetRef loading<commit_after>import os
import json
from binascii import hexlify
from io import BytesIO
from uuid import UUID
from .utils import BinaryReader

SIGNATURE_WEB = "UnityWeb"
SIGNATURE_RAW = "UnityRaw"

with open(os.path.join(os.path.dirname(__file__), "strings.dat"), "rb") as f:
	STRINGS_DAT = f.read()

with open(os.path.join(os.path.dirname(__file__), "classes.json"), "r") as f:
	UNITY_CLASSES = json.load(f)

def UnityClass(i):
	return UNITY_CLASSES[str(i)]

class TypeTree:
	NULL = "(null)"

	def __init__(self):
		self.children = []

	def __repr__(self):
		return "<%s %s (size=%r, index=%r, is_array=%r, flags=%r)>" % (
			self.type, self.name, self.size, self.index, self.is_array, self.flags
		)

	@property
	def post_align(self):
		return self.flags & 0x4000

	def load_blob(self, buf):
		self.num_nodes = buf.read_uint()
		self.buffer_bytes = buf.read_uint()
		node_data = BytesIO(buf.read(24 * self.num_nodes))
		self.data = buf.read(self.buffer_bytes)

		parents = [self]

		buf = BinaryReader(node_data)

		for i in range(self.num_nodes):
			version = buf.read_int16()
			depth = buf.read_byte()
			is_array = buf.read_byte()
			type_offset = buf.read_int()
			name_offset = buf.read_int()
			size = buf.read_int()
			index = buf.read_uint()
			flags = buf.read_int()
			type = self.NULL
			name = self.NULL

			type = self.get_string(type_offset)
			name = self.get_string(name_offset)

			if depth == 0:
				curr = self
			else:
				while len(parents) > depth:
					parents.pop()
				curr = TypeTree()
				parents[-1].children.append(curr)
				parents.append(curr)

			curr.type = type
			curr.name = name
			curr.size = size
			curr.index = index
			curr.is_array = is_array
			curr.version = version
			curr.flags = flags

	def get_string(self, offset):
		if offset < 0:
			offset &= 0x7fffffff
			data = STRINGS_DAT
		elif offset < self.buffer_bytes:
			data = self.data
		else:
			return self.NULL
		return data[offset:].partition(b"\0")[0].decode("utf-8")

class TypeMetadata:
	def __init__(self):
		self.type_trees = {}
		self.hashes = {}

	def load(self, buf):
		offset = buf.tell()
		self.generator_version = buf.read_string()
		self.target_platform = buf.read_uint()

		assert self.target_platform == 5  # Windows. RuntimePlatform?

		# if format >= 13
		self.has_type_trees = bool(buf.read_byte())
		self.num_types = buf.read_int()

		for i in range(self.num_types):
			class_id = buf.read_int()  # TODO get unity class
			if class_id < 0:
				hash = buf.read(0x20)
			else:
				hash = buf.read(0x10)

			self.hashes[class_id] = hash

			if self.has_type_trees:
				tree = TypeTree()
				tree.load_blob(buf)
				self.type_trees[UnityClass(class_id)] = tree
				self.type_trees

class ObjectInfo:
	def __init__(self, asset):
		self.asset = asset

	def __repr__(self):
		return "<%s %i>" % (self.type, self.class_id)

	def bytes(self):
		self.asset.data.seek(self.asset.data_offset + self.data_offset)
		return self.asset.data.read(self.size)

	def load(self, buf):
		self.data_offset = buf.read_uint() + self.asset.data_offset
		self.size = buf.read_uint()
		self.type = UnityClass(buf.read_uint())
		self.class_id = buf.read_int16()

		if self.asset.format <= 10:
			self.is_destroyed = bool(buf.read_int16())
		elif self.asset.format >= 11:
			self.unk0 = buf.read_int16()

			if self.asset.format >= 15:
				self.unk1 = buf.read_byte()

	def read(self):
		type = self.asset.types[self.type]
		buf = BinaryReader(self.asset.data)
		buf.seek(self.data_offset)
		return self.read_value(type, buf)

	def read_value(self, type, buf):
		align = False
		t = type.type
		first_child = type.children[0] if type.children else TypeTree()
		if t == "string":
			size = buf.read_uint()
			result = buf.read_string(size)
			align = type.children[0].post_align
		else:
			if type.is_array:
				first_child = type

			if first_child and first_child.is_array:
				align = first_child.post_align
				size = buf.read_uint()
				array_type = first_child.children[1]
				if array_type.type in ("char", "UInt8"):
					data = buf.read(size)
					result = repr(data)
				else:
					result = []
					for i in range(size):
						result.append(self.read_value(array_type, buf))

			else:
				assert t == "TextAsset", t
				result = {}
				for child in type.children:
					result[child.name] = self.read_value(child, buf)

		if align or type.post_align:
			buf.align()

		return result

class Asset:
	def __init__(self):
		self.objects = {}
		self.adds = []
		self.asset_refs = []
		self.types = {}

	def __repr__(self):
		return "<%s %s>" % (self.__class__.__name__, self.name)

	def load(self, buf):
		offset = buf.tell()
		self.name = buf.read_string()
		self.header_size = buf.read_uint()
		self.size = buf.read_uint()

		buf.seek(offset + self.header_size - 4)
		self.data = BytesIO(buf.read(self.size))
		self.prepare()

	def prepare(self):
		buf = BinaryReader(self.data, endian=">")

		self.metadata_size = buf.read_uint()
		self.file_size = buf.read_uint()
		self.format = buf.read_uint()
		self.data_offset = buf.read_uint()
		self.endianness = buf.read_uint()

		if self.endianness == 0:
			buf.endian = "<"

		assert self.format >= 9

		self.tree = TypeMetadata()
		self.tree.load(buf)

		self.num_objects = buf.read_uint()
		for i in range(self.num_objects):
			if self.format >= 14:
				buf.align()
				path_id = buf.read_int64()
			else:
				path_id = buf.read_int()

			obj = ObjectInfo(self)
			obj.load(buf)

			if obj.type in self.tree.type_trees:
				self.types[obj.type] = self.tree.type_trees[obj.type]
			elif obj.type not in self.types:
				self.types[obj.type] = TypeMetadata.default().type_trees[obj.type]

			if path_id in self.objects:
				raise ValueError("Duplicate asset object: %r" % (obj))

			self.objects[path_id] = obj

		if self.format >= 11:
			num_adds = buf.read_uint()
			for i in range(num_adds):
				if self.format >= 14:
					buf.align()
					id = buf.read_int64()
				else:
					id = buf.read_int()
				self.adds.append((id, buf.read_int()))

		self.num_refs = buf.read_uint()
		for i in range(self.num_refs):
			ref = AssetRef()
			ref.load(buf)
			self.asset_refs.append(ref)

		unk_string = buf.read_string()
		assert not unk_string, unk_string

class AssetRef:
	def load(self, buf):
		self.asset_path = buf.read_string()
		self.guid = UUID(hexlify(buf.read(16)).decode("utf-8"))
		self.type = buf.read_int()
		self.file_path = buf.read_string()

	def __repr__(self):
		return "<%s (asset_path=%r, guid=%r, type=%r, file_path=%r)>" % (
			self.__class__.__name__, self.asset_path, self.guid, self.type, self.file_path
		)

class AssetBundle:
	def __init__(self):
		self.assets = []

	def load(self, file):
		buf = BinaryReader(file, endian=">")

		self.signature = buf.read_string()
		self.format_version = buf.read_int()
		self.unity_version = buf.read_string()
		self.generator_version = buf.read_string()
		self.file_size = buf.read_uint()
		self.header_size = buf.read_int()

		self.file_count = buf.read_int()
		self.bundle_count = buf.read_int()

		if self.format_version >= 2:
			self.complete_file_size = buf.read_uint()

			if self.format_version >= 3:
				self.data_header_size = buf.read_uint()

		if self.header_size >= 60:
			self.uncompressed_file_size = buf.read_uint()
			self.bundle_header_size = buf.read_uint()

		assert self.signature == SIGNATURE_RAW

		# Preload assets
		buf.seek(self.header_size)
		self.num_assets = buf.read_int()
		for i in range(self.num_assets):
			asset = Asset()
			asset.load(buf)
			self.assets.append(asset)

def load(file):
	ret = AssetBundle()
	ret.load(file)
	return ret
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2018 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

def run_natality_tutorial(override_values={}):
    # [START bigquery_query_natality_tutorial]
    """Create a Google BigQuery linear regression input table.

    In the code below, the following actions are taken:
    * A new dataset is created "natality_regression."
    * A query is run against the public dataset,
        bigquery-public-data.samples.natality, selecting only the data of
        interest to the regression, the output of which is stored in a new
        "regression_input" table.
    * The output table is moved over the wire to the user's default project via
        the built-in BigQuery Connector for Spark that bridges BigQuery and
        Cloud Dataproc.
    """

    from google.cloud import bigquery

    # Create a new Google BigQuery client using Google Cloud Platform project
    # defaults.
    client = bigquery.Client()

    # Prepare a reference to a new dataset for storing the query results.
    dataset_id = "natality_regression"
    dataset_id_full = "{}.{}".format(client.project, dataset_id)
    # [END bigquery_query_natality_tutorial]
    # To facilitate testing, we replace values with alternatives
    # provided by the testing harness.
    dataset_id = override_values.get("dataset_id", dataset_id)
    dataset_id_full = "{}.{}".format(client.project, dataset_id)
    # [START bigquery_query_natality_tutorial]

    dataset = bigquery.Dataset(dataset_id_full)

    # Create the new BigQuery dataset.
    dataset = client.create_dataset(dataset)

    # In the new BigQuery dataset, create a reference to a new table for
    # storing the query results.
    table_ref = dataset.table("regression_input")

    # Configure the query job.
    job_config = bigquery.QueryJobConfig()

    # Set the destination table to the table reference created above.
    job_config.destination = table_ref

    # Set up a query in Standard SQL, which is the default for the BigQuery
    # Python client library.
    # The query selects the fields of interest.
    query = """
        SELECT
            weight_pounds, mother_age, father_age, gestation_weeks,
            weight_gain_pounds, apgar_5min
        FROM
            `bigquery-public-data.samples.natality`
        WHERE
            weight_pounds IS NOT NULL
            AND mother_age IS NOT NULL
            AND father_age IS NOT NULL
            AND gestation_weeks IS NOT NULL
            AND weight_gain_pounds IS NOT NULL
            AND apgar_5min IS NOT NULL
    """

    # Run the query.
    query_job = client.query(query, job_config=job_config)
    query_job.result()  # Waits for the query to finish
    # [END bigquery_query_natality_tutorial]

if __name__ == "__main__":
    run_natality_tutorial()
<commit_msg>docs: simplify destination table sample with f-strings (#966)<commit_after>#!/usr/bin/env python

# Copyright 2018 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

def run_natality_tutorial(override_values={}):
    # [START bigquery_query_natality_tutorial]
    """Create a Google BigQuery linear regression input table.

    In the code below, the following actions are taken:
    * A new dataset is created "natality_regression."
    * A query is run against the public dataset,
        bigquery-public-data.samples.natality, selecting only the data of
        interest to the regression, the output of which is stored in a new
        "regression_input" table.
    * The output table is moved over the wire to the user's default project via
        the built-in BigQuery Connector for Spark that bridges BigQuery and
        Cloud Dataproc.
    """

    from google.cloud import bigquery

    # Create a new Google BigQuery client using Google Cloud Platform project
    # defaults.
    client = bigquery.Client()

    # Prepare a reference to a new dataset for storing the query results.
    dataset_id = "natality_regression"
    dataset_id_full = f"{client.project}.{dataset_id}"
    # [END bigquery_query_natality_tutorial]
    # To facilitate testing, we replace values with alternatives
    # provided by the testing harness.
    dataset_id = override_values.get("dataset_id", dataset_id)
    dataset_id_full = f"{client.project}.{dataset_id}"
    # [START bigquery_query_natality_tutorial]

    dataset = bigquery.Dataset(dataset_id_full)

    # Create the new BigQuery dataset.
    dataset = client.create_dataset(dataset)

    # Configure the query job.
    job_config = bigquery.QueryJobConfig()

    # Set the destination table to where you want to store query results.
    # As of google-cloud-bigquery 1.11.0, a fully qualified table ID can be
    # used in place of a TableReference.
    job_config.destination = f"{dataset_id_full}.regression_input"

    # Set up a query in Standard SQL, which is the default for the BigQuery
    # Python client library.
    # The query selects the fields of interest.
    query = """
        SELECT
            weight_pounds, mother_age, father_age, gestation_weeks,
            weight_gain_pounds, apgar_5min
        FROM
            `bigquery-public-data.samples.natality`
        WHERE
            weight_pounds IS NOT NULL
            AND mother_age IS NOT NULL
            AND father_age IS NOT NULL
            AND gestation_weeks IS NOT NULL
            AND weight_gain_pounds IS NOT NULL
            AND apgar_5min IS NOT NULL
    """

    # Run the query.
    query_job = client.query(query, job_config=job_config)
    query_job.result()  # Waits for the query to finish
    # [END bigquery_query_natality_tutorial]

if __name__ == "__main__":
    run_natality_tutorial()
<|endoftext|>


<commit_before>import boto3
import datetime
import os
import re
import argparse
import pandas as pd

'''
Pulls and renames both the most recent CARS file and the most recent GTAS aka SF133 file.
By default, does nothing if there was no S3 files found in the past 24 hours,
can be forced to pull the most recent.

File naming convention is s3://gtas-sf133-frb/PE.[CARS or GTAS]_DA-YYYYMM-PP where PP = period and MM = fiscal month
'''

BUCKET_SOURCE = 'gtas-sf133-frb'

# Helper function...if the contents of the row aren't empty and "null" is in it, delete the contents of the row
def replace_null(row, column_name):
    if row[column_name] and 'null' in row[column_name]:
        return None
    return row[column_name]

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--force_pull_latest', action='store_true', help='Pulls the most recent CARS ' +
        'and the most recent GTAS (aka SF133) file, regardless of when it came in.')
    parser.add_argument('--bucket', nargs='?', const='default', default='default', type=str)
    args = parser.parse_args()

    print('\nPulling latest from bucket "{}".'.format(BUCKET_SOURCE))

    if not args.force_pull_latest:
        print('Ignoring files modified before {}'.format(
            (datetime.datetime.now() - datetime.timedelta(hours=24)).strftime("%Y-%m-%d %H:%m:%S"))
        )

    s3 = boto3.resource(service_name='s3', region_name='us-gov-west-1')
    bucket_source = s3.Bucket(BUCKET_SOURCE)

    all_files = bucket_source.objects.all()

    if args.force_pull_latest:
        recent_files = list(all_files)
    else:
        recent_files = [x for x in all_files if (
            datetime.datetime.utcnow() - x.last_modified.replace(tzinfo=None) <
            datetime.timedelta(hours=24))]

    recent_files.sort(key=lambda tup: tup.last_modified, reverse=True)

    try:
        recent_cars = [x for x in recent_files if re.search('PE\.CARS', x.key)][0]
        cars_exists = True
    except Exception as e:
        print('No CARS file posted in the last 24 hours, or no files found.')
        cars_exists = False
        pass

    if cars_exists:
        current_dir = os.getcwd()
        cars_file_name = os.path.join(current_dir, 'files', 'cars_tas.csv')
        os.makedirs('files', exist_ok=True)
        print('Downloading ' + recent_cars.key + ' as cars_tas.csv')
        bucket_source.download_file(recent_cars.key, cars_file_name)
        print('Download successful, beginning file cleanup.')

        # read CGAC values from csv
        data = pd.read_csv(cars_file_name, dtype=str, keep_default_na=False)

        data['End Date'] = data.apply(lambda x: replace_null(x, 'End Date'), axis=1)
        data['Financial Indicator Type 2'] = data.apply(
            lambda x: replace_null(x, 'Financial Indicator Type 2'),
            axis=1
        )

        data.rename(columns={'FR Entity Type Code': 'FR Entity Type',
                             'Financial Indicator Type 2': 'financial_indicator_type2',
                             'Date/Time Established': 'DT_TM_ESTAB',
                             'End Date': 'DT_END'},
                             inplace=True)
        data.to_csv(cars_file_name)

    try:
        recent_gtas = [x for x in recent_files if re.search('PE\.GTAS', x.key)][0]
        gtas_exists = True
    except:
        print('No GTAS file posted in the last 24 hours, or no files found.')
        gtas_exists = False
        pass

    if gtas_exists:
        gtas_year = recent_gtas.key.split('GTAS')[1][4:8]
        gtas_period = recent_gtas.key.split('GTAS')[1][8:10]
        gtas_filename = '_'.join(('sf', '133', gtas_year,gtas_period)) + '.csv'
        os.makedirs('files', exist_ok=True)
        print('Downloading ' + recent_gtas.key + ' as ' + gtas_filename)
        bucket_source.download_file(recent_gtas.key, os.path.join(os.getcwd(), 'files', gtas_filename))
        print('Download successful')

    if not cars_exists and not gtas_exists:
        print('No files in "{}" modified since {}, or no files found.'.format(
            BUCKET_SOURCE,
            (datetime.datetime.now() - datetime.timedelta(hours=24)).strftime("%Y-%m-%d %H:%m:%S"))
        )

if __name__ == '__main__':
    main()
<commit_msg>Updating frb bucket for DTI<commit_after>import boto3
import datetime
import os
import re
import argparse
import pandas as pd

'''
Pulls and renames both the most recent CARS file and the most recent GTAS aka SF133 file.
By default, does nothing if there was no S3 files found in the past 24 hours,
can be forced to pull the most recent.

File naming convention is s3://dti-gtas-sf133-frb-nonprod/PE.[CARS or GTAS]_DA-YYYYMM-PP
where PP = period and MM = fiscal month
'''

BUCKET_SOURCE = 'dti-gtas-sf133-frb-nonprod'

# Helper function...if the contents of the row aren't empty and "null" is in it, delete the contents of the row
def replace_null(row, column_name):
    if row[column_name] and 'null' in row[column_name]:
        return None
    return row[column_name]

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--force_pull_latest', action='store_true', help='Pulls the most recent CARS ' +
        'and the most recent GTAS (aka SF133) file, regardless of when it came in.')
    parser.add_argument('--bucket', nargs='?', const='default', default='default', type=str)
    args = parser.parse_args()

    print('\nPulling latest from bucket "{}".'.format(BUCKET_SOURCE))

    if not args.force_pull_latest:
        print('Ignoring files modified before {}'.format(
            (datetime.datetime.now() - datetime.timedelta(hours=24)).strftime("%Y-%m-%d %H:%m:%S"))
        )

    s3 = boto3.resource(service_name='s3', region_name='us-gov-west-1')
    bucket_source = s3.Bucket(BUCKET_SOURCE)

    all_files = bucket_source.objects.all()

    if args.force_pull_latest:
        recent_files = list(all_files)
    else:
        recent_files = [x for x in all_files if (
            datetime.datetime.utcnow() - x.last_modified.replace(tzinfo=None) <
            datetime.timedelta(hours=24))]

    recent_files.sort(key=lambda tup: tup.last_modified, reverse=True)

    try:
        recent_cars = [x for x in recent_files if re.search('PE\.CARS', x.key)][0]
        cars_exists = True
    except Exception as e:
        print('No CARS file posted in the last 24 hours, or no files found.')
        cars_exists = False
        pass

    if cars_exists:
        current_dir = os.getcwd()
        cars_file_name = os.path.join(current_dir, 'files', 'cars_tas.csv')
        os.makedirs('files', exist_ok=True)
        print('Downloading ' + recent_cars.key + ' as cars_tas.csv')
        bucket_source.download_file(recent_cars.key, cars_file_name)
        print('Download successful, beginning file cleanup.')

        # read CGAC values from csv
        data = pd.read_csv(cars_file_name, dtype=str, keep_default_na=False)

        data['End Date'] = data.apply(lambda x: replace_null(x, 'End Date'), axis=1)
        data['Financial Indicator Type 2'] = data.apply(
            lambda x: replace_null(x, 'Financial Indicator Type 2'),
            axis=1
        )

        data.rename(columns={'FR Entity Type Code': 'FR Entity Type',
                             'Financial Indicator Type 2': 'financial_indicator_type2',
                             'Date/Time Established': 'DT_TM_ESTAB',
                             'End Date': 'DT_END'},
                             inplace=True)
        data.to_csv(cars_file_name)

    try:
        recent_gtas = [x for x in recent_files if re.search('PE\.GTAS', x.key)][0]
        gtas_exists = True
    except:
        print('No GTAS file posted in the last 24 hours, or no files found.')
        gtas_exists = False
        pass

    if gtas_exists:
        gtas_year = recent_gtas.key.split('GTAS')[1][4:8]
        gtas_period = recent_gtas.key.split('GTAS')[1][8:10]
        gtas_filename = '_'.join(('sf', '133', gtas_year,gtas_period)) + '.csv'
        os.makedirs('files', exist_ok=True)
        print('Downloading ' + recent_gtas.key + ' as ' + gtas_filename)
        bucket_source.download_file(recent_gtas.key, os.path.join(os.getcwd(), 'files', gtas_filename))
        print('Download successful')

    if not cars_exists and not gtas_exists:
        print('No files in "{}" modified since {}, or no files found.'.format(
            BUCKET_SOURCE,
            (datetime.datetime.now() - datetime.timedelta(hours=24)).strftime("%Y-%m-%d %H:%m:%S"))
        )

if __name__ == '__main__':
    main()
<|endoftext|>