<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function

import bisect
import getopt
import logging
import os
import random
import re
import string
import sys

from math import sqrt
from threading import currentThread, Thread
from textwrap import dedent
from time import time

try:
    from httplib import HTTPConnection
except ImportError:
    from http.client import HTTPConnection

try:
    from urllib import urlencode
except ImportError:
    from urllib.parse import urlencode

__program__ = 'pyspeedtest'
__script__ = os.path.basename(sys.argv[0])
__version__ = '1.3'

class SpeedTest(object):

    DOWNLOAD_FILES = [
        '/speedtest/random350x350.jpg',
        '/speedtest/random500x500.jpg',
        '/speedtest/random1500x1500.jpg'
    ]

    UPLOAD_FILES = [
        132884,
        493638
    ]

    def __init__(self, host=None, verbose=False, http_debug=0):
        self.host = host
        self.verbose = verbose
        self.http_debug = http_debug

    def downloadthread(self, connection, url):
        connection.request('GET', url, None, {'Connection': 'Keep-Alive'})
        response = connection.getresponse()
        self_thread = currentThread()
        self_thread.downloaded = len(response.read())

    def download(self, runs=2):
        if self.host is None:
            self.host = self.chooseserver()

        total_downloaded = 0
        connections = []
        for run in range(runs):
            connection = HTTPConnection(self.host)
            connection.set_debuglevel(self.http_debug)
            connection.connect()
            connections.append(connection)
        total_start_time = time()
        for current_file in SpeedTest.DOWNLOAD_FILES:
            threads = []
            for run in range(runs):
                thread = Thread(
                    target=self.downloadthread,
                    args=(connections[run],
                          '%s?x=%d' % (current_file, int(time() * 1000))))
                thread.run_number = run + 1
                thread.start()
                threads.append(thread)
            for thread in threads:
                thread.join()
                total_downloaded += thread.downloaded
                logging.info('Run %d for %s finished',
                             thread.run_number, current_file)
        total_ms = (time() - total_start_time) * 1000
        for connection in connections:
            connection.close()
        logging.info('Took %d ms to download %d bytes',
                     total_ms, total_downloaded)
        return total_downloaded * 8000 / total_ms

    def uploadthread(self, connection, data):
        url = '/speedtest/upload.php?x=%s' % random.random()
        connection.request('POST', url, data, {
            'Connection': 'Keep-Alive',
            'Content-Type': 'application/x-www-form-urlencoded'
        })
        response = connection.getresponse()
        reply = response.read().decode('utf-8')
        self_thread = currentThread()
        self_thread.uploaded = int(reply.split('=')[1])

    def upload(self, runs=2):
        if self.host is None:
            self.host = self.chooseserver()

        connections = []
        for run in range(runs):
            connection = HTTPConnection(self.host)
            connection.set_debuglevel(self.http_debug)
            connection.connect()
            connections.append(connection)

        post_data = []
        alphabet = string.digits + string.ascii_letters
        for current_file_size in SpeedTest.UPLOAD_FILES:
            values = {
                'content0': ''.join(
                    random.choice(alphabet) for i in range(current_file_size))
            }
            post_data.append(urlencode(values))

        total_uploaded = 0
        total_start_time = time()
        for data in post_data:
            threads = []
            for run in range(runs):
                thread = Thread(target=self.uploadthread,
                                args=(connections[run], data))
                thread.run_number = run + 1
                thread.start()
                threads.append(thread)
            for thread in threads:
                thread.join()
                logging.info('Run %d for %d bytes finished',
                             thread.run_number, thread.uploaded)
                total_uploaded += thread.uploaded
        total_ms = (time() - total_start_time) * 1000
        for connection in connections:
            connection.close()
        logging.info('Took %d ms to upload %d bytes',
                     total_ms, total_uploaded)
        return total_uploaded * 8000 / total_ms

    def ping(self, server=None):
        if server is None:
            server = self.host

        if server is None:
            raise Exception('No server specified')

        connection = HTTPConnection(server)
        connection.set_debuglevel(self.http_debug)
        connection.connect()
        times = []
        worst = 0
        for _ in range(5):
            total_start_time = time()
            connection.request(
                'GET',
                '/speedtest/latency.txt?x=%d' % random.random(),
                None,
                {'Connection': 'Keep-Alive'})
            response = connection.getresponse()
            response.read()
            total_ms = time() - total_start_time
            times.append(total_ms)
            if total_ms > worst:
                worst = total_ms
        times.remove(worst)
        total_ms = sum(times) * 250  # * 1000 / number of tries (4) = 250
        connection.close()
        logging.info('Latency for %s - %d', server, total_ms)
        return total_ms

    def chooseserver(self):
        connection = HTTPConnection('www.speedtest.net')
        connection.set_debuglevel(self.http_debug)
        connection.connect()
        now = int(time() * 1000)
        extra_headers = {
            'Connection': 'Keep-Alive',
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2',
        }
        connection.request(
            'GET', '/speedtest-config.php?x=%d' % now, None, extra_headers)
        response = connection.getresponse()
        reply = response.read().decode('utf-8')
        match = re.search(
            r'<client ip="([^"]*)" lat="([^"]*)" lon="([^"]*)"', reply)
        location = None
        if match is None:
            logging.info('Failed to retrieve coordinates')
            return None
        location = match.groups()
        logging.info('Your IP: %s\nYour latitude: %s\nYour longitude: %s' %
                     location)
        connection.request(
            'GET', '/speedtest-servers.php?x=%d' % now, None, extra_headers)
        response = connection.getresponse()
        reply = response.read().decode('utf-8')
        server_list = re.findall(
            r'<server url="([^"]*)" lat="([^"]*)" lon="([^"]*)"', reply)
        my_lat = float(location[1])
        my_lon = float(location[2])
        sorted_server_list = []
        for server in server_list:
            s_lat = float(server[1])
            s_lon = float(server[2])
            distance = sqrt(pow(s_lat - my_lat, 2) + pow(s_lon - my_lon, 2))
            bisect.insort_left(sorted_server_list, (distance, server[0]))
        best_server = (999999, '')
        for server in sorted_server_list[:10]:
            logging.info(server[1])
            match = re.search(
                r'http://([^/]+)/speedtest/upload\.php', server[1])
            if match is None:
                continue
            server_host = match.groups()[0]
            latency = self.ping(server_host)
            if latency < best_server[0]:
                best_server = (latency, server_host)
        print('Using server:', best_server[1])
        return best_server[1]

def error(*args):
    print(*args, file=sys.stderr)
    sys.exit(1)

def usage():
    return dedent('''\
        usage: %s [OPTION]...

        Test your bandwidth speed using Speedtest.net servers.

        optional arguments:

          -d L, --debug=L    set http connection debug level (default is 0)
          -m M, --mode=M     test mode: 1 - download
                                        2 - upload
                                        4 - ping
                                        1 + 2 + 4 = 7 - all (default)
          -r N, --runs=N     use N runs (default is 2)
          -s H, --server=H   use specific server
          -v,   --verbose    output additional information

          -h,   --help       display this help and exit
                --version    output version information and exit
        ''' % __script__)

def version():
    return dedent('''\
        %s %s
        ''' % (__program__, __version__))

def parseargs():
    try:
        opts, _ = getopt.getopt(
            sys.argv[1:],
            'hvd:m:r:s:',
            [
                'help',
                'verbose',
                'version',
                'debug=',
                'mode=',
                'runs=',
                'server='
            ]
        )
    except getopt.GetoptError as err:
        error(err.msg + '\n\n' + usage())
    else:
        return opts

def main():
    opts = parseargs()
    speedtest = SpeedTest()

    mode = 7
    runs = 2

    for opt, arg in opts:
        if opt in ('-h', '--help'):
            print(usage())
            sys.exit()
        elif opt == '--version':
            print(version())
            sys.exit()
        elif opt in ('-d', '--debug'):
            try:
                speedtest.http_debug = int(arg)
            except ValueError:
                error('Bad debug level value')
        elif opt in ('-m', '--mode'):
            try:
                mode = int(arg)
                if not 7 >= mode >= 1:
                    raise ValueError
            except ValueError:
                error('Bad mode value')
        elif opt in ('-r', '--runs'):
            try:
                runs = int(arg)
            except ValueError:
                error('Bad runs value')
        elif opt == '-s':
            speedtest.host = arg
        elif opt in ('-v', '--verbose'):
            speedtest.verbose = True

    logging.basicConfig(
        format='%(message)s',
        level=logging.INFO if speedtest.verbose else logging.ERROR)

    if mode & 4 == 4 and speedtest.host is not None:
        print('Ping: %d ms' % speedtest.ping())

    if mode & 1 == 1:
        print('Download speed: %s' % pretty_speed(speedtest.download(runs)))

    if mode & 2 == 2:
        print('Upload speed: %s' % pretty_speed(speedtest.upload(runs)))

def pretty_speed(speed):
    units = ['bps', 'Kbps', 'Mbps', 'Gbps']
    unit = 0
    while speed >= 1024:
        speed /= 1024
        unit += 1
    return '%0.2f %s' % (speed, units[unit])

if __name__ == '__main__':
    main()
<commit_msg>Call error() instead of raising an exception<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function

import bisect
import getopt
import logging
import os
import random
import re
import string
import sys

from math import sqrt
from threading import currentThread, Thread
from textwrap import dedent
from time import time

try:
    from httplib import HTTPConnection
except ImportError:
    from http.client import HTTPConnection

try:
    from urllib import urlencode
except ImportError:
    from urllib.parse import urlencode

__program__ = 'pyspeedtest'
__script__ = os.path.basename(sys.argv[0])
__version__ = '1.3'

class SpeedTest(object):

    DOWNLOAD_FILES = [
        '/speedtest/random350x350.jpg',
        '/speedtest/random500x500.jpg',
        '/speedtest/random1500x1500.jpg'
    ]

    UPLOAD_FILES = [
        132884,
        493638
    ]

    def __init__(self, host=None, verbose=False, http_debug=0):
        self.host = host
        self.verbose = verbose
        self.http_debug = http_debug

    def downloadthread(self, connection, url):
        connection.request('GET', url, None, {'Connection': 'Keep-Alive'})
        response = connection.getresponse()
        self_thread = currentThread()
        self_thread.downloaded = len(response.read())

    def download(self, runs=2):
        if self.host is None:
            self.host = self.chooseserver()

        total_downloaded = 0
        connections = []
        for run in range(runs):
            connection = HTTPConnection(self.host)
            connection.set_debuglevel(self.http_debug)
            connection.connect()
            connections.append(connection)
        total_start_time = time()
        for current_file in SpeedTest.DOWNLOAD_FILES:
            threads = []
            for run in range(runs):
                thread = Thread(
                    target=self.downloadthread,
                    args=(connections[run],
                          '%s?x=%d' % (current_file, int(time() * 1000))))
                thread.run_number = run + 1
                thread.start()
                threads.append(thread)
            for thread in threads:
                thread.join()
                total_downloaded += thread.downloaded
                logging.info('Run %d for %s finished',
                             thread.run_number, current_file)
        total_ms = (time() - total_start_time) * 1000
        for connection in connections:
            connection.close()
        logging.info('Took %d ms to download %d bytes',
                     total_ms, total_downloaded)
        return total_downloaded * 8000 / total_ms

    def uploadthread(self, connection, data):
        url = '/speedtest/upload.php?x=%s' % random.random()
        connection.request('POST', url, data, {
            'Connection': 'Keep-Alive',
            'Content-Type': 'application/x-www-form-urlencoded'
        })
        response = connection.getresponse()
        reply = response.read().decode('utf-8')
        self_thread = currentThread()
        self_thread.uploaded = int(reply.split('=')[1])

    def upload(self, runs=2):
        if self.host is None:
            self.host = self.chooseserver()

        connections = []
        for run in range(runs):
            connection = HTTPConnection(self.host)
            connection.set_debuglevel(self.http_debug)
            connection.connect()
            connections.append(connection)

        post_data = []
        alphabet = string.digits + string.ascii_letters
        for current_file_size in SpeedTest.UPLOAD_FILES:
            values = {
                'content0': ''.join(
                    random.choice(alphabet) for i in range(current_file_size))
            }
            post_data.append(urlencode(values))

        total_uploaded = 0
        total_start_time = time()
        for data in post_data:
            threads = []
            for run in range(runs):
                thread = Thread(target=self.uploadthread,
                                args=(connections[run], data))
                thread.run_number = run + 1
                thread.start()
                threads.append(thread)
            for thread in threads:
                thread.join()
                logging.info('Run %d for %d bytes finished',
                             thread.run_number, thread.uploaded)
                total_uploaded += thread.uploaded
        total_ms = (time() - total_start_time) * 1000
        for connection in connections:
            connection.close()
        logging.info('Took %d ms to upload %d bytes',
                     total_ms, total_uploaded)
        return total_uploaded * 8000 / total_ms

    def ping(self, server=None):
        if server is None:
            server = self.host

        if server is None:
            error('No server specifier')

        connection = HTTPConnection(server)
        connection.set_debuglevel(self.http_debug)
        connection.connect()
        times = []
        worst = 0
        for _ in range(5):
            total_start_time = time()
            connection.request(
                'GET',
                '/speedtest/latency.txt?x=%d' % random.random(),
                None,
                {'Connection': 'Keep-Alive'})
            response = connection.getresponse()
            response.read()
            total_ms = time() - total_start_time
            times.append(total_ms)
            if total_ms > worst:
                worst = total_ms
        times.remove(worst)
        total_ms = sum(times) * 250  # * 1000 / number of tries (4) = 250
        connection.close()
        logging.info('Latency for %s - %d', server, total_ms)
        return total_ms

    def chooseserver(self):
        connection = HTTPConnection('www.speedtest.net')
        connection.set_debuglevel(self.http_debug)
        connection.connect()
        now = int(time() * 1000)
        extra_headers = {
            'Connection': 'Keep-Alive',
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2',
        }
        connection.request(
            'GET', '/speedtest-config.php?x=%d' % now, None, extra_headers)
        response = connection.getresponse()
        reply = response.read().decode('utf-8')
        match = re.search(
            r'<client ip="([^"]*)" lat="([^"]*)" lon="([^"]*)"', reply)
        location = None
        if match is None:
            logging.info('Failed to retrieve coordinates')
            return None
        location = match.groups()
        logging.info('Your IP: %s\nYour latitude: %s\nYour longitude: %s' %
                     location)
        connection.request(
            'GET', '/speedtest-servers.php?x=%d' % now, None, extra_headers)
        response = connection.getresponse()
        reply = response.read().decode('utf-8')
        server_list = re.findall(
            r'<server url="([^"]*)" lat="([^"]*)" lon="([^"]*)"', reply)
        my_lat = float(location[1])
        my_lon = float(location[2])
        sorted_server_list = []
        for server in server_list:
            s_lat = float(server[1])
            s_lon = float(server[2])
            distance = sqrt(pow(s_lat - my_lat, 2) + pow(s_lon - my_lon, 2))
            bisect.insort_left(sorted_server_list, (distance, server[0]))
        best_server = (999999, '')
        for server in sorted_server_list[:10]:
            logging.info(server[1])
            match = re.search(
                r'http://([^/]+)/speedtest/upload\.php', server[1])
            if match is None:
                continue
            server_host = match.groups()[0]
            latency = self.ping(server_host)
            if latency < best_server[0]:
                best_server = (latency, server_host)
        print('Using server:', best_server[1])
        return best_server[1]

def error(*args):
    print(*args, file=sys.stderr)
    sys.exit(1)

def usage():
    return dedent('''\
        usage: %s [OPTION]...

        Test your bandwidth speed using Speedtest.net servers.

        optional arguments:

          -d L, --debug=L    set http connection debug level (default is 0)
          -m M, --mode=M     test mode: 1 - download
                                        2 - upload
                                        4 - ping
                                        1 + 2 + 4 = 7 - all (default)
          -r N, --runs=N     use N runs (default is 2)
          -s H, --server=H   use specific server
          -v,   --verbose    output additional information

          -h,   --help       display this help and exit
                --version    output version information and exit
        ''' % __script__)

def version():
    return dedent('''\
        %s %s
        ''' % (__program__, __version__))

def parseargs():
    try:
        opts, _ = getopt.getopt(
            sys.argv[1:],
            'hvd:m:r:s:',
            [
                'help',
                'verbose',
                'version',
                'debug=',
                'mode=',
                'runs=',
                'server='
            ]
        )
    except getopt.GetoptError as err:
        error(err.msg + '\n\n' + usage())
    else:
        return opts

def main():
    opts = parseargs()
    speedtest = SpeedTest()

    mode = 7
    runs = 2

    for opt, arg in opts:
        if opt in ('-h', '--help'):
            print(usage())
            sys.exit()
        elif opt == '--version':
            print(version())
            sys.exit()
        elif opt in ('-d', '--debug'):
            try:
                speedtest.http_debug = int(arg)
            except ValueError:
                error('Bad debug level value')
        elif opt in ('-m', '--mode'):
            try:
                mode = int(arg)
                if not 7 >= mode >= 1:
                    raise ValueError
            except ValueError:
                error('Bad mode value')
        elif opt in ('-r', '--runs'):
            try:
                runs = int(arg)
            except ValueError:
                error('Bad runs value')
        elif opt == '-s':
            speedtest.host = arg
        elif opt in ('-v', '--verbose'):
            speedtest.verbose = True

    logging.basicConfig(
        format='%(message)s',
        level=logging.INFO if speedtest.verbose else logging.ERROR)

    if mode & 4 == 4 and speedtest.host is not None:
        print('Ping: %d ms' % speedtest.ping())

    if mode & 1 == 1:
        print('Download speed: %s' % pretty_speed(speedtest.download(runs)))

    if mode & 2 == 2:
        print('Upload speed: %s' % pretty_speed(speedtest.upload(runs)))

def pretty_speed(speed):
    units = ['bps', 'Kbps', 'Mbps', 'Gbps']
    unit = 0
    while speed >= 1024:
        speed /= 1024
        unit += 1
    return '%0.2f %s' % (speed, units[unit])

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># coding: utf-8
import certifi
from urllib3 import PoolManager

class BaseAPI(object):
    _conn = None

    def __init__(self, target, token):
        self.target = target
        self.token = token

    @property
    def conn(self):
        """Connection object"""
        if not self._conn:
            self._conn = PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where())
        return self._conn

    @property
    def headers(self):
        """Default headers for all requests"""
        return {
            'Authorization': 'bearer {}'.format(self.token)
        }

    def build_url(self, url):
        return '{}/{}'.format(self.target.rstrip('/'), url.lstrip('/'))

    def request(self, method, path):
        return self.conn.request(method, self.build_url(path))
<commit_msg>Pass headers to all GET requests<commit_after># coding: utf-8
import certifi
from urllib3 import PoolManager

class BaseAPI(object):
    _conn = None

    def __init__(self, target, token):
        self.target = target
        self.token = token

    @property
    def conn(self):
        """Connection object"""
        if not self._conn:
            self._conn = PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where())
        return self._conn

    @property
    def headers(self):
        """Default headers for all requests"""
        return {
            'Authorization': 'bearer {}'.format(self.token)
        }

    def build_url(self, url):
        return '{}/{}'.format(self.target.rstrip('/'), url.lstrip('/'))

    def request(self, method, path):
        return self.conn.request(method, self.build_url(path), headers=self.headers)
<|endoftext|>


<commit_before>import player
import dancemat
from time import sleep
import pygame
import signal
import effect
import sys
import message_passer

filename = 'media/bicycle-ride.mid'
configuration = 'configurations/effects_1.json'

for arg in sys.argv:
    if '.mid' in arg:
        filename = arg
    elif '.json' in arg:
        configuration = arg

# Set up pygame
pygame.init()
pygame.display.init()
# create screen for pygame to draw to
screen = pygame.display.set_mode((1000, 700))
clock = pygame.time.Clock()

mat = dancemat.DanceMat(pygame)
track = player.Track(filename=filename, is_looping=True)

combinator = effect.Combinator(configuration, track)

play = True

# noinspection PyUnusedLocal
def stop(*args):
    global play
    track.stop()
    play = False

signal.signal(signal.SIGINT, stop)

channels = track.channels

writer = message_passer.Writer()

def note_on_listener(msg):
    writer.write(str(msg))

# TODO: Test whether this message passing mechanism is causing the jitter
for channel in channels:
    channel.note_on_listener = note_on_listener

# Function to listen for changes to button state
def listener(status_dict):
    on_buttons = [button for (button, is_on) in status_dict.iteritems() if is_on]
    combinator.apply_for_buttons(on_buttons)

# Attach that listener function to the dancemat
mat.set_button_listener(listener)

track.start()

# Keep reading forever
while play:
    mat.read()
    sleep(0.05)
<commit_msg>main does not show screen<commit_after>import player
import dancemat
from time import sleep
import pygame
import signal
import effect
import sys
import message_passer

filename = 'media/bicycle-ride.mid'
configuration = 'configurations/effects_1.json'

for arg in sys.argv:
    if '.mid' in arg:
        filename = arg
    elif '.json' in arg:
        configuration = arg

# Set up pygame
pygame.init()
pygame.display.init()
# create screen for pygame to draw to
# screen = pygame.display.set_mode((1000, 700))
clock = pygame.time.Clock()

mat = dancemat.DanceMat(pygame)
track = player.Track(filename=filename, is_looping=True)

combinator = effect.Combinator(configuration, track)

play = True

# noinspection PyUnusedLocal
def stop(*args):
    global play
    track.stop()
    play = False

signal.signal(signal.SIGINT, stop)

channels = track.channels

writer = message_passer.Writer()

def note_on_listener(msg):
    writer.write(str(msg))

# TODO: Test whether this message passing mechanism is causing the jitter
for channel in channels:
    channel.note_on_listener = note_on_listener

# Function to listen for changes to button state
def listener(status_dict):
    on_buttons = [button for (button, is_on) in status_dict.iteritems() if is_on]
    combinator.apply_for_buttons(on_buttons)

# Attach that listener function to the dancemat
mat.set_button_listener(listener)

track.start()

# Keep reading forever
while play:
    mat.read()
    sleep(0.05)
<|endoftext|>


<commit_before>from vapory import *
from moviepy.editor import VideoClip
import json
import tweepy
from credentials import *

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)

with open('data.json') as data_file:
    data = json.load(data_file)

obj = []

def make_scene(t, x):
    camera = Camera( 'location', [20,20,-40], 'look_at', [0,0,0])
    return Scene( camera, objects= obj, included=["glass.inc"])

def make_frame(t):
    #return scene.render("test.png", width=1024, height=512, antialiasing = 0.01, quality=100)
    return make_scene(t, 1).render(width=1024, height=512, antialiasing = 0.01, quality=100)

def render(data, length, shrink, empty, x, y, z):
    if isinstance(data, list):
        obj.append(Sphere([x, y, z], 2, Texture(Pigment('color', [0.1,0.1,0.1])))) # Create C node
        for i,item in enumerate(data): # Create New nodes around C
            [x_new, y_new, len_new, empty_new] = direct(len(data), length, shrink, empty, i, x, y, z)
            obj.append(Cylinder([x,y,z], render(item, len_new, shrink, empty_new, x_new, y_new, z), 0.3, Finish('ambient', 0.1, 'diffuse', 0.7), Pigment('color', [0.4,0.4,0.4])))
    elif data == 'h': # Create new h node
        obj.append(Sphere([x, y, z], 2, Texture(Pigment('color', [0.9,0.9,0.9]))))#White color
    return [x,y,z]

# Empty: 0 = left; 1 = up; 2 = right; 3 = down
def direct(datasize, length, shrink, empty, i, x, y, z):
    if datasize != 0:
        if (i + empty) % 4 == 2:
            x_new = x + length; y_new = y
            len_new = length
            empty = 0 #Tested
        elif (i + empty) % 4 == 1:
            x_new = x; y_new = y + length
            len_new = length - shrink
            empty = 1 #Tested
        elif (i + empty) % 4 == 0: # Evtl swap with under
            x_new = x; y_new = y - length
            len_new = length - shrink
            empty = 2 #Tested
        elif (i + empty) % 4 == 3:
            x_new = x - length; y_new = y
            len_new = -length
            empty = 0 #2
    return [x_new, y_new, len_new, empty]

render(data, 15, 10, -1, 0, 0, 0)

obj.append(LightSource( [10, 120, -40], 'color', [1.3, 1.3, 1.3]))
obj.append(Background("color", [1,1,1]))

VideoClip(make_frame, duration=1).write_gif("anim.gif",fps=1)
api.update_with_media("anim.gif", "Haskell Foreign Function Interface fehlt. lol.")
<commit_msg>Added one plot script<commit_after>#!/usr/bin/env python3
from vapory import *
from moviepy.editor import VideoClip
import json
import tweepy
from credentials import *

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)

with open('data.json') as data_file:
    data = json.load(data_file)

obj = []

def make_scene(t, x):
    camera = Camera( 'location', [20,20,-40], 'look_at', [0,0,0])
    return Scene( camera, objects= obj, included=["glass.inc"])

def make_frame(t):
    #return scene.render("test.png", width=1024, height=512, antialiasing = 0.01, quality=100)
    return make_scene(t, 1).render(width=1024, height=512, antialiasing = 0.01, quality=100)

def render(data, length, shrink, empty, x, y, z):
    if isinstance(data, list):
        obj.append(Sphere([x, y, z], 2, Texture(Pigment('color', [0.1,0.1,0.1])))) # Create C node
        for i,item in enumerate(data): # Create New nodes around C
            [x_new, y_new, len_new, empty_new] = direct(len(data), length, shrink, empty, i, x, y, z)
            obj.append(Cylinder([x,y,z], render(item, len_new, shrink, empty_new, x_new, y_new, z), 0.3, Finish('ambient', 0.1, 'diffuse', 0.7), Pigment('color', [0.4,0.4,0.4])))
    elif data == 'h': # Create new h node
        obj.append(Sphere([x, y, z], 2, Texture(Pigment('color', [0.9,0.9,0.9]))))#White color
    return [x,y,z]

# Empty: 0 = left; 1 = up; 2 = right; 3 = down
def direct(datasize, length, shrink, empty, i, x, y, z):
    if datasize != 0:
        if (i + empty) % 4 == 2:
            x_new = x + length; y_new = y
            len_new = length
            empty = 0 #Tested
        elif (i + empty) % 4 == 1:
            x_new = x; y_new = y + length
            len_new = length - shrink
            empty = 1 #Tested
        elif (i + empty) % 4 == 0: # Evtl swap with under
            x_new = x; y_new = y - length
            len_new = length - shrink
            empty = 2 #Tested
        elif (i + empty) % 4 == 3:
            x_new = x - length; y_new = y
            len_new = -length
            empty = 0 #2
    return [x_new, y_new, len_new, empty]

render(data, 15, 10, -1, 0, 0, 0)

obj.append(LightSource( [10, 120, -40], 'color', [1.3, 1.3, 1.3]))
obj.append(Background("color", [1,1,1]))

VideoClip(make_frame, duration=1).write_gif("anim.gif",fps=1)
api.update_with_media("anim.gif", "Haskell Foreign Function Interface fehlt. lol.")
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# coding: utf8
from vapory import *
from moviepy.editor import VideoClip
import json
import tweepy
import numpy as np
import math, os
from math import pi
from credentials import *
from convertBytes import convert_bytes

def backend(len, dep):
    os.chdir('../HcPlot-hs')
    os.system('stack build')
    #os.system('stack exec create ' + str(len) + ' ' + str(dep) + ' > data')
    # data = os.popen('stack exec create ' + str(len) + ' ' + str(dep)).read()
    os.chdir('../python')
    return json.loads(data)

def nameToJSON(name):
    os.chdir('../HcPlot-hs')
    os.system('stack build')
    HSdata = os.popen('stack exec create n ' + str(name))
    print("HS Data: " + str(data))
    os.chdir('../python')
    try:
        return json.loads(HSData)
    except TypeError:
        print("Fehlerhafte JSON")
        return False

#data = backend(5, 2)
data = nameToJSON("1,2-dimethyl-hexan")
print("Data: " + str(data))

lange = 3 #sek. Länge der gif !!! must be int !!!
obj = []
x_cam = []
y_cam = []
z_cam = []
x_max = [0]
x_min = [0]
y_max = [0]
y_min = [0]

def make_scene(t):
    #camera = Camera( 'location', [t*50,20,-((x_max[0]+y_max[0])/1.25)], 'look_at', [x_max[0]/2,0,0])
    t = int(t*25)
    camera = Camera( 'location', [x_cam[t],y_cam[t],z_cam[t]], 'look_at', center)
    return Scene( camera, objects= obj, included=["glass.inc"])

def make_frame(t):
    #return scene.render("test.png", width=1024, height=512, antialiasing = 0.01, quality=100)
    return make_scene(t).render(width=1024, height=512, antialiasing = 0.01, quality=100)

def generate_circle(center=(0,0,0), r=50, n=100):
    return [
        (
            center[0] + (math.cos(2 * pi / n * x) * r),  # x
            center[1] + (math.sin(2 * pi / n * x) * r),  # y
            center[2] + (math.cos(2 * pi / n * x) * r)   # z

        ) for x in range(0, n + 1)]

def render(data, length, shrink, empty, x, y, z):
    if isinstance(data, list):
        obj.append(Sphere([x, y, z], 2, Texture(Pigment('color', [0.1,0.1,0.1])))) # Create C node
        for i,item in enumerate(data): # Create New nodes around C
            [x_new, y_new, len_new, empty_new] = direct(len(data), length, shrink, empty, i, x, y, z)
            if(x_new > x_max[0]):
                x_max[0] = x_new
            if(x_new < x_min[0]):
                x_min[0] = x_new
            if(y_new > y_max[0]):
                y_max[0] = y_new
            if(y_new < y_min[0]):
                y_min[0] = y_new
            obj.append(Cylinder([x,y,z], render(item, len_new, shrink, empty_new, x_new, y_new, z), 0.3, Finish('ambient', 0.1, 'diffuse', 0.7), Pigment('color', [0.4,0.4,0.4])))
    elif data == 'h': # Create new h node
        obj.append(Sphere([x, y, z], 2, Texture(Pigment('color', [0.9,0.9,0.9]))))#White color
    return [x,y,z]

# Empty: 0 = left; 1 = up; 2 = right; 3 = down
def direct(datasize, length, shrink, empty, i, x, y, z):
    if datasize != 0:
        if (i + empty) % 4 == 2:
            x_new = x + length; y_new = y
            len_new = length
            empty = 0 #Tested
        elif (i + empty) % 4 == 1:
            x_new = x; y_new = y + length
            len_new = length - shrink
            empty = 1 #Tested
        elif (i + empty) % 4 == 0: # Evtl swap with under
            x_new = x; y_new = y - length
            len_new = length - shrink
            empty = 2 #Tested
        elif (i + empty) % 4 == 3:
            x_new = x - length; y_new = y
            len_new = -length
            empty = 0 #2
    return [x_new, y_new, len_new, empty]

render(data, 15, 10, -1, 0, 0, 0)
print("X max: " + str(x_max[0]) + ", Y max: " + str(y_max[0]))
print("Kreispunkte: " + str(lange*25+1))
center = ((x_max[0]+x_min[0])/2, (y_max[0]+y_min[0])/2, 0)

print("Kreismitte: " + str(center))
points = generate_circle(center=center, r=(x_max[0]+y_max[0]), n=lange*25+1)
for x,y,z in points:
    x_cam.append(x)
    y_cam.append(z)
    z_cam.append(y)

obj.append(LightSource( [10, 120, -40], 'color', [1.3, 1.3, 1.3]))
obj.append(Background("color", [1,1,1]))

VideoClip(make_frame, duration=lange).write_gif("animation.gif",fps=25)
size = convert_bytes(os.stat("animation.gif").st_size)
print("Filesize: " + size)

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)
api.update_with_media("animation.gif", "Es fehlt noch: Name und Randomness")
<commit_msg>Debug<commit_after>#!/usr/bin/env python3
# coding: utf8
from vapory import *
from moviepy.editor import VideoClip
import json
import tweepy
import numpy as np
import math, os
from math import pi
from credentials import *
from convertBytes import convert_bytes

def backend(len, dep):
    os.chdir('../HcPlot-hs')
    os.system('stack build')
    #os.system('stack exec create ' + str(len) + ' ' + str(dep) + ' > data')
    # data = os.popen('stack exec create ' + str(len) + ' ' + str(dep)).read()
    os.chdir('../python')
    return json.loads(data)

def nameToJSON(name):
    os.chdir('../HcPlot-hs')
    os.system('stack build')
    HSdata = os.popen('stack exec create n ' + str(name))
    print("HS Data: " + str(HSdata))
    os.chdir('../python')
    try:
        return json.loads(HSdata)
    except TypeError:
        print("Fehlerhafte JSON")
        return False

#data = backend(5, 2)
data = nameToJSON("1,2-dimethyl-hexan")
print("Data: " + str(data))

lange = 3 #sek. Länge der gif !!! must be int !!!
obj = []
x_cam = []
y_cam = []
z_cam = []
x_max = [0]
x_min = [0]
y_max = [0]
y_min = [0]

def make_scene(t):
    #camera = Camera( 'location', [t*50,20,-((x_max[0]+y_max[0])/1.25)], 'look_at', [x_max[0]/2,0,0])
    t = int(t*25)
    camera = Camera( 'location', [x_cam[t],y_cam[t],z_cam[t]], 'look_at', center)
    return Scene( camera, objects= obj, included=["glass.inc"])

def make_frame(t):
    #return scene.render("test.png", width=1024, height=512, antialiasing = 0.01, quality=100)
    return make_scene(t).render(width=1024, height=512, antialiasing = 0.01, quality=100)

def generate_circle(center=(0,0,0), r=50, n=100):
    return [
        (
            center[0] + (math.cos(2 * pi / n * x) * r),  # x
            center[1] + (math.sin(2 * pi / n * x) * r),  # y
            center[2] + (math.cos(2 * pi / n * x) * r)   # z

        ) for x in range(0, n + 1)]

def render(data, length, shrink, empty, x, y, z):
    if isinstance(data, list):
        obj.append(Sphere([x, y, z], 2, Texture(Pigment('color', [0.1,0.1,0.1])))) # Create C node
        for i,item in enumerate(data): # Create New nodes around C
            [x_new, y_new, len_new, empty_new] = direct(len(data), length, shrink, empty, i, x, y, z)
            if(x_new > x_max[0]):
                x_max[0] = x_new
            if(x_new < x_min[0]):
                x_min[0] = x_new
            if(y_new > y_max[0]):
                y_max[0] = y_new
            if(y_new < y_min[0]):
                y_min[0] = y_new
            obj.append(Cylinder([x,y,z], render(item, len_new, shrink, empty_new, x_new, y_new, z), 0.3, Finish('ambient', 0.1, 'diffuse', 0.7), Pigment('color', [0.4,0.4,0.4])))
    elif data == 'h': # Create new h node
        obj.append(Sphere([x, y, z], 2, Texture(Pigment('color', [0.9,0.9,0.9]))))#White color
    return [x,y,z]

# Empty: 0 = left; 1 = up; 2 = right; 3 = down
def direct(datasize, length, shrink, empty, i, x, y, z):
    if datasize != 0:
        if (i + empty) % 4 == 2:
            x_new = x + length; y_new = y
            len_new = length
            empty = 0 #Tested
        elif (i + empty) % 4 == 1:
            x_new = x; y_new = y + length
            len_new = length - shrink
            empty = 1 #Tested
        elif (i + empty) % 4 == 0: # Evtl swap with under
            x_new = x; y_new = y - length
            len_new = length - shrink
            empty = 2 #Tested
        elif (i + empty) % 4 == 3:
            x_new = x - length; y_new = y
            len_new = -length
            empty = 0 #2
    return [x_new, y_new, len_new, empty]

render(data, 15, 10, -1, 0, 0, 0)
print("X max: " + str(x_max[0]) + ", Y max: " + str(y_max[0]))
print("Kreispunkte: " + str(lange*25+1))
center = ((x_max[0]+x_min[0])/2, (y_max[0]+y_min[0])/2, 0)

print("Kreismitte: " + str(center))
points = generate_circle(center=center, r=(x_max[0]+y_max[0]), n=lange*25+1)
for x,y,z in points:
    x_cam.append(x)
    y_cam.append(z)
    z_cam.append(y)

obj.append(LightSource( [10, 120, -40], 'color', [1.3, 1.3, 1.3]))
obj.append(Background("color", [1,1,1]))

VideoClip(make_frame, duration=lange).write_gif("animation.gif",fps=25)
size = convert_bytes(os.stat("animation.gif").st_size)
print("Filesize: " + size)

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth)
api.update_with_media("animation.gif", "Es fehlt noch: Name und Randomness")
<|endoftext|>


<commit_before>import h5py
import numpy as np
from PIL import Image
import sys
import os
import errno

def print_hdf5_file_structure(file_name):
    file = h5py.File(file_name, 'r')
    item = file #["/Configure:0000/Run:0000"]
    print_hdf5_item_structure(item)
    file.close()

def print_hdf5_item_structure(g, offset = '    '):
    if isinstance(g, h5py.File):
        print (g.file, '(File)', g.name)

    elif isinstance(g, h5py.Dataset):
        print ('(Dataset)', g.name, '    len =', g.shape)

    elif isinstance(g, h5py.Group):
        print ('(Group)', g.name)

    else:
        print ('Warning: unknown item in file', g.name)
        sys.exit("EXECUTION IS TERMINATED")

    if isinstance(g, h5py.File) or isinstance(g, h5py.Group):
        for key, val in dict(g).items():
            subg = val
            print (offset, key)
            print_hdf5_item_structure(subg, offset+'    ')
dir = '../../SPIMDataTest/'
#dir_n = 'CriWT_H1_TexasRed/Stack0/'
dir_m = 'Drosophila_Membrane02/Stack0/'
mydir = dir_m
name = 'Cam_Right_00030'
file_name =dir + mydir + name + '.h5'
print_hdf5_file_structure(file_name)
file =  h5py.File(file_name, 'r')
data = file['/Data'][()]
thumb = file['/Thumb'][()]

print ('data.shape=', data.shape)
num_frames = data.shape[0]
print ('num_of_frames:', num_frames)

path = mydir+name+'/'
try:
    os.makedirs(path)
except OSError:
    if not os.path.isdir(path):
        raise

for i in range(num_frames):
    #im = Image.fromarray(data, 'I;16')
    im = Image.fromarray(data[i][:][:], 'I;16')
    outfilename = path + ("%5.5d" % i ) + ".tiff"
    print(outfilename)
    im.save(outfilename)
print('Done')
<commit_msg>new output within tiff limit<commit_after>import h5py
import numpy as np
from PIL import Image
import sys
import os
import errno
import glob

def print_hdf5_file_structure(file_name):
    file = h5py.File(file_name, 'r')
    item = file #["/Configure:0000/Run:0000"]
    print_hdf5_item_structure(item)
    file.close()

def print_hdf5_item_structure(g, offset = '    '):
    if isinstance(g, h5py.File):
        print (g.file, '(File)', g.name)

    elif isinstance(g, h5py.Dataset):
        print ('(Dataset)', g.name, '    len =', g.shape)

    elif isinstance(g, h5py.Group):
        print ('(Group)', g.name)

    else:
        print ('Warning: unknown item in file', g.name)
        sys.exit("EXECUTION IS TERMINATED")

    if isinstance(g, h5py.File) or isinstance(g, h5py.Group):
        for key, val in dict(g).items():
            subg = val
            print (offset, key)
            print_hdf5_item_structure(subg, offset+'    ')

def extract_and_save(file_name, dst_dir):
    try:
        os.makedirs(dst_dir)
    except OSError:
        if not os.path.isdir(dst_dir):
            raise

    print(file_name)
    print_hdf5_file_structure(file_name)
    file =  h5py.File(file_name, 'r')
    data = file['/Data'][()]
    thumb = file['/Thumb'][()]

    print ('data.shape=', data.shape)
    num_frames = data.shape[0]
    print ('num_of_frames:', num_frames)

    for i in range(num_frames):
        im = Image.fromarray(data[i][:][:], 'I;16')
        outfilename = dst_dir + "/" + ("%5.5d" % i ) + ".tiff"
        print(outfilename)
        im.save(outfilename)

#srcdir = '../../SPIMDataTest/Drosophila_Membrane02/'

srcdir = "../../SPIMDataTest/Drosophila_mCherry_Membrane_02/2016-01-19_17.47.58/"
#srcdir = "../../SPIMDataTest/Drosophila_mCherry_Membrane_02/beads/2016-01-20_16.09.38/"

stack01dirs = ["Stack_0_Channel_0", "Stack_1_Channel_0"]

for subdir in stack01dirs:
    camfiles = glob.glob(srcdir + subdir +  "/*_00034.h5")
    for cam in camfiles:
        camname = cam.split("/")[-1:]
        dst_dir = subdir + "/" + camname[0]
        extract_and_save(cam, dst_dir)

for subdir in stack01dirs:
    camfiles = glob.glob(subdir +  "/*_00034.h5")
    for cam in camfiles:
        camname = cam.split("/")[-1:][0]
        files =glob.glob(cam + "/*.tiff")
        files = sorted(files)
        dst_tiff = subdir + "_" + camname.replace("h5", "tiff")
        files = " ".join(files)
        os.system("tiffcp " + files + " " + dst_tiff)

#final_tiff = glob.glob("Stack_[0,1]_Channel_[0,1]/*.tiff")
#final_tiff = sorted(final_tiff)

#os.system("tiffcp " + " ".join(final_tiff) + " final.tiff")
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

from ev3dev.auto import *

import ev3robot.logic as logic
import ev3robot.robot as r

import time

if __name__ == "__main__":

    class SumoController(logic.Controller):

        def setup(self):
            self.ptc_color = self.color()

        def loop(self):
            self.normal_speed()
            self.turn(degree=90)
            self.brake()

            if self.has_obstacle(range=200):
                self.forward()
            else:
                self.brake()
                self.turn(degree=10)

            # 0 black -> 100 white
            if self.ptc_color > self.color() + 10:
                self.turn(degree=90)
                self.forward()
                # drive for 1 sec
                time.sleep(1)

    controller = SumoController(right_motor=LargeMotor('outA'), left_motor=LargeMotor('outB'),
                                    gyro=GyroSensor(), ultrasonic=UltrasonicSensor())

    robot = r.Robot(controller)
    robot.start()

    try:
        # wait for input
        name = raw_input("Press Enter to exit: ")
    except (KeyboardInterrupt, SystemExit):
        pass

    # stop robot
    robot.kill()<commit_msg>some refactoring<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

from ev3dev.auto import *

import ev3robot.logic as logic
import ev3robot.robot as r

import time

if __name__ == "__main__":

    class SumoController(logic.Controller):

        def setup(self):
            self.ptc_color = self.color()

        def loop(self):
            self.normal_speed()
            self.turn(degree=90)
            self.brake()

            if self.has_obstacle(range=200):
                self.forward()
            else:
                self.brake()
                self.turn(degree=10)

            # 0 black -> 100 white
            if self.color() + 10 < self.ptc_color:
                self.turn(degree=90)
                self.forward()
                # drive for 1 sec
                time.sleep(1)

    controller = SumoController(right_motor=LargeMotor('outA'), left_motor=LargeMotor('outB'),
                                    gyro=GyroSensor(), ultrasonic=UltrasonicSensor())

    robot = r.Robot(controller)
    robot.start()

    try:
        # wait for input
        name = raw_input("Press Enter to exit: ")
    except (KeyboardInterrupt, SystemExit):
        pass

    # stop robot
    robot.kill()<|endoftext|>


<commit_before># Copyright 2013 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

#!/usr/bin/env python

import os
import fnmatch
import itertools
import re
from licenseck import licenses

directories_to_check = ["ports", "components", "tests", "python"]
filetypes_to_check = [".rs", ".rc", ".cpp", ".c", ".h", ".py"]
reftest_directories = ["tests/ref"]
reftest_filetype = ".list"

ignored_files = [
    # Upstream
    "support/*",
    "tests/wpt/*",
    "python/mach/*",
    "python/mozdebug/*",
    "python/mozinfo/*",
    "python/mozlog/*",
    "python/toml/*",

    # Generated and upstream code combined with our own. Could use cleanup
    "components/script/dom/bindings/codegen/*",
    "components/style/properties/mod.rs",
    "components/servo/target/*",
    "ports/gonk/target/*",
    "ports/gonk/src/native_window_glue.cpp",
    "ports/cef/*",

    # MIT license
    "components/util/deque/mod.rs",
]

def collect_file_names(top_directories):
    for top_directory in top_directories:
        for dirname, dirs, files in os.walk(top_directory):
            for basename in files:
                yield os.path.join(dirname, basename)

def should_check(file_name):
    if ".#" in file_name:
        return False
    if os.path.splitext(file_name)[1] not in filetypes_to_check:
        return False
    for pattern in ignored_files:
        if fnmatch.fnmatch(file_name, pattern):
            return False
    return True

def should_check_reftest(file_name):
    return file_name.endswith(reftest_filetype)

def check_license(contents):
    valid_license = any(contents.startswith(license) for license in licenses)
    acknowledged_bad_license = "xfail-license" in contents[:100]
    if not (valid_license or acknowledged_bad_license):
        yield (1, "incorrect license")

def check_length(idx, line):
    if len(line) >= 120:
        yield (idx + 1, "(much) overlong line")

def check_whatwg_url(idx, line):
    match = re.search(r"https://html\.spec\.whatwg\.org/multipage/[\w-]+\.html#([\w\:-]+)", line)
    if match is not None:
        preferred_link = "https://html.spec.whatwg.org/multipage/#{}".format(match.group(1))
        yield (idx + 1, "link to WHATWG may break in the future, use this format instead: {}".format(preferred_link))

def check_whitespace(idx, line):
    if line[-1] == "\n":
        line = line[:-1]
    else:
        yield (idx + 1, "no newline at EOF")

    if line.endswith(" "):
        yield (idx + 1, "trailing whitespace")

    if "\t" in line:
        yield (idx + 1, "tab on line")

    if "\r" in line:
        yield (idx + 1, "CR on line")

def check_by_line(contents):
    lines = contents.splitlines(True)
    for idx, line in enumerate(lines):
        errors = itertools.chain(
            check_length(idx, line),
            check_whitespace(idx, line),
            check_whatwg_url(idx, line),
        )
        for error in errors:
            yield error

def collect_errors_for_files(files_to_check, checking_functions):
    for file_name in files_to_check:
        with open(file_name, "r") as fp:
            contents = fp.read()
            for check in checking_functions:
                for error in check(contents):
                    # filename, line, message
                    yield (file_name, error[0], error[1])

def check_reftest_order(files_to_check):
    for file_name in files_to_check:
        with open(file_name, "r") as fp:
            split_lines = fp.read().splitlines()
            lines = filter(lambda l: len(l) > 0 and l[0] != '#', split_lines)
            for idx, line in enumerate(lines[:-1]):
                next_line = lines[idx+1]
                current = get_reftest_names(line)
                next = get_reftest_names(next_line)
                if current is not None and next is not None and current > next:
                    yield (file_name, split_lines.index(next_line) + 1, "line not in alphabetical order")

def get_reftest_names(line):
    tokens = line.split()
    if (len(tokens) == 3):
        return tokens[1] + tokens[2]
    if (len(tokens) == 4):
        return tokens[2] + tokens[3]
    return None

def scan():
    all_files = collect_file_names(directories_to_check)
    files_to_check = filter(should_check, all_files)

    checking_functions = [check_license, check_by_line]
    errors = collect_errors_for_files(files_to_check, checking_functions)

    reftest_files = collect_file_names(reftest_directories)
    reftest_to_check = filter(should_check_reftest, reftest_files)
    r_errors = check_reftest_order(reftest_to_check)

    errors = list(itertools.chain(errors, r_errors))

    if errors:
        for error in errors:
            print("{}:{}: {}".format(*error))
        return 1
    else:
        print("tidy reported no errors.")
        return 0
<commit_msg>Auto merge of #6274 - frewsxcv:tidy-all-the-things, r=metajack<commit_after># Copyright 2013 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

#!/usr/bin/env python

import os
import fnmatch
import itertools
import re
from licenseck import licenses

filetypes_to_check = [".rs", ".rc", ".cpp", ".c", ".h", ".py"]
reftest_directories = ["tests/ref"]
reftest_filetype = ".list"

ignored_files = [
    # Upstream
    "support/*",
    "tests/wpt/*",
    "python/mach/*",
    "python/mozdebug/*",
    "python/mozinfo/*",
    "python/mozlog/*",
    "python/toml/*",

    # Generated and upstream code combined with our own. Could use cleanup
    "components/script/dom/bindings/codegen/*",
    "components/style/properties/mod.rs",
    "components/servo/target/*",
    "ports/gonk/target/*",
    "ports/gonk/src/native_window_glue.cpp",
    "ports/cef/*",

    # MIT license
    "components/util/deque/mod.rs",

    # Hidden files/directories
    ".*",
]

def collect_file_names(top_directories=None):
    if top_directories is None:
        top_directories = os.listdir(".")
    for top_directory in top_directories:
        for dirname, dirs, files in os.walk(top_directory):
            for basename in files:
                yield os.path.join(dirname, basename)

def should_check(file_name):
    if ".#" in file_name:
        return False
    if os.path.splitext(file_name)[1] not in filetypes_to_check:
        return False
    for pattern in ignored_files:
        if fnmatch.fnmatch(file_name, pattern):
            return False
    return True

def should_check_reftest(file_name):
    return file_name.endswith(reftest_filetype)

def check_license(contents):
    valid_license = any(contents.startswith(license) for license in licenses)
    acknowledged_bad_license = "xfail-license" in contents[:100]
    if not (valid_license or acknowledged_bad_license):
        yield (1, "incorrect license")

def check_length(idx, line):
    if len(line) >= 120:
        yield (idx + 1, "(much) overlong line")

def check_whatwg_url(idx, line):
    match = re.search(r"https://html\.spec\.whatwg\.org/multipage/[\w-]+\.html#([\w\:-]+)", line)
    if match is not None:
        preferred_link = "https://html.spec.whatwg.org/multipage/#{}".format(match.group(1))
        yield (idx + 1, "link to WHATWG may break in the future, use this format instead: {}".format(preferred_link))

def check_whitespace(idx, line):
    if line[-1] == "\n":
        line = line[:-1]
    else:
        yield (idx + 1, "no newline at EOF")

    if line.endswith(" "):
        yield (idx + 1, "trailing whitespace")

    if "\t" in line:
        yield (idx + 1, "tab on line")

    if "\r" in line:
        yield (idx + 1, "CR on line")

def check_by_line(contents):
    lines = contents.splitlines(True)
    for idx, line in enumerate(lines):
        errors = itertools.chain(
            check_length(idx, line),
            check_whitespace(idx, line),
            check_whatwg_url(idx, line),
        )
        for error in errors:
            yield error

def collect_errors_for_files(files_to_check, checking_functions):
    for file_name in files_to_check:
        with open(file_name, "r") as fp:
            contents = fp.read()
            for check in checking_functions:
                for error in check(contents):
                    # filename, line, message
                    yield (file_name, error[0], error[1])

def check_reftest_order(files_to_check):
    for file_name in files_to_check:
        with open(file_name, "r") as fp:
            split_lines = fp.read().splitlines()
            lines = filter(lambda l: len(l) > 0 and l[0] != '#', split_lines)
            for idx, line in enumerate(lines[:-1]):
                next_line = lines[idx+1]
                current = get_reftest_names(line)
                next = get_reftest_names(next_line)
                if current is not None and next is not None and current > next:
                    yield (file_name, split_lines.index(next_line) + 1, "line not in alphabetical order")

def get_reftest_names(line):
    tokens = line.split()
    if (len(tokens) == 3):
        return tokens[1] + tokens[2]
    if (len(tokens) == 4):
        return tokens[2] + tokens[3]
    return None

def scan():
    all_files = collect_file_names()
    files_to_check = filter(should_check, all_files)

    checking_functions = [check_license, check_by_line]
    errors = collect_errors_for_files(files_to_check, checking_functions)

    reftest_files = collect_file_names(reftest_directories)
    reftest_to_check = filter(should_check_reftest, reftest_files)
    r_errors = check_reftest_order(reftest_to_check)

    errors = list(itertools.chain(errors, r_errors))

    if errors:
        for error in errors:
            print("{}:{}: {}".format(*error))
        return 1
    else:
        print("tidy reported no errors.")
        return 0
<|endoftext|>


<commit_before>from __future__ import print_function

import codecs
import gzip
import os
from collections import defaultdict
from typing import Iterable, List, Tuple, Dict, IO, Iterator
import itertools
import logging
import re

import pyanyzip

CHECK_NON_ASCII = False
SKIP_COMMENTS = True
VALIDATE_ALL_LINES_SAME_NUMBER_OF_FIELDS = True
USE_ANY_FORMAT = True
SANITIZE = True
CLEAN_EDGES = True
SUB_TRAILING = True
REMOVE_NON_ASCII = True
LOWER_CASE = True
CHECK_NUM_FIELDS = True
CONVERT_TO_STRING = True
DO_GZIP = False
FILENAME_DETECT = True
DEFAULT_ENCODING = 'utf-8'

def clean(
        text,
        clean_edges=True,
        sub_trailing=True,
        remove_non_ascii=True,
        lower_case=True,
):
    # type: (str, bool, bool, bool, bool) -> str
    if sub_trailing:
        # replace all manner of whitespace (consecutive or not)
        # with s single space
        text = re.sub(r"[\r\t\n\v ]+", " ", text)
    if remove_non_ascii:
        text = text.encode('ascii', errors='ignore').decode()
    if clean_edges:
        # remove space from the left and right
        text = text.strip()
    if lower_case:
        text = text.lower()
    return text

def aggregate(
        input_file_names,
        match_columns,
        aggregate_columns,
        output_file_name,
        unlink=True,
):
    # type: (Iterable[str], List[int], List[int], str, bool) -> None
    """
    This function aggregates a bunch of input files by integers.
    :param input_file_names:
    :param match_columns:
    :param aggregate_columns:
    :param output_file_name:
    :param unlink:
    :return:
    """
    counts = dict()  # type: Dict[Tuple[str], List[int]]
    logger = logging.getLogger(__name__)
    for input_file_name in input_file_names:
        logger.debug("working on file [%s]", input_file_name)
        with TsvReader(filename=input_file_name) as input_handle:
            for parts in input_handle:
                # noinspection PyTypeChecker
                match = tuple([parts[i] for i in match_columns])  # type: Tuple[str]
                if match not in counts:
                    counts[match] = [int(0)] * len(aggregate_columns)
                for i, aggregate_column in enumerate(aggregate_columns):
                    counts[match][i] += int(parts[aggregate_column])
    # notice that we create a writer that does not check the number
    # of fields because the check requires a len(fields) to be available
    # and it is not in this case because of itertools.chain
    with TsvWriter(
        filename=output_file_name,
        check_num_fields=False,
        sanitize=False,
    ) as output_file_handle:
        for match, aggregates in counts.items():
            to_write = itertools.chain(match, aggregates)  # type: List[str]
            output_file_handle.write(to_write)
    if unlink:
        for input_file_name in input_file_names:
            os.unlink(input_file_name)

def write_data(data, output_file_name):
    # type: (List[List[str]], str) -> None
    with TsvWriter(
            filename=output_file_name,
            mode="at",
    ) as output_file_handle:
        for d in data:
            output_file_handle.write(d)

def group_by(
        input_file_names,
        group_by_columns,
        collect_columns,
        output_file_template,
        unlink=True,
):
    # type: (Iterable[str], List[int], List[int], str, bool) -> List[str]
    all_data = defaultdict(list)  # type: Dict[str, List[List[str]]]
    limit = 10000
    logger = logging.getLogger(__name__)
    for input_file_name in input_file_names:
        logger.debug("working on file [%s]", input_file_name)
        with open(input_file_name, 'rt') as file_handle:
            for line in file_handle:
                line = line.rstrip()
                parts = line.split("\t")  # type: List[str]
                match = tuple([parts[i] for i in group_by_columns])  # type: Tuple[str]
                match = "_".join(match)
                data_to_append = [parts[i] for i in collect_columns]
                all_data[match].append(data_to_append)
                if len(all_data[match]) > limit:
                    write_data(all_data[match], output_file_template.format(match=match))
                    all_data[match] = list()
    # write the rest for the data
    for match, data in all_data.items():
        write_data(data, output_file_template.format(match=match))
    # remove the input files
    if unlink:
        for input_file_name in input_file_names:
            os.unlink(input_file_name)
    return [output_file_template.format(match=match) for match in all_data]

def is_ascii(s):
    # type: (str) -> bool
    return all(ord(c) < 128 for c in s)

class TsvWriter(object):
    def __init__(
            self,
            filename,
            mode="wt",
            throw_exceptions=False,

            sanitize=SANITIZE,
            fields_to_clean=None,
            clean_edges=CLEAN_EDGES,
            sub_trailing=SUB_TRAILING,
            remove_non_ascii=REMOVE_NON_ASCII,
            lower_case=LOWER_CASE,

            check_num_fields=CHECK_NUM_FIELDS,
            num_fields=None,
            convert_to_string=CONVERT_TO_STRING,

            do_gzip=DO_GZIP,
            filename_detect=FILENAME_DETECT,
            encoding=DEFAULT_ENCODING,
    ):
        # type: (str, str, bool, bool, List[int], bool, bool, bool, bool, bool, int, bool, bool, bool) -> None
        if filename_detect:
            found = False
            if filename.endswith(".tsv.gz"):
                self.io = gzip.open(filename, mode=mode)  # type: IO[str]
                found = True
            if filename.endswith(".tsv"):
                self.io = open(filename, mode=mode)  # type: IO[str]
                found = True
            assert found, "file name unknown"
        else:
            if do_gzip:
                self.io = gzip.open(filename, mode=mode)  # type: IO[str]
            else:
                self.io = open(filename, mode=mode)  # type: IO[str]
        # python 2.7
        self.io = codecs.getwriter(encoding=encoding)(self.io)
        self.throw_exceptions = throw_exceptions

        self.sanitize = sanitize
        self.fields_to_clean = fields_to_clean
        if self.fields_to_clean is None:
            self.fields_to_clean = []
        self.clean_edges = clean_edges
        self.sub_trailing = sub_trailing
        self.remove_non_ascii = remove_non_ascii
        self.lower_case = lower_case

        self.check_num_fields = check_num_fields
        self.num_fields = num_fields
        self.convert_to_string = convert_to_string

    def _sanitize(self, l):
        # type: (List[str]) -> None
        if self.sanitize:
            for field in self.fields_to_clean:
                l[field] = clean(
                    text=l[field],
                    clean_edges=self.clean_edges,
                    sub_trailing=self.sub_trailing,
                    remove_non_ascii=self.remove_non_ascii,
                    lower_case=self.lower_case,
                )

    def _convert(self, l):
        # type: (List[str]) -> Iterator[str]
        if self.convert_to_string:
            for i, t in enumerate(l):
                if type(t) in (int, float, type(None)):
                    yield str(t)
                else:
                    yield t
        else:
            for x in l:
                yield x

    def write(self, l):
        # type: (List[str]) -> None
        self._sanitize(l)
        if self.check_num_fields:
            if self.num_fields is None:
                self.num_fields = len(l)
            else:
                assert len(l) == self.num_fields, "wrong number of fields in {}".format(l)
        print("\t".join(self._convert(l)), file=self.io)

    def close(self):
        # type: () -> None
        self.io.close()

    def __enter__(self):
        """ method needed to be a context manager """
        return self

    def __exit__(self, itype, value, traceback):
        """ method needed to be a context manager """
        self.close()

class TsvReader:
    def __init__(
            self,
            filename,
            mode="rt",
            use_any_format=USE_ANY_FORMAT,
            validate_all_lines_same_number_of_fields=VALIDATE_ALL_LINES_SAME_NUMBER_OF_FIELDS,
            num_fields=None,
            skip_comments=SKIP_COMMENTS,
            check_non_ascii=CHECK_NON_ASCII,
    ):
        # type: (str, str, bool, bool, int, bool, bool) -> None
        if use_any_format:
            self.io = pyanyzip.open(name=filename, mode=mode)
        else:
            self.io = open(filename, mode=mode)
        self.validate_all_lines_same_number_of_fields = validate_all_lines_same_number_of_fields
        self.num_fields = num_fields
        self.skip_comments = skip_comments
        self.check_non_ascii = check_non_ascii

        self.line_number = -1

    def next(self):
        return self.__next__()

    def __next__(self):
        """ method needed to be an iterator """
        self.line_number += 1
        # python2
        line = self.io.next()
        # python3
        # line = self.io.readline()
        if not line:
            raise StopIteration
        if self.skip_comments:
            while line.startswith("#"):
                # python2
                line = self.io.next()
                # python3
                # line = self.io.readline()
                if not line:
                    raise StopIteration
        line = line.rstrip('\r\n')
        if self.check_non_ascii:
            assert is_ascii(line), "non ascii characters in line [{}]".format(self.line_number)
        fields = line.split('\t')
        if self.validate_all_lines_same_number_of_fields:
            if self.num_fields is None:
                self.num_fields = len(fields)
            else:
                assert len(fields) == self.num_fields, "wrong number of fields in line number {} {} {}".format(
                    self.line_number,
                    self.num_fields,
                    len(fields),
                )
        if self.check_non_ascii:
            assert is_ascii(line)
        return fields

    def __iter__(self):
        """ method needed to be an iterator """
        return self

    def __enter__(self):
        """ method needed to be a context manager """
        return self

    def __exit__(self, itype, value, traceback):
        """ method needed to be a context manager """
        self.close()

    def close(self):
        # type: () -> None
        self.io.close()

def write_dict(filename=None, d=None):
    # type: (str, Dict[str,str]) -> None
    with TsvWriter(filename=filename, num_fields=2, fields_to_clean=[0]) as output_handle:
        for k, v in d.items():
            output_handle.write([k, v])
<commit_msg>minor change<commit_after>from __future__ import print_function

import codecs
import gzip
import os
from collections import defaultdict
from typing import Iterable, List, Tuple, Dict, IO, Iterator
import itertools
import logging
import re

import pyanyzip

CHECK_NON_ASCII = False
SKIP_COMMENTS = True
VALIDATE_ALL_LINES_SAME_NUMBER_OF_FIELDS = True
USE_ANY_FORMAT = True
SANITIZE = True
CLEAN_EDGES = True
SUB_TRAILING = True
REMOVE_NON_ASCII = True
LOWER_CASE = True
CHECK_NUM_FIELDS = True
CONVERT_TO_STRING = True
DO_GZIP = False
FILENAME_DETECT = True
DEFAULT_ENCODING = 'utf-8'

def clean(
        text,
        clean_edges=True,
        sub_trailing=True,
        remove_non_ascii=True,
        lower_case=True,
):
    # type: (str, bool, bool, bool, bool) -> str
    if sub_trailing:
        # replace all manner of whitespace (consecutive or not)
        # with s single space
        text = re.sub(r"[\r\t\n\v ]+", " ", text)
    if remove_non_ascii:
        text = text.encode('ascii', errors='ignore').decode()
    if clean_edges:
        # remove space from the left and right
        text = text.strip()
    if lower_case:
        text = text.lower()
    return text

def aggregate(
        input_file_names,
        match_columns,
        aggregate_columns,
        output_file_name,
        unlink=True,
):
    # type: (Iterable[str], List[int], List[int], str, bool) -> None
    """
    This function aggregates a bunch of input files by integers.
    :param input_file_names:
    :param match_columns:
    :param aggregate_columns:
    :param output_file_name:
    :param unlink:
    :return:
    """
    counts = dict()  # type: Dict[Tuple[str], List[int]]
    logger = logging.getLogger(__name__)
    for input_file_name in input_file_names:
        logger.debug("working on file [%s]", input_file_name)
        with TsvReader(filename=input_file_name) as input_handle:
            for parts in input_handle:
                # noinspection PyTypeChecker
                match = tuple([parts[i] for i in match_columns])  # type: Tuple[str]
                if match not in counts:
                    counts[match] = [int(0)] * len(aggregate_columns)
                for i, aggregate_column in enumerate(aggregate_columns):
                    counts[match][i] += int(parts[aggregate_column])
    # notice that we create a writer that does not check the number
    # of fields because the check requires a len(fields) to be available
    # and it is not in this case because of itertools.chain
    with TsvWriter(
        filename=output_file_name,
        check_num_fields=False,
        sanitize=False,
    ) as output_file_handle:
        for match, aggregates in counts.items():
            to_write = itertools.chain(match, aggregates)  # type: List[str]
            output_file_handle.write(to_write)
    if unlink:
        for input_file_name in input_file_names:
            os.unlink(input_file_name)

def write_data(data, output_file_name):
    # type: (List[List[str]], str) -> None
    with TsvWriter(
            filename=output_file_name,
            mode="at",
    ) as output_file_handle:
        for d in data:
            output_file_handle.write(d)

def group_by(
        input_file_names,
        group_by_columns,
        collect_columns,
        output_file_template,
        unlink=True,
):
    # type: (Iterable[str], List[int], List[int], str, bool) -> List[str]
    all_data = defaultdict(list)  # type: Dict[str, List[List[str]]]
    limit = 10000
    logger = logging.getLogger(__name__)
    for input_file_name in input_file_names:
        logger.debug("working on file [%s]", input_file_name)
        with open(input_file_name, 'rt') as file_handle:
            for line in file_handle:
                line = line.rstrip()
                parts = line.split("\t")  # type: List[str]
                match = tuple([parts[i] for i in group_by_columns])  # type: Tuple[str]
                match = "_".join(match)
                data_to_append = [parts[i] for i in collect_columns]
                all_data[match].append(data_to_append)
                if len(all_data[match]) > limit:
                    write_data(all_data[match], output_file_template.format(match=match))
                    all_data[match] = list()
    # write the rest for the data
    for match, data in all_data.items():
        write_data(data, output_file_template.format(match=match))
    # remove the input files
    if unlink:
        for input_file_name in input_file_names:
            os.unlink(input_file_name)
    return [output_file_template.format(match=match) for match in all_data]

def is_ascii(s):
    # type: (str) -> bool
    return all(ord(c) < 128 for c in s)

class TsvWriter(object):
    def __init__(
            self,
            filename,
            mode="wt",
            throw_exceptions=False,

            sanitize=SANITIZE,
            fields_to_clean=None,
            clean_edges=CLEAN_EDGES,
            sub_trailing=SUB_TRAILING,
            remove_non_ascii=REMOVE_NON_ASCII,
            lower_case=LOWER_CASE,

            check_num_fields=CHECK_NUM_FIELDS,
            num_fields=None,
            convert_to_string=CONVERT_TO_STRING,

            do_gzip=DO_GZIP,
            filename_detect=FILENAME_DETECT,
            encoding=DEFAULT_ENCODING,
    ):
        # type: (str, str, bool, bool, List[int], bool, bool, bool, bool, bool, int, bool, bool, bool) -> None
        if filename_detect:
            found = False
            if filename.endswith(".tsv.gz"):
                self.io = gzip.open(filename, mode=mode)  # type: IO[str]
                found = True
            if filename.endswith(".tsv"):
                self.io = open(filename, mode=mode)  # type: IO[str]
                found = True
            assert found, "file name unknown"
        else:
            if do_gzip:
                self.io = gzip.open(filename, mode=mode)  # type: IO[str]
            else:
                self.io = open(filename, mode=mode)  # type: IO[str]
        # python 2.7
        self.io = codecs.getwriter(encoding=encoding)(self.io)
        self.throw_exceptions = throw_exceptions

        self.sanitize = sanitize
        self.fields_to_clean = fields_to_clean
        if self.fields_to_clean is None:
            self.fields_to_clean = []
        self.clean_edges = clean_edges
        self.sub_trailing = sub_trailing
        self.remove_non_ascii = remove_non_ascii
        self.lower_case = lower_case

        self.check_num_fields = check_num_fields
        self.num_fields = num_fields
        self.convert_to_string = convert_to_string

    def _sanitize(self, l):
        # type: (List[str]) -> None
        if self.sanitize:
            for field in self.fields_to_clean:
                l[field] = clean(
                    text=l[field],
                    clean_edges=self.clean_edges,
                    sub_trailing=self.sub_trailing,
                    remove_non_ascii=self.remove_non_ascii,
                    lower_case=self.lower_case,
                )

    def _convert(self, l):
        # type: (List[str]) -> Iterator[str]
        if self.convert_to_string:
            for i, t in enumerate(l):
                if type(t) in (int, float, type(None)):
                    yield str(t)
                else:
                    yield t
        else:
            for x in l:
                yield x

    def write(self, l):
        # type: (List[str]) -> None
        self._sanitize(l)
        if self.check_num_fields:
            if self.num_fields is None:
                self.num_fields = len(l)
            else:
                assert len(l) == self.num_fields, "wrong number of fields in {}".format(l)
        print("\t".join(self._convert(l)), file=self.io)

    def close(self):
        # type: () -> None
        self.io.close()

    def __enter__(self):
        """ method needed to be a context manager """
        return self

    def __exit__(self, itype, value, traceback):
        """ method needed to be a context manager """
        self.close()

class TsvReader:
    def __init__(
            self,
            filename,
            mode="rt",
            use_any_format=USE_ANY_FORMAT,
            validate_all_lines_same_number_of_fields=VALIDATE_ALL_LINES_SAME_NUMBER_OF_FIELDS,
            num_fields=None,
            skip_comments=SKIP_COMMENTS,
            check_non_ascii=CHECK_NON_ASCII,
    ):
        # type: (str, str, bool, bool, int, bool, bool) -> None
        if use_any_format:
            self.io = pyanyzip.open(name=filename, mode=mode)
        else:
            self.io = open(filename, mode=mode)
        self.validate_all_lines_same_number_of_fields = validate_all_lines_same_number_of_fields
        self.num_fields = num_fields
        self.skip_comments = skip_comments
        self.check_non_ascii = check_non_ascii

        self.line_number = -1

    def next(self):
        return self.__next__()

    def __next__(self):
        # type: () -> List[unicode]
        """ method needed to be an iterator """
        self.line_number += 1
        # python2
        line = self.io.next()
        # python3
        # line = self.io.readline()
        if not line:
            raise StopIteration
        if self.skip_comments:
            while line.startswith("#"):
                # python2
                line = self.io.next()
                # python3
                # line = self.io.readline()
                if not line:
                    raise StopIteration
        line = line.rstrip('\r\n')
        if self.check_non_ascii:
            assert is_ascii(line), "non ascii characters in line [{}]".format(self.line_number)
        fields = line.split('\t')
        if self.validate_all_lines_same_number_of_fields:
            if self.num_fields is None:
                self.num_fields = len(fields)
            else:
                assert len(fields) == self.num_fields, "wrong number of fields in line number {} {} {}".format(
                    self.line_number,
                    self.num_fields,
                    len(fields),
                )
        if self.check_non_ascii:
            assert is_ascii(line)
        return fields

    def __iter__(self):
        """ method needed to be an iterator """
        return self

    def __enter__(self):
        """ method needed to be a context manager """
        return self

    def __exit__(self, itype, value, traceback):
        """ method needed to be a context manager """
        self.close()

    def close(self):
        # type: () -> None
        self.io.close()

def write_dict(filename=None, d=None):
    # type: (str, Dict[str,str]) -> None
    with TsvWriter(filename=filename, num_fields=2, fields_to_clean=[0]) as output_handle:
        for k, v in d.items():
            output_handle.write([k, v])
<|endoftext|>


<commit_before>import threading
from . import VEXObject

_libvex_lock = threading.Lock()

class IRSB(VEXObject):
    """
    The IRSB is the primary interface to pyvex. Constructing one of these will make a call into LibVEX to perform a
    translation.

    IRSB stands for *Intermediate Representation Super-Block*. An IRSB in VEX is a single-entry, multiple-exit code
    block.

    :ivar arch:             The architecture this block is lifted under
    :vartype arch:          :class:`archinfo.Arch`
    :ivar statements:       The statements in this block
    :vartype statements:    list of :class:`IRStmt`
    :ivar next:             The expression for the default exit target of this block
    :vartype next:          :class:`IRExpr`
    :ivar int offsIP:       The offset of the instruction pointer in the VEX guest state
    :ivar int stmts_used:   The number of statements in this IRSB
    :ivar str jumpkind:     The type of this block's default jump (call, boring, syscall, etc) as a VEX enum string
    :ivar bool direct_next: Whether this block ends with a direct (not indirect) jump or branch
    """

    __slots__ = ['c_irsb', 'arch', 'statements', 'next', 'tyenv', 'offsIP', 'stmts_used', 'jumpkind', '_addr',
                 'direct_next'
                 ]

    def __init__(self, data, mem_addr, arch, num_inst=None, num_bytes=None, bytes_offset=0,
                 traceflags=0):  # pylint:disable=redefined-builtin
        """
        :param data:            The bytes to lift. Can be either a string of bytes or a cffi buffer object.
        :type data:             str or bytes or cffi.FFI.CData
        :param int mem_addr:    The address to lift the data at.
        :param arch:            The architecture to lift the data as.
        :type arch:             :class:`archinfo.Arch`

        The following are optional:
        :param num_inst:        The maximum number of instructions to lift. Max 99.
        :param num_bytes:       The maximum number of bytes to use. Max 400.
        :param bytes_offset:    The offset into `data` to start lifting at.
        :param traceflags:      The libVEX traceflags, controlling VEX debug prints.
        """
        try:
            _libvex_lock.acquire()

            VEXObject.__init__(self)

            if isinstance(data, (str, bytes)):
                num_bytes = len(data) if num_bytes is None else num_bytes
                c_bytes = ffi.new('char [%d]' % (num_bytes + 8), data + '\0' * 8)
            else:
                if not num_bytes:
                    raise PyVEXError("C-backed bytes must have the length specified by num_bytes")
                c_bytes = data

            if num_bytes == 0:
                raise PyVEXError("No bytes provided")
            pvc.vta.traceflags = traceflags

            vex_arch = getattr(pvc, arch.vex_arch)

            arch.vex_archinfo['hwcache_info']['caches'] = ffi.NULL

            if num_inst is not None:
                c_irsb = pvc.vex_block_inst(vex_arch, arch.vex_archinfo, c_bytes + bytes_offset, mem_addr, num_inst)
            else:
                c_irsb = pvc.vex_block_bytes(vex_arch, arch.vex_archinfo, c_bytes + bytes_offset, mem_addr, num_bytes,
                                             1)

            if c_irsb == ffi.NULL:
                raise PyVEXError(ffi.string(pvc.last_error) if pvc.last_error != ffi.NULL else "unknown error")

            # We must use a pickle value, CData objects are not pickeable so not ffi.NULL
            arch.vex_archinfo['hwcache_info']['caches'] = None

            self.c_irsb = c_irsb
            self.arch = arch
            self.statements = [stmt.IRStmt._translate(c_irsb.stmts[i], self) for i in range(c_irsb.stmts_used)]
            self.next = expr.IRExpr._translate(c_irsb.next, self)
            self.tyenv = IRTypeEnv(c_irsb.tyenv)
            self.offsIP = c_irsb.offsIP
            self.stmts_used = c_irsb.stmts_used
            self.jumpkind = ints_to_enums[c_irsb.jumpkind]

            self._addr = mem_addr
            self.direct_next = self._is_defaultexit_direct_jump()

            del self.c_irsb
        finally:
            _libvex_lock.release()

    def pp(self):
        """
        Pretty-print the IRSB to stdout.
        """
        print self._pp_str()

    def _pp_str(self):
        """
        Return the pretty-printed IRSB.

        :rtype: str
        """
        sa = []
        sa.append("IRSB {")
        sa.append("   %s" % self.tyenv)
        sa.append("")
        for i, s in enumerate(self.statements):
            sa.append("   %02d | %s" % (i, s))
        sa.append(
            "   NEXT: PUT(%s) = %s; %s" % (self.arch.translate_register_name(self.offsIP), self.next, self.jumpkind))
        sa.append("}")
        return '\n'.join(sa)

    @property
    def expressions(self):
        """
        A list of all expressions contained in the IRSB.
        """
        expressions = []
        for s in self.statements:
            expressions.extend(s.expressions)
        expressions.append(self.next)
        return expressions

    @property
    def instructions(self):
        """
        The number of instructions in this block
        """
        return len([s.addr for s in self.statements if isinstance(s, stmt.IMark)])

    @property
    def size(self):
        """
        The size of this block, in bytes
        """
        return sum([s.len for s in self.statements if isinstance(s, stmt.IMark)])

    @property
    def operations(self):
        """
        A list of all operations done by the IRSB, as libVEX enum names
        """
        ops = []
        for e in self.expressions:
            if hasattr(e, 'op'):
                ops.append(e.op)
        return ops

    @property
    def all_constants(self):
        """
        Returns all constants in the block (including incrementing of the program counter) as :class:`pyvex.const.IRConst`.
        """
        return sum((e.constants for e in self.expressions), [])

    @property
    def constants(self):
        """
        The constants (excluding updates of the program counter) in the IRSB as :class:`pyvex.const.IRConst`.
        """
        return sum(
            (s.constants for s in self.statements if not (isinstance(s, stmt.Put) and s.offset == self.offsIP)), [])

    @property
    def constant_jump_targets(self):
        """
        A set of the static jump targets of the basic block.
        """
        exits = set()
        for s in self.statements:
            if isinstance(s, stmt.Exit):
                exits.add(s.dst.value)

        default_target = self._get_defaultexit_target()
        if default_target is not None:
            exits.add(default_target)

        return exits

    @property
    def constant_jump_targets_and_jumpkinds(self):
        """
        A dict of the static jump targets of the basic block to their jumpkind.
        """
        exits = dict()
        for s in self.statements:
            if isinstance(s, stmt.Exit):
                exits[s.dst.value] = s.jumpkind

        default_target = self._get_defaultexit_target()
        if default_target is not None:
            exits[default_target] = self.jumpkind

        return exits

    def _get_defaultexit_target(self):
        """
        The default exit target, if it is constant, or None.
        """
        if isinstance(self.next, expr.Const):
            return self.next.con.value

        if not isinstance(self.next, expr.RdTmp):
            raise PyVEXError("unexpected self.next type: %s", self.next.__class__.__name__)

        tmp_next = self.next.tmp
        reg_next = None
        reg_next_size = None
        for stat in reversed(self.statements):
            if isinstance(stat, stmt.WrTmp) and stat.tmp == tmp_next:
                data = stat.data
            elif isinstance(stat, stmt.Put) and stat.offset == reg_next:
                data = stat.data
                if data.result_size != reg_next_size:
                    return None
            elif isinstance(stat, stmt.LoadG) and stat.dst == tmp_next:
                return None
            else:
                continue

            if isinstance(data, expr.Const):
                return data.con.value
            elif isinstance(data, expr.RdTmp):
                tmp_next = data.tmp
                reg_next = None
            elif isinstance(data, expr.Get):
                tmp_next = None
                reg_next = data.offset
                reg_next_size = data.result_size
            else:
                return None

        if tmp_next is not None:
            raise PyVEXError('Malformed IRSB at address #%x. Please report to Fish.' % self._addr)
        return None

    def _is_defaultexit_direct_jump(self):
        """
        Checks if the default of this IRSB a direct jump or not.
        """
        if not (self.jumpkind == 'Ijk_Boring' or self.jumpkind == 'Ijk_Call'):
            return False

        target = self._get_defaultexit_target()
        return target is not None

class IRTypeEnv(VEXObject):
    """
    An IR type environment.

    :ivar types:        A list of the types of all the temporaries in this block as VEX enum strings.
                        `types[3]` is the type of t3.
    :vartype types:     list of str
    """
    def __init__(self, tyenv):
        VEXObject.__init__(self)
        self.types = [ints_to_enums[tyenv.types[t]] for t in xrange(tyenv.types_used)]
        self.types_used = tyenv.types_used

    def __str__(self):
        return ' '.join(("t%d:%s" % (i, t)) for i, t in enumerate(self.types))

from . import expr, stmt, ffi, pvc
from .enums import ints_to_enums
from .errors import PyVEXError
<commit_msg>Add slots to IRTypeEnv. Use xrange instead of range in IRSB.__init__().<commit_after>import threading
from . import VEXObject

_libvex_lock = threading.Lock()

class IRSB(VEXObject):
    """
    The IRSB is the primary interface to pyvex. Constructing one of these will make a call into LibVEX to perform a
    translation.

    IRSB stands for *Intermediate Representation Super-Block*. An IRSB in VEX is a single-entry, multiple-exit code
    block.

    :ivar arch:             The architecture this block is lifted under
    :vartype arch:          :class:`archinfo.Arch`
    :ivar statements:       The statements in this block
    :vartype statements:    list of :class:`IRStmt`
    :ivar next:             The expression for the default exit target of this block
    :vartype next:          :class:`IRExpr`
    :ivar int offsIP:       The offset of the instruction pointer in the VEX guest state
    :ivar int stmts_used:   The number of statements in this IRSB
    :ivar str jumpkind:     The type of this block's default jump (call, boring, syscall, etc) as a VEX enum string
    :ivar bool direct_next: Whether this block ends with a direct (not indirect) jump or branch
    """

    __slots__ = ['c_irsb', 'arch', 'statements', 'next', 'tyenv', 'offsIP', 'stmts_used', 'jumpkind', '_addr',
                 'direct_next'
                 ]

    def __init__(self, data, mem_addr, arch, num_inst=None, num_bytes=None, bytes_offset=0,
                 traceflags=0):  # pylint:disable=redefined-builtin
        """
        :param data:            The bytes to lift. Can be either a string of bytes or a cffi buffer object.
        :type data:             str or bytes or cffi.FFI.CData
        :param int mem_addr:    The address to lift the data at.
        :param arch:            The architecture to lift the data as.
        :type arch:             :class:`archinfo.Arch`

        The following are optional:
        :param num_inst:        The maximum number of instructions to lift. Max 99.
        :param num_bytes:       The maximum number of bytes to use. Max 400.
        :param bytes_offset:    The offset into `data` to start lifting at.
        :param traceflags:      The libVEX traceflags, controlling VEX debug prints.
        """
        try:
            _libvex_lock.acquire()

            VEXObject.__init__(self)

            if isinstance(data, (str, bytes)):
                num_bytes = len(data) if num_bytes is None else num_bytes
                c_bytes = ffi.new('char [%d]' % (num_bytes + 8), data + '\0' * 8)
            else:
                if not num_bytes:
                    raise PyVEXError("C-backed bytes must have the length specified by num_bytes")
                c_bytes = data

            if num_bytes == 0:
                raise PyVEXError("No bytes provided")
            pvc.vta.traceflags = traceflags

            vex_arch = getattr(pvc, arch.vex_arch)

            arch.vex_archinfo['hwcache_info']['caches'] = ffi.NULL

            if num_inst is not None:
                c_irsb = pvc.vex_block_inst(vex_arch, arch.vex_archinfo, c_bytes + bytes_offset, mem_addr, num_inst)
            else:
                c_irsb = pvc.vex_block_bytes(vex_arch, arch.vex_archinfo, c_bytes + bytes_offset, mem_addr, num_bytes,
                                             1)

            if c_irsb == ffi.NULL:
                raise PyVEXError(ffi.string(pvc.last_error) if pvc.last_error != ffi.NULL else "unknown error")

            # We must use a pickle value, CData objects are not pickeable so not ffi.NULL
            arch.vex_archinfo['hwcache_info']['caches'] = None

            self.c_irsb = c_irsb
            self.arch = arch
            self.statements = [stmt.IRStmt._translate(c_irsb.stmts[i], self) for i in xrange(c_irsb.stmts_used)]
            self.next = expr.IRExpr._translate(c_irsb.next, self)
            self.tyenv = IRTypeEnv(c_irsb.tyenv)
            self.offsIP = c_irsb.offsIP
            self.stmts_used = c_irsb.stmts_used
            self.jumpkind = ints_to_enums[c_irsb.jumpkind]

            self._addr = mem_addr
            self.direct_next = self._is_defaultexit_direct_jump()

            del self.c_irsb
        finally:
            _libvex_lock.release()

    def pp(self):
        """
        Pretty-print the IRSB to stdout.
        """
        print self._pp_str()

    def _pp_str(self):
        """
        Return the pretty-printed IRSB.

        :rtype: str
        """
        sa = []
        sa.append("IRSB {")
        sa.append("   %s" % self.tyenv)
        sa.append("")
        for i, s in enumerate(self.statements):
            sa.append("   %02d | %s" % (i, s))
        sa.append(
            "   NEXT: PUT(%s) = %s; %s" % (self.arch.translate_register_name(self.offsIP), self.next, self.jumpkind))
        sa.append("}")
        return '\n'.join(sa)

    @property
    def expressions(self):
        """
        A list of all expressions contained in the IRSB.
        """
        expressions = []
        for s in self.statements:
            expressions.extend(s.expressions)
        expressions.append(self.next)
        return expressions

    @property
    def instructions(self):
        """
        The number of instructions in this block
        """
        return len([s.addr for s in self.statements if isinstance(s, stmt.IMark)])

    @property
    def size(self):
        """
        The size of this block, in bytes
        """
        return sum([s.len for s in self.statements if isinstance(s, stmt.IMark)])

    @property
    def operations(self):
        """
        A list of all operations done by the IRSB, as libVEX enum names
        """
        ops = []
        for e in self.expressions:
            if hasattr(e, 'op'):
                ops.append(e.op)
        return ops

    @property
    def all_constants(self):
        """
        Returns all constants in the block (including incrementing of the program counter) as :class:`pyvex.const.IRConst`.
        """
        return sum((e.constants for e in self.expressions), [])

    @property
    def constants(self):
        """
        The constants (excluding updates of the program counter) in the IRSB as :class:`pyvex.const.IRConst`.
        """
        return sum(
            (s.constants for s in self.statements if not (isinstance(s, stmt.Put) and s.offset == self.offsIP)), [])

    @property
    def constant_jump_targets(self):
        """
        A set of the static jump targets of the basic block.
        """
        exits = set()
        for s in self.statements:
            if isinstance(s, stmt.Exit):
                exits.add(s.dst.value)

        default_target = self._get_defaultexit_target()
        if default_target is not None:
            exits.add(default_target)

        return exits

    @property
    def constant_jump_targets_and_jumpkinds(self):
        """
        A dict of the static jump targets of the basic block to their jumpkind.
        """
        exits = dict()
        for s in self.statements:
            if isinstance(s, stmt.Exit):
                exits[s.dst.value] = s.jumpkind

        default_target = self._get_defaultexit_target()
        if default_target is not None:
            exits[default_target] = self.jumpkind

        return exits

    def _get_defaultexit_target(self):
        """
        The default exit target, if it is constant, or None.
        """
        if isinstance(self.next, expr.Const):
            return self.next.con.value

        if not isinstance(self.next, expr.RdTmp):
            raise PyVEXError("unexpected self.next type: %s", self.next.__class__.__name__)

        tmp_next = self.next.tmp
        reg_next = None
        reg_next_size = None
        for stat in reversed(self.statements):
            if isinstance(stat, stmt.WrTmp) and stat.tmp == tmp_next:
                data = stat.data
            elif isinstance(stat, stmt.Put) and stat.offset == reg_next:
                data = stat.data
                if data.result_size != reg_next_size:
                    return None
            elif isinstance(stat, stmt.LoadG) and stat.dst == tmp_next:
                return None
            else:
                continue

            if isinstance(data, expr.Const):
                return data.con.value
            elif isinstance(data, expr.RdTmp):
                tmp_next = data.tmp
                reg_next = None
            elif isinstance(data, expr.Get):
                tmp_next = None
                reg_next = data.offset
                reg_next_size = data.result_size
            else:
                return None

        if tmp_next is not None:
            raise PyVEXError('Malformed IRSB at address #%x. Please report to Fish.' % self._addr)
        return None

    def _is_defaultexit_direct_jump(self):
        """
        Checks if the default of this IRSB a direct jump or not.
        """
        if not (self.jumpkind == 'Ijk_Boring' or self.jumpkind == 'Ijk_Call'):
            return False

        target = self._get_defaultexit_target()
        return target is not None

class IRTypeEnv(VEXObject):
    """
    An IR type environment.

    :ivar types:        A list of the types of all the temporaries in this block as VEX enum strings.
                        `types[3]` is the type of t3.
    :vartype types:     list of str
    """

    __slots__ = [ 'types', 'types_used' ]

    def __init__(self, tyenv):
        VEXObject.__init__(self)
        self.types = [ints_to_enums[tyenv.types[t]] for t in xrange(tyenv.types_used)]
        self.types_used = tyenv.types_used

    def __str__(self):
        return ' '.join(("t%d:%s" % (i, t)) for i, t in enumerate(self.types))

from . import expr, stmt, ffi, pvc
from .enums import ints_to_enums
from .errors import PyVEXError
<|endoftext|>