<commit_before>import pybel
from pybel import *
import copy_reg
import numpy as np
from openbabel import OBAtomAtomIter,OBTypeTable
from oddt.spatial import angle, angle_2v, dihedral

backend = 'ob'
# setup typetable to translate atom types
typetable = OBTypeTable()
typetable.SetFromType('INT')
typetable.SetToType('SYB')

# hash OB!
pybel.ob.obErrorLog.StopLogging()

class Molecule(pybel.Molecule):
    def __init__(self, OBMol, protein = False):
        # call parent constructor
        super(Molecule,self).__init__(OBMol)

        self.protein = protein

        #ob.DeterminePeptideBackbone(molecule.OBMol)
        # percieve chains in residues
        #if len(res_dict) > 1 and not molecule.OBMol.HasChainsPerceived():
        #    print "Dirty HACK"
        #    molecule = pybel.readstring('pdb', molecule.write('pdb'))

        self._atom_dict = None
        self._res_dict = None
        self._ring_dict = None
        self._coords = None
        self._charges = None

    # cache frequently used properties and cache them in prefixed [_] variables
    @property
    def coords(self):
        if self._coords is None:
            self._coords = np.array([atom.coords for atom in self.atoms])
        return self._coords

    @property
    def charges(self):
        if self._charges is None:
            self._charges = np.array([atom.partialcharge for atom in self.atoms])
        return self._charges

    #### Custom ODDT properties ####
    def __getattr__(self, attr):
        for desc in pybel._descdict.keys():
            if attr.lower() == desc.lower():
                return self.calcdesc([desc])[desc]
        raise AttributeError('Molecule has no such property: %s' % attr)

    @property
    def num_rotors(self):
        return self.OBMol.NumRotors()

    @property
    def atom_dict(self):
        # check cache and generate dicts
        if self._atom_dict is None:
            self._dicts()
        return self._atom_dict

    @property
    def res_dict(self):
        # check cache and generate dicts
        if self._res_dict is None:
            self._dicts()
        return self._res_dict

    @property
    def ring_dict(self):
        # check cache and generate dicts
        if self._ring_dict is None:
            self._dicts()
        return self._ring_dict

    def _dicts(self):
        # Atoms
        atom_dtype = [('id', 'int16'),
                 # atom info
                 ('coords', 'float16', 3),
                 ('charge', 'float16'),
                 ('atomicnum', 'int8'),
                 ('atomtype','a4'),
                 ('hybridization', 'int8'),
                 ('neighbors', 'float16', (4,3)), # non-H neighbors coordinates for angles (max of 6 neighbors should be enough)
                 # residue info
                 ('resid', 'int16'),
                 ('resname', 'a3'),
                 ('isbackbone', 'bool'),
                 # atom properties
                 ('isacceptor', 'bool'),
                 ('isdonor', 'bool'),
                 ('isdonorh', 'bool'),
                 ('ismetal', 'bool'),
                 ('ishydrophobe', 'bool'),
                 ('isaromatic', 'bool'),
                 ('isminus', 'bool'),
                 ('isplus', 'bool'),
                 ('ishalogen', 'bool'),
                 # secondary structure
                 ('isalpha', 'bool'),
                 ('isbeta', 'bool')
                 ]

        a = []
        atom_dict = np.empty(self.OBMol.NumAtoms(), dtype=atom_dtype)
        metals = [3,4,11,12,13,19,20,21,22,23,24,25,26,27,28,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,87,88,89,90,91,
    92,93,94,95,96,97,98,99,100,101,102,103]
        for i, atom in enumerate(self.atoms):

            atomicnum = atom.atomicnum
            # skip non-polar hydrogens for performance
#            if atomicnum == 1 and atom.OBAtom.IsNonPolarHydrogen():
#                continue
            atomtype = typetable.Translate(atom.type) # sybyl atom type
            partialcharge = atom.partialcharge
            coords = atom.coords

            if self.protein:
                residue = pybel.Residue(atom.OBAtom.GetResidue())
            else:
                residue = False

            # get neighbors, but only for those atoms which realy need them
            neighbors = np.empty(4, dtype=[('coords', 'float16', 3),('atomicnum', 'int8')])
            neighbors.fill(np.nan)
            for n, nbr_atom in enumerate(atom.neighbors):
                # concider raising neighbors list to 6, but must do some benchmarks
                if n > 3:
                    break
                nbr_atomicnum = nbr_atom.atomicnum
                neighbors[n] = (nbr_atom.coords, nbr_atomicnum)
            atom_dict[i] = (atom.idx,
                      coords,
                      partialcharge,
                      atomicnum,
                      atomtype,
                      atom.OBAtom.GetHyb(),
                      neighbors['coords'], #n_coords,
                      # residue info
                      residue.idx if residue else 0,
                      residue.name if residue else '',
                      residue.OBResidue.GetAtomProperty(atom.OBAtom, 2) if residue else False, # is backbone
                      # atom properties
                      atom.OBAtom.IsHbondAcceptor(),
                      atom.OBAtom.IsHbondDonor(),
                      atom.OBAtom.IsHbondDonorH(),
                      atomicnum in metals,
                      atomicnum == 6 and not (np.in1d(neighbors['atomicnum'], [6,1])).any(), #hydrophobe #doble negation, since nan gives False
                      atom.OBAtom.IsAromatic(),
                      atomtype in ['O3-', '02-' 'O-'], # is charged (minus)
                      atomtype in ['N3+', 'N2+', 'Ng+'], # is charged (plus)
                      atomicnum in [9,17,35,53], # is halogen?
                      False, # alpha
                      False # beta
                      )

        if self.protein:
            # Protein Residues (alpha helix and beta sheet)
            res_dtype = [('id', 'int16'),
                         ('resname', 'a3'),
                         ('N', 'float16', 3),
                         ('CA', 'float16', 3),
                         ('C', 'float16', 3),
                         ('isalpha', 'bool'),
                         ('isbeta', 'bool')
                         ] # N, CA, C

            b = []
            for residue in self.residues:
                backbone = {}
                for atom in residue:
                    if residue.OBResidue.GetAtomProperty(atom.OBAtom,1):
                        if atom.atomicnum == 7:
                            backbone['N'] = atom.coords
                        elif atom.atomicnum == 6:
                            if atom.type == 'C3':
                                backbone['CA'] = atom.coords
                            else:
                                backbone['C'] = atom.coords
                if len(backbone.keys()) == 3:
                    b.append((residue.idx, residue.name, backbone['N'],  backbone['CA'], backbone['C'], False, False))
            res_dict = np.array(b, dtype=res_dtype)

            # detect secondary structure by phi and psi angles
            first = res_dict[:-1]
            second = res_dict[1:]
            psi = dihedral(first['N'], first['CA'], first['C'], second['N'])
            phi = dihedral(first['C'], second['N'], second['CA'], second['C'])
            # mark atoms belonging to alpha and beta
            res_mask_alpha = np.where(((phi > -145) & (phi < -35) & (psi > -70) & (psi < 50))) # alpha
            res_dict['isalpha'][res_mask_alpha] = True
            for i in res_dict[res_mask_alpha]['id']:
                atom_dict['isalpha'][atom_dict['resid'] == i] = True

            res_mask_beta = np.where(((phi >= -180) & (phi < -40) & (psi <= 180) & (psi > 90)) | ((phi >= -180) & (phi < -70) & (psi <= -165))) # beta
            res_dict['isbeta'][res_mask_beta] = True
            atom_dict['isbeta'][np.in1d(atom_dict['resid'], res_dict[res_mask_beta]['id'])] = True

        # Aromatic Rings
        r = []
        for ring in self.sssr:
            if ring.IsAromatic():
                path = ring._path
                atom = atom_dict[atom_dict['id'] == path[0]]
                coords = atom_dict[np.in1d(atom_dict['id'], path)]['coords']
                centroid = coords.mean(axis=0)
                # get vector perpendicular to ring
                vector = np.cross(coords - np.vstack((coords[1:],coords[:1])), np.vstack((coords[1:],coords[:1])) - np.vstack((coords[2:],coords[:2]))).mean(axis=0) - centroid
                r.append((centroid, vector, atom['isalpha'], atom['isbeta']))
        ring_dict = np.array(r, dtype=[('centroid', 'float16', 3),('vector', 'float16', 3),('isalpha', 'bool'),('isbeta', 'bool'),])

        self._atom_dict = atom_dict
        self._ring_dict = ring_dict
        if self.protein:
            self._res_dict = res_dict

### Extend pybel.Molecule
pybel.Molecule = Molecule

class Atom(pybel.Atom):
    @property
    def neighbors(self):
        return [Atom(a) for a in OBAtomAtomIter(self.OBAtom)]

pybel.Atom = Atom

class Fingerprint(pybel.Fingerprint):
    @property
    def raw(self):
        return _unrollbits(self.fp, pybel.ob.OBFingerprint.Getbitsperint())

def _unrollbits(fp, bitsperint):
    """ Unroll unsigned int fingerprint to bool """
    ans = np.zeros(len(fp)*bitsperint)
    start = 1
    for x in fp:
        i = start
        while x > 0:
            ans[i] = x % 2
            x >>= 1
            i += 1
        start += bitsperint
    return ans

pybel.Fingerprint = Fingerprint

### Monkeypatch pybel objects pickling
pickle_format = 'mol2'
def pickle_mol(self):
    return unpickle_mol, (self.write(pickle_format), dict(self.data.items()))

def unpickle_mol(string, data):
    mol = readstring(pickle_format, string)
    mol.data.update(data)
    return mol
copy_reg.pickle(Molecule, pickle_mol, unpickle_mol)
<commit_msg>Display html representation of OB molecules in iPython<commit_after>import pybel
from pybel import *
import copy_reg
import numpy as np
from openbabel import OBAtomAtomIter,OBTypeTable
from oddt.spatial import angle, angle_2v, dihedral

backend = 'ob'
# setup typetable to translate atom types
typetable = OBTypeTable()
typetable.SetFromType('INT')
typetable.SetToType('SYB')

# hash OB!
pybel.ob.obErrorLog.StopLogging()

class Molecule(pybel.Molecule):
    def __init__(self, OBMol, protein = False):
        # call parent constructor
        super(Molecule,self).__init__(OBMol)

        self.protein = protein

        #ob.DeterminePeptideBackbone(molecule.OBMol)
        # percieve chains in residues
        #if len(res_dict) > 1 and not molecule.OBMol.HasChainsPerceived():
        #    print "Dirty HACK"
        #    molecule = pybel.readstring('pdb', molecule.write('pdb'))

        self._atom_dict = None
        self._res_dict = None
        self._ring_dict = None
        self._coords = None
        self._charges = None

    # cache frequently used properties and cache them in prefixed [_] variables
    @property
    def coords(self):
        if self._coords is None:
            self._coords = np.array([atom.coords for atom in self.atoms])
        return self._coords

    @property
    def charges(self):
        if self._charges is None:
            self._charges = np.array([atom.partialcharge for atom in self.atoms])
        return self._charges

    #### Custom ODDT properties ####
    def __getattr__(self, attr):
        for desc in pybel._descdict.keys():
            if attr.lower() == desc.lower():
                return self.calcdesc([desc])[desc]
        raise AttributeError('Molecule has no such property: %s' % attr)

    @property
    def num_rotors(self):
        return self.OBMol.NumRotors()

    def _repr_html_():
        return self.write('svg')

    @property
    def atom_dict(self):
        # check cache and generate dicts
        if self._atom_dict is None:
            self._dicts()
        return self._atom_dict

    @property
    def res_dict(self):
        # check cache and generate dicts
        if self._res_dict is None:
            self._dicts()
        return self._res_dict

    @property
    def ring_dict(self):
        # check cache and generate dicts
        if self._ring_dict is None:
            self._dicts()
        return self._ring_dict

    def _dicts(self):
        # Atoms
        atom_dtype = [('id', 'int16'),
                 # atom info
                 ('coords', 'float16', 3),
                 ('charge', 'float16'),
                 ('atomicnum', 'int8'),
                 ('atomtype','a4'),
                 ('hybridization', 'int8'),
                 ('neighbors', 'float16', (4,3)), # non-H neighbors coordinates for angles (max of 6 neighbors should be enough)
                 # residue info
                 ('resid', 'int16'),
                 ('resname', 'a3'),
                 ('isbackbone', 'bool'),
                 # atom properties
                 ('isacceptor', 'bool'),
                 ('isdonor', 'bool'),
                 ('isdonorh', 'bool'),
                 ('ismetal', 'bool'),
                 ('ishydrophobe', 'bool'),
                 ('isaromatic', 'bool'),
                 ('isminus', 'bool'),
                 ('isplus', 'bool'),
                 ('ishalogen', 'bool'),
                 # secondary structure
                 ('isalpha', 'bool'),
                 ('isbeta', 'bool')
                 ]

        a = []
        atom_dict = np.empty(self.OBMol.NumAtoms(), dtype=atom_dtype)
        metals = [3,4,11,12,13,19,20,21,22,23,24,25,26,27,28,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,87,88,89,90,91,
    92,93,94,95,96,97,98,99,100,101,102,103]
        for i, atom in enumerate(self.atoms):

            atomicnum = atom.atomicnum
            # skip non-polar hydrogens for performance
#            if atomicnum == 1 and atom.OBAtom.IsNonPolarHydrogen():
#                continue
            atomtype = typetable.Translate(atom.type) # sybyl atom type
            partialcharge = atom.partialcharge
            coords = atom.coords

            if self.protein:
                residue = pybel.Residue(atom.OBAtom.GetResidue())
            else:
                residue = False

            # get neighbors, but only for those atoms which realy need them
            neighbors = np.empty(4, dtype=[('coords', 'float16', 3),('atomicnum', 'int8')])
            neighbors.fill(np.nan)
            for n, nbr_atom in enumerate(atom.neighbors):
                # concider raising neighbors list to 6, but must do some benchmarks
                if n > 3:
                    break
                nbr_atomicnum = nbr_atom.atomicnum
                neighbors[n] = (nbr_atom.coords, nbr_atomicnum)
            atom_dict[i] = (atom.idx,
                      coords,
                      partialcharge,
                      atomicnum,
                      atomtype,
                      atom.OBAtom.GetHyb(),
                      neighbors['coords'], #n_coords,
                      # residue info
                      residue.idx if residue else 0,
                      residue.name if residue else '',
                      residue.OBResidue.GetAtomProperty(atom.OBAtom, 2) if residue else False, # is backbone
                      # atom properties
                      atom.OBAtom.IsHbondAcceptor(),
                      atom.OBAtom.IsHbondDonor(),
                      atom.OBAtom.IsHbondDonorH(),
                      atomicnum in metals,
                      atomicnum == 6 and not (np.in1d(neighbors['atomicnum'], [6,1])).any(), #hydrophobe #doble negation, since nan gives False
                      atom.OBAtom.IsAromatic(),
                      atomtype in ['O3-', '02-' 'O-'], # is charged (minus)
                      atomtype in ['N3+', 'N2+', 'Ng+'], # is charged (plus)
                      atomicnum in [9,17,35,53], # is halogen?
                      False, # alpha
                      False # beta
                      )

        if self.protein:
            # Protein Residues (alpha helix and beta sheet)
            res_dtype = [('id', 'int16'),
                         ('resname', 'a3'),
                         ('N', 'float16', 3),
                         ('CA', 'float16', 3),
                         ('C', 'float16', 3),
                         ('isalpha', 'bool'),
                         ('isbeta', 'bool')
                         ] # N, CA, C

            b = []
            for residue in self.residues:
                backbone = {}
                for atom in residue:
                    if residue.OBResidue.GetAtomProperty(atom.OBAtom,1):
                        if atom.atomicnum == 7:
                            backbone['N'] = atom.coords
                        elif atom.atomicnum == 6:
                            if atom.type == 'C3':
                                backbone['CA'] = atom.coords
                            else:
                                backbone['C'] = atom.coords
                if len(backbone.keys()) == 3:
                    b.append((residue.idx, residue.name, backbone['N'],  backbone['CA'], backbone['C'], False, False))
            res_dict = np.array(b, dtype=res_dtype)

            # detect secondary structure by phi and psi angles
            first = res_dict[:-1]
            second = res_dict[1:]
            psi = dihedral(first['N'], first['CA'], first['C'], second['N'])
            phi = dihedral(first['C'], second['N'], second['CA'], second['C'])
            # mark atoms belonging to alpha and beta
            res_mask_alpha = np.where(((phi > -145) & (phi < -35) & (psi > -70) & (psi < 50))) # alpha
            res_dict['isalpha'][res_mask_alpha] = True
            for i in res_dict[res_mask_alpha]['id']:
                atom_dict['isalpha'][atom_dict['resid'] == i] = True

            res_mask_beta = np.where(((phi >= -180) & (phi < -40) & (psi <= 180) & (psi > 90)) | ((phi >= -180) & (phi < -70) & (psi <= -165))) # beta
            res_dict['isbeta'][res_mask_beta] = True
            atom_dict['isbeta'][np.in1d(atom_dict['resid'], res_dict[res_mask_beta]['id'])] = True

        # Aromatic Rings
        r = []
        for ring in self.sssr:
            if ring.IsAromatic():
                path = ring._path
                atom = atom_dict[atom_dict['id'] == path[0]]
                coords = atom_dict[np.in1d(atom_dict['id'], path)]['coords']
                centroid = coords.mean(axis=0)
                # get vector perpendicular to ring
                vector = np.cross(coords - np.vstack((coords[1:],coords[:1])), np.vstack((coords[1:],coords[:1])) - np.vstack((coords[2:],coords[:2]))).mean(axis=0) - centroid
                r.append((centroid, vector, atom['isalpha'], atom['isbeta']))
        ring_dict = np.array(r, dtype=[('centroid', 'float16', 3),('vector', 'float16', 3),('isalpha', 'bool'),('isbeta', 'bool'),])

        self._atom_dict = atom_dict
        self._ring_dict = ring_dict
        if self.protein:
            self._res_dict = res_dict

### Extend pybel.Molecule
pybel.Molecule = Molecule

class Atom(pybel.Atom):
    @property
    def neighbors(self):
        return [Atom(a) for a in OBAtomAtomIter(self.OBAtom)]

pybel.Atom = Atom

class Fingerprint(pybel.Fingerprint):
    @property
    def raw(self):
        return _unrollbits(self.fp, pybel.ob.OBFingerprint.Getbitsperint())

def _unrollbits(fp, bitsperint):
    """ Unroll unsigned int fingerprint to bool """
    ans = np.zeros(len(fp)*bitsperint)
    start = 1
    for x in fp:
        i = start
        while x > 0:
            ans[i] = x % 2
            x >>= 1
            i += 1
        start += bitsperint
    return ans

pybel.Fingerprint = Fingerprint

### Monkeypatch pybel objects pickling
pickle_format = 'mol2'
def pickle_mol(self):
    return unpickle_mol, (self.write(pickle_format), dict(self.data.items()))

def unpickle_mol(string, data):
    mol = readstring(pickle_format, string)
    mol.data.update(data)
    return mol
copy_reg.pickle(Molecule, pickle_mol, unpickle_mol)
<|endoftext|>


<commit_before>
import generals

# 1v1
g = generals.Generals('your userid', 'your username', '1v1')

# ffa
# g = generals.Generals('your userid', 'your username', 'ffa')

# private game
# g = generals.Generals('your userid', 'your username', 'private', 'your gameid')

# 2v2 game
# g = generals.Generals('your userid', 'your username', 'team')

for update in g.get_updates():

    # get position of your general
    pi = update['player_index']
    y, x = update['generals'][pi]

    # move units from general to arbitrary square
    for dy, dx in [(0, 1), (0, -1), (1, 0), (-1, 0)]:
        if (0 <= y+dy < update['rows'] and 0 <= x+dx < update['cols']
                and update['tile_grid'][y+dy][x+dx] != generals.MOUNTAIN):
            g.move(y, x, y+dy, x+dx)
            break
<commit_msg>make bot executable for easier testing<commit_after>#!/bin/python
import generals

# 1v1
g = generals.Generals('your userid', 'your username', '1v1')

# ffa
# g = generals.Generals('your userid', 'your username', 'ffa')

# private game
# g = generals.Generals('your userid', 'your username', 'private', 'your gameid')

# 2v2 game
# g = generals.Generals('your userid', 'your username', 'team')

for update in g.get_updates():

    # get position of your general
    pi = update['player_index']
    y, x = update['generals'][pi]

    # move units from general to arbitrary square
    for dy, dx in [(0, 1), (0, -1), (1, 0), (-1, 0)]:
        if (0 <= y+dy < update['rows'] and 0 <= x+dx < update['cols']
                and update['tile_grid'][y+dy][x+dx] != generals.MOUNTAIN):
            g.move(y, x, y+dy, x+dx)
            break
<|endoftext|>


<commit_before>import argparse
import os

from core.tape_parser import parse_tape, load_tape
from file_export.document_variable_factory import DocumentVariableFactory
from file_export.template_engine import TemplateEngine
from file_import.parser import parse_file
from utils.os_utils import os_open_command, open_with

class ExecuteTM:
    '''
    Hilfklasse zum ausführen von Turing maschinen.
    '''
    def __init__(self, turing_machine, observer=None):
        self.__turing_machine = turing_machine
        self.__observer = observer if observer else []

    def add_observer(self, observer):
        self.__observer.append(observer)

    def __init_observer(self, tape):
        states = self.__turing_machine.states
        for observer in self.__observer:
            observer.register_state(tape, states)

    def __notify_observer(self, transition_event=None, transition_target=None):
        for observer in self.__observer:
            observer.register_iteration(transition_event, transition_target)

    def execute_TM(self, input_string):

        tape = load_tape(input_string) if os.path.isfile(input_string) else parse_tape(input_string)
        self.__turing_machine.assert_charset(tape)
        self.__init_observer(tape)

        for transition_event, transition_target in self.__turing_machine.as_iterator(tape):
            self.__notify_observer(transition_event, transition_target)

    @staticmethod
    def _parse_file(filename):
        turing_machine = parse_file(filename)
        return ExecuteTM(turing_machine)

class VisualizeTM:
    def __init__(self, template_engine, viewer):
        self.__template_engine = template_engine
        self.__viewer = viewer

        self.__tape_item_limit = 12
        self.__doc_factory = None
        self.__generated_file = None

    @staticmethod
    def create(template_path, viewer=os_open_command()):
        template_engine = TemplateEngine.load(template_path)
        return VisualizeTM(template_engine, viewer)

    def register_state(self, tape, states):
        self.__doc_factory = DocumentVariableFactory(tape, states, self.__tape_item_limit)
        self.__doc_factory.add_iteration(None, None)

    def register_iteration(self, transition_event, transition_target):
        if transition_target:
            self.__doc_factory.add_iteration(transition_event, transition_target)
            self.__doc_factory.set_remark('Input is valid.')
        else:
            self.__doc_factory.set_remark('Input is invalid.')

    def set_viewer(self, viewername):
        self.__viewer = viewername

    def set_tape_item_limit(self, tape_item_limit):
        self.__tape_item_limit = tape_item_limit

    def write_file(self, output_dir='./target/'):
        if not self.__doc_factory:
            raise RuntimeError('No state registered.')

        if self.__doc_factory.empty():
            raise RuntimeError('No iterations registered.')

        os.makedirs(output_dir, exist_ok=True)
        doc_vars = self.__doc_factory.document_variables()
        self.__generated_file = self.__template_engine.create_document(doc_vars, output_dir)

    def visualize(self):
        if not self.__generated_file:
            self.write_file()

        open_with(self.__generated_file, self.__viewer)

class PrintTM:
    def register_state(self, tape, states):
        print('setting state of turing machine:\n tape=', tape, '\n states=', states, '\n')

    def _format_event(self, transition_event):
        return '{} read: {}'.format(transition_event.current_state.name, transition_event.read_chars)

    def _format_target(self, transition_target):
        return '{} write: {} move: {}'.format(transition_target.new_state.name,
                                              transition_target.new_chars,
                                              transition_target.move_directions)

    def register_iteration(self, transition_event, transition_target):
        if transition_event and transition_target:
            event_str = self._format_event(transition_event)
            target_str = self._format_target(transition_target)
            print(event_str, ' > ', target_str)
        else:
            print('Input string is invalid.')

'''
TODO parse commands
'''
parser = argparse.ArgumentParser(description='Visualizes Turing Machines')
parser.add_argument('--turing_program', help='the path to the turing program')
parser.add_argument('--tape', help='the path to the tape to run')
parser.add_argument('--out_dir', help='the path to the tape to run')

parser.add_argument('--interactive', help='the path to the turing program')
parser.add_argument('--template', help='the path to the turing program')
parser.add_argument('--tape_item_limit', help='the path to the turing program')
parser.add_argument('--verbose', help='the path to the turing program')

file_to_tm = '/home/martin_jutzi/Temp/divisiontest7.txt'
tm_executor = ExecuteTM._parse_file(file_to_tm)

template_path = '/home/martin_jutzi/PycharmProjects/TikzTuringSimulator/templates/latex'
tape_item_limit = 12
viewer = ''

visual_executor = VisualizeTM.create(template_path)
visual_executor.set_tape_item_limit(tape_item_limit)
# visual_executor.set_viewer(viewer)

tm_executor.add_observer(PrintTM())
tm_executor.add_observer(visual_executor)

# 0', '0', '1', '1', '1
tape_str = '1,1,1'
tm_executor.execute_TM(tape_str)

output_dir = '/home/martin_jutzi/Temp'
visual_executor.write_file(output_dir)
<commit_msg>Commit check<commit_after>import argparse
import os

from core.tape_parser import parse_tape, load_tape
from file_export.document_variable_factory import DocumentVariableFactory
from file_export.template_engine import TemplateEngine
from file_import.parser import parse_file
from utils.os_utils import os_open_command, open_with

class ExecuteTM:
    '''
    Hilfklasse zum ausführen von Turing maschinen.
    '''
    def __init__(self, turing_machine, observer=None):
        self.__turing_machine = turing_machine
        self.__observer = observer if observer else []

    def add_observer(self, observer):
        self.__observer.append(observer)

    def __init_observer(self, tape):
        states = self.__turing_machine.states
        for observer in self.__observer:
            observer.register_state(tape, states)

    def __notify_observer(self, transition_event=None, transition_target=None):
        for observer in self.__observer:
            observer.register_iteration(transition_event, transition_target)

    def execute_TM(self, input_string):

        tape = load_tape(input_string) if os.path.isfile(input_string) else parse_tape(input_string)
        self.__turing_machine.assert_charset(tape)
        self.__init_observer(tape)

        for transition_event, transition_target in self.__turing_machine.as_iterator(tape):
            self.__notify_observer(transition_event, transition_target)

    @staticmethod
    def _parse_file(filename):
        turing_machine = parse_file(filename)
        return ExecuteTM(turing_machine)

class VisualizeTM:
    def __init__(self, template_engine, viewer):
        self.__template_engine = template_engine
        self.__viewer = viewer

        self.__tape_item_limit = 12
        self.__doc_factory = None
        self.__generated_file = None

    @staticmethod
    def create(template_path, viewer=os_open_command()):
        template_engine = TemplateEngine.load(template_path)
        return VisualizeTM(template_engine, viewer)

    def register_state(self, tape, states):
        self.__doc_factory = DocumentVariableFactory(tape, states, self.__tape_item_limit)
        self.__doc_factory.add_iteration(None, None)

    def register_iteration(self, transition_event, transition_target):
        if transition_target:
            self.__doc_factory.add_iteration(transition_event, transition_target)
            self.__doc_factory.set_remark('Input is valid.')
        else:
            self.__doc_factory.set_remark('Input is invalid.')

    def set_viewer(self, viewername):
        self.__viewer = viewername

    def set_tape_item_limit(self, tape_item_limit):
        self.__tape_item_limit = tape_item_limit

    def write_file(self, output_dir='./target/'):
        if not self.__doc_factory:
            raise RuntimeError('No state registered.')

        if self.__doc_factory.empty():
            raise RuntimeError('No iterations registered.')

        os.makedirs(output_dir, exist_ok=True)
        doc_vars = self.__doc_factory.document_variables()
        self.__generated_file = self.__template_engine.create_document(doc_vars, output_dir)

    def visualize(self):
        if not self.__generated_file:
            self.write_file()

        open_with(self.__generated_file, self.__viewer)

class PrintTM:
    def register_state(self, tape, states):
        print('setting state of turing machine:\n tape=', tape, '\n states=', states, '\n')

    def _format_event(self, transition_event):
        return '{} read: {}'.format(transition_event.current_state.name, transition_event.read_chars)

    def _format_target(self, transition_target):
        return '{} write: {} move: {}'.format(transition_target.new_state.name,
                                              transition_target.new_chars,
                                              transition_target.move_directions)

    def register_iteration(self, transition_event, transition_target):
        if transition_event and transition_target:
            event_str = self._format_event(transition_event)
            target_str = self._format_target(transition_target)
            print(event_str, ' > ', target_str)
        else:
            print('Input string is invalid.')

'''
TODO debug commands
'''
parser = argparse.ArgumentParser(description='Visualizes Turing Machines')
parser.add_argument('--turing_program', help='the path to the turing program')
parser.add_argument('--tape', help='the path to the tape to run')
parser.add_argument('--out_dir', help='the path to the tape to run')

parser.add_argument('--interactive', help='the path to the turing program')
parser.add_argument('--template', help='the path to the turing program')
parser.add_argument('--tape_item_limit', help='the path to the turing program')
parser.add_argument('--verbose', help='the path to the turing program')

file_to_tm = '/home/martin_jutzi/Temp/divisiontest7.txt'
tm_executor = ExecuteTM._parse_file(file_to_tm)

template_path = '/home/martin_jutzi/PycharmProjects/TikzTuringSimulator/templates/latex'
tape_item_limit = 12
viewer = ''

visual_executor = VisualizeTM.create(template_path)
visual_executor.set_tape_item_limit(tape_item_limit)
# visual_executor.set_viewer(viewer)

tm_executor.add_observer(PrintTM())
tm_executor.add_observer(visual_executor)

# 0', '0', '1', '1', '1
tape_str = '1,1,1'
tm_executor.execute_TM(tape_str)

output_dir = '/home/martin_jutzi/Temp'
visual_executor.write_file(output_dir)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import json
import os
import glob
import time
import re
from terminaltables import AsciiTable
from colorclass import Color

from tasks import Tasks
from runner import Runner
from tool_parameters import ToolParametersHelper

root_dir = os.path.dirname(os.path.abspath(__file__))
src_dir = root_dir + '/src'

def get_builds(out_prefix):
    """Returns all the paths of all the builds in the build directory."""
    build_dir = os.path.join(root_dir, out_prefix)
    builds = []
    for content in os.listdir(build_dir):
        if os.path.isdir(os.path.join(build_dir, content)):
            builds.append(content)

    return builds

def print_summary_table(out_prefix, build_type, build_nr=None):
    """Prints a summary table of the outcome of each test."""
    builds = get_builds(out_prefix)
    table_data = [
        [
            'Project', 'Toolchain', 'Family', 'Part', 'Board', 'Build Type',
            'Build N.', 'Options'
        ]
    ]
    passed = failed = 0
    build_count = 0
    for build in sorted(builds):
        # Split directory name into columns
        # Example: oneblink_vpr_xc7_a35tcsg326-1_arty_generic-build_0_options
        pattern = ''
        for i in range(0, len(table_data[0]) - 1):
            pattern += '([^_]*)_'
        pattern += '(.*)'

        row = list(re.match(pattern, build).groups())

        if build_type != row[5] or (build_nr and int(build_nr) != int(row[6])):
            continue

        # Check if metadata was generated
        # It is created for successful builds only
        if os.path.exists(os.path.join(root_dir, out_prefix, build,
                                       'meta.json')):
            row.append(Color('{autogreen}passed{/autogreen}'))
            passed += 1
        else:
            row.append(Color('{autored}failed{/autored}'))
            failed += 1
        table_data.append(row)
        build_count += 1

    table_data.append(
        [
            Color('{autogreen}Passed:{/autogreen}'), passed,
            Color('{autored}Failed:{/autored}'), failed, '', '', '', '',
            '{}%'.format(int(passed / build_count * 100))
        ]
    )
    table = AsciiTable(table_data)
    table.inner_footing_row_border = True
    print(table.table)

    return failed == 0

def main():
    import argparse
    parser = argparse.ArgumentParser(
        description='Exhaustively try project-toolchain combinations'
    )
    parser.add_argument(
        '--project',
        default=None,
        nargs="+",
        help='run given project only (default: all)'
    )
    parser.add_argument(
        '--toolchain',
        default=None,
        nargs="+",
        help='run given toolchain only (default: all)'
    )
    parser.add_argument(
        '--out-prefix',
        default='build',
        help='output directory prefix (default: build)'
    )
    parser.add_argument(
        '--build_type',
        default='generic',
        help=
        'Type of build that is performed (e.g. regression test, multiple options, etc.)'
    )
    parser.add_argument('--build', default=None, help='Build number')
    parser.add_argument(
        '--parameters', default=None, help='Tool parameters json file'
    )
    parser.add_argument('--fail', action='store_true', help='fail on error')
    parser.add_argument(
        '--verbose', action='store_true', help='verbose output'
    )

    args = parser.parse_args()

    tasks = Tasks(src_dir)

    args_dict = {"project": args.project, "toolchain": args.toolchain}

    task_list = tasks.get_tasks(args_dict)

    params_file = args.parameters
    params_strings = [None]
    if params_file:
        assert len(
            args.toolchain
        ) == 1, "A single toolchain can be selected when running multiple params."

        params_helper = ToolParametersHelper(args.toolchain[0], params_file)
        for params in params_helper.get_all_params_combinations():
            params_strings.append(" ".join(params))

    runner = Runner(
        task_list, args.verbose, args.out_prefix, root_dir, args.build_type,
        args.build, params_strings
    )
    runner.run()
    runner.collect_results()

    result = print_summary_table(args.out_prefix, args.build_type, args.build)

    if not result and args.fail:
        print("ERROR: some tests have failed.")
        exit(1)

if __name__ == '__main__':
    main()
<commit_msg>exhaust: reset parameters list when the parameter type build is selected<commit_after>#!/usr/bin/env python3

import json
import os
import glob
import time
import re
from terminaltables import AsciiTable
from colorclass import Color

from tasks import Tasks
from runner import Runner
from tool_parameters import ToolParametersHelper

root_dir = os.path.dirname(os.path.abspath(__file__))
src_dir = root_dir + '/src'

def get_builds(out_prefix):
    """Returns all the paths of all the builds in the build directory."""
    build_dir = os.path.join(root_dir, out_prefix)
    builds = []
    for content in os.listdir(build_dir):
        if os.path.isdir(os.path.join(build_dir, content)):
            builds.append(content)

    return builds

def print_summary_table(out_prefix, build_type, build_nr=None):
    """Prints a summary table of the outcome of each test."""
    builds = get_builds(out_prefix)
    table_data = [
        [
            'Project', 'Toolchain', 'Family', 'Part', 'Board', 'Build Type',
            'Build N.', 'Options'
        ]
    ]
    passed = failed = 0
    build_count = 0
    for build in sorted(builds):
        # Split directory name into columns
        # Example: oneblink_vpr_xc7_a35tcsg326-1_arty_generic-build_0_options
        pattern = ''
        for i in range(0, len(table_data[0]) - 1):
            pattern += '([^_]*)_'
        pattern += '(.*)'

        row = list(re.match(pattern, build).groups())

        if build_type != row[5] or (build_nr and int(build_nr) != int(row[6])):
            continue

        # Check if metadata was generated
        # It is created for successful builds only
        if os.path.exists(os.path.join(root_dir, out_prefix, build,
                                       'meta.json')):
            row.append(Color('{autogreen}passed{/autogreen}'))
            passed += 1
        else:
            row.append(Color('{autored}failed{/autored}'))
            failed += 1
        table_data.append(row)
        build_count += 1

    table_data.append(
        [
            Color('{autogreen}Passed:{/autogreen}'), passed,
            Color('{autored}Failed:{/autored}'), failed, '', '', '', '',
            '{}%'.format(int(passed / build_count * 100))
        ]
    )
    table = AsciiTable(table_data)
    table.inner_footing_row_border = True
    print(table.table)

    return failed == 0

def main():
    import argparse
    parser = argparse.ArgumentParser(
        description='Exhaustively try project-toolchain combinations'
    )
    parser.add_argument(
        '--project',
        default=None,
        nargs="+",
        help='run given project only (default: all)'
    )
    parser.add_argument(
        '--toolchain',
        default=None,
        nargs="+",
        help='run given toolchain only (default: all)'
    )
    parser.add_argument(
        '--out-prefix',
        default='build',
        help='output directory prefix (default: build)'
    )
    parser.add_argument(
        '--build_type',
        default='generic',
        help=
        'Type of build that is performed (e.g. regression test, multiple options, etc.)'
    )
    parser.add_argument('--build', default=None, help='Build number')
    parser.add_argument(
        '--parameters', default=None, help='Tool parameters json file'
    )
    parser.add_argument('--fail', action='store_true', help='fail on error')
    parser.add_argument(
        '--verbose', action='store_true', help='verbose output'
    )

    args = parser.parse_args()

    tasks = Tasks(src_dir)

    args_dict = {"project": args.project, "toolchain": args.toolchain}

    task_list = tasks.get_tasks(args_dict)

    params_file = args.parameters
    params_strings = [None]
    if params_file:
        params_strings = []
        assert len(
            args.toolchain
        ) == 1, "A single toolchain can be selected when running multiple params."

        params_helper = ToolParametersHelper(args.toolchain[0], params_file)
        for params in params_helper.get_all_params_combinations():
            params_strings.append(" ".join(params))

    runner = Runner(
        task_list, args.verbose, args.out_prefix, root_dir, args.build_type,
        args.build, params_strings
    )
    runner.run()
    runner.collect_results()

    result = print_summary_table(args.out_prefix, args.build_type, args.build)

    if not result and args.fail:
        print("ERROR: some tests have failed.")
        exit(1)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
#
# SPDX-License-Identifier: MIT

import argparse

import os
import struct
import sys

sys.path.append("bitmap_font")
sys.path.append("../../tools/bitmap_font")

from adafruit_bitmap_font import bitmap_font

parser = argparse.ArgumentParser(description='Generate USB descriptors.')
parser.add_argument('--font', type=str,
                    help='Font path', required=True)
parser.add_argument('--extra_characters', type=str,
                    help='Unicode string of extra characters')
parser.add_argument('--sample_file', type=argparse.FileType('r'),
                    help='Text file that includes strings to support.')
parser.add_argument('--output_c_file', type=argparse.FileType('w'), required=True)

args = parser.parse_args()

class BitmapStub:
    def __init__(self, width, height, color_depth):
        self.width = width
        self.rows = [b''] * height

    def _load_row(self, y, row):
        self.rows[y] = bytes(row)

f = bitmap_font.load_font(args.font, BitmapStub)

# Load extra characters from the sample file.
sample_characters = set()
if args.sample_file:
    for line in args.sample_file:
        # Skip comments because we add additional characters in our huffman comments.
        if line.startswith("//"):
            continue
        for c in line.strip():
            sample_characters.add(c)

# Merge visible ascii, sample characters and extra characters.
visible_ascii = bytes(range(0x20, 0x7f)).decode("utf-8")
all_characters = visible_ascii
for c in sample_characters:
    if c not in all_characters:
        all_characters += c
if args.extra_characters:
    all_characters.extend(args.extra_characters)
all_characters = "".join(sorted(set(all_characters)))
filtered_characters = all_characters

# Try to pre-load all of the glyphs. Misses will still be slow later.
f.load_glyphs(set(ord(c) for c in all_characters))

missing = 0
# Get each glyph.
for c in set(all_characters):
    if ord(c) not in f._glyphs:
        missing += 1
        filtered_characters = filtered_characters.replace(c, "")
        continue
    g = f.get_glyph(ord(c))
    if g["shift"][1] != 0:
        raise RuntimeError("y shift")

if missing > 0:
    print("Font missing", missing, "characters", file=sys.stderr)

x, y, dx, dy = f.get_bounding_box()
tile_x, tile_y = x - dx, y - dy
total_bits = tile_x * len(all_characters)
total_bits += 32 - total_bits % 32
bytes_per_row = total_bits // 8
b = bytearray(bytes_per_row * tile_y)

for x, c in enumerate(filtered_characters):
    g = f.get_glyph(ord(c))
    start_bit = x * tile_x + g["bounds"][2]
    start_y = (tile_y - 2) - (g["bounds"][1] + g["bounds"][3])
    for y, row in enumerate(g["bitmap"].rows):
        for i in range(g["bounds"][0]):
            byte = i // 8
            bit = i % 8
            if row[byte] & (1 << (7-bit)) != 0:
                overall_bit = start_bit + (start_y + y) * bytes_per_row * 8 + i
                b[overall_bit // 8] |= 1 << (7 - (overall_bit % 8))

extra_characters = ""
for c in filtered_characters:
    if c not in visible_ascii:
        extra_characters += c

c_file = args.output_c_file

c_file.write("""\

#include "shared-bindings/displayio/Palette.h"
#include "supervisor/shared/display.h"

""")

c_file.write("""\
_displayio_color_t terminal_colors[2] = {
    {
        .rgb888 = 0x000000,
        .rgb565 = 0x0000,
        .luma = 0x00,
        .chroma = 0
    },
    {
        .rgb888 = 0xffffff,
        .rgb565 = 0xffff,
        .luma = 0xff,
        .chroma = 0
    },
};

displayio_palette_t supervisor_terminal_color = {
    .base = {.type = &displayio_palette_type },
    .colors = terminal_colors,
    .color_count = 2,
    .needs_refresh = false
};
""")

c_file.write("""\
displayio_tilegrid_t supervisor_terminal_text_grid = {{
    .base = {{ .type = &displayio_tilegrid_type }},
    .bitmap = (displayio_bitmap_t*) &supervisor_terminal_font_bitmap,
    .pixel_shader = &supervisor_terminal_color,
    .x = 16,
    .y = 0,
    .pixel_width = {1},
    .pixel_height = {2},
    .bitmap_width_in_tiles = {0},
    .tiles_in_bitmap = {0},
    .width_in_tiles = 1,
    .height_in_tiles = 1,
    .tile_width = {1},
    .tile_height = {2},
    .tiles = NULL,
    .partial_change = false,
    .full_change = false,
    .hidden = false,
    .hidden_by_parent = false,
    .moved = false,
    .inline_tiles = false,
    .in_group = true
}};
""".format(len(all_characters), tile_x, tile_y))

c_file.write("""\
const uint32_t font_bitmap_data[{}] = {{
""".format(bytes_per_row * tile_y // 4))

for i, word in enumerate(struct.iter_unpack(">I", b)):
    c_file.write("0x{:08x}, ".format(word[0]))
    if (i + 1) % (bytes_per_row // 4) == 0:
        c_file.write("\n")

c_file.write("""\
};
""")

c_file.write("""\
displayio_bitmap_t supervisor_terminal_font_bitmap = {{
    .base = {{.type = &displayio_bitmap_type }},
    .width = {},
    .height = {},
    .data = (size_t*) font_bitmap_data,
    .stride = {},
    .bits_per_value = 1,
    .x_shift = 5,
    .x_mask = 0x1f,
    .bitmask = 0x1,
    .read_only = true
}};
""".format(len(all_characters) * tile_x, tile_y, bytes_per_row / 4))

c_file.write("""\
const fontio_builtinfont_t supervisor_terminal_font = {{
    .base = {{.type = &fontio_builtinfont_type }},
    .bitmap = &supervisor_terminal_font_bitmap,
    .width = {},
    .height = {},
    .unicode_characters = (const uint8_t*) "{}",
    .unicode_characters_len = {}
}};
""".format(tile_x, tile_y, extra_characters, len(extra_characters.encode("utf-8"))))

c_file.write("""\
terminalio_terminal_obj_t supervisor_terminal = {
    .base = { .type = &terminalio_terminal_type },
    .font = &supervisor_terminal_font,
    .cursor_x = 0,
    .cursor_y = 0,
    .tilegrid = &supervisor_terminal_text_grid
};
""")
<commit_msg>fix bitmap_font path in display resource gen<commit_after># SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
#
# SPDX-License-Identifier: MIT

import argparse

import os
import struct
import sys

sys.path.insert(0, "bitmap_font")
sys.path.insert(0, "../../tools/bitmap_font")

from adafruit_bitmap_font import bitmap_font

parser = argparse.ArgumentParser(description='Generate USB descriptors.')
parser.add_argument('--font', type=str,
                    help='Font path', required=True)
parser.add_argument('--extra_characters', type=str,
                    help='Unicode string of extra characters')
parser.add_argument('--sample_file', type=argparse.FileType('r'),
                    help='Text file that includes strings to support.')
parser.add_argument('--output_c_file', type=argparse.FileType('w'), required=True)

args = parser.parse_args()

class BitmapStub:
    def __init__(self, width, height, color_depth):
        self.width = width
        self.rows = [b''] * height

    def _load_row(self, y, row):
        self.rows[y] = bytes(row)

f = bitmap_font.load_font(args.font, BitmapStub)

# Load extra characters from the sample file.
sample_characters = set()
if args.sample_file:
    for line in args.sample_file:
        # Skip comments because we add additional characters in our huffman comments.
        if line.startswith("//"):
            continue
        for c in line.strip():
            sample_characters.add(c)

# Merge visible ascii, sample characters and extra characters.
visible_ascii = bytes(range(0x20, 0x7f)).decode("utf-8")
all_characters = visible_ascii
for c in sample_characters:
    if c not in all_characters:
        all_characters += c
if args.extra_characters:
    all_characters.extend(args.extra_characters)
all_characters = "".join(sorted(set(all_characters)))
filtered_characters = all_characters

# Try to pre-load all of the glyphs. Misses will still be slow later.
f.load_glyphs(set(ord(c) for c in all_characters))

missing = 0
# Get each glyph.
for c in set(all_characters):
    if ord(c) not in f._glyphs:
        missing += 1
        filtered_characters = filtered_characters.replace(c, "")
        continue
    g = f.get_glyph(ord(c))
    if g["shift"][1] != 0:
        raise RuntimeError("y shift")

if missing > 0:
    print("Font missing", missing, "characters", file=sys.stderr)

x, y, dx, dy = f.get_bounding_box()
tile_x, tile_y = x - dx, y - dy
total_bits = tile_x * len(all_characters)
total_bits += 32 - total_bits % 32
bytes_per_row = total_bits // 8
b = bytearray(bytes_per_row * tile_y)

for x, c in enumerate(filtered_characters):
    g = f.get_glyph(ord(c))
    start_bit = x * tile_x + g["bounds"][2]
    start_y = (tile_y - 2) - (g["bounds"][1] + g["bounds"][3])
    for y, row in enumerate(g["bitmap"].rows):
        for i in range(g["bounds"][0]):
            byte = i // 8
            bit = i % 8
            if row[byte] & (1 << (7-bit)) != 0:
                overall_bit = start_bit + (start_y + y) * bytes_per_row * 8 + i
                b[overall_bit // 8] |= 1 << (7 - (overall_bit % 8))

extra_characters = ""
for c in filtered_characters:
    if c not in visible_ascii:
        extra_characters += c

c_file = args.output_c_file

c_file.write("""\

#include "shared-bindings/displayio/Palette.h"
#include "supervisor/shared/display.h"

""")

c_file.write("""\
_displayio_color_t terminal_colors[2] = {
    {
        .rgb888 = 0x000000,
        .rgb565 = 0x0000,
        .luma = 0x00,
        .chroma = 0
    },
    {
        .rgb888 = 0xffffff,
        .rgb565 = 0xffff,
        .luma = 0xff,
        .chroma = 0
    },
};

displayio_palette_t supervisor_terminal_color = {
    .base = {.type = &displayio_palette_type },
    .colors = terminal_colors,
    .color_count = 2,
    .needs_refresh = false
};
""")

c_file.write("""\
displayio_tilegrid_t supervisor_terminal_text_grid = {{
    .base = {{ .type = &displayio_tilegrid_type }},
    .bitmap = (displayio_bitmap_t*) &supervisor_terminal_font_bitmap,
    .pixel_shader = &supervisor_terminal_color,
    .x = 16,
    .y = 0,
    .pixel_width = {1},
    .pixel_height = {2},
    .bitmap_width_in_tiles = {0},
    .tiles_in_bitmap = {0},
    .width_in_tiles = 1,
    .height_in_tiles = 1,
    .tile_width = {1},
    .tile_height = {2},
    .tiles = NULL,
    .partial_change = false,
    .full_change = false,
    .hidden = false,
    .hidden_by_parent = false,
    .moved = false,
    .inline_tiles = false,
    .in_group = true
}};
""".format(len(all_characters), tile_x, tile_y))

c_file.write("""\
const uint32_t font_bitmap_data[{}] = {{
""".format(bytes_per_row * tile_y // 4))

for i, word in enumerate(struct.iter_unpack(">I", b)):
    c_file.write("0x{:08x}, ".format(word[0]))
    if (i + 1) % (bytes_per_row // 4) == 0:
        c_file.write("\n")

c_file.write("""\
};
""")

c_file.write("""\
displayio_bitmap_t supervisor_terminal_font_bitmap = {{
    .base = {{.type = &displayio_bitmap_type }},
    .width = {},
    .height = {},
    .data = (size_t*) font_bitmap_data,
    .stride = {},
    .bits_per_value = 1,
    .x_shift = 5,
    .x_mask = 0x1f,
    .bitmask = 0x1,
    .read_only = true
}};
""".format(len(all_characters) * tile_x, tile_y, bytes_per_row / 4))

c_file.write("""\
const fontio_builtinfont_t supervisor_terminal_font = {{
    .base = {{.type = &fontio_builtinfont_type }},
    .bitmap = &supervisor_terminal_font_bitmap,
    .width = {},
    .height = {},
    .unicode_characters = (const uint8_t*) "{}",
    .unicode_characters_len = {}
}};
""".format(tile_x, tile_y, extra_characters, len(extra_characters.encode("utf-8"))))

c_file.write("""\
terminalio_terminal_obj_t supervisor_terminal = {
    .base = { .type = &terminalio_terminal_type },
    .font = &supervisor_terminal_font,
    .cursor_x = 0,
    .cursor_y = 0,
    .tilegrid = &supervisor_terminal_text_grid
};
""")
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding:utf-8 -*-

from setuptools import setup, find_packages
import sys
import os

if sys.argv[-1] == 'publish':
    os.system("python setup.py bdist_wheel --python-tag cp35 -p win-amd64")
    os.system("twine upload dist/*")
    sys.exit()

hku_name = "Hikyuu"
hku_version = "1.0.3"
hku_author = "fasiondog"
hku_author_email = "fasiondog@sina.com"

hku_license = "MIT"
hku_keywords = ("hikyuu", "quant", "trade", "System Trading", "backtester", "量化", "程序化交易", "量化交易", "系统交易")
hku_platforms = "Independant"
hku_url = "http://hikyuu.org/"

hku_description = "Hikyuu Quant Framework for System Trading Analysis and backtester"
with open("README.rst", encoding='utf-8') as f:
    hku_long_description = f.read()

if sys.platform == 'win32':
    hku_data_files = [('Lib/site-packages/hikyuu', ['hikyuu/hikyuu.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/hikyuu_utils.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/sqlite3.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/importdata.exe']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/boost_chrono-vc100-mt-1_56.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/boost_date_time-vc100-mt-1_56.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/boost_filesystem-vc100-mt-1_56.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/boost_python-vc100-mt-1_56.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/boost_serialization-vc100-mt-1_56.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/boost_system-vc100-mt-1_56.dll']),
                      #('Lib/site-packages/hikyuu', ['hikyuu/boost_thread-vc100-mt-1_56.dll']),
                      #('Lib/site-packages/hikyuu', ['hikyuu/boost_unit_test_framework-vc100-mt-1_56.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/log4cplus.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/libmysql.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/hdf5dll.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/hdf5_hldll.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/hdf5_cppdll.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/hdf5_hl_cppdll.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/szip.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/zlib.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/msvcp100.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/msvcr100.dll'])]
else:
    hku_data_files = []

setup(
        name = hku_name,
        version = hku_version,
        description = hku_description,
        long_description = hku_long_description,
        author = hku_author,
        author_email = hku_author_email,

        license = hku_license,
        keywords = hku_keywords,
        platforms = hku_platforms,
        url = hku_url,

        packages = find_packages(),
        zip_safe = False,
        include_package_data=True,
        package_data = {
            '': ['*.rst', '*.pyd', '*.ini', '*.sql', '*.properties', '*.xml'],
        },

        data_files = hku_data_files,

        classifiers=[
            # How mature is this project? Common values are
            #   3 - Alpha
            #   4 - Beta
            #   5 - Production/Stable
            'Development Status :: 3 - Alpha',

            # Indicate who your project is intended for
            'Intended Audience :: Developers',
            'Topic :: Software Development :: Libraries :: Python Modules',
            'Topic :: Office/Business :: Financial',
            'Topic :: Office/Business :: Financial :: Investment',
            'Topic :: Scientific/Engineering :: Mathematics',

            # Pick your license as you wish (should match "license" above)
             'License :: OSI Approved :: MIT License',

            'Operating System :: Microsoft :: Windows',

            # Specify the Python versions you support here. In particular, ensure
            # that you indicate whether you support Python 2, Python 3 or both.
            'Programming Language :: Python :: 3',
            'Programming Language :: Python :: 3.5',
        ],

        install_requires=[
            'numpy>=1.10.4',
            'matplotlib>=1.5.0',
            'pandas>=0.17.1',
            'tushare>=0.8.2',
            'pyperclip>=1.5.27',
        ],
        )<commit_msg>update setup.py<commit_after>#!/usr/bin/env python
#-*- coding:utf-8 -*-

from setuptools import setup, find_packages
import sys
import os

if sys.argv[-1] == 'publish':
    os.system("python setup.py bdist_wheel --python-tag cp35 -p win-amd64")
    os.system("twine upload dist/*")
    sys.exit()

hku_name = "Hikyuu"
hku_version = "1.0.3"
hku_author = "fasiondog"
hku_author_email = "fasiondog@sina.com"

hku_license = "MIT"
hku_keywords = ("hikyuu", "quant", "trade", "System Trading", "backtester", "量化", "程序化交易", "量化交易", "系统交易")
hku_platforms = "Independant"
hku_url = "http://hikyuu.org/"

hku_description = "Hikyuu Quant Framework for System Trading Analysis and backtester"
with open("README.rst", encoding='utf-8') as f:
    hku_long_description = f.read()

if sys.platform == 'win32':
    hku_data_files = [('Lib/site-packages/hikyuu', ['hikyuu/hikyuu.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/hikyuu_utils.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/sqlite3.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/importdata.exe']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/boost_chrono-vc100-mt-1_56.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/boost_date_time-vc100-mt-1_56.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/boost_filesystem-vc100-mt-1_56.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/boost_python-vc100-mt-1_56.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/boost_serialization-vc100-mt-1_56.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/boost_system-vc100-mt-1_56.dll']),
                      #('Lib/site-packages/hikyuu', ['hikyuu/boost_thread-vc100-mt-1_56.dll']),
                      #('Lib/site-packages/hikyuu', ['hikyuu/boost_unit_test_framework-vc100-mt-1_56.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/log4cplus.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/libmysql.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/hdf5dll.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/hdf5_hldll.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/hdf5_cppdll.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/hdf5_hl_cppdll.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/szip.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/zlib.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/msvcp100.dll']),
                      ('Lib/site-packages/hikyuu', ['hikyuu/msvcr100.dll'])]
else:
    hku_data_files = []

setup(
        name = hku_name,
        version = hku_version,
        description = hku_description,
        long_description = hku_long_description,
        author = hku_author,
        author_email = hku_author_email,

        license = hku_license,
        keywords = hku_keywords,
        platforms = hku_platforms,
        url = hku_url,

        packages = find_packages(),
        zip_safe = False,
        include_package_data=True,
        package_data = {
            '': ['*.rst', '*.pyd', '*.ini', '*.sql', '*.properties', '*.xml'],
        },

        data_files = hku_data_files,

        classifiers=[
            # How mature is this project? Common values are
            #   3 - Alpha
            #   4 - Beta
            #   5 - Production/Stable
            'Development Status :: 3 - Alpha',

            # Indicate who your project is intended for
            'Intended Audience :: Developers',
            'Topic :: Software Development :: Libraries :: Python Modules',
            'Topic :: Office/Business :: Financial',
            'Topic :: Office/Business :: Financial :: Investment',
            'Topic :: Scientific/Engineering :: Mathematics',

            # Pick your license as you wish (should match "license" above)
             'License :: OSI Approved :: MIT License',

            'Operating System :: Microsoft :: Windows',

            # Specify the Python versions you support here. In particular, ensure
            # that you indicate whether you support Python 2, Python 3 or both.
            'Programming Language :: Python :: 3',
            'Programming Language :: Python :: 3.5',
        ],

        install_requires=[
            #'numpy>=1.10.4',
            #'matplotlib>=1.5.0',
            #'pandas>=0.17.1',
            #'tushare>=0.8.2',
            'pyperclip>=1.5.27',
        ],
        )<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os

from django.core.management.base import BaseCommand
from django.conf import settings
from django.core.files import File

from legacy.models import legacyLeaflet, legacyParty

from leaflets.models import Leaflet, LeafletImage
from constituencies.models import Constituency
from uk_political_parties.models import Party

class Command(BaseCommand):

    def clean_legacy_leaflet(self, legacy_leaflet, party=None, constituency=None):
        data = legacy_leaflet.__dict__.copy()
        del data['publisher_party_id']
        del data['_publisher_party_cache']
        del data['_state']
        data['publisher_party'] = party
        data['constituency'] = constituency
        if data.get('live'):
            data['status'] = 'live'
        else:
            data['status'] = 'removed'
        del data['live']
        return data

    def clean_legacy_leaflet_image(self, legacy_image):
        data = {}
        key = "%s.jpg" % legacy_image.image_key
        if getattr(settings, 'IMAGE_LOCAL_CACHE'):
            image_path = os.path.join(
                settings.IMAGE_LOCAL_CACHE,
                key
            )

            if os.path.exists(image_path):
                print "Exists"
                f = open(image_path, 'r')
                data['image'] = File(f)
            else:
                image_path = os.path.join(
                    settings.IMAGE_LOCAL_CACHE,
                    'large',
                    key
                )
                if os.path.exists(image_path):
                    print "Exists"
                    f = open(image_path, 'r')
                    data['image'] = File(f)
                else:
                    print "Doesn't exist"
            return data

    def clean_constituency(self, con):
        con_name = con.constituency.name
        if con_name == "Ynys Mon":
            con_name = "Ynys Môn"
        if con_name == "Cotswold":
            con_name = "The Cotswolds"
        if con_name == "Taunton":
            con_name = "Taunton Deane"
        try:
            con = Constituency.objects.get(name__iexact=con_name)
        except Constituency.DoesNotExist:
            con_name = ", ".join(con_name.split(' ', 1))
            con = Constituency.objects.get(name=con_name)
        return con

    def handle(self, **options):
        for legacy_leaflet in legacyLeaflet.objects.all():
            if not legacy_leaflet.date_uploaded:
                if legacy_leaflet.date_delivered:
                    legacy_leaflet.date_uploaded = legacy_leaflet.date_delivered

            if legacy_leaflet.date_uploaded:
                if not bool(legacy_leaflet.publisher_party_id):
                    party = None
                else:
                    party = Party.objects.find_party_by_name(
                        legacy_leaflet.publisher_party.name)

                    cons = legacy_leaflet.legacyleafletconstituency_set.all()
                    con = None
                    if cons:
                        con = cons[0]
                        con = self.clean_constituency(con)

                    # import ipdb
                    # ipdb.set_trace()
                    new_leaflet, created = Leaflet.objects.update_or_create(
                        pk=legacy_leaflet.pk,
                        defaults=self.clean_legacy_leaflet(
                            legacy_leaflet,
                            party,
                            constituency=con
                            ))

                    # for legacy_image in legacy_leaflet.images.all():
                    #     new_image, created = LeafletImage.objects.update_or_create(
                    #     leaflet=new_leaflet,
                    #     legacy_image_key=legacy_image.image_key,
                    #     defaults=self.clean_legacy_leaflet_image(legacy_image))
                    #
                    #<commit_msg>Import images with leaflets<commit_after># -*- coding: utf-8 -*-
import os

from django.core.management.base import BaseCommand
from django.conf import settings
from django.core.files import File

from legacy.models import legacyLeaflet, legacyParty

from leaflets.models import Leaflet, LeafletImage
from constituencies.models import Constituency
from uk_political_parties.models import Party

class Command(BaseCommand):

    def clean_legacy_leaflet(self, legacy_leaflet, party=None, constituency=None):
        data = legacy_leaflet.__dict__.copy()
        del data['publisher_party_id']
        del data['_publisher_party_cache']
        del data['_state']
        data['publisher_party'] = party
        data['constituency'] = constituency
        if data.get('live'):
            data['status'] = 'live'
        else:
            data['status'] = 'removed'
        del data['live']
        return data

    def clean_legacy_leaflet_image(self, legacy_image):
        data = {}
        key = "%s.jpg" % legacy_image.image_key
        if getattr(settings, 'IMAGE_LOCAL_CACHE'):
            image_path = os.path.join(
                settings.IMAGE_LOCAL_CACHE,
                key
            )

            if os.path.exists(image_path):
                print "Exists"
                f = open(image_path, 'r')
                data['image'] = File(f)
            else:
                image_path = os.path.join(
                    settings.IMAGE_LOCAL_CACHE,
                    'large',
                    key
                )
                if os.path.exists(image_path):
                    print "Exists"
                    f = open(image_path, 'r')
                    data['image'] = File(f)
                else:
                    print "Doesn't exist"
            return data

    def clean_constituency(self, con):
        con_name = con.constituency.name
        if con_name == "Ynys Mon":
            con_name = "Ynys Môn"
        if con_name == "Cotswold":
            con_name = "The Cotswolds"
        if con_name == "Taunton":
            con_name = "Taunton Deane"
        try:
            con = Constituency.objects.get(name__iexact=con_name)
        except Constituency.DoesNotExist:
            con_name = ", ".join(con_name.split(' ', 1))
            con = Constituency.objects.get(name=con_name)
        return con

    def handle(self, **options):
        for legacy_leaflet in legacyLeaflet.objects.all():
            if not legacy_leaflet.date_uploaded:
                if legacy_leaflet.date_delivered:
                    legacy_leaflet.date_uploaded = legacy_leaflet.date_delivered

            if legacy_leaflet.date_uploaded:
                if not bool(legacy_leaflet.publisher_party_id):
                    party = None
                else:
                    party = Party.objects.find_party_by_name(
                        legacy_leaflet.publisher_party.name)

                    cons = legacy_leaflet.legacyleafletconstituency_set.all()
                    con = None
                    if cons:
                        con = cons[0]
                        con = self.clean_constituency(con)

                    # import ipdb
                    # ipdb.set_trace()
                    new_leaflet, created = Leaflet.objects.update_or_create(
                        pk=legacy_leaflet.pk,
                        defaults=self.clean_legacy_leaflet(
                            legacy_leaflet,
                            party,
                            constituency=con
                            ))

                    for legacy_image in legacy_leaflet.images.all():
                        new_image, created = LeafletImage.objects.update_or_create(
                        leaflet=new_leaflet,
                        legacy_image_key=legacy_image.image_key,
                        defaults=self.clean_legacy_leaflet_image(legacy_image))
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import argparse
import sys
import os
import fsb5

parser = argparse.ArgumentParser(description='Extract audio samples from FSB5 files')
parser.add_argument('fsb_file', nargs='*', type=argparse.FileType('rb'), default=[sys.stdin.buffer.raw ],
						help='FSB5 container to extract audio from (defaults to stdin)')
parser.add_argument('-o', '--output-directory', default='out',
						help='output directory to write extracted samples into')
parser.add_argument('-p', '--prefix-samples', action='store_true',
						help='prefix extracted samples with the filename of the FSB container they were extracted from')
parser.add_argument('-q', '--quiet', action='store_true',
						help='suppress output of header and sample information (samples that failed to decode will still be printed)')
parser.add_argument('-r', '--resource', action='store_true',
						help='read multiple FSB5 files packed into the same file (e.g. Unity3D\'s .resource files)')

def write_to_file(directory, filename_prefix, filename, extension, contents):
	if not os.path.exists(directory):
		os.makedirs(directory)

	if filename_prefix:
		path = os.path.join(directory, '{0}.{1}.{2}'.format(filename_prefix, filename, extension))
	else:
		path = os.path.join(directory, '{0}.{1}'.format(filename, extension))

	with open(path, 'wb') as f:
		written = f.write(contents)
	return path

def extract_fsb(name, data, prefix, output_directory, printq):
	fsb = fsb5.load(data)
	ext = fsb.get_sample_extension()

	printq('''Header:
	Version: 5.{header.version}
	Number of samples: {header.numSamples}
	Samples have names: {has_names}
	Sound format: {sound_format}'''.format(
		header=fsb.header,
		has_names='Yes' if fsb.header.nameTableSize else 'No',
		sound_format=fsb.header.mode.name.lower().capitalize()))

	failed, written = [], []
	printq('Samples: ')
	for sample in fsb.samples:
		printq('''\t{sample.name}.{extension}:
		Frequency: {sample.frequency}
		Channels: {sample.channels}
		Samples: {sample.samples}'''.format(sample=sample, extension=ext))
		if sample.metadata:
			for meta_type, meta_value in sample.metadata.items():
				if type(meta_type) is fsb5.MetadataChunkType:
					contents = str(meta_value)
					if len(contents) > 45:
						contents = contents[:45] + '... )'
					printq('\t\t{type}: {contents}'.format(type=meta_type.name, contents=contents))
				else:
					printq('\t\t<unknown metadata type: {0}'.format(meta_type))

		sample_fakepath = '{0}:{1}.{2}'.format(name, sample.name, ext)
		try:
			outpath = write_to_file(output_directory, prefix, sample.name, ext, fsb.rebuild_sample(sample))
			written.append((sample_fakepath, outpath))
		except ValueError as e:
			failed.append((sample_fakepath, e))
			printq(e)
	printq()
	printq()

	return failed, written, fsb.raw_size

def main():
	args = parser.parse_args()
	if args.fsb_file[0].fileno() == 0 and args.prefix_samples:
		parser.error('Cannot prefix samples with filename when input is stdin')
	def printq(*a):
		if args.quiet:
			return
		else:
			print(*a)

	failed, written = [], []
	for f in args.fsb_file:
		data = f.read()
		prefix = os.path.splitext(os.path.basename(f.name))[0] if args.prefix_samples else ''

		printq('Reading FSB5 container: {0}'.format(f.name))

		if args.resource:
			index = 0
			while data:
				nfailed, nwritten, raw_size = extract_fsb(
					'{0}:{1}'.format(f.name, index),
					data,
					'{0}.{1}'.format(prefix, index) if prefix else str(index),
					args.output_directory,
					printq
				)
				data = data[raw_size:]
				index += 1
				failed += nfailed
				written += nwritten
		else:
			nfailed, nwritten, _ = extract_fsb(f.name, data, prefix, args.output_directory, printq)
			failed += nfailed
			written += nwritten

	print('The following filed were extracted:')
	for sample_fakepath, outpath in written:
		print('\t' + sample_fakepath, '->', outpath)

	if failed:
		print('The following samples failed to be decoded:')
		for sample_fakepath, reason in failed:
			print('\t' + sample_fakepath, reason)

if __name__ == '__main__':
	main()
<commit_msg>Rewrite extractor as a class<commit_after>#!/usr/bin/env python3

import argparse
import sys
import os
import fsb5

class FSBExtractor:
	description = 'Extract audio samples from FSB5 files'

	def __init__(self):
		self.parser = self.init_parser()

	def init_parser(self):
		parser = argparse.ArgumentParser(description=self.description)
		parser.add_argument('fsb_file',
			nargs='*', type=argparse.FileType('rb'), default=[sys.stdin.buffer.raw],
			help='FSB5 container to extract audio from (defaults to stdin)'
		)
		parser.add_argument('-o', '--output-directory', default='out/',
			help='output directory to write extracted samples into'
		)
		parser.add_argument('-p', '--prefix-samples', action='store_true',
			help='prefix extracted samples with the filename of the FSB container they were extracted from'
		)
		parser.add_argument('-q', '--quiet', action='store_true',
			help='suppress most output'
		)
		parser.add_argument('-r', '--resource', action='store_true',
			help="read multiple FSB5 files packed into the same file (e.g. Unity3D's .resource files)"
		)

		return parser

	def print(self, *args):
		if not self.args.quiet:
			print(*args)

	def error(self, *args):
		print(*args, file=sys.stderr)

	def write_to_file(self, filename_prefix, filename, extension, contents):
		directory = self.args.output_directory

		if not os.path.exists(directory):
			os.makedirs(directory)

		if filename_prefix:
			path = os.path.join(directory, '{0}.{1}.{2}'.format(filename_prefix, filename, extension))
		else:
			path = os.path.join(directory, '{0}.{1}'.format(filename, extension))

		with open(path, 'wb') as f:
			written = f.write(contents)
		return path

	def extract_fsb(self, name, data, prefix):
		fsb = fsb5.load(data)
		ext = fsb.get_sample_extension()

		self.print('Header:')
		self.print('\tVersion: 5.%s' % (fsb.header.version))
		self.print('\tSample count: %i' % (fsb.header.numSamples))
		self.print('\tNamed samples: %s' % ('Yes' if fsb.header.nameTableSize else 'No'))
		self.print('\tSound format: %s' % (fsb.header.mode.name.capitalize()))

		failed, written = [], []
		self.print('Samples:')
		for sample in fsb.samples:
			self.print('\t%s.%s' % (sample.name, ext))
			self.print('Frequency: %iHz' % (sample.frequency))
			self.print('Channels: %i' % (sample.channels))
			self.print('Samples %r' % (sample.samples))

			if sample.metadata:
				for meta_type, meta_value in sample.metadata.items():
					if type(meta_type) is fsb5.MetadataChunkType:
						contents = str(meta_value)
						if len(contents) > 45:
							contents = contents[:45] + '... )'
						self.print('\t%s: %s' % (meta_type.name, contents))
					else:
						self.print('\t<unknown metadata type: %r>' % (meta_type))

			sample_fakepath = '{0}:{1}.{2}'.format(name, sample.name, ext)
			try:
				outpath = self.write_to_file(prefix, sample.name, ext, fsb.rebuild_sample(sample))
				written.append((sample_fakepath, outpath))
			except ValueError as e:
				failed.append((sample_fakepath, e))
				self.error(e)

		self.print('\n')

		return failed, written, fsb.raw_size

	def handle_file(self, f):
		data = f.read()
		prefix = os.path.splitext(os.path.basename(f.name))[0] if self.args.prefix_samples else ''

		self.print('Reading FSB5 container: %s' % (f.name))

		if self.args.resource:
			index = 0
			while data:
				nfailed, nwritten, raw_size = self.extract_fsb(
					'{0}:{1}'.format(f.name, index),
					data,
					'{0}.{1}'.format(prefix, index) if prefix else str(index)
				)
				data = data[raw_size:]
				index += 1
		else:
			nfailed, nwritten, _ = self.extract_fsb(f.name, data, prefix)

		return nfailed, nwritten

	def run(self, args):
		self.args = self.parser.parse_args(args)

		if self.args.fsb_file[0].fileno() == 0 and self.args.prefix_samples:
			parser.error('Cannot prefix samples with filename when input is stdin')

		failed, written = [], []
		for f in self.args.fsb_file:
			nfailed, nwritten = self.handle_file(f)
			failed += nfailed
			written += nwritten

		print('The following files were extracted:')
		for sample_fakepath, outpath in written:
			print('\t' + sample_fakepath, '->', outpath)

		if failed:
			print('The following samples failed to be decoded:')
			for sample_fakepath, reason in failed:
				print('\t' + sample_fakepath, reason)

		return 0

def main():
	app = FSBExtractor()
	exit(app.run(sys.argv[1:]))

if __name__ == '__main__':
	main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

print '["http://box.scraperwiki.com/box/foobar/1234567890/http/spreadsheet.xls"]'<commit_msg>python extraction script turns sqlite api response into xlsx file<commit_after>#!/usr/bin/env python

import requests
from openpyxl.workbook import Workbook
from openpyxl.writer.excel import ExcelWriter

debug = False

def sqlite(dataset_box_url, q):
    if debug: print 'GET %s/sqlite?q=%s' % (dataset_box_url, q)
    return requests.get(dataset_box_url + '/sqlite', params={'q': q})

def extract(dataset_box_url):
    r = sqlite(dataset_box_url, 'select name from sqlite_master where type="table"')
    tables = [ x['name'] for x in r.json ]

    wb = Workbook()
    wb.remove_sheet(wb.get_active_sheet())

    for table in tables:
        if debug: print table
        ws = wb.create_sheet(title=table)
        r = sqlite(dataset_box_url, 'select * from [%s]' % table)
        data = r.json
        ws.append(data[0].keys())
        for row in data:
            ws.append(row.values())
        ws.cell(row=1, column=1).value = table

    wb.save(filename='http/spreadsheet.xlsx')
    if not debug:
        print '["spreadsheet.xlsx"]'

try:
    extract('https://box.scraperwiki.com/zarino.lastfm/f2cf672ac2b8405')
except Exception as e:
    print '"We encountered a Python error while extracting your dataset: %s"' % e
    if debug: raise
<|endoftext|>


<commit_before>from optparse import OptionParser
import commands
import os
from os import system
import socket

parser = OptionParser()
parser.add_option("-e", "--exp", action="store", type="string", dest="exp", default="contention")
parser.add_option("-s", "--short", action="store_true", dest="short", default=False)
parser.add_option("-p", "--print", action="store_true", dest="dprint", default=False)
parser.add_option("-a", "--allocate", action="store_true", dest="allocate", default=False)
parser.add_option("-n", "--ncores", action="store", type="int", dest="default_ncores", default=8)
parser.add_option("-c", "--contention", action="store", type="int", dest="default_contention", default=100000)
parser.add_option("-r", "--rr", action="store", type="int", dest="read_rate", default=50)
parser.add_option("-l", "--latency", action="store_true", dest="latency", default=False)
parser.add_option("-x", "--rlock", action="store_false", dest="rlock", default=True)
parser.add_option("-m", "--scp", action="store_true", dest="scp", default=False)

(options, args) = parser.parse_args()

ben_list_cpus = "socket@0,1,2,7,3-6"

LATENCY_PART = " -latency=%s" % options.latency

BASE_CMD = "GOGC=500 numactl -C `list-cpus seq -n %d %s` ./buy -ngo %d -nprocs %d -nsec %d -contention %d -rr %d -allocate=%s -sys=%d -rlock=%s" + LATENCY_PART

def run_one(fn, cmd):
    if options.dprint:
        print cmd
    status, output = commands.getstatusoutput(cmd)
    if status != 0:
        print "Bad status", status, output
        exit(1)
    if options.dprint:
        print output
    x = output.split()[23].rstrip(',')
    tps = float(x)
    fn.write("%0.2f\t" % tps)

def get_cpus(host):
    ncpus = [1, 2, 4, 8]
    if host == "mat":
        ncpus = [1, 2, 4, 8, 12, 24]
    elif host == "tbilisi":
        ncpus = [1, 2, 4, 8, 12]
    elif host == "tom":
        ncpus = [1, 2, 6, 12, 18, 24, 30, 42, 48]
    elif host == "ben":
        ncpus = [1, 4, 10, 20, 30, 40, 50, 60, 70, 80]
    if options.short:
        ncpus=[2, 4]
    return ncpus

def fill_cmd(rr, contention, ncpus, systype, cpus_arg=""):
    nsec = 5
    if options.short:
        nsec = 1
    cmd = BASE_CMD % (ncpus, cpus_arg, ncpus, ncpus, nsec, contention, rr, options.allocate, systype, options.rlock)
    return cmd

def do(f, rr, contention, ncpu, list_cpus, sys):
    cmd = fill_cmd(rr, contention, ncpu, sys, list_cpus)
    run_one(f, cmd)
    f.write("\t")

# x-axis is # cores
def contention_exp(fnpath, host, contention, rr):
    fnn = '%s-scalability-%d-%d-False.data' % (host, contention, rr)
    filename=os.path.join(fnpath, fnn)
    f = open(filename, 'w')
    cpus = get_cpus(host)
    f.write("#Doppel\tOCC\n")
    cpu_args = ""
    if host == "ben":
        cpu_args = ben_list_cpus

    for i in cpus:
        f.write("%d"% i)
        f.write("\t")
        do(f, rr, contention, i, cpu_args, 0)
        do(f, rr, contention, i, cpu_args, 1)
        f.write("\n")
    f.close()
    if options.scp:
        system("scp %s tbilisi.csail.mit.edu:/home/neha/src/txn/src/txn/data/" % filename)
        system("scp %s tbilisi.csail.mit.edu:/home/neha/doc/ddtxn-doc/graphs/" % filename)

def rw_exp(fnpath, host, contention, ncores):
    fnn = '%s-rw-%d-%d-False.data' % (host, contention, ncores)
    filename=os.path.join(fnpath, fnn)
    f = open(filename, 'w')
    rr = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100]
    if options.short:
        rr = [0, 50, 100]
    cpu_args = ""
    if host == "ben":
        cpu_args = ben_list_cpus
    f.write("#Doppel\tOCC\n")
    for i in rr:
        f.write("%d"% i)
        f.write("\t")
        do(f, i, contention, ncores, cpu_args, 0)
        do(f, i, contention, ncores, cpu_args, 1)
        f.write("\n")
    f.close()
    if options.scp:
        system("scp %s tbilisi.csail.mit.edu:/home/neha/src/txn/src/txn/data/" % filename)
        system("scp %s tbilisi.csail.mit.edu:/home/neha/doc/ddtxn-doc/graphs/" % filename)

def products_exp(fnpath, host, rr, ncores):
    fnn = '%s-products-%d-%d-True.data' % (host, rr, ncores)
    filename=os.path.join(fnpath, fnn)
    f = open(filename, 'w')
    cont = [1, 10, 100, 1000, 5000, 10000, 100000, 200000, 500000, 1000000]
    if options.short:
        cont = [100, 100000]
    cpu_args = ""
    if host == "ben":
        cpu_args = ben_list_cpus

    f.write("#Doppel\tOCC\n")
    for i in cont:
        f.write("%d"% i)
        f.write("\t")
        do(f, rr, i, ncores, cpu_args, 0)
        do(f, rr, i, ncores, cpu_args, 1)
        f.write("\n")
    f.close()
    if options.scp:
        system("scp %s tbilisi.csail.mit.edu:/home/neha/src/txn/src/txn/data/" % filename)
        system("scp %s tbilisi.csail.mit.edu:/home/neha/doc/ddtxn-doc/graphs/" % filename)

def print_output(output, prefix, sys):
    x = output.split("Read ")[1]
    y = x.split(":")
    s = prefix + "-" + sys
    s += "\t & "
    for i, thing in enumerate(y):
        if i%2 == 0:
            continue
        thing = thing[:-4]
        thing = str(int(thing)/1000.0)
        s = s + thing
        s = s + "\\textmu s"
        s = s + " & "
    print s

def run_latency(cmd, prefix, sys):
    if options.dprint:
        print cmd
    status, output = commands.getstatusoutput(cmd)
    if status != 0:
        print "Bad status", status, output
        exit(1)
    if options.dprint:
        print output
    print_output(output)

def latency():
    pass

if __name__ == "__main__":
    host = socket.gethostname()
    if len(host.split(".")) > 1:
        host = host.split(".")[0]
    fnpath = 'tmp/'
    if not os.path.exists(fnpath):
        os.mkdir(fnpath)
    if options.exp == "contention":
        contention_exp(fnpath, host, options.default_contention, options.read_rate)
    elif options.exp == "rw":
        rw_exp(fnpath, host, options.default_contention, options.default_ncores)
    elif options.exp == "products":
        products_exp(fnpath, host, options.read_rate, options.default_ncores)
    elif options.exp == "all":
        options.dynamic = True
        if host == "ben":
            options.default_ncores = 40
        elif host == "mat":
            options.default_ncores = 24
        elif host == "tom":
            options.default_ncores = 48
        contention_exp(fnpath, host, options.default_contention, 90)
        contention_exp(fnpath, host, options.default_contention, 10)
        contention_exp(fnpath, host, options.default_contention, 50)
        rw_exp(fnpath, host, options.default_contention, options.default_ncores)
        products_exp(fnpath, host, options.read_rate, options.default_ncores)
<commit_msg>extend running time<commit_after>from optparse import OptionParser
import commands
import os
from os import system
import socket

parser = OptionParser()
parser.add_option("-e", "--exp", action="store", type="string", dest="exp", default="contention")
parser.add_option("-s", "--short", action="store_true", dest="short", default=False)
parser.add_option("-p", "--print", action="store_true", dest="dprint", default=False)
parser.add_option("-a", "--allocate", action="store_true", dest="allocate", default=False)
parser.add_option("-n", "--ncores", action="store", type="int", dest="default_ncores", default=8)
parser.add_option("-c", "--contention", action="store", type="int", dest="default_contention", default=100000)
parser.add_option("-r", "--rr", action="store", type="int", dest="read_rate", default=50)
parser.add_option("-l", "--latency", action="store_true", dest="latency", default=False)
parser.add_option("-x", "--rlock", action="store_false", dest="rlock", default=True)
parser.add_option("-m", "--scp", action="store_true", dest="scp", default=False)

(options, args) = parser.parse_args()

ben_list_cpus = "socket@0,1,2,7,3-6"

LATENCY_PART = " -latency=%s" % options.latency

BASE_CMD = "GOGC=500 numactl -C `list-cpus seq -n %d %s` ./buy -ngo %d -nprocs %d -nsec %d -contention %d -rr %d -allocate=%s -sys=%d -rlock=%s" + LATENCY_PART

def run_one(fn, cmd):
    if options.dprint:
        print cmd
    status, output = commands.getstatusoutput(cmd)
    if status != 0:
        print "Bad status", status, output
        exit(1)
    if options.dprint:
        print output
    x = output.split()[23].rstrip(',')
    tps = float(x)
    fn.write("%0.2f\t" % tps)

def get_cpus(host):
    ncpus = [1, 2, 4, 8]
    if host == "mat":
        ncpus = [1, 2, 4, 8, 12, 24]
    elif host == "tbilisi":
        ncpus = [1, 2, 4, 8, 12]
    elif host == "tom":
        ncpus = [1, 2, 6, 12, 18, 24, 30, 42, 48]
    elif host == "ben":
        ncpus = [1, 4, 10, 20, 30, 40, 50, 60, 70, 80]
    if options.short:
        ncpus=[2, 4]
    return ncpus

def fill_cmd(rr, contention, ncpus, systype, cpus_arg=""):
    nsec = 10
    if options.short:
        nsec = 1
    cmd = BASE_CMD % (ncpus, cpus_arg, ncpus, ncpus, nsec, contention, rr, options.allocate, systype, options.rlock)
    return cmd

def do(f, rr, contention, ncpu, list_cpus, sys):
    cmd = fill_cmd(rr, contention, ncpu, sys, list_cpus)
    run_one(f, cmd)
    f.write("\t")

# x-axis is # cores
def contention_exp(fnpath, host, contention, rr):
    fnn = '%s-scalability-%d-%d-False.data' % (host, contention, rr)
    filename=os.path.join(fnpath, fnn)
    f = open(filename, 'w')
    cpus = get_cpus(host)
    f.write("#Doppel\tOCC\n")
    cpu_args = ""
    if host == "ben":
        cpu_args = ben_list_cpus

    for i in cpus:
        f.write("%d"% i)
        f.write("\t")
        do(f, rr, contention, i, cpu_args, 0)
        do(f, rr, contention, i, cpu_args, 1)
        f.write("\n")
    f.close()
    if options.scp:
        system("scp %s tbilisi.csail.mit.edu:/home/neha/src/txn/src/txn/data/" % filename)
        system("scp %s tbilisi.csail.mit.edu:/home/neha/doc/ddtxn-doc/graphs/" % filename)

def rw_exp(fnpath, host, contention, ncores):
    fnn = '%s-rw-%d-%d-False.data' % (host, contention, ncores)
    filename=os.path.join(fnpath, fnn)
    f = open(filename, 'w')
    rr = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100]
    if options.short:
        rr = [0, 50, 100]
    cpu_args = ""
    if host == "ben":
        cpu_args = ben_list_cpus
    f.write("#Doppel\tOCC\n")
    for i in rr:
        f.write("%d"% i)
        f.write("\t")
        do(f, i, contention, ncores, cpu_args, 0)
        do(f, i, contention, ncores, cpu_args, 1)
        f.write("\n")
    f.close()
    if options.scp:
        system("scp %s tbilisi.csail.mit.edu:/home/neha/src/txn/src/txn/data/" % filename)
        system("scp %s tbilisi.csail.mit.edu:/home/neha/doc/ddtxn-doc/graphs/" % filename)

def products_exp(fnpath, host, rr, ncores):
    fnn = '%s-products-%d-%d-True.data' % (host, rr, ncores)
    filename=os.path.join(fnpath, fnn)
    f = open(filename, 'w')
    cont = [1, 10, 100, 1000, 5000, 10000, 100000, 200000, 500000, 1000000]
    if options.short:
        cont = [100, 100000]
    cpu_args = ""
    if host == "ben":
        cpu_args = ben_list_cpus

    f.write("#Doppel\tOCC\n")
    for i in cont:
        f.write("%d"% i)
        f.write("\t")
        do(f, rr, i, ncores, cpu_args, 0)
        do(f, rr, i, ncores, cpu_args, 1)
        f.write("\n")
    f.close()
    if options.scp:
        system("scp %s tbilisi.csail.mit.edu:/home/neha/src/txn/src/txn/data/" % filename)
        system("scp %s tbilisi.csail.mit.edu:/home/neha/doc/ddtxn-doc/graphs/" % filename)

def print_output(output, prefix, sys):
    x = output.split("Read ")[1]
    y = x.split(":")
    s = prefix + "-" + sys
    s += "\t & "
    for i, thing in enumerate(y):
        if i%2 == 0:
            continue
        thing = thing[:-4]
        thing = str(int(thing)/1000.0)
        s = s + thing
        s = s + "\\textmu s"
        s = s + " & "
    print s

def run_latency(cmd, prefix, sys):
    if options.dprint:
        print cmd
    status, output = commands.getstatusoutput(cmd)
    if status != 0:
        print "Bad status", status, output
        exit(1)
    if options.dprint:
        print output
    print_output(output)

def latency():
    pass

if __name__ == "__main__":
    host = socket.gethostname()
    if len(host.split(".")) > 1:
        host = host.split(".")[0]
    fnpath = 'tmp/'
    if not os.path.exists(fnpath):
        os.mkdir(fnpath)
    if options.exp == "contention":
        contention_exp(fnpath, host, options.default_contention, options.read_rate)
    elif options.exp == "rw":
        rw_exp(fnpath, host, options.default_contention, options.default_ncores)
    elif options.exp == "products":
        products_exp(fnpath, host, options.read_rate, options.default_ncores)
    elif options.exp == "all":
        options.dynamic = True
        if host == "ben":
            options.default_ncores = 40
        elif host == "mat":
            options.default_ncores = 24
        elif host == "tom":
            options.default_ncores = 48
        contention_exp(fnpath, host, options.default_contention, 90)
        contention_exp(fnpath, host, options.default_contention, 10)
        contention_exp(fnpath, host, options.default_contention, 50)
        rw_exp(fnpath, host, options.default_contention, options.default_ncores)
        products_exp(fnpath, host, options.read_rate, options.default_ncores)
<|endoftext|>


<commit_before>
bfs = basisset(h2,'sto3g')
solver = rhf(h2,bfs)
ens = solver.converge()
<commit_msg>updated plotting driver. still doesn't work.<commit_after>from pyquante2 import basisset,rhf,h2
from pyquante2.graphics.vtk import vtk_orbital

bfs = basisset(h2,'sto3g')
solver = rhf(h2,bfs)
ens = solver.converge()

# Note: these orbitals are not coming out symmetric. Why not??
print solver
print solver.orbs
vtk_orbital(h2,solver.orbs,bfs)

<|endoftext|>