<commit_before>
import httplib2
import os
import os.path
import random
import re
import subprocess
import StringIO
import sys
import tempfile
import time
import urllib2

vm_image_info = {
    'ttylinux': {
        'id': 'BN1EEkPiBx87_uLj2-sdybSI-Xb',
        'email': 'airaj@lal.in2p3.fr',
        'mkplace-uri': 'https://marketplace.stratuslab.eu/metadata/BN1EEkPiBx87_uLj2-sdybSI-Xb/airaj@lal.in2p3.fr/2013-01-14T08:06:16Z'
        'url': 'http://appliances.stratuslab.eu/images/base/ttylinux-14.0-x86_64-base/1.0/ttylinux-14.0-x86_64-base-1.0.img.gz'
        },
    'ubuntu': {
        'id': 'ObFsK5hQ2CPRBvzEBdW8DYB-2G4',
        'email': 'hudson.builder@stratuslab.eu',
        },
    'cernvm': {
        'id': 'FPG7anShWeLV3mbCD2wZHKIr8CA',
        'email': 'loomis@lal.in2p3.fr',
        },
    'centos': {
        'id': 'LfjImFaziqhStKNVGQ11OJVJUjw',
        'email': 'hudson.builder@stratuslab.eu',
        },
    'data': {
        'id': 'GPAUQFkojP5dMQJNdJ4qD_62mCo',
        'email': 'loomis@lal.in2p3.fr',
        }
    }

def getVmImageInfo():
    return vm_image_info

def readRemoteFile(url):
    fd = wget(url)
    return fd.read()

def wget(url):
    return urllib2.urlopen(url)

def which(file):
    for path in os.environ["PATH"].split(os.pathsep):
        if file in os.listdir(path):
            return os.path.join(path, file)

def stratuslabBinDir():
    location = which('stratus-describe-instance')
    return os.path.dirname(location)

def execute(cmd, returnType=None, exit=True, quiet=False, shell=False):
    printCmd(' '.join(cmd))
    if quiet:
        devNull = open(os.devnull, 'w')
        stdout = devNull
        stderr = devNull
    else:
        stdout = subprocess.PIPE
        stderr = subprocess.PIPE
    p = subprocess.Popen(cmd, stdout=stdout, stderr=stderr, shell=shell)
    p.wait()
    if quiet:
        devNull.close()
    if returnType:
        return p.returncode
    else:
        out = p.stdout.read()
        err = p.stderr.read()
        if p.returncode == 0:
            if not quiet:
                if out:
                    printAndFlush(out + '\n')
                if err:
                    printAndFlush(err + '\n')
            return out
        else:
            printAndFlush('  [ERROR] Error executing command!\n')
            if out:
                printAndFlush(out + '\n')
            if err:
                printAndFlush(err + '\n')
            if exit:
                raise Exception

def printAndFlush(msg):
    sys.stdout.flush()
    print msg,
    sys.stdout.flush()

def printAction(msg):
    printAndFlush('\n :::%s:::\n' % (':' *len(msg)))
    printAndFlush(' :: %s ::\n' % msg)
    printAndFlush(' :::%s:::\n' % (':' *len(msg)))

def printStep(msg):
    printAndFlush(' :: %s\n' % msg)

def printCmd(msg):
    printAndFlush('  [Executing] %s\n' % msg)

def stratusRunInstance(image, persistentDisk=None, options=None):
    cmd = ["stratus-run-instance", "--quiet", "--no-check-image-url",
           "--cpu", "2", "--ram", "2048", "--swap", "2048",
           image]
    if persistentDisk:
        cmd.extend(["--persistent-disk", persistentDisk ])
    if options:
        cmd.extend(options)
    response = execute(cmd)
    return extractIdAndIp(response)

def stratusDescribeInstance(vmId):
    return execute(["stratus-describe-instance", str(vmId)])

def stratusShutdownInstance(vmId):
    printStep('Shutting down instance: %s' % str(vmId))
    return execute(["stratus-shutdown-instance", str(vmId)])

def stratusKillInstance(vmId):
    if vmId:
        execute(["stratus-kill-instance", str(vmId)])

def stratusPrepareContext(args):
    cmd = ['stratus-prepare-context']
    cmd.extend(args)
    return execute(cmd)

def getVmState(vmId):
    lines = stratusDescribeInstance(vmId).split('\n')
    if len(lines) >= 2:
        return _extractState(lines[1])
    else:
        raise Exception('wrong number of lines (%d) from stratus-describe-instance' % len(lines))

def _extractState(line):
    trimmed = line.strip()
    fields = re.split('\s+', trimmed)
    return fields[1]

def waitVmRunningOrTimeout(vmId, timeout=(2*60), sleepInterval=5):
    start = time.time()
    printStep('Started waiting for VM to be up at: %s' % start)
    state = ''
    while state != 'Running' and ((time.time() - start) < timeout):
        state = getVmState(vmId)
        print "\tStatus of VM '%s' is  '%s'" % (vmId, state)
        if state == 'Failed':
            raise Exception('VM (%s) failed to start' % vmId)
        time.sleep(sleepInterval)

    if state != 'Running':
        raise Exception('VM (%s) timed out' % vmId)

    return state

def stratusCreateVolume(size=1, tag='test-disk'):
    response = execute(["stratus-create-volume", "--size=%s" % size, "--tag=%s" % tag])
    return response.split(" ")[1].replace('\n', '')

def stratusDeleteVolume(uuid):
    if (uuid):
        execute(["stratus-delete-volume", uuid])

def stratusDescribeVolumes(uuid=None):
    cmd = ["stratus-describe-volumes"]
    if uuid:
        cmd.append(uuid)
    return execute(cmd)

def stratusDescribeVolumesWithFilter(tag='dummy'):
    filter = "--filter=tag:%s" % tag
    cmd = ["stratus-describe-volumes", filter]
    return execute(cmd)

def stratusAttachVolume(vmId, uuid):
    return execute(["stratus-attach-volume", "-i", str(vmId), uuid])

def stratusDetachVolume(vmId, uuid):
    return execute(["stratus-detach-volume", "-i", str(vmId), uuid])

def stratusUpdateVolume(uuid, options=[]):
    cmd = ['stratus-update-volume']
    for option in options:
        cmd.append(option)
    cmd.append(uuid)
    return execute(cmd)

def createDummyImage(digits=1024):
    file_descriptor, filename = tempfile.mkstemp()

    file = None
    try:
        file = open(filename, 'w')
        for i in range(0, digits):
            file.write(randomDigit())
    except Exception as e:
        print e
        raise e
    finally:
        closeFileReliably(file)

    return (file_descriptor, filename)

def randomDigit():
    return str(random.choice(xrange(0,10)))

def removeFile(file):
    if (file):
        try:
            os.remove(file)
        except Exception as e:
            print e

def closeFileReliably(file):
    if (file):
        try:
            file.close()
        except Exception as e:
            print e

def closeFileDescriptorReliably(file_descriptor):
    if (file_descriptor):
        try:
            os.close(file_descriptor)
        except Exception as e:
            print e

def stratusBuildMetadata(image):
    execute(["stratus-build-metadata",
             "--author=Alice Smith",
             "--os=dummyos",
             "--os-version=0.0",
             "--os-arch=i686",
             "--image-version=1.0",
             image])

def expectedMetadataFilename():
    return "%s-%s-%s-base-%s.xml" % ("dummyos", "0.0", "i686", "1.0")

def stratusSignMetadata(image):
    execute(["stratus-sign-metadata", image])

def stratusValidateMetadata(metadata):
    execute(["stratus-validate-metadata", "-vvv", metadata])

def stratusUploadMetadata(metadata):
    return execute(["stratus-upload-metadata", metadata])

def stratusDeprecateMetadata(identifier, email, reason="Just For Fun"):
    return execute(["stratus-deprecate-metadata",
                    "--email", email,
                    "--reason", reason,
                    identifier
                    ])

def stratusUploadImage(image):
    return execute(["stratus-upload-image",
                    "--image-only",
                    image])

def stratusDownloadImage(url, filename):
    return execute(["stratus-download-image",
                    "--output", filename,
                    url])

def findImageDiskOrTimeout(tag='dummy', timeout=(10*60), sleepInterval=10):
    start = time.time()
    printStep('Started trying to find image disk: %s' % start)
    while ((time.time() - start) < timeout):
        try:
            diskinfo = stratusDescribeVolumesWithFilter(tag=tag)
            uuid, identifier = extractUuidAndIdentifierFromDisk(diskinfo)
            printStep('Disk uuid and identifier: %s, %s' % (uuid, identifier))
            return (uuid, identifier)
        except Exception as e:
            time.sleep(sleepInterval)
    raise Exception('timeout exceeded while trying to find image disk')

def getUrlOrTimeout(url, timeout=(5*60), sleepInterval=10):
    start = time.time()
    printStep('Started trying to get %s at: %s' % (url, start))
    while ((time.time() - start) < timeout):
        try:
            h = httplib2.Http()
            resp, content = h.request(url, "GET")

            printStep('Response and content: %s\n%s' % (resp, content))
            return (resp, content)
        except Exception as e:
            printStep('Exception: %s' % e)
            time.sleep(sleepInterval)

    raise Exception('timeout exceeded while trying to get URL: %s' % url)

def extractUuidAndIdentifierFromDisk(diskinfo):
    uuid = None
    identifier = None
    try:
        f = StringIO.StringIO(diskinfo)
        for line in f:
            match = re.match('.*DISK\s+([^\s]+).*', line)
            if match:
                uuid = match.group(1)
            match = re.match('.*identifier:\s+([^\s]+).*', line)
            if match:
                identifier = match.group(1)
    finally:
        f.close()

    if (uuid is None):
        raise Exception("uuid not found")
    else:
        return uuid, identifier

def extractIdAndIp(runinfo):
    try:
        f = StringIO.StringIO(runinfo)
        for line in f:
            match = re.match('\s*(\d+)\s*,\s*([^\s]+)\s*', line)
            if match:
                return (match.group(1), match.group(2))
    finally:
        f.close()

    return (None, None)

def ssh(ip='localhost', cmd='/bin/true', user='root'):
    ssh_id = "%s@%s" % (user, ip)
    ssh_cmd = ['ssh', ssh_id, '-q',
               '-o', 'ConnectTimeout=5',
               '-o', 'StrictHostKeyChecking=false',
               cmd]
    execute(ssh_cmd)

def sshConnectionOrTimeout(ip='localhost', user='root', timeout=(2*60), sleepInterval=5):
    start = time.time()
    printStep('Started trying to SSH to VM: %s' % start)
    while ((time.time() - start) < timeout):
        try:
            ssh(ip, '/bin/true', user)
            return
        except Exception:
            time.sleep(sleepInterval)
    raise Exception('timeout exceeded while trying SSH')
<commit_msg>fix syntax error<commit_after>
import httplib2
import os
import os.path
import random
import re
import subprocess
import StringIO
import sys
import tempfile
import time
import urllib2

vm_image_info = {
    'ttylinux': {
        'id': 'BN1EEkPiBx87_uLj2-sdybSI-Xb',
        'email': 'airaj@lal.in2p3.fr',
        'mkplace-uri': 'https://marketplace.stratuslab.eu/metadata/BN1EEkPiBx87_uLj2-sdybSI-Xb/airaj@lal.in2p3.fr/2013-01-14T08:06:16Z',
        'url': 'http://appliances.stratuslab.eu/images/base/ttylinux-14.0-x86_64-base/1.0/ttylinux-14.0-x86_64-base-1.0.img.gz'
        },
    'ubuntu': {
        'id': 'ObFsK5hQ2CPRBvzEBdW8DYB-2G4',
        'email': 'hudson.builder@stratuslab.eu',
        },
    'cernvm': {
        'id': 'FPG7anShWeLV3mbCD2wZHKIr8CA',
        'email': 'loomis@lal.in2p3.fr',
        },
    'centos': {
        'id': 'LfjImFaziqhStKNVGQ11OJVJUjw',
        'email': 'hudson.builder@stratuslab.eu',
        },
    'data': {
        'id': 'GPAUQFkojP5dMQJNdJ4qD_62mCo',
        'email': 'loomis@lal.in2p3.fr',
        }
    }

def getVmImageInfo():
    return vm_image_info

def readRemoteFile(url):
    fd = wget(url)
    return fd.read()

def wget(url):
    return urllib2.urlopen(url)

def which(file):
    for path in os.environ["PATH"].split(os.pathsep):
        if file in os.listdir(path):
            return os.path.join(path, file)

def stratuslabBinDir():
    location = which('stratus-describe-instance')
    return os.path.dirname(location)

def execute(cmd, returnType=None, exit=True, quiet=False, shell=False):
    printCmd(' '.join(cmd))
    if quiet:
        devNull = open(os.devnull, 'w')
        stdout = devNull
        stderr = devNull
    else:
        stdout = subprocess.PIPE
        stderr = subprocess.PIPE
    p = subprocess.Popen(cmd, stdout=stdout, stderr=stderr, shell=shell)
    p.wait()
    if quiet:
        devNull.close()
    if returnType:
        return p.returncode
    else:
        out = p.stdout.read()
        err = p.stderr.read()
        if p.returncode == 0:
            if not quiet:
                if out:
                    printAndFlush(out + '\n')
                if err:
                    printAndFlush(err + '\n')
            return out
        else:
            printAndFlush('  [ERROR] Error executing command!\n')
            if out:
                printAndFlush(out + '\n')
            if err:
                printAndFlush(err + '\n')
            if exit:
                raise Exception

def printAndFlush(msg):
    sys.stdout.flush()
    print msg,
    sys.stdout.flush()

def printAction(msg):
    printAndFlush('\n :::%s:::\n' % (':' *len(msg)))
    printAndFlush(' :: %s ::\n' % msg)
    printAndFlush(' :::%s:::\n' % (':' *len(msg)))

def printStep(msg):
    printAndFlush(' :: %s\n' % msg)

def printCmd(msg):
    printAndFlush('  [Executing] %s\n' % msg)

def stratusRunInstance(image, persistentDisk=None, options=None):
    cmd = ["stratus-run-instance", "--quiet", "--no-check-image-url",
           "--cpu", "2", "--ram", "2048", "--swap", "2048",
           image]
    if persistentDisk:
        cmd.extend(["--persistent-disk", persistentDisk ])
    if options:
        cmd.extend(options)
    response = execute(cmd)
    return extractIdAndIp(response)

def stratusDescribeInstance(vmId):
    return execute(["stratus-describe-instance", str(vmId)])

def stratusShutdownInstance(vmId):
    printStep('Shutting down instance: %s' % str(vmId))
    return execute(["stratus-shutdown-instance", str(vmId)])

def stratusKillInstance(vmId):
    if vmId:
        execute(["stratus-kill-instance", str(vmId)])

def stratusPrepareContext(args):
    cmd = ['stratus-prepare-context']
    cmd.extend(args)
    return execute(cmd)

def getVmState(vmId):
    lines = stratusDescribeInstance(vmId).split('\n')
    if len(lines) >= 2:
        return _extractState(lines[1])
    else:
        raise Exception('wrong number of lines (%d) from stratus-describe-instance' % len(lines))

def _extractState(line):
    trimmed = line.strip()
    fields = re.split('\s+', trimmed)
    return fields[1]

def waitVmRunningOrTimeout(vmId, timeout=(2*60), sleepInterval=5):
    start = time.time()
    printStep('Started waiting for VM to be up at: %s' % start)
    state = ''
    while state != 'Running' and ((time.time() - start) < timeout):
        state = getVmState(vmId)
        print "\tStatus of VM '%s' is  '%s'" % (vmId, state)
        if state == 'Failed':
            raise Exception('VM (%s) failed to start' % vmId)
        time.sleep(sleepInterval)

    if state != 'Running':
        raise Exception('VM (%s) timed out' % vmId)

    return state

def stratusCreateVolume(size=1, tag='test-disk'):
    response = execute(["stratus-create-volume", "--size=%s" % size, "--tag=%s" % tag])
    return response.split(" ")[1].replace('\n', '')

def stratusDeleteVolume(uuid):
    if (uuid):
        execute(["stratus-delete-volume", uuid])

def stratusDescribeVolumes(uuid=None):
    cmd = ["stratus-describe-volumes"]
    if uuid:
        cmd.append(uuid)
    return execute(cmd)

def stratusDescribeVolumesWithFilter(tag='dummy'):
    filter = "--filter=tag:%s" % tag
    cmd = ["stratus-describe-volumes", filter]
    return execute(cmd)

def stratusAttachVolume(vmId, uuid):
    return execute(["stratus-attach-volume", "-i", str(vmId), uuid])

def stratusDetachVolume(vmId, uuid):
    return execute(["stratus-detach-volume", "-i", str(vmId), uuid])

def stratusUpdateVolume(uuid, options=[]):
    cmd = ['stratus-update-volume']
    for option in options:
        cmd.append(option)
    cmd.append(uuid)
    return execute(cmd)

def createDummyImage(digits=1024):
    file_descriptor, filename = tempfile.mkstemp()

    file = None
    try:
        file = open(filename, 'w')
        for i in range(0, digits):
            file.write(randomDigit())
    except Exception as e:
        print e
        raise e
    finally:
        closeFileReliably(file)

    return (file_descriptor, filename)

def randomDigit():
    return str(random.choice(xrange(0,10)))

def removeFile(file):
    if (file):
        try:
            os.remove(file)
        except Exception as e:
            print e

def closeFileReliably(file):
    if (file):
        try:
            file.close()
        except Exception as e:
            print e

def closeFileDescriptorReliably(file_descriptor):
    if (file_descriptor):
        try:
            os.close(file_descriptor)
        except Exception as e:
            print e

def stratusBuildMetadata(image):
    execute(["stratus-build-metadata",
             "--author=Alice Smith",
             "--os=dummyos",
             "--os-version=0.0",
             "--os-arch=i686",
             "--image-version=1.0",
             image])

def expectedMetadataFilename():
    return "%s-%s-%s-base-%s.xml" % ("dummyos", "0.0", "i686", "1.0")

def stratusSignMetadata(image):
    execute(["stratus-sign-metadata", image])

def stratusValidateMetadata(metadata):
    execute(["stratus-validate-metadata", "-vvv", metadata])

def stratusUploadMetadata(metadata):
    return execute(["stratus-upload-metadata", metadata])

def stratusDeprecateMetadata(identifier, email, reason="Just For Fun"):
    return execute(["stratus-deprecate-metadata",
                    "--email", email,
                    "--reason", reason,
                    identifier
                    ])

def stratusUploadImage(image):
    return execute(["stratus-upload-image",
                    "--image-only",
                    image])

def stratusDownloadImage(url, filename):
    return execute(["stratus-download-image",
                    "--output", filename,
                    url])

def findImageDiskOrTimeout(tag='dummy', timeout=(10*60), sleepInterval=10):
    start = time.time()
    printStep('Started trying to find image disk: %s' % start)
    while ((time.time() - start) < timeout):
        try:
            diskinfo = stratusDescribeVolumesWithFilter(tag=tag)
            uuid, identifier = extractUuidAndIdentifierFromDisk(diskinfo)
            printStep('Disk uuid and identifier: %s, %s' % (uuid, identifier))
            return (uuid, identifier)
        except Exception as e:
            time.sleep(sleepInterval)
    raise Exception('timeout exceeded while trying to find image disk')

def getUrlOrTimeout(url, timeout=(5*60), sleepInterval=10):
    start = time.time()
    printStep('Started trying to get %s at: %s' % (url, start))
    while ((time.time() - start) < timeout):
        try:
            h = httplib2.Http()
            resp, content = h.request(url, "GET")

            printStep('Response and content: %s\n%s' % (resp, content))
            return (resp, content)
        except Exception as e:
            printStep('Exception: %s' % e)
            time.sleep(sleepInterval)

    raise Exception('timeout exceeded while trying to get URL: %s' % url)

def extractUuidAndIdentifierFromDisk(diskinfo):
    uuid = None
    identifier = None
    try:
        f = StringIO.StringIO(diskinfo)
        for line in f:
            match = re.match('.*DISK\s+([^\s]+).*', line)
            if match:
                uuid = match.group(1)
            match = re.match('.*identifier:\s+([^\s]+).*', line)
            if match:
                identifier = match.group(1)
    finally:
        f.close()

    if (uuid is None):
        raise Exception("uuid not found")
    else:
        return uuid, identifier

def extractIdAndIp(runinfo):
    try:
        f = StringIO.StringIO(runinfo)
        for line in f:
            match = re.match('\s*(\d+)\s*,\s*([^\s]+)\s*', line)
            if match:
                return (match.group(1), match.group(2))
    finally:
        f.close()

    return (None, None)

def ssh(ip='localhost', cmd='/bin/true', user='root'):
    ssh_id = "%s@%s" % (user, ip)
    ssh_cmd = ['ssh', ssh_id, '-q',
               '-o', 'ConnectTimeout=5',
               '-o', 'StrictHostKeyChecking=false',
               cmd]
    execute(ssh_cmd)

def sshConnectionOrTimeout(ip='localhost', user='root', timeout=(2*60), sleepInterval=5):
    start = time.time()
    printStep('Started trying to SSH to VM: %s' % start)
    while ((time.time() - start) < timeout):
        try:
            ssh(ip, '/bin/true', user)
            return
        except Exception:
            time.sleep(sleepInterval)
    raise Exception('timeout exceeded while trying SSH')
<|endoftext|>


<commit_before>#!/usr/bin/python

import os

import config
import copy
import messages as msg

# pcbmode modules
import utils
import messages as msg
from shape import Shape
from style import Style
from footprint import Footprint

class Component():
    """
    """

    def __init__(self, refdef, component):
        """
        """

        self._refdef = refdef
        self._layer = component.get('layer') or 'top'

        self._rotate = component.get('rotate') or 0
        self._rotate_point = utils.toPoint(component.get('rotate-point') or [0, 0])
        self._scale = component.get('scale') or 1
        self._location = component.get('location') or [0, 0]

        # Get footprint definition and shapes
        try:
            self._footprint_name = component['footprint']
        except:
            msg.error("Cannot find a 'footprint' name for refdef %s." % refdef)

        fname = os.path.join(config.cfg['base-dir'],
                             config.cfg['locations']['components'],
                             self._footprint_name + '.json')
        footprint_dict = utils.dictFromJsonFile(fname)
        footprint = Footprint(footprint_dict)
        footprint_shapes = footprint.getShapes()

        #------------------------------------------------
        # Apply component-specific modifiers to footprint
        #------------------------------------------------
        #for sheet in ['conductor', 'soldermask', 'solderpaste', 'silkscreen', 'assembly', 'drills']:
        for sheet in ['conductor', 'silkscreen', 'assembly', 'drills']:
            for layer in config.stk['layer-names']:
                for shape in footprint_shapes[sheet].get(layer) or []:

                    # In order to apply the rotation we need to adust the location
                    shape.rotateLocation(self._rotate, self._rotate_point)

                    shape.transformPath(self._scale,
                                        self._rotate,
                                        self._rotate_point,
                                        False,
                                        True)

        #--------------------------------------
        # Remove silkscreen and assembly shapes
        #--------------------------------------
        for sheet in ['silkscreen','assembly']:

            try:
                shapes_dict = component[sheet].get('shapes') or {}
            except:
                shapes_dict = {}

            # If the setting is to not show silkscreen shapes for the
            # component, delete the shapes from the shapes' dictionary
            if shapes_dict.get('show') == False:
                for pcb_layer in utils.getSurfaceLayers():
                    footprint_shapes[sheet][pcb_layer] = []

        #----------------------------------------------------------
        # Add silkscreen and assembly reference designator (refdef)
        #----------------------------------------------------------
        for sheet in ['silkscreen','assembly']:

            try:
                refdef_dict = component[sheet].get('refdef') or {}
            except:
                refdef_dict = {}

            if refdef_dict.get('show') != False:
                layer = refdef_dict.get('layer') or 'top'

                # Rotate the refdef; if unspecified the rotation is the same as
                # the rotation of the component
                refdef_dict['rotate'] = refdef_dict.get('rotate') or 0

                # Sometimes you'd want to keep all refdefs at the same angle
                # and not rotated with the component
                if refdef_dict.get('rotate-with-component') != False:
                    refdef_dict['rotate'] += self._rotate

                refdef_dict['rotate-point'] = utils.toPoint(refdef_dict.get('rotate-point')) or self._rotate_point

                refdef_dict['location'] = refdef_dict.get('location') or [0, 0]
                refdef_dict['type'] = 'text'
                refdef_dict['value'] = refdef_dict.get('value') or refdef
                refdef_dict['font-family'] = (refdef_dict.get('font-family') or
                                              config.stl['layout'][sheet]['refdef'].get('font-family') or
                                              config.stl['defaults']['font-family'])
                refdef_dict['font-size'] = (refdef_dict.get('font-size') or
                                            config.stl['layout'][sheet]['refdef'].get('font-size') or
                                            "2mm")
                refdef_shape = Shape(refdef_dict)

                refdef_shape.is_refdef = True
                refdef_shape.rotateLocation(self._rotate, self._rotate_point)
                style = Style(refdef_dict, sheet, 'refdef')
                refdef_shape.setStyle(style)

                # Add the refdef to the silkscreen/assembly list. It's
                # important that this is added at the very end since the
                # placement process assumes the refdef is last
                footprint_shapes[sheet][layer].append(refdef_shape)

        #------------------------------------------------------
        # Invert 'top' and 'bottom' if layer is on the 'bottom'
        #------------------------------------------------------
        if self._layer == 'bottom':
            layers = config.stk['layer-names']
            #layers_rev = copy.copy(reversed(config.stk['layer-names'])

            for sheet in ['conductor', 'soldermask', 'solderpaste', 'silkscreen', 'assembly']:
                sheet_dict = footprint_shapes[sheet]
                #sheet_dict_copy = copy.copy(sheet_dict)
                sheet_dict_new = {}
                for i, pcb_layer in enumerate(layers):
                    #print len(layers)-i
                    #print layers[len(layers)-i]
                    try:
                        sheet_dict_new[layers[len(layers)-i-1]] = copy.copy(sheet_dict[pcb_layer])
                    except:
                        continue

                footprint_shapes[sheet] = copy.copy(sheet_dict_new)

        self._footprint_shapes = footprint_shapes

    def getShapes(self):
        """
        """
        return self._footprint_shapes

    def getLocation(self):
        """
        """
        return self._location

    def getRefdef(self):
        return self._refdef

    def getPlacementLayer(self):
        return self._layer

    def getFootprintName(self):
        return self._footprint_name

    def getRotation(self):
        return self._rotate
<commit_msg>Comments<commit_after>#!/usr/bin/python

import os

import config
import copy
import messages as msg

# pcbmode modules
import utils
import messages as msg
from shape import Shape
from style import Style
from footprint import Footprint

class Component():
    """
    """

    def __init__(self, refdef, component):
        """
        """

        self._refdef = refdef
        self._layer = component.get('layer') or 'top'

        self._rotate = component.get('rotate') or 0
        self._rotate_point = utils.toPoint(component.get('rotate-point') or [0, 0])
        self._scale = component.get('scale') or 1
        self._location = component.get('location') or [0, 0]

        # Get footprint definition and shapes
        try:
            self._footprint_name = component['footprint']
        except:
            msg.error("Cannot find a 'footprint' name for refdef %s." % refdef)

        fname = os.path.join(config.cfg['base-dir'],
                             config.cfg['locations']['components'],
                             self._footprint_name + '.json')
        footprint_dict = utils.dictFromJsonFile(fname)
        footprint = Footprint(footprint_dict)
        footprint_shapes = footprint.getShapes()

        #------------------------------------------------
        # Apply component-specific modifiers to footprint
        #------------------------------------------------
        for sheet in ['conductor', 'silkscreen', 'assembly', 'drills']:
            for layer in config.stk['layer-names']:
                for shape in footprint_shapes[sheet].get(layer) or []:

                    # In order to apply the rotation we need to adust the location
                    shape.rotateLocation(self._rotate, self._rotate_point)

                    shape.transformPath(self._scale,
                                        self._rotate,
                                        self._rotate_point,
                                        False,
                                        True)

        #--------------------------------------------------------------
        # Remove silkscreen and assembly shapes
        #--------------------------------------------------------------
        # If the 'show' flag is 'false then remove these items from the
        # shapes dictionary
        #--------------------------------------------------------------
        for sheet in ['silkscreen','assembly']:

            try:
                shapes_dict = component[sheet].get('shapes') or {}
            except:
                shapes_dict = {}

            # If the setting is to not show silkscreen shapes for the
            # component, delete the shapes from the shapes' dictionary
            if shapes_dict.get('show') == False:
                for pcb_layer in utils.getSurfaceLayers():
                    footprint_shapes[sheet][pcb_layer] = []

        #----------------------------------------------------------
        # Add silkscreen and assembly reference designator (refdef)
        #----------------------------------------------------------
        for sheet in ['silkscreen','assembly']:

            try:
                refdef_dict = component[sheet].get('refdef') or {}
            except:
                refdef_dict = {}

            if refdef_dict.get('show') != False:
                layer = refdef_dict.get('layer') or 'top'

                # Rotate the refdef; if unspecified the rotation is the same as
                # the rotation of the component
                refdef_dict['rotate'] = refdef_dict.get('rotate') or 0

                # Sometimes you'd want to keep all refdefs at the same angle
                # and not rotated with the component
                if refdef_dict.get('rotate-with-component') != False:
                    refdef_dict['rotate'] += self._rotate

                refdef_dict['rotate-point'] = utils.toPoint(refdef_dict.get('rotate-point')) or self._rotate_point

                refdef_dict['location'] = refdef_dict.get('location') or [0, 0]
                refdef_dict['type'] = 'text'
                refdef_dict['value'] = refdef_dict.get('value') or refdef
                refdef_dict['font-family'] = (refdef_dict.get('font-family') or
                                              config.stl['layout'][sheet]['refdef'].get('font-family') or
                                              config.stl['defaults']['font-family'])
                refdef_dict['font-size'] = (refdef_dict.get('font-size') or
                                            config.stl['layout'][sheet]['refdef'].get('font-size') or
                                            "2mm")
                refdef_shape = Shape(refdef_dict)

                refdef_shape.is_refdef = True
                refdef_shape.rotateLocation(self._rotate, self._rotate_point)
                style = Style(refdef_dict, sheet, 'refdef')
                refdef_shape.setStyle(style)

                # Add the refdef to the silkscreen/assembly list. It's
                # important that this is added at the very end since the
                # placement process assumes the refdef is last
                footprint_shapes[sheet][layer].append(refdef_shape)

        #------------------------------------------------------
        # Invert 'top' and 'bottom' if layer is on the 'bottom'
        #------------------------------------------------------
        if self._layer == 'bottom':
            layers = config.stk['layer-names']
            #layers_rev = copy.copy(reversed(config.stk['layer-names'])

            for sheet in ['conductor', 'soldermask', 'solderpaste', 'silkscreen', 'assembly']:
                sheet_dict = footprint_shapes[sheet]
                #sheet_dict_copy = copy.copy(sheet_dict)
                sheet_dict_new = {}
                for i, pcb_layer in enumerate(layers):
                    #print len(layers)-i
                    #print layers[len(layers)-i]
                    try:
                        sheet_dict_new[layers[len(layers)-i-1]] = copy.copy(sheet_dict[pcb_layer])
                    except:
                        continue

                footprint_shapes[sheet] = copy.copy(sheet_dict_new)

        self._footprint_shapes = footprint_shapes

    def getShapes(self):
        """
        """
        return self._footprint_shapes

    def getLocation(self):
        """
        """
        return self._location

    def getRefdef(self):
        return self._refdef

    def getPlacementLayer(self):
        return self._layer

    def getFootprintName(self):
        return self._footprint_name

    def getRotation(self):
        return self._rotate
<|endoftext|>


<commit_before><commit_msg>Copying swiftshader over if available.<commit_after><|endoftext|>


<commit_before>from joblib import Parallel, delayed
import multiprocessing
import tra

num_cores = multiprocessing.cpu_count()

import rethinkdb as r

def run_count(i, theid, api_url, app_token):

    count_url = '%s?$select=count(*)&$$app_token=%s' % (api_url, app_token)
    try:
        count_data = requests.get(count_url, verify=False).json()
        number_of_rows = count_data[0]['count']
    except Exception, err:
        print count_url
        print traceback.print_exc()
    conn = r.connect( "localhost", 28015).repl()
    r.db('public').table('datasets').get(theid).update({"number_of_rows": number_of_rows}).run(conn)
    print i
    return number_of_rows

def do():
    import rethinkdb as r
    import traceback
    r.connect( "localhost", 28015).repl()
    import requests
    app_token = r.db('nonpublic').table('third_party_creds').get('socrata').run()['app_token']
    results = []
    for i in range(10):
        results.extend(requests.get('http://api.us.socrata.com/api/catalog/v1?only=datasets&limit=10000&offset='+str(10000*i)).json()['results'])
    data = results
    modified_data = []
    inputs = []
    for i, row in enumerate(data):
        d = {}
        d.update(row)
        d.update(row['resource'])
        d.update(row['resource']['view_count'])
        d.update(row['classification'])
        # https://data.cityofnewyork.us/Public-Safety/Disposition-Of-Offensive-Language-Allegations-2007/xah7-gu5w
        # https://data.cityofnewyork.us/resource/xah7-gu5w.json
        # use permalink
        d['api_url'] = d['permalink'].replace('/d/', '/resource/') + '.json'
        # ?$select=count(*)&$$app_token=%s'
        inputs.append([i, d['id'], d['api_url'], app_token])
        modified_data.append(d)
    print r.db('public').table('datasets').insert(modified_data).run(conflict='update')
    results = Parallel(n_jobs=num_cores)(delayed(run_count)(*inp) for inp in inputs)
while True:
    do()<commit_msg>auto commit<commit_after>from joblib import Parallel, delayed
import multiprocessing
import traceback

num_cores = multiprocessing.cpu_count()

import rethinkdb as r

def run_count(i, theid, api_url, app_token):

    count_url = '%s?$select=count(*)&$$app_token=%s' % (api_url, app_token)
    try:
        count_data = requests.get(count_url, verify=False).json()
        number_of_rows = count_data[0]['count']
    except Exception, err:
        print count_url
        print traceback.print_exc()
    conn = r.connect( "localhost", 28015).repl()
    r.db('public').table('datasets').get(theid).update({"number_of_rows": number_of_rows}).run(conn)
    print i
    return number_of_rows

def do():
    import rethinkdb as r
    import traceback
    r.connect( "localhost", 28015).repl()
    import requests
    app_token = r.db('nonpublic').table('third_party_creds').get('socrata').run()['app_token']
    results = []
    for i in range(10):
        results.extend(requests.get('http://api.us.socrata.com/api/catalog/v1?only=datasets&limit=10000&offset='+str(10000*i)).json()['results'])
    data = results
    modified_data = []
    inputs = []
    for i, row in enumerate(data):
        d = {}
        d.update(row)
        d.update(row['resource'])
        d.update(row['resource']['view_count'])
        d.update(row['classification'])
        # https://data.cityofnewyork.us/Public-Safety/Disposition-Of-Offensive-Language-Allegations-2007/xah7-gu5w
        # https://data.cityofnewyork.us/resource/xah7-gu5w.json
        # use permalink
        d['api_url'] = d['permalink'].replace('/d/', '/resource/') + '.json'
        # ?$select=count(*)&$$app_token=%s'
        inputs.append([i, d['id'], d['api_url'], app_token])
        modified_data.append(d)
    print r.db('public').table('datasets').insert(modified_data).run(conflict='update')
    results = Parallel(n_jobs=num_cores)(delayed(run_count)(*inp) for inp in inputs)
while True:
    do()<|endoftext|>


<commit_before>"""Parse PAD rule sets.

The general syntax for PAD rules is (on one line):

<type> <name> <value>

Various options can be defined for a rule and they get bundled up using
the name as unique identifier.
"""

from __future__ import absolute_import

from builtins import dict
from builtins import object

import re
import os
import warnings
import contextlib
import collections
import locale

import oa.config
import oa.errors
import oa.context
import oa.plugins
import oa.rules.uri
import oa.rules.body
import oa.rules.meta
import oa.rules.full
import oa.rules.eval_
import oa.rules.header
import oa.rules.ruleset

from oa.regex import Regex

# Simple protection against recursion with "include".
MAX_RECURSION = 10

# Rules that require 2 arguments
KNOWN_2_RTYPE = frozenset(
        (
            "score",  # Specifies the score adjustment if the rule matches
            "priority",  # Specifies the priority of the rule
            "describe",  # Specifies a comment describing the rule
            "full",  # Specifies a FullRule
            "body",  # Specifies a BodyRule
            "rawbody",  # Specifies a RawBodyRule
            "uri",  # Specifies a URIRule
            "header",  # Specifies a HeaderRule
            "mimeheader",  # Specifies a MimeHeaderRule
            "meta",  # Specifies a MetaRule
            "eval",  # Specifies a EvalRule
            "lang",  # Specifies a language
            "tflags", #Specifies a TflagRule
        )
)
# Rules that require 1 arguments
KNOWN_1_RTYPE = frozenset(
        (
            "report",  # Add some text to the report template
            "unsafe_report", # Add some text to the unsafe report template
            "add_header",  # Adds a header to the message
            "remove_header",  # Remove header from message
            "include",  # Include another file in the current one
            "ifplugin",  # Check if plugin is loaded.
            "loadplugin",  # Load a plugin.
            "require_version",  # Only load this file if the version matches
            "required_score",  # Set the required score for this ruleset
            "report_safe",  # Set the method of reporting spam
            "report_contact",  # Set the contact address
            "required_score",  # Set the required score (default 5)
        )
)

# These are the types of rules that we know how to interpret, ignore anything
# else. These include rule types and option types.
KNOWN_RTYPE = KNOWN_1_RTYPE | KNOWN_2_RTYPE

# These types define rules and not options. Map against their specific class.
RULES = {
    "full": oa.rules.full.FullRule,
    "body": oa.rules.body.BodyRule,
    "rawbody": oa.rules.body.RawBodyRule,
    "uri": oa.rules.uri.URIRule,
    "meta": oa.rules.meta.MetaRule,
    "header": oa.rules.header.HeaderRule,
    "mimeheader": oa.rules.header.MimeHeaderRule,
    "eval": oa.rules.eval_.EvalRule,
}

_COMMENT_P = Regex(r"((?<=[^\\])#.*)")

class PADParser(object):
    """Parses PAD ruleset and extracts and combines the relevant data.

    Note that this is not thread-safe.
    """

    def __init__(self, paranoid=False, ignore_unknown=True, lazy_mode=True):
        self.ctxt = oa.context.GlobalContext(paranoid=paranoid,
                                             ignore_unknown=ignore_unknown,
                                             lazy_mode=lazy_mode)
        # XXX This could be a default OrderedDict
        self.results = collections.OrderedDict()
        self.ruleset = oa.rules.ruleset.RuleSet(self.ctxt)
        self._ignore = False

    @contextlib.contextmanager
    def _paranoid(self, *exceptions):
        """If not paranoid ignore the specified exceptions."""
        try:
            yield
        except exceptions as e:
            self.ctxt.err(e)
            if self.ctxt.paranoid:
                raise

    def parse_file(self, filename, _depth=0):
        """Parses a single PAD ruleset file."""
        if _depth > MAX_RECURSION:
            raise oa.errors.MaxRecursionDepthExceeded()
        self.ctxt.log.debug("Parsing file: %s", filename)
        if not os.path.isfile(filename):
            self.ctxt.log.warn("Ignoring %s, not a file", filename)
            return
        with open(filename, "rb") as rulef:
            for line_no, line in enumerate(rulef):
                try:
                    with self._paranoid(oa.errors.InvalidSyntax):
                        self._handle_line(filename, line, line_no + 1, _depth)
                except oa.errors.PluginLoadError as e:
                    warnings.warn(str(e))
                    self.ctxt.log.warn("%s", e)

    def _handle_line(self, filename, line, line_no, _depth=0):
        """Handles a single line."""
        try:
            line = line.decode("iso-8859-1").strip()
        except UnicodeDecodeError as e:
            raise oa.errors.InvalidSyntax(filename, line_no, line,
                                           "Decoding Error: %s" % e)
        if line.startswith("if can"):
            # XXX We don't support for this check, simply
            # XXX skip everything for now.
            self._ignore = True
            return

        if line.startswith("endif"):
            self._ignore = False
            return

        if line.startswith("else"):
            if self._ignore:
                self._ignore = False
            else:
                self._ignore = True
            return

        if line.startswith("require_version"):
            # XXX We don't really have any use for this now
            # XXX Just skip it.
            return

        if not line or line.startswith("#") or self._ignore:
            return

        # Remove any comments
        line = _COMMENT_P.sub("", line).strip()

        try:
            rtype, value = line.split(None, 1)
        except ValueError:
            # Some plugin might know how to handle this line
            rtype, value = line, ""

        if rtype == "include":
            dirname = None
            if os.path.isabs(filename):
                dirname = os.path.dirname(filename)
            self._handle_include(value, line, line_no, _depth,
                                 dirname=dirname)
        elif rtype == "ifplugin":
            self._handle_ifplugin(value)
        elif rtype == "loadplugin":
            self._handle_loadplugin(value)
        elif rtype in KNOWN_2_RTYPE or rtype in self.ctxt.cmds:
            try:
                rtype, name, value = line.split(None, 2)
            except ValueError:
                raise oa.errors.InvalidSyntax(filename, line_no, line,
                                               "Missing argument")
            if rtype == "tflags":
                value = value.split()

            if rtype == "lang":
                locale.setlocale(locale.LC_ALL, '')
                locale_language = locale.getlocale(locale.LC_MESSAGES)[0]
                if not locale_language.startswith(name):
                    self.ctxt.log.debug("Lang argument does not"
                                        "correspond with locales")
                    return

                if "report" in value:
                    try:
                        rtype, value = value.split(None, 1)
                    except ValueError:
                        raise oa.errors.InvalidSyntax(filename, line_no, line,
                                                       "Missing argument")

                    if not self.ctxt.hook_parse_config(rtype, value):
                        self.ctxt.err("%s:%s Ignoring unknown"
                                      "configuration line: %s",
                                      filename, line_no, line)
                    return

                try:
                    rtype, name, value = value.split(None, 2)
                except ValueError:
                    raise oa.errors.InvalidSyntax(filename, line_no, line,
                                                   "Missing argument")

            if name not in self.results:
                self.results[name] = dict()

            if rtype in RULES or rtype in self.ctxt.cmds:
                if value.startswith("eval:"):
                    # This is for compatibility with SA ruleset
                    self.results[name]["target"] = rtype
                    rtype = "eval"
                self.results[name]["type"] = rtype
                self.results[name]["value"] = value
            else:
                if rtype == 'priority':
                    try:
                        int(value)
                    except ValueError:
                        self.ctxt.err("%s:%s Invalid type for priority value "
                                      "in configuration line: %s, setting it by"
                                      " default to 0", filename, line_no, line)
                self.results[name][rtype] = value

        else:
            if not self.ctxt.hook_parse_config(rtype, value):
                self.ctxt.err("%s:%s Ignoring unknown configuration line: %s",
                              filename, line_no, line)

    def _handle_include(self, value, line, line_no, _depth=0,
                        dirname=None):
        """Handles the 'include' keyword."""
        filename = value.strip()
        if not os.path.isabs(filename) and dirname is not None:
            filename = os.path.join(dirname, filename)
        try:
            self.parse_file(filename, _depth=_depth + 1)
        except oa.errors.MaxRecursionDepthExceeded as e:
            e.add_call(filename, line_no, line)
            raise e

    def _handle_ifplugin(self, value):
        """Handles the 'ifplugin' keyword."""
        plugin_name = oa.plugins.REIMPLEMENTED_PLUGINS.get(value, value)
        try:
            plugin_name = plugin_name.rsplit(".", 1)[1]
        except IndexError:
            pass
        if plugin_name not in self.ctxt.plugins:
            self.ctxt.log.debug("Plugin %s not loaded, skipping.", plugin_name)
            self._ignore = True

    def _handle_loadplugin(self, value):
        """Handles the 'loadplugin' keyword."""
        try:
            plugin_name, path = value.split(None, 1)
        except ValueError:
            plugin_name, path = value, None
        if "::" in plugin_name:
            plugin_name = oa.plugins.REIMPLEMENTED_PLUGINS.get(plugin_name)
        if plugin_name:
            self.ctxt.load_plugin(plugin_name, path)
        else:
            self.ctxt.log.warn("Plugin not available: %s", value)

    def get_ruleset(self):
        """Create and return the corresponding ruleset for the parsed files."""
        self.ctxt.hook_parsing_start(self.results)
        for name, data in self.results.items():
            try:
                rule_type = data["type"]
            except KeyError:
                e = oa.errors.InvalidRule(name, "No rule type defined.")
                self.ctxt.err(e)
                if self.ctxt.paranoid:
                    raise e
            else:
                with self._paranoid(oa.errors.InvalidRule,
                                    oa.errors.InvalidRegex):
                    try:
                        rule_class = RULES[rule_type]
                    except KeyError:
                        # A plugin might have been loaded that
                        # can handle this.
                        rule_class = self.ctxt.cmds[rule_type]
                    self.ctxt.log.debug("Adding rule %s with: %s", name, data)
                    rule = rule_class.get_rule(name, data)
                    self.ruleset.add_rule(rule)
        self.ctxt.hook_parsing_end(self.ruleset)
        self.ctxt.log.info("%s rules loaded", len(self.ruleset.checked))
        self.ruleset.post_parsing()
        return self.ruleset

def parse_pad_rules(files, paranoid=False, ignore_unknown=True):
    """Parse a list of PAD rules and returns the corresponding ruleset.

    'files' - a list of file paths.

    Returns a dictionary that maps rule names to a dictionary of rule options.
    Every rule will contain "type" and "value" which corresponds to the
    mandatory line for defining a rule. For example (type, name value):

    body LOCAL_DEMONSTRATION_RULE   /test/

    Other options may be included such as "score", "describe".
    """
    parser = PADParser(paranoid=paranoid, ignore_unknown=ignore_unknown)
    for filename in files:
        parser.parse_file(filename)

    return parser
<commit_msg>Typos.<commit_after>"""Parse OA rule sets.

The general syntax for OA rules is (on one line):

<type> <name> <value>

Various options can be defined for a rule and they get bundled up using
the name as unique identifier.
"""

from __future__ import absolute_import

from builtins import dict
from builtins import object

import re
import os
import warnings
import contextlib
import collections
import locale

import oa.config
import oa.errors
import oa.context
import oa.plugins
import oa.rules.uri
import oa.rules.body
import oa.rules.meta
import oa.rules.full
import oa.rules.eval_
import oa.rules.header
import oa.rules.ruleset

from oa.regex import Regex

# Simple protection against recursion with "include".
MAX_RECURSION = 10

# Rules that require 2 arguments
KNOWN_2_RTYPE = frozenset(
        (
            "score",  # Specifies the score adjustment if the rule matches
            "priority",  # Specifies the priority of the rule
            "describe",  # Specifies a comment describing the rule
            "full",  # Specifies a FullRule
            "body",  # Specifies a BodyRule
            "rawbody",  # Specifies a RawBodyRule
            "uri",  # Specifies a URIRule
            "header",  # Specifies a HeaderRule
            "mimeheader",  # Specifies a MimeHeaderRule
            "meta",  # Specifies a MetaRule
            "eval",  # Specifies a EvalRule
            "lang",  # Specifies a language
            "tflags", #Specifies a TflagRule
        )
)
# Rules that require 1 arguments
KNOWN_1_RTYPE = frozenset(
        (
            "report",  # Add some text to the report template
            "unsafe_report", # Add some text to the unsafe report template
            "add_header",  # Adds a header to the message
            "remove_header",  # Remove header from message
            "include",  # Include another file in the current one
            "ifplugin",  # Check if plugin is loaded.
            "loadplugin",  # Load a plugin.
            "require_version",  # Only load this file if the version matches
            "required_score",  # Set the required score for this ruleset
            "report_safe",  # Set the method of reporting spam
            "report_contact",  # Set the contact address
            "required_score",  # Set the required score (default 5)
        )
)

# These are the types of rules that we know how to interpret, ignore anything
# else. These include rule types and option types.
KNOWN_RTYPE = KNOWN_1_RTYPE | KNOWN_2_RTYPE

# These types define rules and not options. Map against their specific class.
RULES = {
    "full": oa.rules.full.FullRule,
    "body": oa.rules.body.BodyRule,
    "rawbody": oa.rules.body.RawBodyRule,
    "uri": oa.rules.uri.URIRule,
    "meta": oa.rules.meta.MetaRule,
    "header": oa.rules.header.HeaderRule,
    "mimeheader": oa.rules.header.MimeHeaderRule,
    "eval": oa.rules.eval_.EvalRule,
}

_COMMENT_P = Regex(r"((?<=[^\\])#.*)")

class PADParser(object):
    """Parses PAD ruleset and extracts and combines the relevant data.

    Note that this is not thread-safe.
    """

    def __init__(self, paranoid=False, ignore_unknown=True, lazy_mode=True):
        self.ctxt = oa.context.GlobalContext(paranoid=paranoid,
                                             ignore_unknown=ignore_unknown,
                                             lazy_mode=lazy_mode)
        # XXX This could be a default OrderedDict
        self.results = collections.OrderedDict()
        self.ruleset = oa.rules.ruleset.RuleSet(self.ctxt)
        self._ignore = False

    @contextlib.contextmanager
    def _paranoid(self, *exceptions):
        """If not paranoid ignore the specified exceptions."""
        try:
            yield
        except exceptions as e:
            self.ctxt.err(e)
            if self.ctxt.paranoid:
                raise

    def parse_file(self, filename, _depth=0):
        """Parses a single PAD ruleset file."""
        if _depth > MAX_RECURSION:
            raise oa.errors.MaxRecursionDepthExceeded()
        self.ctxt.log.debug("Parsing file: %s", filename)
        if not os.path.isfile(filename):
            self.ctxt.log.warn("Ignoring %s, not a file", filename)
            return
        with open(filename, "rb") as rulef:
            for line_no, line in enumerate(rulef):
                try:
                    with self._paranoid(oa.errors.InvalidSyntax):
                        self._handle_line(filename, line, line_no + 1, _depth)
                except oa.errors.PluginLoadError as e:
                    warnings.warn(str(e))
                    self.ctxt.log.warn("%s", e)

    def _handle_line(self, filename, line, line_no, _depth=0):
        """Handles a single line."""
        try:
            line = line.decode("iso-8859-1").strip()
        except UnicodeDecodeError as e:
            raise oa.errors.InvalidSyntax(filename, line_no, line,
                                           "Decoding Error: %s" % e)
        if line.startswith("if can"):
            # XXX We don't support for this check, simply
            # XXX skip everything for now.
            self._ignore = True
            return

        if line.startswith("endif"):
            self._ignore = False
            return

        if line.startswith("else"):
            if self._ignore:
                self._ignore = False
            else:
                self._ignore = True
            return

        if line.startswith("require_version"):
            # XXX We don't really have any use for this now
            # XXX Just skip it.
            return

        if not line or line.startswith("#") or self._ignore:
            return

        # Remove any comments
        line = _COMMENT_P.sub("", line).strip()

        try:
            rtype, value = line.split(None, 1)
        except ValueError:
            # Some plugin might know how to handle this line
            rtype, value = line, ""

        if rtype == "include":
            dirname = None
            if os.path.isabs(filename):
                dirname = os.path.dirname(filename)
            self._handle_include(value, line, line_no, _depth,
                                 dirname=dirname)
        elif rtype == "ifplugin":
            self._handle_ifplugin(value)
        elif rtype == "loadplugin":
            self._handle_loadplugin(value)
        elif rtype in KNOWN_2_RTYPE or rtype in self.ctxt.cmds:
            try:
                rtype, name, value = line.split(None, 2)
            except ValueError:
                raise oa.errors.InvalidSyntax(filename, line_no, line,
                                               "Missing argument")
            if rtype == "tflags":
                value = value.split()

            if rtype == "lang":
                locale.setlocale(locale.LC_ALL, '')
                locale_language = locale.getlocale(locale.LC_MESSAGES)[0]
                if not locale_language.startswith(name):
                    self.ctxt.log.debug("Lang argument does not"
                                        "correspond with locales")
                    return

                if "report" in value:
                    try:
                        rtype, value = value.split(None, 1)
                    except ValueError:
                        raise oa.errors.InvalidSyntax(filename, line_no, line,
                                                       "Missing argument")

                    if not self.ctxt.hook_parse_config(rtype, value):
                        self.ctxt.err("%s:%s Ignoring unknown"
                                      "configuration line: %s",
                                      filename, line_no, line)
                    return

                try:
                    rtype, name, value = value.split(None, 2)
                except ValueError:
                    raise oa.errors.InvalidSyntax(filename, line_no, line,
                                                   "Missing argument")

            if name not in self.results:
                self.results[name] = dict()

            if rtype in RULES or rtype in self.ctxt.cmds:
                if value.startswith("eval:"):
                    # This is for compatibility with SA ruleset
                    self.results[name]["target"] = rtype
                    rtype = "eval"
                self.results[name]["type"] = rtype
                self.results[name]["value"] = value
            else:
                if rtype == 'priority':
                    try:
                        int(value)
                    except ValueError:
                        self.ctxt.err("%s:%s Invalid type for priority value "
                                      "in configuration line: %s, setting it by"
                                      " default to 0", filename, line_no, line)
                self.results[name][rtype] = value

        else:
            if not self.ctxt.hook_parse_config(rtype, value):
                self.ctxt.err("%s:%s Ignoring unknown configuration line: %s",
                              filename, line_no, line)

    def _handle_include(self, value, line, line_no, _depth=0,
                        dirname=None):
        """Handles the 'include' keyword."""
        filename = value.strip()
        if not os.path.isabs(filename) and dirname is not None:
            filename = os.path.join(dirname, filename)
        try:
            self.parse_file(filename, _depth=_depth + 1)
        except oa.errors.MaxRecursionDepthExceeded as e:
            e.add_call(filename, line_no, line)
            raise e

    def _handle_ifplugin(self, value):
        """Handles the 'ifplugin' keyword."""
        plugin_name = oa.plugins.REIMPLEMENTED_PLUGINS.get(value, value)
        try:
            plugin_name = plugin_name.rsplit(".", 1)[1]
        except IndexError:
            pass
        if plugin_name not in self.ctxt.plugins:
            self.ctxt.log.debug("Plugin %s not loaded, skipping.", plugin_name)
            self._ignore = True

    def _handle_loadplugin(self, value):
        """Handles the 'loadplugin' keyword."""
        try:
            plugin_name, path = value.split(None, 1)
        except ValueError:
            plugin_name, path = value, None
        if "::" in plugin_name:
            plugin_name = oa.plugins.REIMPLEMENTED_PLUGINS.get(plugin_name)
        if plugin_name:
            self.ctxt.load_plugin(plugin_name, path)
        else:
            self.ctxt.log.warn("Plugin not available: %s", value)

    def get_ruleset(self):
        """Create and return the corresponding ruleset for the parsed files."""
        self.ctxt.hook_parsing_start(self.results)
        for name, data in self.results.items():
            try:
                rule_type = data["type"]
            except KeyError:
                e = oa.errors.InvalidRule(name, "No rule type defined.")
                self.ctxt.err(e)
                if self.ctxt.paranoid:
                    raise e
            else:
                with self._paranoid(oa.errors.InvalidRule,
                                    oa.errors.InvalidRegex):
                    try:
                        rule_class = RULES[rule_type]
                    except KeyError:
                        # A plugin might have been loaded that
                        # can handle this.
                        rule_class = self.ctxt.cmds[rule_type]
                    self.ctxt.log.debug("Adding rule %s with: %s", name, data)
                    rule = rule_class.get_rule(name, data)
                    self.ruleset.add_rule(rule)
        self.ctxt.hook_parsing_end(self.ruleset)
        self.ctxt.log.info("%s rules loaded", len(self.ruleset.checked))
        self.ruleset.post_parsing()
        return self.ruleset

def parse_pad_rules(files, paranoid=False, ignore_unknown=True):
    """Parse a list of PAD rules and returns the corresponding ruleset.

    'files' - a list of file paths.

    Returns a dictionary that maps rule names to a dictionary of rule options.
    Every rule will contain "type" and "value" which corresponds to the
    mandatory line for defining a rule. For example (type, name value):

    body LOCAL_DEMONSTRATION_RULE   /test/

    Other options may be included such as "score", "describe".
    """
    parser = PADParser(paranoid=paranoid, ignore_unknown=ignore_unknown)
    for filename in files:
        parser.parse_file(filename)

    return parser
<|endoftext|>


<commit_before>#!/usr/bin/python

from __future__ import print_function

import argparse as ap
import subprocess
from collections import namedtuple, defaultdict
import io

from run_control import RunControl, NotSpecified, parse_rc

TypeInfo = namedtuple('TypeInfo', ['t', 'bytes'])

h5ts = {
    'H5T_IEEE_F64LE': TypeInfo('float', 8),
    'H5T_IEEE_F64BE': TypeInfo('float', 8),
    'H5T_IEEE_F32LE': TypeInfo('float', 4),
    'H5T_IEEE_F32BE': TypeInfo('float', 4),
    'H5T_STD_I64LE': TypeInfo('integer', 8),
    'H5T_STD_I64BE': TypeInfo('integer', 8),
    'H5T_STD_I32LE': TypeInfo('integer', 4),
    'H5T_STD_I32BE': TypeInfo('integer', 4),
    'H5T_STD_I16LE': TypeInfo('integer', 2),
    'H5T_STD_I16BE': TypeInfo('integer', 2),
    'H5T_STD_B8LE': TypeInfo('integer bitfield', 1),
    'H5T_STD_B8BE': TypeInfo('integer bitfield', 1),
    'H5T_STRING': 'string',
    'H5T_ARRAY': 'array',
}

tex_replace = {
    '_',
}

_tbl_template = """\\begin{{table}}[{options}]
\centering
\label{{{label}}}
\caption{{{caption}}}
\\begin{{tabular}}{{{layout}}}
\hline
{header}
{entries}
\end{{tabular}}
\end{{table}}
"""

def first_idx(x, lst):
    return lst.index(next(_ for _ in lst if x in _))

def all_idxs(x, lst):
    return [i for i, _ in enumerate(lst) if x in _]

def tex_clean(s):
    for x in tex_replace:
        s = s.replace(x, '\{0}'.format(x))
    return s

def read_type(x):
    name = x[-1].split("\"")[1]
    try:
        h5t = h5ts[x[0].strip().split()[0]]
    except KeyError:
        raise KeyError("H5T for {0} not supported. Line read: {1}".format(name, x))

    if h5t is 'array':
        x = x[0].split()
        h5t = h5ts[x[3]]
        n = x[2][1:-1]
        dtstr = '{0}-length array of {1}-byte {2}s'.format(n, h5t.bytes, h5t.t)
    elif h5t is 'string':
        h5t = TypeInfo(h5t, int(x[1].strip().split()[1][:-1]))
        dtstr = '{0}-character {1}'.format(h5t.bytes, h5t.t)
    else:
        dtstr = '{0}-byte {1}'.format(h5t.bytes, h5t.t)
    return name, dtstr

def datasets(f):
    result = subprocess.check_output(['h5ls', '-r', f]).split('\n')
    return [x.split()[0] for x in result if 'Dataset' in x]

def insert_single_lines(idxs, l):
    # insert in the middle
    added = 0
    for i in range(1, len(idxs)):
        diff  = idxs[i][0] - idxs[i - 1][1] - 1
        if diff != 0:
            add = [(x, x) for x in range(idxs[i-1][1] + 1, idxs[i][0])]
            for x in reversed(add):
                idxs.insert(i + added, x)
            added += len(add)

    # insert in the front
    add = [(x, x) for x in range(idxs[0][0])]
    for x in reversed(add):
        idxs.insert(0, x)

    # insert in the back
    pos = len(idxs)
    add = [(x, x) for x in range(idxs[-1][1] + 1, l)]
    for x in reversed(add):
        idxs.insert(pos, x)

def datatypes(f, dataset):
    result = subprocess.check_output(['h5dump', '-d', dataset, f]).split('\n')
    typelines = result[
        first_idx('DATATYPE', result) + 1:first_idx('DATASPACE', result) - 1]
    idxs = zip(all_idxs('{', typelines), all_idxs('}', typelines))
    if len(idxs) > 0:
        insert_single_lines(idxs, len(typelines))
    typelines = [typelines[i:j + 1] for i, j in idxs]
    return [read_type(x) for x in typelines]

def gen_parser():
    p = ap.ArgumentParser("h5tex", add_help=True)

    h = "The HDF5 database to translate."
    p.add_argument('db', help=h)
    h = "A runcontrol file."
    p.add_argument('--rc', default=None, help=h)
    h = "The path to a specific dataset to print."
    p.add_argument('-d', '--dataset', default=None, help=h)
    h = "A latex table template file."
    p.add_argument('-t', '--template', default=None, help=h)
    h = "Print out debug statements."
    p.add_argument('--debug', action='store_true', default=False, help=h)

    return p

def main():
    p = gen_parser()
    args = p.parse_args()

    db = args.db
    sets = [args.dataset] if args.dataset else datasets(db)
    rc = parse_rc(args.rc, default_nones=True) if args.rc else \
        RunControl(default_nones=True)
    if args.template:
        with io.open(args.template, 'r') as f:
            template = f.read()
    else:
        template = _tbl_template

    for s in sets:
        if args.debug:
            print(s)
        name = s.split('/')[-1]
        options = rc.options or ''
        label = rc.label or 'tbl:{0}'
        caption = rc.caption or \
            'Datatype description of the {0} dataset.'
        layout = rc.layout or '|c|c|c|'
        header = rc.header or ("\\textbf{Name} & \\textbf{Data Type} "
                               "& \\textbf{Description}       \\\\ \\hline")
        entry_template = rc.entry_template or "{0} & {1} & {2} \\\\ \\hline\n"
        descriptions = rc.descriptions or defaultdict(str)
        entries = ''
        for name, dtstr in datatypes(db, s):
            entries += entry_template.format(tex_clean(name),
                                             dtstr, tex_clean(descriptions[name]))
        tbl = template.format(options=options,
                              label=label.format(name),
                              caption=caption.format(name),
                              layout=layout,
                              header=header,
                              entries=entries.strip())
        print()
        print(tbl)
        print()

if __name__ == '__main__':
    main()
<commit_msg>fixed bug where col names took over a var<commit_after>#!/usr/bin/python

from __future__ import print_function

import argparse as ap
import subprocess
from collections import namedtuple, defaultdict
import io

from run_control import RunControl, NotSpecified, parse_rc

TypeInfo = namedtuple('TypeInfo', ['t', 'bytes'])

h5ts = {
    'H5T_IEEE_F64LE': TypeInfo('float', 8),
    'H5T_IEEE_F64BE': TypeInfo('float', 8),
    'H5T_IEEE_F32LE': TypeInfo('float', 4),
    'H5T_IEEE_F32BE': TypeInfo('float', 4),
    'H5T_STD_I64LE': TypeInfo('integer', 8),
    'H5T_STD_I64BE': TypeInfo('integer', 8),
    'H5T_STD_I32LE': TypeInfo('integer', 4),
    'H5T_STD_I32BE': TypeInfo('integer', 4),
    'H5T_STD_I16LE': TypeInfo('integer', 2),
    'H5T_STD_I16BE': TypeInfo('integer', 2),
    'H5T_STD_B8LE': TypeInfo('integer bitfield', 1),
    'H5T_STD_B8BE': TypeInfo('integer bitfield', 1),
    'H5T_STRING': 'string',
    'H5T_ARRAY': 'array',
}

tex_replace = {
    '_',
}

_tbl_template = """\\begin{{table}}[{options}]
\centering
\label{{{label}}}
\caption{{{caption}}}
\\begin{{tabular}}{{{layout}}}
\hline
{header}
{entries}
\end{{tabular}}
\end{{table}}
"""

def first_idx(x, lst):
    return lst.index(next(_ for _ in lst if x in _))

def all_idxs(x, lst):
    return [i for i, _ in enumerate(lst) if x in _]

def tex_clean(s):
    for x in tex_replace:
        s = s.replace(x, '\{0}'.format(x))
    return s

def read_type(x):
    name = x[-1].split("\"")[1]
    try:
        h5t = h5ts[x[0].strip().split()[0]]
    except KeyError:
        raise KeyError("H5T for {0} not supported. Line read: {1}".format(name, x))

    if h5t is 'array':
        x = x[0].split()
        h5t = h5ts[x[3]]
        n = x[2][1:-1]
        dtstr = '{0}-length array of {1}-byte {2}s'.format(n, h5t.bytes, h5t.t)
    elif h5t is 'string':
        h5t = TypeInfo(h5t, int(x[1].strip().split()[1][:-1]))
        dtstr = '{0}-character {1}'.format(h5t.bytes, h5t.t)
    else:
        dtstr = '{0}-byte {1}'.format(h5t.bytes, h5t.t)
    return name, dtstr

def datasets(f):
    result = subprocess.check_output(['h5ls', '-r', f]).split('\n')
    return [x.split()[0] for x in result if 'Dataset' in x]

def insert_single_lines(idxs, l):
    # insert in the middle
    added = 0
    for i in range(1, len(idxs)):
        diff  = idxs[i][0] - idxs[i - 1][1] - 1
        if diff != 0:
            add = [(x, x) for x in range(idxs[i-1][1] + 1, idxs[i][0])]
            for x in reversed(add):
                idxs.insert(i + added, x)
            added += len(add)

    # insert in the front
    add = [(x, x) for x in range(idxs[0][0])]
    for x in reversed(add):
        idxs.insert(0, x)

    # insert in the back
    pos = len(idxs)
    add = [(x, x) for x in range(idxs[-1][1] + 1, l)]
    for x in reversed(add):
        idxs.insert(pos, x)

def datatypes(f, dataset):
    result = subprocess.check_output(['h5dump', '-d', dataset, f]).split('\n')
    typelines = result[
        first_idx('DATATYPE', result) + 1:first_idx('DATASPACE', result) - 1]
    idxs = zip(all_idxs('{', typelines), all_idxs('}', typelines))
    if len(idxs) > 0:
        insert_single_lines(idxs, len(typelines))
    typelines = [typelines[i:j + 1] for i, j in idxs]
    return [read_type(x) for x in typelines]

def gen_parser():
    p = ap.ArgumentParser("h5tex", add_help=True)

    h = "The HDF5 database to translate."
    p.add_argument('db', help=h)
    h = "A runcontrol file."
    p.add_argument('--rc', default=None, help=h)
    h = "The path to a specific dataset to print."
    p.add_argument('-d', '--dataset', default=None, help=h)
    h = "A latex table template file."
    p.add_argument('-t', '--template', default=None, help=h)
    h = "Print out debug statements."
    p.add_argument('--debug', action='store_true', default=False, help=h)

    return p

def main():
    p = gen_parser()
    args = p.parse_args()

    db = args.db
    sets = [args.dataset] if args.dataset else datasets(db)
    rc = parse_rc(args.rc, default_nones=True) if args.rc else \
        RunControl(default_nones=True)
    if args.template:
        with io.open(args.template, 'r') as f:
            template = f.read()
    else:
        template = _tbl_template

    for s in sets:
        if args.debug:
            print(s)
        name = s.split('/')[-1]
        options = rc.options or ''
        label = rc.label or 'tbl:{0}'
        caption = rc.caption or \
            'Datatype description of the {0} dataset.'
        layout = rc.layout or '|c|c|c|'
        header = rc.header or ("\\textbf{Name} & \\textbf{Data Type} "
                               "& \\textbf{Description}       \\\\ \\hline")
        entry_template = rc.entry_template or "{0} & {1} & {2} \\\\ \\hline\n"
        descriptions = rc.descriptions or defaultdict(str)
        entries = ''
        for col, dtstr in datatypes(db, s):
            entries += entry_template.format(tex_clean(col),
                                             dtstr, tex_clean(descriptions[col]))
        tbl = template.format(options=options,
                              label=label.format(name),
                              caption=caption.format(name),
                              layout=layout,
                              header=header,
                              entries=entries.strip())
        print()
        print(tbl)
        print()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from six import add_metaclass

from rx.observable import Observable
from rx.subjects import ReplaySubject
from rx.internal import ExtensionMethod

@add_metaclass(ExtensionMethod)
class ObservableRetry(Observable):
    """Uses a meta class to extend Observable with the methods in this class"""

    def replay(self, selector, buffer_size, window, scheduler):
        """Returns an observable sequence that is the result of invoking the
        selector on a connectable observable sequence that shares a single
        subscription to the underlying sequence replaying notifications subject
        to a maximum time length for the replay buffer.

        This operator is a specialization of Multicast using a ReplaySubject.

        Example:
        res = source.replay(buffer_size=3)
        res = source.replay(buffer_size=3, window=500)
        res = source.replay(None, 3, 500, scheduler)
        res = source.replay(lambda x: x.take(6).repeat(), 3, 500, scheduler)

        Keyword arguments:
        selector -- [Optional] Selector function which can use the multicasted
            source sequence as many times as needed, without causing multiple
            subscriptions to the source sequence. Subscribers to the given
            source will receive all the notifications of the source subject to
            the specified replay buffer trimming policy.
        buffer_size -- [Optional] Maximum element count of the replay buffer.
        window -- [Optional] Maximum time length of the replay buffer.
        scheduler -- [Optional] Scheduler where connected observers within the
            selector function will be invoked on.

        Returns {Observable} An observable sequence that contains the elements
        of a sequence produced by multicasting the source sequence within a
        selector function."""

        if callable(selector):
            def subject_selector():
                return ReplaySubject(buffer_size, window, scheduler)
            return self.multicast(subject_selector, selector)
        else:
            return self.multicast(ReplaySubject(buffer_size, window, scheduler))

<commit_msg>Bugfix, name parameters<commit_after>from six import add_metaclass

from rx.observable import Observable
from rx.subjects import ReplaySubject
from rx.internal import ExtensionMethod

@add_metaclass(ExtensionMethod)
class ObservableRetry(Observable):
    """Uses a meta class to extend Observable with the methods in this class"""

    def replay(self, selector, buffer_size=None, window=None, scheduler=None):
        """Returns an observable sequence that is the result of invoking the
        selector on a connectable observable sequence that shares a single
        subscription to the underlying sequence replaying notifications subject
        to a maximum time length for the replay buffer.

        This operator is a specialization of Multicast using a ReplaySubject.

        Example:
        res = source.replay(buffer_size=3)
        res = source.replay(buffer_size=3, window=500)
        res = source.replay(None, 3, 500, scheduler)
        res = source.replay(lambda x: x.take(6).repeat(), 3, 500, scheduler)

        Keyword arguments:
        selector -- [Optional] Selector function which can use the multicasted
            source sequence as many times as needed, without causing multiple
            subscriptions to the source sequence. Subscribers to the given
            source will receive all the notifications of the source subject to
            the specified replay buffer trimming policy.
        buffer_size -- [Optional] Maximum element count of the replay buffer.
        window -- [Optional] Maximum time length of the replay buffer.
        scheduler -- [Optional] Scheduler where connected observers within the
            selector function will be invoked on.

        Returns {Observable} An observable sequence that contains the elements
        of a sequence produced by multicasting the source sequence within a
        selector function."""

        if callable(selector):
            def subject_selector():
                return ReplaySubject(buffer_size, window, scheduler)
            return self.multicast(subject_selector=subject_selector,
                                 selector=selector)
        else:
            return self.multicast(ReplaySubject(buffer_size, window, scheduler))

<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
"""
Module is containing all necessary global variables for the package.

Module also has the ability to read user-defined data from two paths:

- ``$HOME/_SETTINGS_PATH``
- ``/etc/_SETTINGS_PATH``

See :attr:`._SETTINGS_PATH` for details.

Note:
    If the first path is found, other is ignored.

Example of the configuration file (``$HOME/edeposit/ltp.json``)::

    {
        "EXPORT_DIR": "/somedir/somewhere"
    }

Attributes
----------
"""
import json
import os
import os.path

# Module configuration ========================================================
#: Path to the temporary directory, where the packages are built.
TEMP_DIR = "/tmp"

#: Path to the directory for LTP export.
EXPORT_DIR = "/home/ltp/edep2ltp"

#: Path to the directory for LTP import.
IMPORT_DIR = "/home/ltp/ltp2edep"

# User configuration reader (don't edit this ==================================
_ALLOWED = [str, int, float]

_SETTINGS_PATH = "/edeposit/ltp.json"
"""
Path which is appended to default search paths (``$HOME`` and ``/etc``).

Note:
    It has to start with ``/``. Variable is **appended** to the default search
    paths, so this doesn't mean, that the path is absolute!
"""

def get_all_constants():
    """
    Get list of all uppercase, non-private globals (doesn't start with ``_``).

    Returns:
        list: Uppercase names defined in `globals()` (variables from this \
              module).
    """
    return filter(
        lambda key: key.upper() == key and type(globals()[key]) in _ALLOWED,

        filter(                               # filter _PRIVATE variables
            lambda x: not x.startswith("_"),
            globals()
        )
    )

def substitute_globals(config_dict):
    """
    Set global variables to values defined in `config_dict`.

    Args:
        config_dict (dict): dictionary with data, which are used to set \
                            `globals`.

    Note:
        `config_dict` have to be dictionary, or it is ignored. Also all
        variables, that are not already in globals, or are not types defined in
        :attr:`_ALLOWED` (str, int, float) or starts with ``_`` are silently
        ignored.
    """
    constants = get_all_constants()

    if type(config_dict) != dict:
        return

    for key in config_dict:
        if key in constants and type(config_dict[key]) in _ALLOWED:
            globals()[key] = config_dict[key]

# try to read data from configuration paths ($HOME/_SETTINGS_PATH,
# /etc/_SETTINGS_PATH)
if "HOME" in os.environ and os.path.exists(os.environ["HOME"] + _SETTINGS_PATH):
    with open(os.environ["HOME"] + _SETTINGS_PATH) as f:
        substitute_globals(json.loads(f.read()))
elif os.path.exists("/etc" + _SETTINGS_PATH):
    with open("/etc" + _SETTINGS_PATH) as f:
        substitute_globals(json.loads(f.read()))
<commit_msg>settings.py updated.<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
"""
Module is containing all necessary global variables for the package.

Module also has the ability to read user-defined data from two paths:

- ``$HOME/_SETTINGS_PATH``
- ``/etc/_SETTINGS_PATH``

See :attr:`_SETTINGS_PATH` for details.

Note:
    If the first path is found, other is ignored.

Example of the configuration file (``$HOME/edeposit/ltp.json``)::

    {
        "EXPORT_DIR": "/somedir/somewhere"
    }

Attributes
----------
"""
# Imports =====================================================================
import os
import json
import os.path

# Module configuration ========================================================
#: Path to the temporary directory, where the packages are built.
TEMP_DIR = "/tmp"

#: Path to the directory for LTP export.
EXPORT_DIR = "/home/ltp/edep2ltp"

#: Path to the directory for LTP import.
IMPORT_DIR = "/home/ltp/ltp2edep"

# User configuration reader (don't edit this) =================================
_ALLOWED = [str, unicode, int, float, long, bool]  #: Allowed types.
_SETTINGS_PATH = "/edeposit/ltp.json"  #: Appended to default search paths.

def _get_all_constants():
    """
    Get list of all uppercase, non-private globals (doesn't start with ``_``).

    Returns:
        list: Uppercase names defined in `globals()` (variables from this \
              module).
    """
    return [
        key for key in globals().keys()
        if all([
            not key.startswith("_"),          # publicly accesible
            key.upper() == key,               # uppercase
            type(globals()[key]) in _ALLOWED  # and with type from _ALLOWED
        ])
    ]

def _substitute_globals(config_dict):
    """
    Set global variables to values defined in `config_dict`.

    Args:
        config_dict (dict): dict with data, which are used to set `globals`.

    Note:
        `config_dict` have to be dictionary, or it is ignored. Also all
        variables, that are not already in globals, or are not types defined in
        :attr:`_ALLOWED` (str, int, ..) or starts with ``_`` are silently
        ignored.
    """
    constants = _get_all_constants()

    if type(config_dict) != dict:
        return

    for key, val in config_dict.iteritems():
        if key in constants and type(val) in _ALLOWED:
            globals()[key] = val

def _read_from_paths():
    """
    Try to read data from configuration paths ($HOME/_SETTINGS_PATH,
    /etc/_SETTINGS_PATH).
    """
    home = os.environ.get("HOME", "/")
    home_path = os.path.join(home, _SETTINGS_PATH)
    etc_path = os.path.join("/etc", _SETTINGS_PATH)

    read_path = None
    if home and os.path.exists(home_path):
        read_path = home_path
    elif os.path.exists(etc_path):
        read_path = etc_path

    if read_path:
        with open(read_path) as f:
            _substitute_globals(
                json.loads(f.read())
            )

_read_from_paths()
<|endoftext|>


<commit_before>#!/usr/bin/python

from __future__ import print_function

import argparse as ap
import subprocess
from collections import namedtuple, defaultdict
import io

from run_control import RunControl, NotSpecified, parse_rc

TypeInfo = namedtuple('TypeInfo', ['t', 'bytes'])

h5ts = {
    'H5T_IEEE_F64LE': TypeInfo('float', 8),
    'H5T_IEEE_F64BE': TypeInfo('float', 8),
    'H5T_IEEE_F32LE': TypeInfo('float', 4),
    'H5T_IEEE_F32BE': TypeInfo('float', 4),
    'H5T_STD_I64LE': TypeInfo('integer', 8),
    'H5T_STD_I64BE': TypeInfo('integer', 8),
    'H5T_STD_I32LE': TypeInfo('integer', 4),
    'H5T_STD_I32BE': TypeInfo('integer', 4),
    'H5T_STD_I16LE': TypeInfo('integer', 2),
    'H5T_STD_I16BE': TypeInfo('integer', 2),
    'H5T_STD_B8LE': TypeInfo('integer bitfield', 1),
    'H5T_STD_B8BE': TypeInfo('integer bitfield', 1),
    'H5T_STRING': 'string',
    'H5T_ARRAY': 'array',
}

_tbl_template = """\\begin{{table}}[{options}]
\centering
\label{{{label}}}
\caption{{{caption}}}
\\begin{{tabular}}{{{layout}}}
\hline
{header}
{entries}
\end{{tabular}}
\end{{table}}
"""

def first_idx(x, lst):
    return lst.index(next(_ for _ in lst if x in _))

def all_idxs(x, lst):
    return [i for i, _ in enumerate(lst) if x in _]

def read_type(x):
    name = x[-1].split("\"")[1]
    try:
        h5t = h5ts[x[0].strip().split()[0]]
    except KeyError:
        raise KeyError("H5T for {0} not supported. Line read: {1}".format(name, x))

    if h5t is 'array':
        x = x[0].split()
        h5t = h5ts[x[3]]
        n = x[2][1:-1]
        dtstr = '{0}-length array of {1}-byte {2}s'.format(n, h5t.bytes, h5t.t)
    elif h5t is 'string':
        h5t = TypeInfo(h5t, int(x[1].strip().split()[1][:-1]))
        dtstr = '{0}-character {1}'.format(h5t.bytes, h5t.t)
    else:
        dtstr = '{0}-byte {1}'.format(h5t.bytes, h5t.t)
    return name, dtstr

def datasets(f):
    result = subprocess.check_output(['h5ls', '-r', f]).split('\n')
    return [x.split()[0] for x in result if 'Dataset' in x]

def insert_single_lines(idxs, l):
    # insert in the middle
    added = 0
    for i in range(1, len(idxs)):
        diff  = idxs[i][0] - idxs[i - 1][1] - 1
        if diff != 0:
            add = [(x, x) for x in range(idxs[i-1][1] + 1, idxs[i][0])]
            for x in reversed(add):
                idxs.insert(i + added, x)
            added += len(add)

    # insert in the front
    add = [(x, x) for x in range(idxs[0][0])]
    for x in reversed(add):
        idxs.insert(0, x)

    # insert in the back
    pos = len(idxs)
    add = [(x, x) for x in range(idxs[-1][1] + 1, l)]
    for x in reversed(add):
        idxs.insert(pos, x)

def datatypes(f, dataset):
    result = subprocess.check_output(['h5dump', '-d', dataset, f]).split('\n')
    typelines = result[
        first_idx('DATATYPE', result) + 1:first_idx('DATASPACE', result) - 1]
    idxs = zip(all_idxs('{', typelines), all_idxs('}', typelines))
    if len(idxs) > 0:
        insert_single_lines(idxs, len(typelines))
    typelines = [typelines[i:j + 1] for i, j in idxs]
    return [read_type(x) for x in typelines]

def gen_parser():
    p = ap.ArgumentParser("h5tex", add_help=True)

    h = "The HDF5 database to translate."
    p.add_argument('db', help=h)
    h = "A runcontrol file."
    p.add_argument('--rc', default=None, help=h)
    h = "The path to a specific dataset to print."
    p.add_argument('-d', '--dataset', default=None, help=h)
    h = "A latex table template file."
    p.add_argument('-t', '--template', default=None, help=h)
    h = "Print out debug statements."
    p.add_argument('--debug', action='store_true', default=False, help=h)

    return p

def main():
    p = gen_parser()
    args = p.parse_args()

    db = args.db
    sets = [args.dataset] if args.dataset else datasets(db)
    rc = parse_rc(args.rc, default_nones=True) if args.rc else \
        RunControl(default_nones=True)
    if args.template:
        with io.open(args.template, 'r') as f:
            template = f.read()
    else:
        template = _tbl_template

    for s in sets:
        if args.debug:
            print(s)
        name = s.split('/')[-1]
        options = rc.options or ''
        label = rc.label or 'tbl:{0}'
        caption = rc.caption or \
            'Datatype description of the {0} dataset.'
        layout = rc.layout or '|c|c|c|'
        header = rc.header or ("\\textbf{Name} & \\textbf{Data Type} "
                               "& \\textbf{Description}       \\\\ \\hline")
        entry_template = rc.entry_template or "{0} & {1} & {2} \\\\ \\hline\n"
        descriptions = rc.descriptions or defaultdict(str)
        entries = ''
        for name, dtstr in datatypes(db, s):
            entries += entry_template.format(name, dtstr, descriptions[name])
        tbl = template.format(options=options,
                              label=label.format(name),
                              caption=caption.format(name),
                              layout=layout,
                              header=header,
                              entries=entries.strip())
        print()
        print(tbl)
        print()

if __name__ == '__main__':
    main()
<commit_msg>cleaned up _ which needs to be escaped in latex<commit_after>#!/usr/bin/python

from __future__ import print_function

import argparse as ap
import subprocess
from collections import namedtuple, defaultdict
import io

from run_control import RunControl, NotSpecified, parse_rc

TypeInfo = namedtuple('TypeInfo', ['t', 'bytes'])

h5ts = {
    'H5T_IEEE_F64LE': TypeInfo('float', 8),
    'H5T_IEEE_F64BE': TypeInfo('float', 8),
    'H5T_IEEE_F32LE': TypeInfo('float', 4),
    'H5T_IEEE_F32BE': TypeInfo('float', 4),
    'H5T_STD_I64LE': TypeInfo('integer', 8),
    'H5T_STD_I64BE': TypeInfo('integer', 8),
    'H5T_STD_I32LE': TypeInfo('integer', 4),
    'H5T_STD_I32BE': TypeInfo('integer', 4),
    'H5T_STD_I16LE': TypeInfo('integer', 2),
    'H5T_STD_I16BE': TypeInfo('integer', 2),
    'H5T_STD_B8LE': TypeInfo('integer bitfield', 1),
    'H5T_STD_B8BE': TypeInfo('integer bitfield', 1),
    'H5T_STRING': 'string',
    'H5T_ARRAY': 'array',
}

tex_replace = {
    '_',
}

_tbl_template = """\\begin{{table}}[{options}]
\centering
\label{{{label}}}
\caption{{{caption}}}
\\begin{{tabular}}{{{layout}}}
\hline
{header}
{entries}
\end{{tabular}}
\end{{table}}
"""

def first_idx(x, lst):
    return lst.index(next(_ for _ in lst if x in _))

def all_idxs(x, lst):
    return [i for i, _ in enumerate(lst) if x in _]

def tex_clean(s):
    for x in tex_replace:
        s = s.replace(x, '\{0}'.format(x))
    return s

def read_type(x):
    name = x[-1].split("\"")[1]
    try:
        h5t = h5ts[x[0].strip().split()[0]]
    except KeyError:
        raise KeyError("H5T for {0} not supported. Line read: {1}".format(name, x))

    if h5t is 'array':
        x = x[0].split()
        h5t = h5ts[x[3]]
        n = x[2][1:-1]
        dtstr = '{0}-length array of {1}-byte {2}s'.format(n, h5t.bytes, h5t.t)
    elif h5t is 'string':
        h5t = TypeInfo(h5t, int(x[1].strip().split()[1][:-1]))
        dtstr = '{0}-character {1}'.format(h5t.bytes, h5t.t)
    else:
        dtstr = '{0}-byte {1}'.format(h5t.bytes, h5t.t)
    return name, dtstr

def datasets(f):
    result = subprocess.check_output(['h5ls', '-r', f]).split('\n')
    return [x.split()[0] for x in result if 'Dataset' in x]

def insert_single_lines(idxs, l):
    # insert in the middle
    added = 0
    for i in range(1, len(idxs)):
        diff  = idxs[i][0] - idxs[i - 1][1] - 1
        if diff != 0:
            add = [(x, x) for x in range(idxs[i-1][1] + 1, idxs[i][0])]
            for x in reversed(add):
                idxs.insert(i + added, x)
            added += len(add)

    # insert in the front
    add = [(x, x) for x in range(idxs[0][0])]
    for x in reversed(add):
        idxs.insert(0, x)

    # insert in the back
    pos = len(idxs)
    add = [(x, x) for x in range(idxs[-1][1] + 1, l)]
    for x in reversed(add):
        idxs.insert(pos, x)

def datatypes(f, dataset):
    result = subprocess.check_output(['h5dump', '-d', dataset, f]).split('\n')
    typelines = result[
        first_idx('DATATYPE', result) + 1:first_idx('DATASPACE', result) - 1]
    idxs = zip(all_idxs('{', typelines), all_idxs('}', typelines))
    if len(idxs) > 0:
        insert_single_lines(idxs, len(typelines))
    typelines = [typelines[i:j + 1] for i, j in idxs]
    return [read_type(x) for x in typelines]

def gen_parser():
    p = ap.ArgumentParser("h5tex", add_help=True)

    h = "The HDF5 database to translate."
    p.add_argument('db', help=h)
    h = "A runcontrol file."
    p.add_argument('--rc', default=None, help=h)
    h = "The path to a specific dataset to print."
    p.add_argument('-d', '--dataset', default=None, help=h)
    h = "A latex table template file."
    p.add_argument('-t', '--template', default=None, help=h)
    h = "Print out debug statements."
    p.add_argument('--debug', action='store_true', default=False, help=h)

    return p

def main():
    p = gen_parser()
    args = p.parse_args()

    db = args.db
    sets = [args.dataset] if args.dataset else datasets(db)
    rc = parse_rc(args.rc, default_nones=True) if args.rc else \
        RunControl(default_nones=True)
    if args.template:
        with io.open(args.template, 'r') as f:
            template = f.read()
    else:
        template = _tbl_template

    for s in sets:
        if args.debug:
            print(s)
        name = s.split('/')[-1]
        options = rc.options or ''
        label = rc.label or 'tbl:{0}'
        caption = rc.caption or \
            'Datatype description of the {0} dataset.'
        layout = rc.layout or '|c|c|c|'
        header = rc.header or ("\\textbf{Name} & \\textbf{Data Type} "
                               "& \\textbf{Description}       \\\\ \\hline")
        entry_template = rc.entry_template or "{0} & {1} & {2} \\\\ \\hline\n"
        descriptions = rc.descriptions or defaultdict(str)
        entries = ''
        for name, dtstr in datatypes(db, s):
            entries += entry_template.format(tex_clean(name),
                                             dtstr, tex_clean(descriptions[name]))
        tbl = template.format(options=options,
                              label=label.format(name),
                              caption=caption.format(name),
                              layout=layout,
                              header=header,
                              entries=entries.strip())
        print()
        print(tbl)
        print()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
The FVCOM Python toolbox (PyFvcom)

"""

__version__ = '1.3'
__author__ = 'Pierre Cazenave'
__credits__ = ['Pierre Cazenave']
__license__ = 'MIT'
__maintainer__ = 'Pierre Cazenave'
__email__ = 'pica@pml.ac.uk'

import inspect
from warnings import warn

# Import everything!
from PyFVCOM import buoy_tools
from PyFVCOM import cst_tools
from PyFVCOM import ctd_tools
from PyFVCOM import grid_tools
from PyFVCOM import ll2utm
from PyFVCOM import ocean_tools
from PyFVCOM import stats_tools
from PyFVCOM import tide_tools
from PyFVCOM import tidal_ellipse
from PyFVCOM import process_results
from PyFVCOM import read_results

# External TAPPY now instead of my bundled version. Requires my forked version
# of TAPPY from https://github.com/pwcazenave/tappy or
# http://gitlab.em.pml.ac.uk/pica/tappy.
from tappy import tappy

# For backwards-compatibility.
process_FVCOM_results = process_results
read_FVCOM_results = read_results<commit_msg>Fix the name in the __init__.py.<commit_after>"""
The FVCOM Python toolbox (PyFVCOM)

"""

__version__ = '1.3'
__author__ = 'Pierre Cazenave'
__credits__ = ['Pierre Cazenave']
__license__ = 'MIT'
__maintainer__ = 'Pierre Cazenave'
__email__ = 'pica@pml.ac.uk'

import inspect
from warnings import warn

# Import everything!
from PyFVCOM import buoy_tools
from PyFVCOM import cst_tools
from PyFVCOM import ctd_tools
from PyFVCOM import grid_tools
from PyFVCOM import ll2utm
from PyFVCOM import ocean_tools
from PyFVCOM import stats_tools
from PyFVCOM import tide_tools
from PyFVCOM import tidal_ellipse
from PyFVCOM import process_results
from PyFVCOM import read_results

# External TAPPY now instead of my bundled version. Requires my forked version
# of TAPPY from https://github.com/pwcazenave/tappy or
# http://gitlab.em.pml.ac.uk/pica/tappy.
from tappy import tappy

# For backwards-compatibility.
process_FVCOM_results = process_results
read_FVCOM_results = read_results<|endoftext|>