<commit_before># Ben Jones
# Fall 2013
# HTPT Pluggable Transport

import unittest

from htpt import frame

class TestFrame(unittest.TestCase):
  """Test the validity of the framing module in htpt"""

  def setUp(self):
    self.frame = frame.Framer(self.callback)

  def callback(data):
    """Dummy callback function to test the framing modules ability to
    send data up"""

    self.uploadedData = data

  def test_isSeqNumInBuffer(self):
    """Test whether this function correctly identifies when sequence
    numbers are in or out of the scope of the buffer"""

    #test 1-> no wrapping min < num < max
    self.frame.minAcceptableSeqNum = 1
    self.frame.maxAcceptableSeqNum = 10
    seqNum = 5
    self.assertEqual(True, self.frame.isSeqNumInBuffer(seqNum))
    #test 2-> wrap max, max < min < num
    self.frame.minAcceptableSeqNum = 65000
    self.frame.maxAcceptableSeqNum = 1
    seqNum = 65005
    self.assertEqual(True, self.frame.isSeqNumInBuffer(seqNum))
    #test 3-> wrap max and num, num < max < min
    self.frame.minAcceptableSeqNum = 65000
    self.frame.maxAcceptableSeqNum = 20
    seqNum = 3
    self.assertEqual(True, self.frame.isSeqNumInBuffer(seqNum))
    #test 4-> too low-> num < min < max
    self.frame.minAcceptableSeqNum = 60000
    self.frame.maxAcceptableSeqNum = 60002
    seqNum = 3
    self.assertEqual(False, self.frame.isSeqNumInBuffer(seqNum))
    #test 5-> too high-> min < max < num
    self.frame.minAcceptableSeqNum = 60000
    self.frame.maxAcceptableSeqNum = 65002
    seqNum = 65005
    self.assertEqual(False, self.frame.isSeqNumInBuffer(seqNum))
    #test 6-> bad seq num-> max < min < num, where num > max_seq_num
    self.frame.minAcceptableSeqNum = 65000
    self.frame.maxAcceptableSeqNum = 5
    seqNum = 68000
    self.assertEqual(False, self.frame.isSeqNumInBuffer(seqNum))
<commit_msg>Added stubs for all the unit tests<commit_after># Ben Jones
# Fall 2013
# HTPT Pluggable Transport

import unittest

from htpt import frame

class TestFrame(unittest.TestCase):
  """Test the validity of the framing module in htpt"""

  def setUp(self):
    self.frame = frame.Framer(self.callback)

  def callback(data):
    """Dummy callback function to test the framing modules ability to
    send data up"""

    self.uploadedData = data

  def test_init(self):
    pass

  def test_getSequenceAndIncrement(self):
    pass

  def test_FramerConstructor(self):
    pass

  def test_addFrameToBuffer(self):
    pass

  def test_flushBuffer(self):
    pass

  def test_recvFrame(self):
    pass

  def test_isSeqNumInBuffer(self):
    """Test whether this function correctly identifies when sequence
    numbers are in or out of the scope of the buffer"""

    #test 1-> no wrapping min < num < max
    self.frame.minAcceptableSeqNum = 1
    self.frame.maxAcceptableSeqNum = 10
    seqNum = 5
    self.assertEqual(True, self.frame.isSeqNumInBuffer(seqNum))
    #test 2-> wrap max, max < min < num
    self.frame.minAcceptableSeqNum = 65000
    self.frame.maxAcceptableSeqNum = 1
    seqNum = 65005
    self.assertEqual(True, self.frame.isSeqNumInBuffer(seqNum))
    #test 3-> wrap max and num, num < max < min
    self.frame.minAcceptableSeqNum = 65000
    self.frame.maxAcceptableSeqNum = 20
    seqNum = 3
    self.assertEqual(True, self.frame.isSeqNumInBuffer(seqNum))
    #test 4-> too low-> num < min < max
    self.frame.minAcceptableSeqNum = 60000
    self.frame.maxAcceptableSeqNum = 60002
    seqNum = 3
    self.assertEqual(False, self.frame.isSeqNumInBuffer(seqNum))
    #test 5-> too high-> min < max < num
    self.frame.minAcceptableSeqNum = 60000
    self.frame.maxAcceptableSeqNum = 65002
    seqNum = 65005
    self.assertEqual(False, self.frame.isSeqNumInBuffer(seqNum))
    #test 6-> bad seq num-> max < min < num, where num > max_seq_num
    self.frame.minAcceptableSeqNum = 65000
    self.frame.maxAcceptableSeqNum = 5
    seqNum = 68000
    self.assertEqual(False, self.frame.isSeqNumInBuffer(seqNum))
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2020 The TF-Agents Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Define TF Agents version information."""

# We follow Semantic Versioning (https://semver.org/)
_MAJOR_VERSION = '0'
_MINOR_VERSION = '12'
_PATCH_VERSION = '0'

# When building releases, we can update this value on the release branch to
# reflect the current release candidate ('rc0', 'rc1') or, finally, the official
# stable release (indicated by `_REL_SUFFIX = ''`). Outside the context of a
# release branch, the current version is by default assumed to be a
# 'development' version, labeled 'dev'.
_DEV_SUFFIX = 'dev'
_REL_SUFFIX = 'rc0'

# Example, '0.10.0rc0'
__version__ = '.'.join([
    _MAJOR_VERSION,
    _MINOR_VERSION,
    _PATCH_VERSION,
])
__dev_version__ = '{}.{}'.format(__version__, _DEV_SUFFIX)
__rel_version__ = '{}{}'.format(__version__, _REL_SUFFIX)
<commit_msg>Moves nightly to 0.13.x.<commit_after># coding=utf-8
# Copyright 2020 The TF-Agents Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Define TF Agents version information."""

# We follow Semantic Versioning (https://semver.org/)
_MAJOR_VERSION = '0'
_MINOR_VERSION = '13'
_PATCH_VERSION = '0'

# When building releases, we can update this value on the release branch to
# reflect the current release candidate ('rc0', 'rc1') or, finally, the official
# stable release (indicated by `_REL_SUFFIX = ''`). Outside the context of a
# release branch, the current version is by default assumed to be a
# 'development' version, labeled 'dev'.
_DEV_SUFFIX = 'dev'
_REL_SUFFIX = 'rc0'

# Example, '0.10.0rc0'
__version__ = '.'.join([
    _MAJOR_VERSION,
    _MINOR_VERSION,
    _PATCH_VERSION,
])
__dev_version__ = '{}.{}'.format(__version__, _DEV_SUFFIX)
__rel_version__ = '{}{}'.format(__version__, _REL_SUFFIX)
<|endoftext|>


<commit_before>__version__ = '1.0.1'

default_app_config = 'tg_react.apps.TgReactConfig'
<commit_msg>Update __init__.py<commit_after>__version__ = '1.0.2'

default_app_config = 'tg_react.apps.TgReactConfig'
<|endoftext|>


<commit_before># coding: utf-8
import requests

# django classes
from logging import getLogger
from django.core.cache import caches

# django_th classes
from django_th.services.services import ServicesMgr
from django_th.models import TriggerService
from th_slack.models import Slack

logger = getLogger('django_th.trigger_happy')

cache = caches['th_slack']

class ServiceSlack(ServicesMgr):
    """
        Service Slack
    """
    def __init__(self, token=None, **kwargs):
        super(ServiceSlack, self).__init__(token, **kwargs)

    def read_data(self, **kwargs):
        """
            get the data from the service

            :param kwargs: contain keyword args : trigger_id and model name
            :type kwargs: dict
            :rtype: dict
        """
        data = ()
        return data

    def save_data(self, trigger_id, **data):
        """
            get the data from the service

            :param trigger_id: id of the trigger
            :params data, dict
            :rtype: dict
        """
        status = False
        service = TriggerService.objects.get(id=trigger_id)
        desc = service.description

        slack = Slack.objects.get(trigger_id=trigger_id)

        title = self.set_title(data)
        if title is None:
            title = data.get('subject')
        type_action = data.get('type_action')

        # set the bot username of Slack to the name of the
        # provider service
        username = service.provider.name.name.split('Service')[1]
        # 'build' a link
        title_link = ''
        if data.get('permalink'):
            title_link = ': <' + data.get('permalink') + '|' + title + '>'
        data = '*' + desc + '*: ' + type_action + title_link

        payload = {'username': username,
                   'text': data}

        r = requests.post(slack.webhook_url, json=payload)

        if r.status_code == requests.codes.ok:
            status = True
        # return the data
        return status
<commit_msg>Replaced cache variable<commit_after># coding: utf-8
import requests

# django classes
from logging import getLogger
from django.core.cache import caches

# django_th classes
from django_th.services.services import ServicesMgr
from django_th.models import TriggerService
from th_slack.models import Slack

logger = getLogger('django_th.trigger_happy')

cache = caches['django_th']

class ServiceSlack(ServicesMgr):
    """
        Service Slack
    """
    def __init__(self, token=None, **kwargs):
        super(ServiceSlack, self).__init__(token, **kwargs)

    def read_data(self, **kwargs):
        """
            get the data from the service

            :param kwargs: contain keyword args : trigger_id and model name
            :type kwargs: dict
            :rtype: dict
        """
        data = ()
        return data

    def save_data(self, trigger_id, **data):
        """
            get the data from the service

            :param trigger_id: id of the trigger
            :params data, dict
            :rtype: dict
        """
        status = False
        service = TriggerService.objects.get(id=trigger_id)
        desc = service.description

        slack = Slack.objects.get(trigger_id=trigger_id)

        title = self.set_title(data)
        if title is None:
            title = data.get('subject')
        type_action = data.get('type_action')

        # set the bot username of Slack to the name of the
        # provider service
        username = service.provider.name.name.split('Service')[1]
        # 'build' a link
        title_link = ''
        if data.get('permalink'):
            title_link = ': <' + data.get('permalink') + '|' + title + '>'
        data = '*' + desc + '*: ' + type_action + title_link

        payload = {'username': username,
                   'text': data}

        r = requests.post(slack.webhook_url, json=payload)

        if r.status_code == requests.codes.ok:
            status = True
        # return the data
        return status
<|endoftext|>


<commit_before>#!/usr/bin/python
import Adafruit_PCA9685
import time
import random
import sys
import socket
import json
from socket import error as socket_error

# Initialise the PCA9685 using the default address (0x40).
pwm = Adafruit_PCA9685.PCA9685(0x40)
pwm.set_pwm_freq(60)

SRV_OPTIONS = []
ACTIONS = {}

STATUS = ""
try:
    chknet = sys.argv[1]
    print("Chknet: %s" % chknet)
    if int(chknet) == 1:
        NETWORK = 0
    else:
        NETWORK = 1
except:
    NETWORK = 1

thingfile = "/home/pi/pimeup/thingbox/thing.json"
thingactionfile = "/home/pi/pimeup/thingbox/thingactions.json"
STATUS_OPT = [ 'LIDUP', 'HANDUPLIDUP', 'HANDDOWNLIDUP', 'HANDDOWNLIDDOWN' ]
DEBUG = 0
NET_DEBUG = 1

UDP_IP = '192.168.1.110'
UDP_PORT = 30000
UDP_BUFFER_SIZE = 5

def main():
    global SRV_OPTIONS
    global ACTIONS
    global STATUS

    SRV_OPTIONS = loadfile(thingfile)
    ACTIONS = loadfile(thingactionfile)

    cur_finger = -1
    ACT_SHORT = []
    upact = ""
    downact = ""
    for x in ACTIONS:
        if x['KEY'] == "U":
            upact = x['ACTION']
        if x['KEY'] == "P":
            downact = x['ACTION']
        ACT_SHORT.append(x['KEY'])

    if NETWORK:
        print("Listening on %s:%s" % (UDP_IP, UDP_PORT))
        sock = socket.socket(socket.AF_INET, # Internet
                        socket.SOCK_DGRAM) # UDP
        sock.bind((UDP_IP, UDP_PORT))
    try:
        while True:
            if NETWORK:
                data, addr = sock.recvfrom(UDP_BUFFER_SIZE)
            else:
                data = raw_input("Please Enter Raw Command: ")
            if not data:
                continue
            else:
                if DEBUG or NET_DEBUG:
                    print("Recieved Data Update: %s" % data)

                tdata = data.split(":")
                if len(tdata) !=2 and len(tdata) != 4:
                    print("Ignoring Bad Data: %s" % data)
                    continue
                else:
                    cmdkey = tdata[0]
                    cmdval = tdata[1]
                    if str(cmdkey) == "A" and cmdval in ACT_SHORT:
                        processAction(cmdval)
                    elif str(cmdkey) == "S" and cmdval != "":
                        if cmdval in STATUS_OPT:
                            STATUS = cmdval
                        else:
                            print("Status needs to be in %s" % STATUS_OPT)
                    else:
                        try:
                            cmdkey = int(cmdkey)
                            cmdval = int(cmdval)
                        except:
                            print("cmdkey must be A or an integer")
                            continue
                        setfingerperc(cmdkey, cmdval)
    except socket_error:
        exitGracefully()
    except KeyboardInterrupt:
        exitGracefully()

def setfingerperc(cmdkey, cmdorigval, ignorestatus=False):
    global SRV_OPTIONS
    global STATUS
    if STATUS.find("HANDUP") >= 0 or ignorestatus:
        if SRV_OPTIONS[cmdkey]["INVERT"] == True:
            cmdval = abs(cmdorigval - 100)
        else:
            cmdval = cmdorigval
        setval = (cmdval * (SRV_OPTIONS[cmdkey]['RANGE_MAX'] - SRV_OPTIONS[cmdkey]['RANGE_MIN']) / 100) + SRV_OPTIONS[cmdkey]['RANGE_MIN']
        if DEBUG or NET_DEBUG:
            print("Setting Servo: %s (%s) to %s percent - (%s)" % (cmdkey, SRV_OPTIONS[cmdkey]['DESC'], cmdorigval, setval))
        pwm.set_pwm(cmdkey, 0, setval)
    else:
        print("Will not preform commands due to STATUS: %s" % STATUS)

def processAction(actKey):
    global STATUS
    act = {}
    bfound = False
    for x in ACTIONS:
        if actKey == x['KEY']:
            act = x
            bfound = True
    if bfound == True:
        new_status = act['STATUS']
        req_status = act['REQ_STATUS']
        actStr = act['ACTION']
        if req_status != "":
            if STATUS.find(req_status) < 0:
                print("Can't do it")
                print("STATUS: %s" % STATUS)
                print("req_status: %s" % req_status)
                return
        print("Running Action: %s" % act['NAME'])
        for action in actStr.split(","):
            tval = action.split(":")
            act = tval[0]
            val = tval[1]
            if act == "P":
                val = float(val)
                time.sleep(val)
            elif act == "A":
                shutdown = False
                try:
                    val = int(val)
                    if val == 0:
                        shutdown = True
                except:
                    shutdown = False
                if shutdown == True:
                    for x in range(len(SRV_OPTIONS)):
                        pwm.set_pwm(x, 4096, 0)
                else:
                    processAction(val)
            else:
                act = int(act)
                val = int(val)
                setfingerperc(act, val, True)
        if new_status != "":
            STATUS = new_status

def loadfile(f):
    o = open(f, "rb")
    tj = o.read()
    o.close()

    pj = ""
    for line in tj.split("\n"):
        if line.strip() == "" or line.strip().find("#") == 0:
            pass
        else:
            pj += line.strip() + "\n"
    print(pj)
    return json.loads(pj)

def exitGracefully():
    print("")
    print("ok Bye")
    pwm.set_pwm(0, 4096, 0)
    pwm.set_pwm(1, 4096, 0)
    pwm.set_pwm(2, 4096, 0)
    pwm.set_pwm(3, 4096, 0)
    pwm.set_pwm(4, 4096, 0)
    pwm.set_pwm(5, 4096, 0)
    pwm.set_pwm(6, 4096, 0)
    pwm.set_pwm(7, 4096, 0)
    pwm.set_pwm(8, 4096, 0)
    sys.exit(0)

def setServoPulse(channel, pulse):
  pulseLength = 1000000                   # 1,000,000 us per second
  pulseLength /= 60                       # 60 Hz
  print "%d us per period" % pulseLength
  pulseLength /= 4096                     # 12 bits of resolution
  print "%d us per bit" % pulseLength
  pulse *= 1000
  pulse /= pulseLength
  pwm.set_pwm(channel, 0, pulse)

if __name__ == "__main__":
    main()
<commit_msg>Updates<commit_after>#!/usr/bin/python
import Adafruit_PCA9685
import time
import random
import sys
import socket
import json
from socket import error as socket_error

# Initialise the PCA9685 using the default address (0x40).
pwm = Adafruit_PCA9685.PCA9685(0x40)
pwm.set_pwm_freq(60)

SRV_OPTIONS = []
ACTIONS = {}
HOMENET = 0
NETWORK = 1
STATUS = ""
try:
    chknet = sys.argv[1]
    print("Chknet: %s" % chknet)
    if int(chknet) == 2: # No network command line interface
        NETWORK = 0
    elif int(chknet) == 1:  # Use home net ip of 192.168.0.130
        HOMENET = 1
    else:
        NETWORK = 1
except:
    NETWORK = 1

thingfile = "/home/pi/pimeup/thingbox/thing.json"
thingactionfile = "/home/pi/pimeup/thingbox/thingactions.json"
STATUS_OPT = [ 'LIDUP', 'HANDUPLIDUP', 'HANDDOWNLIDUP', 'HANDDOWNLIDDOWN' ]
DEBUG = 0
NET_DEBUG = 1
if HOMENET == 1:
    UDP_IP = '192.168.0.130'
else:
    UDP_IP = '192.168.1.110'
UDP_PORT = 30000
UDP_BUFFER_SIZE = 5

def main():
    global SRV_OPTIONS
    global ACTIONS
    global STATUS

    SRV_OPTIONS = loadfile(thingfile)
    ACTIONS = loadfile(thingactionfile)

    cur_finger = -1
    ACT_SHORT = []
    upact = ""
    downact = ""
    for x in ACTIONS:
        if x['KEY'] == "U":
            upact = x['ACTION']
        if x['KEY'] == "P":
            downact = x['ACTION']
        ACT_SHORT.append(x['KEY'])

    if NETWORK:
        print("Listening on %s:%s" % (UDP_IP, UDP_PORT))
        sock = socket.socket(socket.AF_INET, # Internet
                        socket.SOCK_DGRAM) # UDP
        sock.bind((UDP_IP, UDP_PORT))
    try:
        while True:
            if NETWORK:
                data, addr = sock.recvfrom(UDP_BUFFER_SIZE)
            else:
                data = raw_input("Please Enter Raw Command: ")
            if not data:
                continue
            else:
                if DEBUG or NET_DEBUG:
                    print("Recieved Data Update: %s" % data)

                tdata = data.split(":")
                if len(tdata) !=2 and len(tdata) != 4:
                    print("Ignoring Bad Data: %s" % data)
                    continue
                else:
                    cmdkey = tdata[0]
                    cmdval = tdata[1]
                    if str(cmdkey) == "A" and cmdval in ACT_SHORT:
                        processAction(cmdval)
                    elif str(cmdkey) == "S" and cmdval != "":
                        if cmdval in STATUS_OPT:
                            STATUS = cmdval
                        else:
                            print("Status needs to be in %s" % STATUS_OPT)
                    else:
                        try:
                            cmdkey = int(cmdkey)
                            cmdval = int(cmdval)
                        except:
                            print("cmdkey must be A or an integer")
                            continue
                        setfingerperc(cmdkey, cmdval)
    except socket_error:
        exitGracefully()
    except KeyboardInterrupt:
        exitGracefully()

def setfingerperc(cmdkey, cmdorigval, ignorestatus=False):
    global SRV_OPTIONS
    global STATUS
    if STATUS.find("HANDUP") >= 0 or ignorestatus:
        if SRV_OPTIONS[cmdkey]["INVERT"] == True:
            cmdval = abs(cmdorigval - 100)
        else:
            cmdval = cmdorigval
        setval = (cmdval * (SRV_OPTIONS[cmdkey]['RANGE_MAX'] - SRV_OPTIONS[cmdkey]['RANGE_MIN']) / 100) + SRV_OPTIONS[cmdkey]['RANGE_MIN']
        if DEBUG or NET_DEBUG:
            print("Setting Servo: %s (%s) to %s percent - (%s)" % (cmdkey, SRV_OPTIONS[cmdkey]['DESC'], cmdorigval, setval))
        pwm.set_pwm(cmdkey, 0, setval)
    else:
        print("Will not preform commands due to STATUS: %s" % STATUS)

def processAction(actKey):
    global STATUS
    act = {}
    bfound = False
    for x in ACTIONS:
        if actKey == x['KEY']:
            act = x
            bfound = True
    if bfound == True:
        new_status = act['STATUS']
        req_status = act['REQ_STATUS']
        actStr = act['ACTION']
        if req_status != "":
            if STATUS.find(req_status) < 0:
                print("Can't do it")
                print("STATUS: %s" % STATUS)
                print("req_status: %s" % req_status)
                return
        print("Running Action: %s" % act['NAME'])
        for action in actStr.split(","):
            tval = action.split(":")
            act = tval[0]
            val = tval[1]
            if act == "P":
                val = float(val)
                time.sleep(val)
            elif act == "A":
                shutdown = False
                try:
                    val = int(val)
                    if val == 0:
                        shutdown = True
                except:
                    shutdown = False
                if shutdown == True:
                    for x in range(len(SRV_OPTIONS)):
                        pwm.set_pwm(x, 4096, 0)
                else:
                    processAction(val)
            else:
                act = int(act)
                val = int(val)
                setfingerperc(act, val, True)
        if new_status != "":
            STATUS = new_status

def loadfile(f):
    o = open(f, "rb")
    tj = o.read()
    o.close()

    pj = ""
    for line in tj.split("\n"):
        if line.strip() == "" or line.strip().find("#") == 0:
            pass
        else:
            pj += line.strip() + "\n"
    print(pj)
    return json.loads(pj)

def exitGracefully():
    print("")
    print("ok Bye")
    pwm.set_pwm(0, 4096, 0)
    pwm.set_pwm(1, 4096, 0)
    pwm.set_pwm(2, 4096, 0)
    pwm.set_pwm(3, 4096, 0)
    pwm.set_pwm(4, 4096, 0)
    pwm.set_pwm(5, 4096, 0)
    pwm.set_pwm(6, 4096, 0)
    pwm.set_pwm(7, 4096, 0)
    pwm.set_pwm(8, 4096, 0)
    sys.exit(0)

def setServoPulse(channel, pulse):
  pulseLength = 1000000                   # 1,000,000 us per second
  pulseLength /= 60                       # 60 Hz
  print "%d us per period" % pulseLength
  pulseLength /= 4096                     # 12 bits of resolution
  print "%d us per bit" % pulseLength
  pulse *= 1000
  pulse /= pulseLength
  pwm.set_pwm(channel, 0, pulse)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import uuid
import os

from django.db.models import ImageField as DjangoImageField

from .backends import metadata, storage
from .files import ThumbnailedImageFile
from . import processors, post_processors

class ImageField(DjangoImageField):
    attr_class = ThumbnailedImageFile

    def __init__(self, *args, **kwargs):
        self.resize_source_to = kwargs.pop('resize_source_to', None)
        if kwargs.get('storage'):
            raise ValueError('Please define storage backend in settings.py instead on the field itself')
        kwargs['storage'] = storage.get_backend()
        self.metadata_backend = metadata.get_backend()
        super(ImageField, self).__init__(self, *args, **kwargs)

    def __unicode__(self):
        return self.attname

    def pre_save(self, model_instance, add):
        """
        Process the source image through the defined processors.
        """
        file = getattr(model_instance, self.attname)

        if file and not file._committed:
            image_file = file
            if self.resize_source_to:
                file.seek(0)
                image_file = processors.process(file, self.resize_source_to)
                image_file = post_processors.process(image_file, self.resize_source_to)
            filename = str(uuid.uuid4()) + os.path.splitext(file.name)[1]
            file.save(filename, image_file, save=False)
        return file

    def south_field_triple(self):
        """
        Return a suitable description of this field for South.
        Taken from smiley chris' easy_thumbnails
        """
        from south.modelsinspector import introspector
        field_class = 'django.db.models.fields.files.ImageField'
        args, kwargs = introspector(self)
        return (field_class, args, kwargs)
<commit_msg>Fixed typo in ImageField's super call.<commit_after>import uuid
import os

from django.db.models import ImageField as DjangoImageField

from .backends import metadata, storage
from .files import ThumbnailedImageFile
from . import processors, post_processors

class ImageField(DjangoImageField):
    attr_class = ThumbnailedImageFile

    def __init__(self, *args, **kwargs):
        self.resize_source_to = kwargs.pop('resize_source_to', None)
        if kwargs.get('storage'):
            raise ValueError('Please define storage backend in settings.py instead on the field itself')
        kwargs['storage'] = storage.get_backend()
        self.metadata_backend = metadata.get_backend()
        super(ImageField, self).__init__(*args, **kwargs)

    def __unicode__(self):
        return self.attname

    def pre_save(self, model_instance, add):
        """
        Process the source image through the defined processors.
        """
        file = getattr(model_instance, self.attname)

        if file and not file._committed:
            image_file = file
            if self.resize_source_to:
                file.seek(0)
                image_file = processors.process(file, self.resize_source_to)
                image_file = post_processors.process(image_file, self.resize_source_to)
            filename = str(uuid.uuid4()) + os.path.splitext(file.name)[1]
            file.save(filename, image_file, save=False)
        return file

    def south_field_triple(self):
        """
        Return a suitable description of this field for South.
        Taken from smiley chris' easy_thumbnails
        """
        from south.modelsinspector import introspector
        field_class = 'django.db.models.fields.files.ImageField'
        args, kwargs = introspector(self)
        return (field_class, args, kwargs)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.

# Gets or updates a content shell (a nearly headless build of chrome). This is
# used for running browser tests of client applications.

import json
import optparse
import os
import platform
import shutil
import subprocess
import sys
import tempfile
import zipfile

import utils

def NormJoin(path1, path2):
  return os.path.normpath(os.path.join(path1, path2))

# Change into the dart directory as we want the project to be rooted here.
dart_src = NormJoin(os.path.dirname(sys.argv[0]), os.pardir)
os.chdir(dart_src)

GSUTIL_DIR = os.path.join('third_party', 'gsutil')
GSUTIL = GSUTIL_DIR + '/gsutil'

DRT_DIR = os.path.join('client', 'tests', 'drt')
DRT_VERSION = os.path.join(DRT_DIR, 'LAST_VERSION')
DRT_LATEST_PATTERN = (
    'gs://dartium-archive/latest/drt-%(osname)s-%(bot)s-*.zip')
DRT_PERMANENT_PATTERN = ('gs://dartium-archive/drt-%(osname)s-%(bot)s/drt-'
                         '%(osname)s-%(bot)s-%(num1)s.%(num2)s.zip')

DARTIUM_DIR = os.path.join('client', 'tests', 'dartium')
DARTIUM_VERSION = os.path.join(DARTIUM_DIR, 'LAST_VERSION')
DARTIUM_LATEST_PATTERN = (
    'gs://dartium-archive/latest/dartium-%(osname)s-%(bot)s-*.zip')
DARTIUM_PERMANENT_PATTERN = ('gs://dartium-archive/dartium-%(osname)s-%(bot)s/'
                             'dartium-%(osname)s-%(bot)s-%(num1)s.%(num2)s.zip')

SDK_DIR = os.path.join(utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32'),
    'dart-sdk')
SDK_VERSION = os.path.join(SDK_DIR, 'LAST_VERSION')
SDK_LATEST_PATTERN = 'gs://dart-archive/channels/dev/raw/latest/VERSION'
# TODO(efortuna): Once the x64 VM also is optimized, select the version
# based on whether we are running on a 32-bit or 64-bit system.
SDK_PERMANENT = ('gs://dart-archive/channels/dev/raw/%(version_num)s/sdk/' +
    'dartsdk-%(osname)s-ia32-release.zip')

# Dictionary storing the earliest revision of each download we have stored.
LAST_VALID = {'dartium': 4285, 'chromedriver': 7823, 'sdk': 9761, 'drt': 5342}

sys.path.append(os.path.join(GSUTIL_DIR, 'third_party', 'boto'))
import boto

def ExecuteCommand(*cmd):
  """Execute a command in a subprocess."""
  pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  output, error = pipe.communicate()
  return pipe.returncode, output

def ExecuteCommandVisible(*cmd):
  """Execute a command in a subprocess, but show stdout/stderr."""
  result = subprocess.call(cmd, stdout=sys.stdout, stderr=sys.stderr,
                           stdin=sys.stdin)
  if result != 0:
    raise Exception('Execution of "%s" failed' % ' '.join(cmd))

def Gsutil(*cmd):
  return ExecuteCommand('python', GSUTIL, *cmd)

def GsutilVisible(*cmd):
  ExecuteCommandVisible('python', GSUTIL, *cmd)

def HasBotoConfig():
  """Returns true if boto config exists."""

  config_paths = boto.pyami.config.BotoConfigLocations
  if 'AWS_CREDENTIAL_FILE' in os.environ:
    config_paths.append(os.environ['AWS_CREDENTIAL_FILE'])
  for config_path in config_paths:
    if os.path.exists(config_path):
      return True

  return False

def InRunhooks():
  """True if this script was called by "gclient runhooks" or "gclient sync\""""
  return 'runhooks' in sys.argv

def EnsureConfig():
  # If ~/.boto doesn't exist, tell the user to run "gsutil config"
  if not HasBotoConfig():
    print >>sys.stderr, '''
*******************************************************************************
* WARNING: Can't download content shell! This is required to test client apps.
* You need to do a one-time configuration step to access Google Storage.
* Please run this command and follow the instructions:
*     %s config
*
* NOTE: When prompted you can leave "project-id" blank. Just hit enter.
*******************************************************************************
''' % GSUTIL
    sys.exit(1)

def GetDartiumRevision(name, bot, directory, version_file, latest_pattern,
    permanent_prefix, revision_num=None):
  """Get the latest binary that is stored in the dartium archive.

  Args:
    name: the name of the desired download.
    directory: target directory (recreated) to install binary
    version_file: name of file with the current version stamp
    latest_pattern: the google store url pattern pointing to the latest binary
    permanent_prefix: stable google store folder used to download versions
    revision_num: The desired revision number to retrieve. If revision_num is
        None, we return the latest revision. If the revision number is specified
        but unavailable, find the nearest older revision and use that instead.
  """
  osdict = {'Darwin':'mac', 'Linux':'lucid64', 'Windows':'win'}

  def FindPermanentUrl(out, osname, the_revision_num):
    output_lines = out.split()
    latest = output_lines[-1]
    if not the_revision_num:
      latest = (permanent_prefix[:permanent_prefix.rindex('/')] % { 'osname' :
          osname, 'bot' : bot } + latest[latest.rindex('/'):])
    else:
      latest = (permanent_prefix % { 'osname' : osname, 'num1' : the_revision_num,
          'num2' : the_revision_num, 'bot' : bot })
      foundURL = False
      while not foundURL:
        # Test to ensure this URL exists because the dartium-archive builds can
        # have unusual numbering (a range of CL numbers) sometimes.
        result, out = Gsutil('ls', permanent_prefix % {'osname' : osname,
            'num1': the_revision_num, 'num2': '*', 'bot': bot })
        if result == 0:
          # First try to find one with the the second number the same as the
          # requested number.
          latest = out.split()[0]
          # Now test that the permissions are correct so you can actually
          # download it.
          temp_dir = tempfile.mkdtemp()
          temp_zip = os.path.join(temp_dir, 'foo.zip')
          returncode, out = Gsutil('cp', latest, 'file://' + temp_zip)
          if returncode == 0:
            foundURL = True
          else:
            # Unable to download this item (most likely because something went
            # wrong on the upload and the permissions are bad). Keep looking for
            # a different URL.
            the_revision_num = int(the_revision_num) - 1
          shutil.rmtree(temp_dir)
        else:
          # Now try to find one with a nearby CL num.
          the_revision_num = int(the_revision_num) - 1
          if the_revision_num <= 0:
            TooEarlyError()
    return latest

  GetFromGsutil(name, directory, version_file, latest_pattern, osdict,
                  FindPermanentUrl, revision_num, bot)

def GetSdkRevision(name, directory, version_file, latest_pattern,
    permanent_prefix, revision_num):
  """Get a revision of the SDK from the editor build archive.

  Args:
    name: the name of the desired download
    directory: target directory (recreated) to install binary
    version_file: name of file with the current version stamp
    latest_pattern: the google store url pattern pointing to the latest binary
    permanent_prefix: stable google store folder used to download versions
    revision_num: the desired revision number, or None for the most recent
  """
  osdict = {'Darwin':'macos', 'Linux':'linux', 'Windows':'win32'}
  def FindPermanentUrl(out, osname, not_used):
    rev_num = revision_num
    if not rev_num:
      temp_file = tempfile.NamedTemporaryFile(delete=False)
      temp_file.close()
      temp_file_url = 'file://' + temp_file.name
      Gsutil('cp', latest_pattern % {'osname' : osname }, temp_file_url)
      temp_file = open(temp_file.name)
      temp_file.seek(0)
      version_info = temp_file.read()
      temp_file.close()
      os.unlink(temp_file.name)
      if version_info != '':
        rev_num = json.loads(version_info)['revision']
      else:
        print 'Unable to get latest version information.'
        return ''
    latest = (permanent_prefix % { 'osname' : osname, 'version_num': rev_num})
    return latest

  GetFromGsutil(name, directory, version_file, latest_pattern, osdict,
                  FindPermanentUrl, revision_num)

def GetFromGsutil(name, directory, version_file, latest_pattern,
    os_name_dict, get_permanent_url, revision_num = '', bot = None):
  """Download and unzip the desired file from Google Storage.
    Args:
    name: the name of the desired download
    directory: target directory (recreated) to install binary
    version_file: name of file with the current version stamp
    latest_pattern: the google store url pattern pointing to the latest binary
    os_name_dict: a dictionary of operating system names and their corresponding
        strings on the google storage site.
    get_permanent_url: a function that accepts a listing of available files
        and the os name, and returns a permanent URL for downloading.
    revision_num: the desired revision number to get (if not supplied, we get
        the latest revision)
  """
  system = platform.system()
  try:
    osname = os_name_dict[system]
  except KeyError:
    print >>sys.stderr, ('WARNING: platform "%s" does not support'
        '%s.') % (system, name)
    return 0

  EnsureConfig()

  # Query for the latest version
  pattern = latest_pattern  % { 'osname' : osname, 'bot' : bot }
  result, out = Gsutil('ls', pattern)
  if result == 0:
    # use permanent link instead, just in case the latest zip entry gets deleted
    # while we are downloading it.
    latest = get_permanent_url(out, osname, revision_num)
  else: # e.g. no access
    print "Couldn't download %s: %s\n%s" % (name, pattern, out)
    if not os.path.exists(version_file):
      print "Using %s will not work. Please try again later." % name
    return 0

  # Check if we need to update the file
  if os.path.exists(version_file):
    v = open(version_file, 'r').read()
    if v == latest:
      if not InRunhooks():
        print name + ' is up to date.\nVersion: ' + latest
      return 0 # up to date

  if os.path.exists(directory):
    print 'Removing old %s tree %s' % (name, directory)
    shutil.rmtree(directory)

  # download the zip file to a temporary path, and unzip to the target location
  temp_dir = tempfile.mkdtemp()
  try:
    temp_zip = os.path.join(temp_dir, 'drt.zip')
    temp_zip_url = 'file://' + temp_zip
    # It's nice to show download progress
    GsutilVisible('cp', latest, temp_zip_url)

    if platform.system() != 'Windows':
      # The Python zip utility does not preserve executable permissions, but
      # this does not seem to be a problem for Windows, which does not have a
      # built in zip utility. :-/
      result, out = ExecuteCommand('unzip', temp_zip, '-d', temp_dir)
      if result != 0:
        raise Exception('Execution of "unzip %s -d %s" failed: %s' %
                        (temp_zip, temp_dir, str(out)))
      unzipped_dir = temp_dir + '/' + os.path.basename(latest)[:-len('.zip')]
    else:
      z = zipfile.ZipFile(temp_zip)
      z.extractall(temp_dir)
      unzipped_dir = os.path.join(temp_dir,
                                  os.path.basename(latest)[:-len('.zip')])
      z.close()
    if directory == SDK_DIR:
      unzipped_dir = os.path.join(temp_dir, 'dart-sdk')
    if os.path.exists(directory):
      raise Exception(
          'Removal of directory %s failed. Is the executable running?' %
          directory)
    shutil.move(unzipped_dir, directory)
  finally:
    shutil.rmtree(temp_dir)

  # create the version stamp
  v = open(version_file, 'w')
  v.write(latest)
  v.close()

  print 'Successfully downloaded to %s' % directory
  return 0

def TooEarlyError():
  """Quick shortcutting function, to return early if someone requests a revision
  that is smaller than the earliest stored. This saves us from doing repeated
  requests until we get down to 0."""
  print ('Unable to download requested revision because it is earlier than the '
      'earliest revision stored.')
  sys.exit(1)

def CopyDrtFont(drt_dir):
  if platform.system() != 'Windows':
    return
  shutil.copy('third_party/drt_resources/AHEM____.TTF', drt_dir)

def main():
  parser = optparse.OptionParser(usage='usage: %prog [options] download_name')
  parser.add_option('-r', '--revision', dest='revision',
                    help='Desired revision number to retrieve for the SDK. If '
                    'unspecified, retrieve the latest SDK build.',
                    action='store', default=None)
  parser.add_option('-d', '--debug', dest='debug',
                    help='Download a debug archive instead of a release.',
                    action='store_true', default=False)
  args, positional = parser.parse_args()

  if args.revision and int(args.revision) < LAST_VALID[positional[0]]:
    return TooEarlyError()

  # Use the incremental release bot ('dartium-*-inc-be') by default.
  # Issue 13399 Quick fix, update with channel support.
  bot = 'inc-be'
  if args.debug:
    bot = 'debug-be'

  if positional[0] == 'dartium':
    GetDartiumRevision('Dartium', bot, DARTIUM_DIR, DARTIUM_VERSION,
                         DARTIUM_LATEST_PATTERN, DARTIUM_PERMANENT_PATTERN,
                         args.revision)
  elif positional[0] == 'sdk':
    GetSdkRevision('sdk', SDK_DIR, SDK_VERSION, SDK_LATEST_PATTERN,
        SDK_PERMANENT, args.revision)
  elif positional[0] == 'drt':
    GetDartiumRevision('content_shell', bot, DRT_DIR, DRT_VERSION,
                         DRT_LATEST_PATTERN, DRT_PERMANENT_PATTERN,
                         args.revision)
    CopyDrtFont(DRT_DIR)
  else:
    print ('Please specify the target you wish to download from Google Storage '
        '("drt", "dartium", "chromedriver", or "sdk")')

if __name__ == '__main__':
  sys.exit(main())
<commit_msg>Don't check for .boto file when getting drt.<commit_after>#!/usr/bin/env python
#
# Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.

# Gets or updates a content shell (a nearly headless build of chrome). This is
# used for running browser tests of client applications.

import json
import optparse
import os
import platform
import shutil
import subprocess
import sys
import tempfile
import zipfile

import utils

def NormJoin(path1, path2):
  return os.path.normpath(os.path.join(path1, path2))

# Change into the dart directory as we want the project to be rooted here.
dart_src = NormJoin(os.path.dirname(sys.argv[0]), os.pardir)
os.chdir(dart_src)

GSUTIL_DIR = os.path.join('third_party', 'gsutil')
GSUTIL = GSUTIL_DIR + '/gsutil'

DRT_DIR = os.path.join('client', 'tests', 'drt')
DRT_VERSION = os.path.join(DRT_DIR, 'LAST_VERSION')
DRT_LATEST_PATTERN = (
    'gs://dartium-archive/latest/drt-%(osname)s-%(bot)s-*.zip')
DRT_PERMANENT_PATTERN = ('gs://dartium-archive/drt-%(osname)s-%(bot)s/drt-'
                         '%(osname)s-%(bot)s-%(num1)s.%(num2)s.zip')

DARTIUM_DIR = os.path.join('client', 'tests', 'dartium')
DARTIUM_VERSION = os.path.join(DARTIUM_DIR, 'LAST_VERSION')
DARTIUM_LATEST_PATTERN = (
    'gs://dartium-archive/latest/dartium-%(osname)s-%(bot)s-*.zip')
DARTIUM_PERMANENT_PATTERN = ('gs://dartium-archive/dartium-%(osname)s-%(bot)s/'
                             'dartium-%(osname)s-%(bot)s-%(num1)s.%(num2)s.zip')

SDK_DIR = os.path.join(utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32'),
    'dart-sdk')
SDK_VERSION = os.path.join(SDK_DIR, 'LAST_VERSION')
SDK_LATEST_PATTERN = 'gs://dart-archive/channels/dev/raw/latest/VERSION'
# TODO(efortuna): Once the x64 VM also is optimized, select the version
# based on whether we are running on a 32-bit or 64-bit system.
SDK_PERMANENT = ('gs://dart-archive/channels/dev/raw/%(version_num)s/sdk/' +
    'dartsdk-%(osname)s-ia32-release.zip')

# Dictionary storing the earliest revision of each download we have stored.
LAST_VALID = {'dartium': 4285, 'chromedriver': 7823, 'sdk': 9761, 'drt': 5342}

sys.path.append(os.path.join(GSUTIL_DIR, 'third_party', 'boto'))
import boto

def ExecuteCommand(*cmd):
  """Execute a command in a subprocess."""
  pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  output, error = pipe.communicate()
  return pipe.returncode, output

def ExecuteCommandVisible(*cmd):
  """Execute a command in a subprocess, but show stdout/stderr."""
  result = subprocess.call(cmd, stdout=sys.stdout, stderr=sys.stderr,
                           stdin=sys.stdin)
  if result != 0:
    raise Exception('Execution of "%s" failed' % ' '.join(cmd))

def Gsutil(*cmd):
  return ExecuteCommand('python', GSUTIL, *cmd)

def GsutilVisible(*cmd):
  ExecuteCommandVisible('python', GSUTIL, *cmd)

def HasBotoConfig():
  """Returns true if boto config exists."""

  config_paths = boto.pyami.config.BotoConfigLocations
  if 'AWS_CREDENTIAL_FILE' in os.environ:
    config_paths.append(os.environ['AWS_CREDENTIAL_FILE'])
  for config_path in config_paths:
    if os.path.exists(config_path):
      return True

  return False

def InRunhooks():
  """True if this script was called by "gclient runhooks" or "gclient sync\""""
  return 'runhooks' in sys.argv

def GetDartiumRevision(name, bot, directory, version_file, latest_pattern,
    permanent_prefix, revision_num=None):
  """Get the latest binary that is stored in the dartium archive.

  Args:
    name: the name of the desired download.
    directory: target directory (recreated) to install binary
    version_file: name of file with the current version stamp
    latest_pattern: the google store url pattern pointing to the latest binary
    permanent_prefix: stable google store folder used to download versions
    revision_num: The desired revision number to retrieve. If revision_num is
        None, we return the latest revision. If the revision number is specified
        but unavailable, find the nearest older revision and use that instead.
  """
  osdict = {'Darwin':'mac', 'Linux':'lucid64', 'Windows':'win'}

  def FindPermanentUrl(out, osname, the_revision_num):
    output_lines = out.split()
    latest = output_lines[-1]
    if not the_revision_num:
      latest = (permanent_prefix[:permanent_prefix.rindex('/')] % { 'osname' :
          osname, 'bot' : bot } + latest[latest.rindex('/'):])
    else:
      latest = (permanent_prefix % { 'osname' : osname, 'num1' : the_revision_num,
          'num2' : the_revision_num, 'bot' : bot })
      foundURL = False
      while not foundURL:
        # Test to ensure this URL exists because the dartium-archive builds can
        # have unusual numbering (a range of CL numbers) sometimes.
        result, out = Gsutil('ls', permanent_prefix % {'osname' : osname,
            'num1': the_revision_num, 'num2': '*', 'bot': bot })
        if result == 0:
          # First try to find one with the the second number the same as the
          # requested number.
          latest = out.split()[0]
          # Now test that the permissions are correct so you can actually
          # download it.
          temp_dir = tempfile.mkdtemp()
          temp_zip = os.path.join(temp_dir, 'foo.zip')
          returncode, out = Gsutil('cp', latest, 'file://' + temp_zip)
          if returncode == 0:
            foundURL = True
          else:
            # Unable to download this item (most likely because something went
            # wrong on the upload and the permissions are bad). Keep looking for
            # a different URL.
            the_revision_num = int(the_revision_num) - 1
          shutil.rmtree(temp_dir)
        else:
          # Now try to find one with a nearby CL num.
          the_revision_num = int(the_revision_num) - 1
          if the_revision_num <= 0:
            TooEarlyError()
    return latest

  GetFromGsutil(name, directory, version_file, latest_pattern, osdict,
                  FindPermanentUrl, revision_num, bot)

def GetSdkRevision(name, directory, version_file, latest_pattern,
    permanent_prefix, revision_num):
  """Get a revision of the SDK from the editor build archive.

  Args:
    name: the name of the desired download
    directory: target directory (recreated) to install binary
    version_file: name of file with the current version stamp
    latest_pattern: the google store url pattern pointing to the latest binary
    permanent_prefix: stable google store folder used to download versions
    revision_num: the desired revision number, or None for the most recent
  """
  osdict = {'Darwin':'macos', 'Linux':'linux', 'Windows':'win32'}
  def FindPermanentUrl(out, osname, not_used):
    rev_num = revision_num
    if not rev_num:
      temp_file = tempfile.NamedTemporaryFile(delete=False)
      temp_file.close()
      temp_file_url = 'file://' + temp_file.name
      Gsutil('cp', latest_pattern % {'osname' : osname }, temp_file_url)
      temp_file = open(temp_file.name)
      temp_file.seek(0)
      version_info = temp_file.read()
      temp_file.close()
      os.unlink(temp_file.name)
      if version_info != '':
        rev_num = json.loads(version_info)['revision']
      else:
        print 'Unable to get latest version information.'
        return ''
    latest = (permanent_prefix % { 'osname' : osname, 'version_num': rev_num})
    return latest

  GetFromGsutil(name, directory, version_file, latest_pattern, osdict,
                  FindPermanentUrl, revision_num)

def GetFromGsutil(name, directory, version_file, latest_pattern,
    os_name_dict, get_permanent_url, revision_num = '', bot = None):
  """Download and unzip the desired file from Google Storage.
    Args:
    name: the name of the desired download
    directory: target directory (recreated) to install binary
    version_file: name of file with the current version stamp
    latest_pattern: the google store url pattern pointing to the latest binary
    os_name_dict: a dictionary of operating system names and their corresponding
        strings on the google storage site.
    get_permanent_url: a function that accepts a listing of available files
        and the os name, and returns a permanent URL for downloading.
    revision_num: the desired revision number to get (if not supplied, we get
        the latest revision)
  """
  system = platform.system()
  try:
    osname = os_name_dict[system]
  except KeyError:
    print >>sys.stderr, ('WARNING: platform "%s" does not support'
        '%s.') % (system, name)
    return 0

  # Query for the latest version
  pattern = latest_pattern  % { 'osname' : osname, 'bot' : bot }
  result, out = Gsutil('ls', pattern)
  if result == 0:
    # use permanent link instead, just in case the latest zip entry gets deleted
    # while we are downloading it.
    latest = get_permanent_url(out, osname, revision_num)
  else: # e.g. no access
    print "Couldn't download %s: %s\n%s" % (name, pattern, out)
    if not os.path.exists(version_file):
      print "Using %s will not work. Please try again later." % name
    return 0

  # Check if we need to update the file
  if os.path.exists(version_file):
    v = open(version_file, 'r').read()
    if v == latest:
      if not InRunhooks():
        print name + ' is up to date.\nVersion: ' + latest
      return 0 # up to date

  if os.path.exists(directory):
    print 'Removing old %s tree %s' % (name, directory)
    shutil.rmtree(directory)

  # download the zip file to a temporary path, and unzip to the target location
  temp_dir = tempfile.mkdtemp()
  try:
    temp_zip = os.path.join(temp_dir, 'drt.zip')
    temp_zip_url = 'file://' + temp_zip
    # It's nice to show download progress
    GsutilVisible('cp', latest, temp_zip_url)

    if platform.system() != 'Windows':
      # The Python zip utility does not preserve executable permissions, but
      # this does not seem to be a problem for Windows, which does not have a
      # built in zip utility. :-/
      result, out = ExecuteCommand('unzip', temp_zip, '-d', temp_dir)
      if result != 0:
        raise Exception('Execution of "unzip %s -d %s" failed: %s' %
                        (temp_zip, temp_dir, str(out)))
      unzipped_dir = temp_dir + '/' + os.path.basename(latest)[:-len('.zip')]
    else:
      z = zipfile.ZipFile(temp_zip)
      z.extractall(temp_dir)
      unzipped_dir = os.path.join(temp_dir,
                                  os.path.basename(latest)[:-len('.zip')])
      z.close()
    if directory == SDK_DIR:
      unzipped_dir = os.path.join(temp_dir, 'dart-sdk')
    if os.path.exists(directory):
      raise Exception(
          'Removal of directory %s failed. Is the executable running?' %
          directory)
    shutil.move(unzipped_dir, directory)
  finally:
    shutil.rmtree(temp_dir)

  # create the version stamp
  v = open(version_file, 'w')
  v.write(latest)
  v.close()

  print 'Successfully downloaded to %s' % directory
  return 0

def TooEarlyError():
  """Quick shortcutting function, to return early if someone requests a revision
  that is smaller than the earliest stored. This saves us from doing repeated
  requests until we get down to 0."""
  print ('Unable to download requested revision because it is earlier than the '
      'earliest revision stored.')
  sys.exit(1)

def CopyDrtFont(drt_dir):
  if platform.system() != 'Windows':
    return
  shutil.copy('third_party/drt_resources/AHEM____.TTF', drt_dir)

def main():
  parser = optparse.OptionParser(usage='usage: %prog [options] download_name')
  parser.add_option('-r', '--revision', dest='revision',
                    help='Desired revision number to retrieve for the SDK. If '
                    'unspecified, retrieve the latest SDK build.',
                    action='store', default=None)
  parser.add_option('-d', '--debug', dest='debug',
                    help='Download a debug archive instead of a release.',
                    action='store_true', default=False)
  args, positional = parser.parse_args()

  if args.revision and int(args.revision) < LAST_VALID[positional[0]]:
    return TooEarlyError()

  # Use the incremental release bot ('dartium-*-inc-be') by default.
  # Issue 13399 Quick fix, update with channel support.
  bot = 'inc-be'
  if args.debug:
    bot = 'debug-be'

  if positional[0] == 'dartium':
    GetDartiumRevision('Dartium', bot, DARTIUM_DIR, DARTIUM_VERSION,
                         DARTIUM_LATEST_PATTERN, DARTIUM_PERMANENT_PATTERN,
                         args.revision)
  elif positional[0] == 'sdk':
    GetSdkRevision('sdk', SDK_DIR, SDK_VERSION, SDK_LATEST_PATTERN,
        SDK_PERMANENT, args.revision)
  elif positional[0] == 'drt':
    GetDartiumRevision('content_shell', bot, DRT_DIR, DRT_VERSION,
                         DRT_LATEST_PATTERN, DRT_PERMANENT_PATTERN,
                         args.revision)
    CopyDrtFont(DRT_DIR)
  else:
    print ('Please specify the target you wish to download from Google Storage '
        '("drt", "dartium", "chromedriver", or "sdk")')

if __name__ == '__main__':
  sys.exit(main())
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2018-2019 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import argparse
import os

from fragments import FragmentFileModel
from sdkconfig import SDKConfig
from generation import GenerationModel, TemplateModel, SectionsInfo

def main():

    argparser = argparse.ArgumentParser(description = "ESP-IDF linker script generator")

    argparser.add_argument(
        "--input", "-i",
        help = "Linker template file",
        type = argparse.FileType("r"))

    argparser.add_argument(
        "--fragments", "-f",
        type = argparse.FileType("r"),
        help = "Input fragment files",
        nargs = "+")

    argparser.add_argument(
        "--sections", "-s",
        type = argparse.FileType("r"),
        help = "Library sections info",
        nargs = "+")

    argparser.add_argument(
        "--output", "-o",
        help = "Output linker script",
        type = argparse.FileType("w"))

    argparser.add_argument(
        "--config", "-c",
        help = "Project configuration",
        type = argparse.FileType("r"))

    argparser.add_argument(
        "--kconfig", "-k",
        help = "IDF Kconfig file",
        type = argparse.FileType("r"))

    argparser.add_argument(
        "--env", "-e",
        action='append', default=[],
        help='Environment to set when evaluating the config file', metavar='NAME=VAL')

    args = argparser.parse_args()

    input_file = args.input
    fragment_files = [] if not args.fragments else args.fragments
    config_file = args.config
    output_file = args.output
    sections_info_files = [] if not args.sections else args.sections
    kconfig_file = args.kconfig

    try:
        sections_infos = SectionsInfo()

        for sections_info_file in sections_info_files:
            sections_infos.add_sections_info(sections_info_file)

        generation_model = GenerationModel()

        for fragment_file in fragment_files:
            fragment_file = FragmentFileModel(fragment_file)
            generation_model.add_fragments_from_file(fragment_file)

        sdkconfig = SDKConfig(kconfig_file, config_file, args.env)
        mapping_rules = generation_model.generate_rules(sdkconfig, sections_infos)

        script_model = TemplateModel(input_file)
        script_model.fill(mapping_rules, sdkconfig)

        script_model.write(output_file)

    except Exception, e:
        print("linker script generation failed for %s\nERROR: %s" % (input_file.name, e.message))
        # Delete the file so the entire build will fail; and not use an outdated script.
        os.remove(output_file.name)

if __name__ == "__main__":
    main()<commit_msg>ldgen: catch exception with python3 compatible style<commit_after>#!/usr/bin/env python
#
# Copyright 2018-2019 Espressif Systems (Shanghai) PTE LTD
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import argparse
import os

from fragments import FragmentFileModel
from sdkconfig import SDKConfig
from generation import GenerationModel, TemplateModel, SectionsInfo

def main():

    argparser = argparse.ArgumentParser(description = "ESP-IDF linker script generator")

    argparser.add_argument(
        "--input", "-i",
        help = "Linker template file",
        type = argparse.FileType("r"))

    argparser.add_argument(
        "--fragments", "-f",
        type = argparse.FileType("r"),
        help = "Input fragment files",
        nargs = "+")

    argparser.add_argument(
        "--sections", "-s",
        type = argparse.FileType("r"),
        help = "Library sections info",
        nargs = "+")

    argparser.add_argument(
        "--output", "-o",
        help = "Output linker script",
        type = argparse.FileType("w"))

    argparser.add_argument(
        "--config", "-c",
        help = "Project configuration",
        type = argparse.FileType("r"))

    argparser.add_argument(
        "--kconfig", "-k",
        help = "IDF Kconfig file",
        type = argparse.FileType("r"))

    argparser.add_argument(
        "--env", "-e",
        action='append', default=[],
        help='Environment to set when evaluating the config file', metavar='NAME=VAL')

    args = argparser.parse_args()

    input_file = args.input
    fragment_files = [] if not args.fragments else args.fragments
    config_file = args.config
    output_file = args.output
    sections_info_files = [] if not args.sections else args.sections
    kconfig_file = args.kconfig

    try:
        sections_infos = SectionsInfo()

        for sections_info_file in sections_info_files:
            sections_infos.add_sections_info(sections_info_file)

        generation_model = GenerationModel()

        for fragment_file in fragment_files:
            fragment_file = FragmentFileModel(fragment_file)
            generation_model.add_fragments_from_file(fragment_file)

        sdkconfig = SDKConfig(kconfig_file, config_file, args.env)
        mapping_rules = generation_model.generate_rules(sdkconfig, sections_infos)

        script_model = TemplateModel(input_file)
        script_model.fill(mapping_rules, sdkconfig)

        script_model.write(output_file)
    except Exception as e:
        print("linker script generation failed for %s\nERROR: %s" % (input_file.name, e.message))
        # Delete the file so the entire build will fail; and not use an outdated script.
        os.remove(output_file.name)

if __name__ == "__main__":
    main()<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright (c) 2019 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Run fuzz test targets.
"""

import argparse
import configparser
import os
import sys
import subprocess
import logging

# Fuzzers known to lack a seed corpus in https://github.com/bitcoin-core/qa-assets/tree/master/fuzz_seed_corpus
FUZZERS_MISSING_CORPORA = [
    "addr_info_deserialize",
    "asmap",
    "base_encode_decode",
    "block",
    "block_file_info_deserialize",
    "block_filter_deserialize",
    "block_header_and_short_txids_deserialize",
    "decode_tx",
    "fee_rate_deserialize",
    "flat_file_pos_deserialize",
    "hex",
    "integer",
    "key_origin_info_deserialize",
    "merkle_block_deserialize",
    "out_point_deserialize",
    "parse_hd_keypath",
    "parse_numbers",
    "parse_script",
    "parse_univalue",
    "partial_merkle_tree_deserialize",
    "partially_signed_transaction_deserialize",
    "prefilled_transaction_deserialize",
    "psbt_input_deserialize",
    "psbt_output_deserialize",
    "pub_key_deserialize",
    "script_deserialize",
    "strprintf",
    "sub_net_deserialize",
    "tx_in",
    "tx_in_deserialize",
    "tx_out",
]

def main():
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        "-l",
        "--loglevel",
        dest="loglevel",
        default="INFO",
        help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.",
    )
    parser.add_argument(
        '--export_coverage',
        action='store_true',
        help='If true, export coverage information to files in the seed corpus',
    )
    parser.add_argument(
        'seed_dir',
        help='The seed corpus to run on (must contain subfolders for each fuzz target).',
    )
    parser.add_argument(
        'target',
        nargs='*',
        help='The target(s) to run. Default is to run all targets.',
    )

    args = parser.parse_args()

    # Set up logging
    logging.basicConfig(
        format='%(message)s',
        level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(),
    )

    # Read config generated by configure.
    config = configparser.ConfigParser()
    configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
    config.read_file(open(configfile, encoding="utf8"))

    if not config["components"].getboolean("ENABLE_FUZZ"):
        logging.error("Must have fuzz targets built")
        sys.exit(1)

    # Build list of tests
    test_list_all = parse_test_list(makefile=os.path.join(config["environment"]["SRCDIR"], 'src', 'Makefile.test.include'))

    if not test_list_all:
        logging.error("No fuzz targets found")
        sys.exit(1)

    logging.info("Fuzz targets found: {}".format(test_list_all))

    args.target = args.target or test_list_all  # By default run all
    test_list_error = list(set(args.target).difference(set(test_list_all)))
    if test_list_error:
        logging.error("Unknown fuzz targets selected: {}".format(test_list_error))
    test_list_selection = list(set(test_list_all).intersection(set(args.target)))
    if not test_list_selection:
        logging.error("No fuzz targets selected")
    logging.info("Fuzz targets selected: {}".format(test_list_selection))

    try:
        help_output = subprocess.run(
            args=[
                os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]),
                '-help=1',
            ],
            timeout=10,
            check=True,
            stderr=subprocess.PIPE,
            universal_newlines=True,
        ).stderr
        if "libFuzzer" not in help_output:
            logging.error("Must be built with libFuzzer")
            sys.exit(1)
    except subprocess.TimeoutExpired:
        logging.error("subprocess timed out: Currently only libFuzzer is supported")
        sys.exit(1)

    run_once(
        corpus=args.seed_dir,
        test_list=test_list_selection,
        build_dir=config["environment"]["BUILDDIR"],
        export_coverage=args.export_coverage,
    )

def run_once(*, corpus, test_list, build_dir, export_coverage):
    for t in test_list:
        corpus_path = os.path.join(corpus, t)
        if t in FUZZERS_MISSING_CORPORA:
            os.makedirs(corpus_path, exist_ok=True)
        args = [
            os.path.join(build_dir, 'src', 'test', 'fuzz', t),
            '-runs=1',
            '-detect_leaks=0',
            corpus_path,
        ]
        logging.debug('Run {} with args {}'.format(t, args))
        result = subprocess.run(args, stderr=subprocess.PIPE, universal_newlines=True)
        output = result.stderr
        logging.debug('Output: {}'.format(output))
        result.check_returncode()
        if not export_coverage:
            continue
        for l in output.splitlines():
            if 'INITED' in l:
                with open(os.path.join(corpus, t + '_coverage'), 'w', encoding='utf-8') as cov_file:
                    cov_file.write(l)
                    break

def parse_test_list(makefile):
    with open(makefile, encoding='utf-8') as makefile_test:
        test_list_all = []
        read_targets = False
        for line in makefile_test.readlines():
            line = line.strip().replace('test/fuzz/', '').replace(' \\', '')
            if read_targets:
                if not line:
                    break
                test_list_all.append(line)
                continue

            if line == 'FUZZ_TARGETS =':
                read_targets = True
    return test_list_all

if __name__ == '__main__':
    main()
<commit_msg>tests: Add --valgrind option to test/fuzz/test_runner.py for running fuzzing test cases under valgrind<commit_after>#!/usr/bin/env python3
# Copyright (c) 2019 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Run fuzz test targets.
"""

import argparse
import configparser
import os
import sys
import subprocess
import logging

# Fuzzers known to lack a seed corpus in https://github.com/bitcoin-core/qa-assets/tree/master/fuzz_seed_corpus
FUZZERS_MISSING_CORPORA = [
    "addr_info_deserialize",
    "asmap",
    "base_encode_decode",
    "block",
    "block_file_info_deserialize",
    "block_filter_deserialize",
    "block_header_and_short_txids_deserialize",
    "decode_tx",
    "fee_rate_deserialize",
    "flat_file_pos_deserialize",
    "hex",
    "integer",
    "key_origin_info_deserialize",
    "merkle_block_deserialize",
    "out_point_deserialize",
    "parse_hd_keypath",
    "parse_numbers",
    "parse_script",
    "parse_univalue",
    "partial_merkle_tree_deserialize",
    "partially_signed_transaction_deserialize",
    "prefilled_transaction_deserialize",
    "psbt_input_deserialize",
    "psbt_output_deserialize",
    "pub_key_deserialize",
    "script_deserialize",
    "strprintf",
    "sub_net_deserialize",
    "tx_in",
    "tx_in_deserialize",
    "tx_out",
]

def main():
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        "-l",
        "--loglevel",
        dest="loglevel",
        default="INFO",
        help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.",
    )
    parser.add_argument(
        '--export_coverage',
        action='store_true',
        help='If true, export coverage information to files in the seed corpus',
    )
    parser.add_argument(
        '--valgrind',
        action='store_true',
        help='If true, run fuzzing binaries under the valgrind memory error detector. Valgrind 3.14 or later required.',
    )
    parser.add_argument(
        'seed_dir',
        help='The seed corpus to run on (must contain subfolders for each fuzz target).',
    )
    parser.add_argument(
        'target',
        nargs='*',
        help='The target(s) to run. Default is to run all targets.',
    )

    args = parser.parse_args()

    # Set up logging
    logging.basicConfig(
        format='%(message)s',
        level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(),
    )

    # Read config generated by configure.
    config = configparser.ConfigParser()
    configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
    config.read_file(open(configfile, encoding="utf8"))

    if not config["components"].getboolean("ENABLE_FUZZ"):
        logging.error("Must have fuzz targets built")
        sys.exit(1)

    # Build list of tests
    test_list_all = parse_test_list(makefile=os.path.join(config["environment"]["SRCDIR"], 'src', 'Makefile.test.include'))

    if not test_list_all:
        logging.error("No fuzz targets found")
        sys.exit(1)

    logging.info("Fuzz targets found: {}".format(test_list_all))

    args.target = args.target or test_list_all  # By default run all
    test_list_error = list(set(args.target).difference(set(test_list_all)))
    if test_list_error:
        logging.error("Unknown fuzz targets selected: {}".format(test_list_error))
    test_list_selection = list(set(test_list_all).intersection(set(args.target)))
    if not test_list_selection:
        logging.error("No fuzz targets selected")
    logging.info("Fuzz targets selected: {}".format(test_list_selection))

    try:
        help_output = subprocess.run(
            args=[
                os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]),
                '-help=1',
            ],
            timeout=10,
            check=True,
            stderr=subprocess.PIPE,
            universal_newlines=True,
        ).stderr
        if "libFuzzer" not in help_output:
            logging.error("Must be built with libFuzzer")
            sys.exit(1)
    except subprocess.TimeoutExpired:
        logging.error("subprocess timed out: Currently only libFuzzer is supported")
        sys.exit(1)

    run_once(
        corpus=args.seed_dir,
        test_list=test_list_selection,
        build_dir=config["environment"]["BUILDDIR"],
        export_coverage=args.export_coverage,
        use_valgrind=args.valgrind,
    )

def run_once(*, corpus, test_list, build_dir, export_coverage, use_valgrind):
    for t in test_list:
        corpus_path = os.path.join(corpus, t)
        if t in FUZZERS_MISSING_CORPORA:
            os.makedirs(corpus_path, exist_ok=True)
        args = [
            os.path.join(build_dir, 'src', 'test', 'fuzz', t),
            '-runs=1',
            '-detect_leaks=0',
            corpus_path,
        ]
        if use_valgrind:
            args = ['valgrind', '--quiet', '--error-exitcode=1', '--exit-on-first-error=yes'] + args
        logging.debug('Run {} with args {}'.format(t, args))
        result = subprocess.run(args, stderr=subprocess.PIPE, universal_newlines=True)
        output = result.stderr
        logging.debug('Output: {}'.format(output))
        result.check_returncode()
        if not export_coverage:
            continue
        for l in output.splitlines():
            if 'INITED' in l:
                with open(os.path.join(corpus, t + '_coverage'), 'w', encoding='utf-8') as cov_file:
                    cov_file.write(l)
                    break

def parse_test_list(makefile):
    with open(makefile, encoding='utf-8') as makefile_test:
        test_list_all = []
        read_targets = False
        for line in makefile_test.readlines():
            line = line.strip().replace('test/fuzz/', '').replace(' \\', '')
            if read_targets:
                if not line:
                    break
                test_list_all.append(line)
                continue

            if line == 'FUZZ_TARGETS =':
                read_targets = True
    return test_list_all

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from scrapy.conf import settings

class ProxyMiddleware(object):
    def process_request(self, request, spider):
        if spider.name == "test_services_spider":
            print "THE SPIDER IS TEST SERVICES SPIDER!"
            request.meta["proxy"] = settings.get("HTTP_PROXY")
<commit_msg>oops. getting rid of silly debug print statement<commit_after>from scrapy.conf import settings

class ProxyMiddleware(object):
    def process_request(self, request, spider):
        if spider.name == "test_services_spider":
            request.meta["proxy"] = settings.get("HTTP_PROXY")
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright (c) 2019 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Run fuzz test targets.
"""

import argparse
import configparser
import os
import sys
import subprocess
import logging

# Fuzzers known to lack a seed corpus in https://github.com/bitcoin-core/qa-assets/tree/master/fuzz_seed_corpus
FUZZERS_MISSING_CORPORA = [
    "addr_info_deserialize",
    "base_encode_decode",
    "block",
    "block_file_info_deserialize",
    "block_filter_deserialize",
    "block_header_and_short_txids_deserialize",
    "decode_tx",
    "fee_rate_deserialize",
    "flat_file_pos_deserialize",
    "hex",
    "integer",
    "key_origin_info_deserialize",
    "merkle_block_deserialize",
    "out_point_deserialize",
    "parse_hd_keypath",
    "parse_numbers",
    "parse_script",
    "parse_univalue",
    "partial_merkle_tree_deserialize",
    "partially_signed_transaction_deserialize",
    "prefilled_transaction_deserialize",
    "psbt_input_deserialize",
    "psbt_output_deserialize",
    "pub_key_deserialize",
    "script_deserialize",
    "sub_net_deserialize",
    "tx_in",
    "tx_in_deserialize",
    "tx_out",
]

def main():
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        "-l",
        "--loglevel",
        dest="loglevel",
        default="INFO",
        help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.",
    )
    parser.add_argument(
        '--export_coverage',
        action='store_true',
        help='If true, export coverage information to files in the seed corpus',
    )
    parser.add_argument(
        'seed_dir',
        help='The seed corpus to run on (must contain subfolders for each fuzz target).',
    )
    parser.add_argument(
        'target',
        nargs='*',
        help='The target(s) to run. Default is to run all targets.',
    )

    args = parser.parse_args()

    # Set up logging
    logging.basicConfig(
        format='%(message)s',
        level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(),
    )

    # Read config generated by configure.
    config = configparser.ConfigParser()
    configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
    config.read_file(open(configfile, encoding="utf8"))

    if not config["components"].getboolean("ENABLE_FUZZ"):
        logging.error("Must have fuzz targets built")
        sys.exit(1)

    # Build list of tests
    test_list_all = parse_test_list(makefile=os.path.join(config["environment"]["SRCDIR"], 'src', 'Makefile.test.include'))

    if not test_list_all:
        logging.error("No fuzz targets found")
        sys.exit(1)

    logging.info("Fuzz targets found: {}".format(test_list_all))

    args.target = args.target or test_list_all  # By default run all
    test_list_error = list(set(args.target).difference(set(test_list_all)))
    if test_list_error:
        logging.error("Unknown fuzz targets selected: {}".format(test_list_error))
    test_list_selection = list(set(test_list_all).intersection(set(args.target)))
    if not test_list_selection:
        logging.error("No fuzz targets selected")
    logging.info("Fuzz targets selected: {}".format(test_list_selection))

    try:
        help_output = subprocess.run(
            args=[
                os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]),
                '-help=1',
            ],
            timeout=10,
            check=True,
            stderr=subprocess.PIPE,
            universal_newlines=True,
        ).stderr
        if "libFuzzer" not in help_output:
            logging.error("Must be built with libFuzzer")
            sys.exit(1)
    except subprocess.TimeoutExpired:
        logging.error("subprocess timed out: Currently only libFuzzer is supported")
        sys.exit(1)

    run_once(
        corpus=args.seed_dir,
        test_list=test_list_selection,
        build_dir=config["environment"]["BUILDDIR"],
        export_coverage=args.export_coverage,
    )

def run_once(*, corpus, test_list, build_dir, export_coverage):
    for t in test_list:
        corpus_path = os.path.join(corpus, t)
        if t in FUZZERS_MISSING_CORPORA:
            os.makedirs(corpus_path, exist_ok=True)
        args = [
            os.path.join(build_dir, 'src', 'test', 'fuzz', t),
            '-runs=1',
            '-detect_leaks=0',
            corpus_path,
        ]
        logging.debug('Run {} with args {}'.format(t, args))
        result = subprocess.run(args, stderr=subprocess.PIPE, universal_newlines=True)
        output = result.stderr
        logging.debug('Output: {}'.format(output))
        result.check_returncode()
        if not export_coverage:
            continue
        for l in output.splitlines():
            if 'INITED' in l:
                with open(os.path.join(corpus, t + '_coverage'), 'w', encoding='utf-8') as cov_file:
                    cov_file.write(l)
                    break

def parse_test_list(makefile):
    with open(makefile, encoding='utf-8') as makefile_test:
        test_list_all = []
        read_targets = False
        for line in makefile_test.readlines():
            line = line.strip().replace('test/fuzz/', '').replace(' \\', '')
            if read_targets:
                if not line:
                    break
                test_list_all.append(line)
                continue

            if line == 'FUZZ_TARGETS =':
                read_targets = True
    return test_list_all

if __name__ == '__main__':
    main()
<commit_msg>tests: Add fuzzer strprintf to FUZZERS_MISSING_CORPORA (temporarily)<commit_after>#!/usr/bin/env python3
# Copyright (c) 2019 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Run fuzz test targets.
"""

import argparse
import configparser
import os
import sys
import subprocess
import logging

# Fuzzers known to lack a seed corpus in https://github.com/bitcoin-core/qa-assets/tree/master/fuzz_seed_corpus
FUZZERS_MISSING_CORPORA = [
    "addr_info_deserialize",
    "base_encode_decode",
    "block",
    "block_file_info_deserialize",
    "block_filter_deserialize",
    "block_header_and_short_txids_deserialize",
    "decode_tx",
    "fee_rate_deserialize",
    "flat_file_pos_deserialize",
    "hex",
    "integer",
    "key_origin_info_deserialize",
    "merkle_block_deserialize",
    "out_point_deserialize",
    "parse_hd_keypath",
    "parse_numbers",
    "parse_script",
    "parse_univalue",
    "partial_merkle_tree_deserialize",
    "partially_signed_transaction_deserialize",
    "prefilled_transaction_deserialize",
    "psbt_input_deserialize",
    "psbt_output_deserialize",
    "pub_key_deserialize",
    "script_deserialize",
    "strprintf",
    "sub_net_deserialize",
    "tx_in",
    "tx_in_deserialize",
    "tx_out",
]

def main():
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        "-l",
        "--loglevel",
        dest="loglevel",
        default="INFO",
        help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.",
    )
    parser.add_argument(
        '--export_coverage',
        action='store_true',
        help='If true, export coverage information to files in the seed corpus',
    )
    parser.add_argument(
        'seed_dir',
        help='The seed corpus to run on (must contain subfolders for each fuzz target).',
    )
    parser.add_argument(
        'target',
        nargs='*',
        help='The target(s) to run. Default is to run all targets.',
    )

    args = parser.parse_args()

    # Set up logging
    logging.basicConfig(
        format='%(message)s',
        level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(),
    )

    # Read config generated by configure.
    config = configparser.ConfigParser()
    configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
    config.read_file(open(configfile, encoding="utf8"))

    if not config["components"].getboolean("ENABLE_FUZZ"):
        logging.error("Must have fuzz targets built")
        sys.exit(1)

    # Build list of tests
    test_list_all = parse_test_list(makefile=os.path.join(config["environment"]["SRCDIR"], 'src', 'Makefile.test.include'))

    if not test_list_all:
        logging.error("No fuzz targets found")
        sys.exit(1)

    logging.info("Fuzz targets found: {}".format(test_list_all))

    args.target = args.target or test_list_all  # By default run all
    test_list_error = list(set(args.target).difference(set(test_list_all)))
    if test_list_error:
        logging.error("Unknown fuzz targets selected: {}".format(test_list_error))
    test_list_selection = list(set(test_list_all).intersection(set(args.target)))
    if not test_list_selection:
        logging.error("No fuzz targets selected")
    logging.info("Fuzz targets selected: {}".format(test_list_selection))

    try:
        help_output = subprocess.run(
            args=[
                os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]),
                '-help=1',
            ],
            timeout=10,
            check=True,
            stderr=subprocess.PIPE,
            universal_newlines=True,
        ).stderr
        if "libFuzzer" not in help_output:
            logging.error("Must be built with libFuzzer")
            sys.exit(1)
    except subprocess.TimeoutExpired:
        logging.error("subprocess timed out: Currently only libFuzzer is supported")
        sys.exit(1)

    run_once(
        corpus=args.seed_dir,
        test_list=test_list_selection,
        build_dir=config["environment"]["BUILDDIR"],
        export_coverage=args.export_coverage,
    )

def run_once(*, corpus, test_list, build_dir, export_coverage):
    for t in test_list:
        corpus_path = os.path.join(corpus, t)
        if t in FUZZERS_MISSING_CORPORA:
            os.makedirs(corpus_path, exist_ok=True)
        args = [
            os.path.join(build_dir, 'src', 'test', 'fuzz', t),
            '-runs=1',
            '-detect_leaks=0',
            corpus_path,
        ]
        logging.debug('Run {} with args {}'.format(t, args))
        result = subprocess.run(args, stderr=subprocess.PIPE, universal_newlines=True)
        output = result.stderr
        logging.debug('Output: {}'.format(output))
        result.check_returncode()
        if not export_coverage:
            continue
        for l in output.splitlines():
            if 'INITED' in l:
                with open(os.path.join(corpus, t + '_coverage'), 'w', encoding='utf-8') as cov_file:
                    cov_file.write(l)
                    break

def parse_test_list(makefile):
    with open(makefile, encoding='utf-8') as makefile_test:
        test_list_all = []
        read_targets = False
        for line in makefile_test.readlines():
            line = line.strip().replace('test/fuzz/', '').replace(' \\', '')
            if read_targets:
                if not line:
                    break
                test_list_all.append(line)
                continue

            if line == 'FUZZ_TARGETS =':
                read_targets = True
    return test_list_all

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""
Example usage of the ODrive python library to monitor and control ODrive devices
"""

from __future__ import print_function

import odrive
import time
import math

# Find a connected ODrive (this will block until you connect one)
my_drive = odrive.find_any()

# Find an ODrive that is connected on the serial port /dev/ttyUSB0
#my_drive = odrive.find_any("serial:/dev/ttyUSB0")

# The above call returns a python object with a dynamically generated type. The
# type hierarchy will correspond to the endpoint list in `MotorControl/protocol.cpp`.
# You can also inspect the object using the dir-function:
#print(dir(my_drive))
#print(dir(my_drive.motor0))
# TODO: maybe provide an introspection method that dumps the whole type hierarchy at once

# To read a value, simply read the property
print("Bus voltage is " + str(my_drive.vbus_voltage) + "V")

# Or to change a value, just assign to the property
my_drive.motor0.pos_setpoint = 3.14
print("Position setpoint is " + str(my_drive.motor0.pos_setpoint))

# And this is how function calls are done:
my_drive.motor0.set_pos_setpoint(0.0, 0.0, 0.0)

# A little sine wave to test
t0 = time.monotonic()
while True:
    setpoint = 10000.0 * math.sin((time.monotonic() - t0)*2)
    print("goto " + str(int(setpoint)))
    my_drive.motor0.set_pos_setpoint(setpoint, 0.0, 0.0)
    time.sleep(0.01)

# Some more things you can try:

# Write to a read-only property:
my_drive.vbus_voltage = 11.0  # fails with `AttributeError: can't set attribute`

# Assign an incompatible value:
my_drive.motor0.pos_setpoint = "I like trains"  # fails with `ValueError: could not convert string to float`
<commit_msg>remove old comments<commit_after>#!/usr/bin/env python3
"""
Example usage of the ODrive python library to monitor and control ODrive devices
"""

from __future__ import print_function

import odrive
import time
import math

# Find a connected ODrive (this will block until you connect one)
my_drive = odrive.find_any()

# Find an ODrive that is connected on the serial port /dev/ttyUSB0
#my_drive = odrive.find_any("serial:/dev/ttyUSB0")

# To read a value, simply read the property
print("Bus voltage is " + str(my_drive.vbus_voltage) + "V")

# Or to change a value, just assign to the property
my_drive.motor0.pos_setpoint = 3.14
print("Position setpoint is " + str(my_drive.motor0.pos_setpoint))

# And this is how function calls are done:
my_drive.motor0.set_pos_setpoint(0.0, 0.0, 0.0)

# A sine wave to test
t0 = time.monotonic()
while True:
    setpoint = 10000.0 * math.sin((time.monotonic() - t0)*2)
    print("goto " + str(int(setpoint)))
    my_drive.motor0.set_pos_setpoint(setpoint, 0.0, 0.0)
    time.sleep(0.01)

# Some more things you can try:

# Write to a read-only property:
my_drive.vbus_voltage = 11.0  # fails with `AttributeError: can't set attribute`

# Assign an incompatible value:
my_drive.motor0.pos_setpoint = "I like trains"  # fails with `ValueError: could not convert string to float`
<|endoftext|>


<commit_before>#!/usr/bin/env python
# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set
# to 1, a combination of tools/oldest_constraints.txt,
# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the
# top level of the package's directory is used, otherwise, a combination of
# certbot-auto's requirements file and tools/dev_constraints.txt is used. The
# other file always takes precedence over tools/dev_constraints.txt. If
# CERTBOT_OLDEST is set, this script must be run with `-e <package-name>` and
# no other arguments.

from __future__ import print_function, absolute_import

import subprocess
import os
import sys
import re
import shutil
import tempfile

import merge_requirements as merge_module
import readlink
import strip_hashes

def find_tools_path():
    return os.path.dirname(readlink.main(__file__))

def certbot_oldest_processing(tools_path, args, test_constraints):
    if args[0] != '-e' or len(args) != 2:
        raise ValueError('When CERTBOT_OLDEST is set, this script must be run '
                         'with a single -e <path> argument.')
    # remove any extras such as [dev]
    pkg_dir = re.sub(r'\[\w+\]', '', args[1])
    requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt')
    shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints)
    # packages like acme don't have any local oldest requirements
    if not os.path.isfile(requirements):
        return None

    return requirements

def certbot_normal_processing(tools_path, test_constraints):
    repo_path = os.path.dirname(tools_path)
    certbot_requirements = os.path.normpath(os.path.join(
        repo_path, 'letsencrypt-auto-source/pieces/dependency-requirements.txt'))
    with open(certbot_requirements, 'r') as fd:
        data = fd.readlines()
    with open(test_constraints, 'w') as fd:
        data = "\n".join(strip_hashes.process_entries(data))
        fd.write(data)

def merge_requirements(tools_path, requirements, test_constraints, all_constraints):
    # Order of the files in the merge function matters.
    # Indeed version retained for a given package will be the last version
    # found when following all requirements in the given order.
    # Here is the order by increasing priority:
    # 1) The general development constraints (tools/dev_constraints.txt)
    # 2) The general tests constraints (oldest_requirements.txt or
    #    certbot-auto's dependency-requirements.txt for the normal processing)
    # 3) The local requirement file, typically local-oldest-requirement in oldest tests
    files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints]
    if requirements:
        files.append(requirements)
    merged_requirements = merge_module.main(*files)
    with open(all_constraints, 'w') as fd:
        fd.write(merged_requirements)

def call_with_print(command, cwd=None):
    print(command)
    subprocess.check_call(command, shell=True, cwd=cwd or os.getcwd())

def main(args):
    tools_path = find_tools_path()
    working_dir = tempfile.mkdtemp()

    if os.environ.get('TRAVIS'):
        # When this script is executed on Travis, the following print will make the log
        # be folded until the end command is printed (see finally section).
        print('travis_fold:start:install_certbot_deps')

    try:
        test_constraints = os.path.join(working_dir, 'test_constraints.txt')
        all_constraints = os.path.join(working_dir, 'all_constraints.txt')

        if os.environ.get('CERTBOT_NO_PIN') == '1':
            # With unpinned dependencies, there is no constraint
            call_with_print('"{0}" -m pip install {1}'
                            .format(sys.executable, ' '.join(args)))
        else:
            # Otherwise, we merge requirements to build the constraints and pin dependencies
            requirements = None
            if os.environ.get('CERTBOT_OLDEST') == '1':
                requirements = certbot_oldest_processing(tools_path, args, test_constraints)
            else:
                certbot_normal_processing(tools_path, test_constraints)

            merge_requirements(tools_path, requirements, test_constraints, all_constraints)
            if requirements:
                call_with_print('"{0}" -m pip install --constraint "{1}" --requirement "{2}"'
                                .format(sys.executable, all_constraints, requirements))

            call_with_print('"{0}" -m pip install --constraint "{1}" {2}'
                            .format(sys.executable, all_constraints, ' '.join(args)))
    finally:
        if os.environ.get('TRAVIS'):
            print('travis_fold:end:install_certbot_deps')
        shutil.rmtree(working_dir)

if __name__ == '__main__':
    main(sys.argv[1:])
<commit_msg>Add --disable-pip-version-check to pip calls. (#6938)<commit_after>#!/usr/bin/env python
# pip installs packages using pinned package versions. If CERTBOT_OLDEST is set
# to 1, a combination of tools/oldest_constraints.txt,
# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the
# top level of the package's directory is used, otherwise, a combination of
# certbot-auto's requirements file and tools/dev_constraints.txt is used. The
# other file always takes precedence over tools/dev_constraints.txt. If
# CERTBOT_OLDEST is set, this script must be run with `-e <package-name>` and
# no other arguments.

from __future__ import print_function, absolute_import

import subprocess
import os
import sys
import re
import shutil
import tempfile

import merge_requirements as merge_module
import readlink
import strip_hashes

def find_tools_path():
    return os.path.dirname(readlink.main(__file__))

def certbot_oldest_processing(tools_path, args, test_constraints):
    if args[0] != '-e' or len(args) != 2:
        raise ValueError('When CERTBOT_OLDEST is set, this script must be run '
                         'with a single -e <path> argument.')
    # remove any extras such as [dev]
    pkg_dir = re.sub(r'\[\w+\]', '', args[1])
    requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt')
    shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints)
    # packages like acme don't have any local oldest requirements
    if not os.path.isfile(requirements):
        return None

    return requirements

def certbot_normal_processing(tools_path, test_constraints):
    repo_path = os.path.dirname(tools_path)
    certbot_requirements = os.path.normpath(os.path.join(
        repo_path, 'letsencrypt-auto-source/pieces/dependency-requirements.txt'))
    with open(certbot_requirements, 'r') as fd:
        data = fd.readlines()
    with open(test_constraints, 'w') as fd:
        data = "\n".join(strip_hashes.process_entries(data))
        fd.write(data)

def merge_requirements(tools_path, requirements, test_constraints, all_constraints):
    # Order of the files in the merge function matters.
    # Indeed version retained for a given package will be the last version
    # found when following all requirements in the given order.
    # Here is the order by increasing priority:
    # 1) The general development constraints (tools/dev_constraints.txt)
    # 2) The general tests constraints (oldest_requirements.txt or
    #    certbot-auto's dependency-requirements.txt for the normal processing)
    # 3) The local requirement file, typically local-oldest-requirement in oldest tests
    files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints]
    if requirements:
        files.append(requirements)
    merged_requirements = merge_module.main(*files)
    with open(all_constraints, 'w') as fd:
        fd.write(merged_requirements)

def call_with_print(command, cwd=None):
    print(command)
    subprocess.check_call(command, shell=True, cwd=cwd or os.getcwd())

def pip_install_with_print(args_str):
    command = '"{0}" -m pip install --disable-pip-version-check {1}'.format(sys.executable,
                                                                            args_str)
    call_with_print(command)

def main(args):
    tools_path = find_tools_path()
    working_dir = tempfile.mkdtemp()

    if os.environ.get('TRAVIS'):
        # When this script is executed on Travis, the following print will make the log
        # be folded until the end command is printed (see finally section).
        print('travis_fold:start:install_certbot_deps')

    try:
        test_constraints = os.path.join(working_dir, 'test_constraints.txt')
        all_constraints = os.path.join(working_dir, 'all_constraints.txt')

        if os.environ.get('CERTBOT_NO_PIN') == '1':
            # With unpinned dependencies, there is no constraint
            pip_install_with_print(' '.join(args))
        else:
            # Otherwise, we merge requirements to build the constraints and pin dependencies
            requirements = None
            if os.environ.get('CERTBOT_OLDEST') == '1':
                requirements = certbot_oldest_processing(tools_path, args, test_constraints)
            else:
                certbot_normal_processing(tools_path, test_constraints)

            merge_requirements(tools_path, requirements, test_constraints, all_constraints)
            if requirements:
                pip_install_with_print('--constraint "{0}" --requirement "{1}"'
                                       .format(all_constraints, requirements))

            pip_install_with_print('--constraint "{0}" {1}'
                                   .format(all_constraints, ' '.join(args)))
    finally:
        if os.environ.get('TRAVIS'):
            print('travis_fold:end:install_certbot_deps')
        shutil.rmtree(working_dir)

if __name__ == '__main__':
    main(sys.argv[1:])
<|endoftext|>