<commit_before>
# helper methods & common code for the uorb message templates msg.{cpp,h}.template

# Another positive effect of having the code here, is that this file will get
# precompiled and thus message generation will be much faster

import genmsg.msgs
import gencpp

type_map = {
    'int8': 'int8_t',
    'int16': 'int16_t',
    'int32': 'int32_t',
    'int64': 'int64_t',
    'uint8': 'uint8_t',
    'uint16': 'uint16_t',
    'uint32': 'uint32_t',
    'uint64': 'uint64_t',
    'float32': 'float',
    'float64': 'double',
    'bool': 'bool',
    'char': 'char',
}

type_serialize_map = {
    'int8': 'SignedChar',
    'int16': 'Short',
    'int32': 'Int',
    'int64': 'Long',
    'uint8': 'UnsignedChar',
    'uint16': 'UnsignedShort',
    'uint32': 'UnsignedInt',
    'uint64': 'UnsignedLong',
    'float32': 'Float',
    'float64': 'Double',
    'bool': 'Bool',
    'char': 'Char',
}

type_idl_map = {
    'int8': 'octet',
    'int16': 'short',
    'int32': 'long',
    'int64': 'long long',
    'uint8': 'octet',
    'uint16': 'unsigned short',
    'uint32': 'unsigned long',
    'uint64': 'unsigned long long',
    'float32': 'float',
    'float64': 'double',
    'bool': 'boolean',
    'char': 'char',
}

msgtype_size_map = {
    'int8': 1,
    'int16': 2,
    'int32': 4,
    'int64': 8,
    'uint8': 1,
    'uint16': 2,
    'uint32': 4,
    'uint64': 8,
    'float32': 4,
    'float64': 8,
    'bool': 1,
    'char': 1,
}

type_printf_map = {
    'int8': '%d',
    'int16': '%d',
    'int32': '%" PRId32 "',
    'int64': '%" PRId64 "',
    'uint8': '%u',
    'uint16': '%u',
    'uint32': '%" PRIu32 "',
    'uint64': '%" PRIu64 "',
    'float32': '%.3f',
    'float64': '%.3f',
    'bool': '%u',
    'char': '%c',
}

def bare_name(msg_type):
    """
    Get bare_name from <dir>/<bare_name>[x] format
    """
    bare = msg_type
    if '/' in msg_type:
        # removing prefix
        bare = (msg_type.split('/'))[1]
    # removing suffix
    return bare.split('[')[0]

def sizeof_field_type(field):
    """
    Get size of a field, used for sorting
    """
    bare_name_str = bare_name(field.type)
    if bare_name_str in msgtype_size_map:
        return msgtype_size_map[bare_name_str]
    return 0 # this is for non-builtin types: sort them at the end

def get_children_fields(base_type, search_path):
    (package, name) = genmsg.names.package_resource_name(base_type)
    tmp_msg_context = genmsg.msg_loader.MsgContext.create_default()
    spec_temp = genmsg.msg_loader.load_msg_by_type(tmp_msg_context, '%s/%s' %(package, name), search_path)
    sorted_fields = sorted(spec_temp.parsed_fields(), key=sizeof_field_type, reverse=True)
    return sorted_fields

def add_padding_bytes(fields, search_path):
    """
    Add padding fields before the embedded types, at the end and calculate the
    struct size
    returns a tuple with the struct size and padding at the end
    """
    struct_size = 8 # account for the timestamp
    align_to = 8 # this is always 8, because of the 64bit timestamp
    i = 0
    padding_idx = 0
    while i < len(fields):
        field = fields[i]
        if not field.is_header:
            a_pos = field.type.find('[')
            array_size = 1
            if field.is_array:
                array_size = field.array_len
            if field.is_builtin:
                field.sizeof_field_type = sizeof_field_type(field)
            else:
                # embedded type: may need to add padding
                num_padding_bytes = align_to - (struct_size % align_to)
                if num_padding_bytes != align_to:
                    padding_field = genmsg.Field('_padding'+str(padding_idx),
                        'uint8['+str(num_padding_bytes)+']')
                    padding_idx += 1
                    padding_field.sizeof_field_type = 1
                    struct_size += num_padding_bytes
                    fields.insert(i, padding_field)
                    i += 1
                children_fields = get_children_fields(field.base_type, search_path)
                field.sizeof_field_type, unused = add_padding_bytes(children_fields,
                        search_path)
            struct_size += field.sizeof_field_type * array_size
        i += 1

    # add padding at the end (necessary for embedded types)
    num_padding_bytes = align_to - (struct_size % align_to)
    if num_padding_bytes == align_to:
        num_padding_bytes = 0
    else:
        padding_field = genmsg.Field('_padding'+str(padding_idx),
                'uint8['+str(num_padding_bytes)+']')
        padding_idx += 1
        padding_field.sizeof_field_type = 1
        struct_size += num_padding_bytes
        fields.append(padding_field)
    return (struct_size, num_padding_bytes)

def convert_type(spec_type):
    """
    Convert from msg type to C type
    """
    bare_type = spec_type
    if '/' in spec_type:
        # removing prefix
        bare_type = (spec_type.split('/'))[1]

    msg_type, is_array, array_length = genmsg.msgs.parse_type(bare_type)
    c_type = msg_type
    if msg_type in type_map:
        c_type = type_map[msg_type]
    if is_array:
        return c_type + "[" + str(array_length) + "]"
    return c_type

def print_field(field):
    """
    Echo printf line
    """

    # skip padding
    if ("_padding" in field.name):
        return

    bare_type = field.type
    if '/' in field.type:
        # removing prefix
        bare_type = (bare_type.split('/'))[1]

    msg_type, is_array, array_length = genmsg.msgs.parse_type(bare_type)

    field_name = ""

    if is_array:
        c_type = "["

        if msg_type in type_map:
            p_type = type_printf_map[msg_type]

        else:
            for i in range(array_length):
                print("printf(\"   " + field.type + " " + field.name + "[" + str(i) + "]\");")
                print(" print_message(message." + field.name + "[" + str(i) + "]);")
            return

        for i in range(array_length):

            if i > 0:
                c_type += ", "
                field_name += ", "

            if "float32" in field.type:
                field_name += "(double)message." + field.name + "[" + str(i) + "]"
            else:
                field_name += "message." + field.name + "[" + str(i) + "]"

            c_type += str(p_type)

        c_type += "]"

    else:
        c_type = msg_type
        if msg_type in type_map:
            c_type = type_printf_map[msg_type]

            field_name = "message." + field.name

            # cast double
            if field.type == "float32":
                field_name = "(double)" + field_name

        else:
            print("printf(\"  " + field.name + "\");")
            print("\tprint_message(message."+ field.name + ");")
            return

    print("printf(\"\t" + field.name + ": " + c_type + "\\n\", " + field_name + ");" )

def print_field_def(field):
    """
    Print the C type from a field
    """
    type_name = field.type
    # detect embedded types
    sl_pos = type_name.find('/')
    type_appendix = ''
    type_prefix = ''
    if (sl_pos >= 0):
        type_name = type_name[sl_pos + 1:]
        type_prefix = 'struct '
        type_appendix = '_s'

    # detect arrays
    a_pos = type_name.find('[')
    array_size = ''
    if (a_pos >= 0):
        # field is array
        array_size = type_name[a_pos:]
        type_name = type_name[:a_pos]

    if type_name in type_map:
        # need to add _t: int8 --> int8_t
        type_px4 = type_map[type_name]
    else:
        type_px4 = type_name

    comment = ''
    if field.name.startswith('_padding'):
        comment = ' // required for logger'

    print('\t%s%s%s %s%s;%s'%(type_prefix, type_px4, type_appendix, field.name,
                array_size, comment))
<commit_msg>generate uorb topic helper ignore _padding properly<commit_after>
# helper methods & common code for the uorb message templates msg.{cpp,h}.template

# Another positive effect of having the code here, is that this file will get
# precompiled and thus message generation will be much faster

import genmsg.msgs
import gencpp

type_map = {
    'int8': 'int8_t',
    'int16': 'int16_t',
    'int32': 'int32_t',
    'int64': 'int64_t',
    'uint8': 'uint8_t',
    'uint16': 'uint16_t',
    'uint32': 'uint32_t',
    'uint64': 'uint64_t',
    'float32': 'float',
    'float64': 'double',
    'bool': 'bool',
    'char': 'char',
}

type_serialize_map = {
    'int8': 'SignedChar',
    'int16': 'Short',
    'int32': 'Int',
    'int64': 'Long',
    'uint8': 'UnsignedChar',
    'uint16': 'UnsignedShort',
    'uint32': 'UnsignedInt',
    'uint64': 'UnsignedLong',
    'float32': 'Float',
    'float64': 'Double',
    'bool': 'Bool',
    'char': 'Char',
}

type_idl_map = {
    'int8': 'octet',
    'int16': 'short',
    'int32': 'long',
    'int64': 'long long',
    'uint8': 'octet',
    'uint16': 'unsigned short',
    'uint32': 'unsigned long',
    'uint64': 'unsigned long long',
    'float32': 'float',
    'float64': 'double',
    'bool': 'boolean',
    'char': 'char',
}

msgtype_size_map = {
    'int8': 1,
    'int16': 2,
    'int32': 4,
    'int64': 8,
    'uint8': 1,
    'uint16': 2,
    'uint32': 4,
    'uint64': 8,
    'float32': 4,
    'float64': 8,
    'bool': 1,
    'char': 1,
}

type_printf_map = {
    'int8': '%d',
    'int16': '%d',
    'int32': '%" PRId32 "',
    'int64': '%" PRId64 "',
    'uint8': '%u',
    'uint16': '%u',
    'uint32': '%" PRIu32 "',
    'uint64': '%" PRIu64 "',
    'float32': '%.3f',
    'float64': '%.3f',
    'bool': '%u',
    'char': '%c',
}

def bare_name(msg_type):
    """
    Get bare_name from <dir>/<bare_name>[x] format
    """
    bare = msg_type
    if '/' in msg_type:
        # removing prefix
        bare = (msg_type.split('/'))[1]
    # removing suffix
    return bare.split('[')[0]

def sizeof_field_type(field):
    """
    Get size of a field, used for sorting
    """
    bare_name_str = bare_name(field.type)
    if bare_name_str in msgtype_size_map:
        return msgtype_size_map[bare_name_str]
    return 0 # this is for non-builtin types: sort them at the end

def get_children_fields(base_type, search_path):
    (package, name) = genmsg.names.package_resource_name(base_type)
    tmp_msg_context = genmsg.msg_loader.MsgContext.create_default()
    spec_temp = genmsg.msg_loader.load_msg_by_type(tmp_msg_context, '%s/%s' %(package, name), search_path)
    sorted_fields = sorted(spec_temp.parsed_fields(), key=sizeof_field_type, reverse=True)
    return sorted_fields

def add_padding_bytes(fields, search_path):
    """
    Add padding fields before the embedded types, at the end and calculate the
    struct size
    returns a tuple with the struct size and padding at the end
    """
    struct_size = 8 # account for the timestamp
    align_to = 8 # this is always 8, because of the 64bit timestamp
    i = 0
    padding_idx = 0
    while i < len(fields):
        field = fields[i]
        if not field.is_header:
            a_pos = field.type.find('[')
            array_size = 1
            if field.is_array:
                array_size = field.array_len
            if field.is_builtin:
                field.sizeof_field_type = sizeof_field_type(field)
            else:
                # embedded type: may need to add padding
                num_padding_bytes = align_to - (struct_size % align_to)
                if num_padding_bytes != align_to:
                    padding_field = genmsg.Field('_padding'+str(padding_idx),
                        'uint8['+str(num_padding_bytes)+']')
                    padding_idx += 1
                    padding_field.sizeof_field_type = 1
                    struct_size += num_padding_bytes
                    fields.insert(i, padding_field)
                    i += 1
                children_fields = get_children_fields(field.base_type, search_path)
                field.sizeof_field_type, unused = add_padding_bytes(children_fields,
                        search_path)
            struct_size += field.sizeof_field_type * array_size
        i += 1

    # add padding at the end (necessary for embedded types)
    num_padding_bytes = align_to - (struct_size % align_to)
    if num_padding_bytes == align_to:
        num_padding_bytes = 0
    else:
        padding_field = genmsg.Field('_padding'+str(padding_idx),
                'uint8['+str(num_padding_bytes)+']')
        padding_idx += 1
        padding_field.sizeof_field_type = 1
        struct_size += num_padding_bytes
        fields.append(padding_field)
    return (struct_size, num_padding_bytes)

def convert_type(spec_type):
    """
    Convert from msg type to C type
    """
    bare_type = spec_type
    if '/' in spec_type:
        # removing prefix
        bare_type = (spec_type.split('/'))[1]

    msg_type, is_array, array_length = genmsg.msgs.parse_type(bare_type)
    c_type = msg_type
    if msg_type in type_map:
        c_type = type_map[msg_type]
    if is_array:
        return c_type + "[" + str(array_length) + "]"
    return c_type

def print_field(field):
    """
    Echo printf line
    """

    # skip padding
    if field.name.startswith('_padding'):
        return

    bare_type = field.type
    if '/' in field.type:
        # removing prefix
        bare_type = (bare_type.split('/'))[1]

    msg_type, is_array, array_length = genmsg.msgs.parse_type(bare_type)

    field_name = ""

    if is_array:
        c_type = "["

        if msg_type in type_map:
            p_type = type_printf_map[msg_type]

        else:
            for i in range(array_length):
                print("printf(\"   " + field.type + " " + field.name + "[" + str(i) + "]\");")
                print(" print_message(message." + field.name + "[" + str(i) + "]);")
            return

        for i in range(array_length):

            if i > 0:
                c_type += ", "
                field_name += ", "

            if "float32" in field.type:
                field_name += "(double)message." + field.name + "[" + str(i) + "]"
            else:
                field_name += "message." + field.name + "[" + str(i) + "]"

            c_type += str(p_type)

        c_type += "]"

    else:
        c_type = msg_type
        if msg_type in type_map:
            c_type = type_printf_map[msg_type]

            field_name = "message." + field.name

            # cast double
            if field.type == "float32":
                field_name = "(double)" + field_name

        else:
            print("printf(\"  " + field.name + "\");")
            print("\tprint_message(message."+ field.name + ");")
            return

    print("printf(\"\t" + field.name + ": " + c_type + "\\n\", " + field_name + ");" )

def print_field_def(field):
    """
    Print the C type from a field
    """
    type_name = field.type
    # detect embedded types
    sl_pos = type_name.find('/')
    type_appendix = ''
    type_prefix = ''
    if (sl_pos >= 0):
        type_name = type_name[sl_pos + 1:]
        type_prefix = 'struct '
        type_appendix = '_s'

    # detect arrays
    a_pos = type_name.find('[')
    array_size = ''
    if (a_pos >= 0):
        # field is array
        array_size = type_name[a_pos:]
        type_name = type_name[:a_pos]

    if type_name in type_map:
        # need to add _t: int8 --> int8_t
        type_px4 = type_map[type_name]
    else:
        type_px4 = type_name

    comment = ''
    if field.name.startswith('_padding'):
        comment = ' // required for logger'

    print('\t%s%s%s %s%s;%s'%(type_prefix, type_px4, type_appendix, field.name,
                array_size, comment))
<|endoftext|>


<commit_before>#!/usr/bin/python
import geojson
import json
import sets

#pip install geojson
#pip install --upgrade geojson
# to test: in terminal:
	#python
	#from geojson import Point

def FeaturePoint(lon, lat, weight, name):
	geometry = { "type" : "Point", "coordinates" : [lon, lat] }
	properties = { "weight" : weight, "name" : name }
	#Feature takes in: id= "", geometry json, property json
	feature = geojson.Feature(name, geometry, properties)
	return feature

def MakeGeoJsonElement(location, existing_locations):
	#lookup x in database
	lon = 0.00
	lat = 0.00

	#weight calculations
	weight = 0
	for y in existing_locations:
		if location == y :
			weight = weight + 1
			print weight
	name = location

	return FeaturePoint(lon,lat,weight,name)

def MakeGeoJsonCollection(locations):
	# --- turn locations into FeaturePoints
	feature_array = []
	for l in locations:
		feature_array.append(MakeGeoJsonElement(l, locations))
	# --- Convert FeaturePoints List to FeatureCollection
	return geojson.FeatureCollection(feature_array )
<commit_msg>added location points<commit_after>#!/usr/bin/python
import geojson
import json
import sets

#pip install geojson
#pip install --upgrade geojson
# to test: in terminal:
	#python
	#from geojson import Point

def FeaturePoint(lon, lat, weight, name):
	geometry = { "type" : "Point", "coordinates" : [lat, lon] }
	properties = { "weight" : weight, "name" : name }
	#Feature takes in: id= "", geometry json, property json
	feature = geojson.Feature(name, geometry, properties)
	return feature

def MakeGeoJsonElement(location, existing_locations):
	#lookup x in database

	#necessary imports
	##should move these to a separate file so that we can reuse them without having to rewrite them.
	from sqlalchemy import create_engine
	from sqlalchemy.orm import sessionmaker
	from models import Location
	import os

	#engine pointing to the db
	engine = create_engine(os.getenv('DATABASE_URL','postgres://postgres@{0}/app'.format(os.getenv('DB_1_PORT_5432_TCP_ADDR'))), echo=True);
	Session = sessionmaker(bind=engine)
	session = Session()

	#Gets the first hit that it gets with the given location name. It only searches the name instead of the other parameters. Also because of first hit, the accuracy is not very good. Will need to add additional logic for checking

	loc = session.query(Location).filter(Location.name == location).first()
	lon = 0.00
	lat = 0.00
	#if there is no match, the locations will be 0,0.
	if not (loc is None):
		lon = loc.longitude
		lat = loc.latitude

	#weight calculations
	weight = 0
	for y in existing_locations:
		if location == y :
			weight = weight + 1
			print weight
	name = location

	return FeaturePoint(lon,lat,weight,name)

def MakeGeoJsonCollection(locations):
	# --- turn locations into FeaturePoints
	feature_array = []
	for l in locations:
		feature_array.append(MakeGeoJsonElement(l, locations))
	# --- Convert FeaturePoints List to FeatureCollection
	return geojson.FeatureCollection(feature_array )
<|endoftext|>


<commit_before>from locu import VenueApiClient
import sys
import unittest

global KEY

class IsOpenTest(object):

	def test_no_data(self):
		venue_client = VenueApiClient(KEY)
		venue_id = "50d70de7fbafb112c2de" ##no data
		self.assertEqual(venue_client.is_open(venue_id,"12:00:00","Monday"))

	def test_open(self):
		venue_client = VenueApiClient(KEY)
		venue_id='b7b1644a6bb10dff58bd' ## open Monday 09:00:00 - 15:00:00
		self,assertTrue(venue_client.is_open(venue_id,"12:00:00","Monday"))

	def test_closed(self):
		venue_client = VenueApiClient(KEY)
		venue_id='b7b1644a6bb10dff58bd' ## open Monday 09:00:00 - 15:00:00
		self.assertFalse(venue_client.is_open(venue_id,"16:00:00","Monday"))

if __name__ == '__main__':
	print "Please enter a locu api key"
	KEY = sys.stdin.readline().strip()
	unittest.main()
<commit_msg>made changes in Jason's comments on pull request<commit_after>from locu import VenueApiClient
import sys
import unittest

global KEY

class IsOpenTest(unittest.TestCase):

	def test_no_data(self):
		venue_client = VenueApiClient(KEY)
		venue_id = "50d70de7fbafb112c2de" ##no data
		self.assertIsNone(venue_client.is_open(venue_id,"12:00:00","Monday"))

	def test_open(self):
		venue_client = VenueApiClient(KEY)
		venue_id='b7b1644a6bb10dff58bd' ## open Monday 09:00:00 - 15:00:00
		self.assertTrue(venue_client.is_open(venue_id,"12:00:00","Monday"))

	def test_closed(self):
		venue_client = VenueApiClient(KEY)
		venue_id='b7b1644a6bb10dff58bd' ## open Monday 09:00:00 - 15:00:00
		self.assertFalse(venue_client.is_open(venue_id,"16:00:00","Monday"))

if __name__ == '__main__':
	print "Please enter a locu api key"
	KEY = sys.stdin.readline().strip()
	print KEY
	unittest.main()
<|endoftext|>


<commit_before>import scrapy
import sys
import inspect
import logging

class BritBoxSpider(scrapy.Spider):
    name = "shows"
    #  Doesn't handle /us/programme, or /us/show/ with only one season
    #  Works for /us/show/ with > 1 seson
    # yapf: disable
    start_urls = [
        'https://www.britbox.com/us/programmes',                              # Top level
        'https://www.britbox.com/us/movie/70_Glorious_Years_13550',           # Movie
        'https://www.britbox.com/us/movie/A_Christmas_Carol_15128',           # Movie
        'https://www.britbox.com/us/show/New_Blood_9275',                     # Show (1 season)
        'https://www.britbox.com/us/show/Till_Death_Us_Do_Part_15810',        # Show (1 season)
        'https://www.britbox.com/us/show/A_History_of_Ancient_Britain_6385',  # Show (1+ seasons)
        'https://www.britbox.com/us/show/A_Bit_of_Fry_and_Laurie_6393',       # Show (+1 seasons)
        'https://www.britbox.com/us/show/The_Jury_9565',                      # Show (2 seasons)
        'https://www.britbox.com/us/season/Vera_S8_15720',                    # Season (2 seasons)
        'https://www.britbox.com/us/episode/Superhomes_S1_E1_9008',           # Episode (1 season)
        'https://www.britbox.com/us/episode/Cold_Blood_S2_E2_10298',          # Episode (2 seasons)
    ]
    # yapf: enable

    def parse(self, response):
        for page in self.start_urls:
            if ("/movie/" in page) or ("/episode/" in page):
                yield scrapy.Request(page, callback=self.parse1Episode)
            elif "/show/" in page:
                seasons = response.css(
                    'div.program-item__title-wrapper p.program-item__program-subtitle::text'
                ).extract_first()
                shortURL = "/" + response.url.split('/', 3)[3]
                if seasons is not None:
                    if "Season " in seasons:
                        logging.info("Found Season in " + seasons + " on " + shortURL)
                        yield scrapy.Request(page, callback=self.parseAllSeasons)
                    else:
                        logging.info("No Season in " + seasons + " on " + shortURL)
                        yield scrapy.Request(page, callback=self.parseAllEpisodes)
            elif "/season/" in page:
                yield scrapy.Request(page, callback=self.parseAllEpisodes)

    def parseAllEpisodes(self, response):
        shortURL = "/" + response.url.split('/', 3)[3]
        for show in response.css('div.program-item'):
            yield {
                'URL': show.css('a.program-item__block::attr(href)').extract(),
                'Title': show.css('h3.program-item__program-title::text').extract(),
                'Subtitle': show.css('p.program-item__program-subtitle::text').extract(),
                'Duration': show.css('span.programme-metadata__duration::text').re(r'(\d+)'),
                'Year': show.css('span.programme-metadata__year::text').extract(),
                'Rating': show.css('span.programme-metadata__classification::text').extract(),
                # 'Description': "-desc-",
                'Description': show.css('p.program-item__program-description::text').extract(),
                'shortURL': shortURL,
                'function': sys._getframe().f_code.co_name,
            }

    def parse1Episode(self, response):
        shortURL = "/" + response.url.split('/', 3)[3]
        URLs = (response.css('div.program-item a.program-item__block::attr(href)').extract())
        for i, item in enumerate(URLs):
            if (item == shortURL):
                URL = URLs[i]
                Title = response.css('div.program-item h3.program-item__program-title::text')[
                    i].extract()
                if (len(
                        response.css('div.program-item p.program-item__program-subtitle::text')
                        .extract()) == 0):
                    Subtitle = ''
                else:
                    Subtitle = response.css('div.program-item p.program-item__program-subtitle::text')[
                        i].extract()
                Duration = response.css('div.program-item span.programme-metadata__duration::text')[
                    i].re(r'(\d+)')
                Year = response.css('div.program-item span.programme-metadata__year::text')[
                    i].extract()
                Rating = response.css(
                    'div.program-item span.programme-metadata__classification::text')[i].extract()
                yield {
                    'URL':
                        URL,
                    'Title':
                        Title,
                    'Subtitle':
                        Subtitle,
                    'Duration':
                        Duration,
                    'Year':
                        Year,
                    'Rating':
                        Rating,
                    # 'Description': "-desc-",
                    'Description':
                        response.css('div.program-item p.program-item__program-description::text')[i]
                        .extract(),
                    'shortURL':
                        shortURL,
                    'function':
                        sys._getframe().f_code.co_name,
                }

    def parseAllSeasons(self, response):
        # for season in response.css('a.brand-season-item'):
        shortURL = "/" + response.url.split('/', 3)[3]
        title = response.css('h1.brand-hero-info__title::text').extract()

        for season in response.css('a.brand-season-item'):
            yield {
                'URL':
                    season.css('a.brand-season-item::attr(href)').extract(),
                'Title':
                    title,
                'SeasonNumber':
                    season.css('h2.program-item__program-title::text').re(r'Season (\d+)'),
                'Year':
                    season.css('p.season-metadata::text').extract(),
                'NumEpisodes':
                    season.css('p.season-metadata span::text')[2].re(r'(\d+)'),
                # 'Description': "-desc-",
                'Description':
                    response.css('div.program-item p.program-item__program-description::text')
                    .extract(),
                'shortURL':
                    shortURL,
                'function':
                    sys._getframe().f_code.co_name,
            }

            for href in response.css('a.brand-season-item::attr(href)'):
                yield response.follow(href, self.parseAllEpisodes)
<commit_msg>Get description from season not episodes<commit_after>import scrapy
import sys
import inspect
import logging

class BritBoxSpider(scrapy.Spider):
    name = "shows"
    #  Doesn't handle /us/programme, or /us/show/ with only one season
    #  Works for /us/show/ with > 1 seson
    # yapf: disable
    start_urls = [
        'https://www.britbox.com/us/programmes',                              # Top level
        'https://www.britbox.com/us/movie/70_Glorious_Years_13550',           # Movie
        'https://www.britbox.com/us/movie/A_Christmas_Carol_15128',           # Movie
        'https://www.britbox.com/us/show/New_Blood_9275',                     # Show (1 season)
        'https://www.britbox.com/us/show/Till_Death_Us_Do_Part_15810',        # Show (1 season)
        'https://www.britbox.com/us/show/A_History_of_Ancient_Britain_6385',  # Show (1+ seasons)
        'https://www.britbox.com/us/show/A_Bit_of_Fry_and_Laurie_6393',       # Show (+1 seasons)
        'https://www.britbox.com/us/show/The_Jury_9565',                      # Show (2 seasons)
        'https://www.britbox.com/us/season/Vera_S8_15720',                    # Season (2 seasons)
        'https://www.britbox.com/us/episode/Superhomes_S1_E1_9008',           # Episode (1 season)
        'https://www.britbox.com/us/episode/Cold_Blood_S2_E2_10298',          # Episode (2 seasons)
    ]
    # yapf: enable

    def parse(self, response):
        for page in self.start_urls:
            if ("/movie/" in page) or ("/episode/" in page):
                yield scrapy.Request(page, callback=self.parse1Episode)
            elif "/show/" in page:
                seasons = response.css(
                    'div.program-item__title-wrapper p.program-item__program-subtitle::text'
                ).extract_first()
                shortURL = "/" + response.url.split('/', 3)[3]
                if seasons is not None:
                    if "Season " in seasons:
                        # logging.info("Found Season in " + seasons + " on " + shortURL)
                        yield scrapy.Request(page, callback=self.parseAllSeasons)
                    else:
                        # logging.info("No Season in " + seasons + " on " + shortURL)
                        yield scrapy.Request(page, callback=self.parseAllEpisodes)
            elif "/season/" in page:
                yield scrapy.Request(page, callback=self.parseAllEpisodes)

    def parseAllEpisodes(self, response):
        shortURL = "/" + response.url.split('/', 3)[3]
        for show in response.css('div.program-item'):
            logging.info("Parsing " + shortURL + " in parseAllEpisodes")
            yield {
                'URL': show.css('a.program-item__block::attr(href)').extract(),
                'Title': show.css('h3.program-item__program-title::text').extract(),
                'Subtitle': show.css('p.program-item__program-subtitle::text').extract(),
                'Duration': show.css('span.programme-metadata__duration::text').re(r'(\d+)'),
                'Year': show.css('span.programme-metadata__year::text').extract(),
                'Rating': show.css('span.programme-metadata__classification::text').extract(),
                # 'Description': "-desc-",
                'Description': show.css('p.program-item__program-description::text').extract(),
                'shortURL': shortURL,
                'function': sys._getframe().f_code.co_name,
            }

    def parse1Episode(self, response):
        shortURL = "/" + response.url.split('/', 3)[3]
        URLs = (response.css('div.program-item a.program-item__block::attr(href)').extract())
        for i, item in enumerate(URLs):
            if (item == shortURL):
                URL = URLs[i]
                Title = response.css('div.program-item h3.program-item__program-title::text')[
                    i].extract()
                if (len(
                        response.css('div.program-item p.program-item__program-subtitle::text')
                        .extract()) == 0):
                    Subtitle = ''
                else:
                    Subtitle = response.css('div.program-item p.program-item__program-subtitle::text')[
                        i].extract()
                Duration = response.css('div.program-item span.programme-metadata__duration::text')[
                    i].re(r'(\d+)')
                Year = response.css('div.program-item span.programme-metadata__year::text')[
                    i].extract()
                Rating = response.css(
                    'div.program-item span.programme-metadata__classification::text')[i].extract()
                logging.info("Parsing " + shortURL + " in parse1Episode")
                yield {
                    'URL':
                        URL,
                    'Title':
                        Title,
                    'Subtitle':
                        Subtitle,
                    'Duration':
                        Duration,
                    'Year':
                        Year,
                    'Rating':
                        Rating,
                    # 'Description': "-desc-",
                    'Description':
                        response.css('div.program-item p.program-item__program-description::text')[i]
                        .extract(),
                    'shortURL':
                        shortURL,
                    'function':
                        sys._getframe().f_code.co_name,
                }

    def parseAllSeasons(self, response):
        # for season in response.css('a.brand-season-item'):
        shortURL = "/" + response.url.split('/', 3)[3]
        title = response.css('h1.brand-hero-info__title::text').extract()

        for season in response.css('a.brand-season-item'):
            description = response.css('p.brand-hero-info__description::text').extract()
            logging.info("Parsing " + shortURL + " in parseAllSeasons")
            yield {
                'URL':
                    season.css('a.brand-season-item::attr(href)').extract(),
                'Title':
                    title,
                'SeasonNumber':
                    season.css('h2.program-item__program-title::text').re(r'Season (\d+)'),
                'Year':
                    season.css('p.season-metadata::text').extract(),
                'NumEpisodes':
                    season.css('p.season-metadata span::text')[2].re(r'(\d+)'),
                # 'Description': "-desc-",
                'Description':
                    description,
                # 'Description':
                #     season.css('p.brand-hero-info__description::text').extract(),
                'shortURL':
                    shortURL,
                'function':
                    sys._getframe().f_code.co_name,
            }

            for href in response.css('a.brand-season-item::attr(href)'):
                yield response.follow(href, self.parseAllEpisodes)
<|endoftext|>


<commit_before>__version__ = '1.5.0'
<commit_msg>Bump to 1.5.1<commit_after>__version__ = '1.5.1'
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import argparse
import re

from biocode import gff, things

'''
This script converts native (GTF) or GFF output (via the --gff3 option) of Augustus
into GFF3 format.

Admittedly, when you run augustus with --gff3=on you will get legal GFF3 with lots of
comments.  What I often need is GFF3 that also follows the gene model graph conventions
defined in the GFF3, which the default output does not.  There are no exon or mRNA
features, for example.

This script reads the native GFF output and makes it conform to the suggested canonical
gene model, defined here:

http://www.sequenceontology.org/gff3.shtml

WARNING: The augustus output format appears to change pretty frequently.  I've provided
an example of the expected input below, and the versions I tested with are 2.7 and 3.0.1

WARNING: Only the gene/transcript/CDS rows are kept (and exon rows are created).  All
others are discarded.

EXPECTED INPUT example:

###
# start gene g2
FO082871        AUGUSTUS        gene    16991   18816   0.04    +       .       ID=g2
FO082871        AUGUSTUS        transcript      16991   18816   0.04    +       .       ID=g2.t1;Parent=g2
FO082871        AUGUSTUS        transcription_start_site        16991   16991   .       +       .       Parent=g2.t1
FO082871        AUGUSTUS        five_prime_utr  16991   17193   0.04    +       .       Parent=g2.t1
FO082871        AUGUSTUS        start_codon     17194   17196   .       +       0       Parent=g2.t1
FO082871        AUGUSTUS        intron  17317   17557   0.15    +       .       Parent=g2.t1
FO082871        AUGUSTUS        CDS     17194   17316   0.15    +       0       ID=g2.t1.cds;Parent=g2.t1
FO082871        AUGUSTUS        CDS     17558   17647   0.15    +       0       ID=g2.t1.cds;Parent=g2.t1
FO082871        AUGUSTUS        stop_codon      17645   17647   .       +       0       Parent=g2.t1
FO082871        AUGUSTUS        three_prime_utr 17648   18816   0.07    +       .       Parent=g2.t1
FO082871        AUGUSTUS        transcription_end_site  18816   18816   .       +       .       Parent=g2.t1
# protein sequence = [MQDNYTDGQPGIHRAIAKLEAIINEVDQQLYDYLLERQIHLSPPSKYWEISDIDELVAKAYVCFYAGLCC]
# Evidence for and against this transcript:
# % of transcript supported by hints (any source): 80
# CDS exons: 2/2
#      W:   2
# CDS introns: 0/1
# 5'UTR exons and introns: 1/1
#      W:   1
# 3'UTR exons and introns: 1/1
#      W:   1
# hint groups fully obeyed: 148
#      W: 148
# incompatible hint groups: 11
#      W:  11
# end gene g2

'''

def main():
    parser = argparse.ArgumentParser( description='Convert native (GTF) or GFF output from Augustus into GFF3 format')

    ## output file to be written
    parser.add_argument('-i', '--input', type=str, required=True, help='Path to a GFF file created by Augustus' )
    parser.add_argument('-o', '--output', type=str, required=True, help='Path to an output file to be created' )
    args = parser.parse_args()

    assemblies = dict()
    current_assembly = None

    gene = None
    mRNAs = dict()
    in_sequence = False
    current_sequence = None
    current_gene_comment_lines = list()

    ## Used for tracking the exon count for each gene (for ID purposes)
    exon_count_by_mRNA = dict()

    fout = open(args.output, mode='wt', encoding='utf-8')
    fout.write("##gff-version 3\n")

    for line in open(args.input):
        if line.startswith("#"):
            current_gene_comment_lines.append(line)

            if line.startswith("# end gene "):
                ## purge the comments, then write the gene
                fout.write( "".join(current_gene_comment_lines) )
                gene.print_as(fh=fout, source='AUGUSTUS', format='gff3')

                gene = None
                mRNAs = dict()
                in_sequence = False
                current_sequence = None
                current_gene_comment_lines = list()

            elif line.startswith("# protein sequence = ["):
                pass
            elif in_sequence is True:
                # build 'current_sequence'
                pass

        else:
            cols = line.split("\t")

            if len(cols) != 9:
                continue

            mol_id = cols[0]
            feat_type = cols[2]

            if feat_type not in ['gene', 'transcript', 'CDS']:
                continue

            ## The output format is GTF by default and (mostly) GFF if the --gff option is used.
            #   If GTF is detected, let's start by transforming the 9th column into GFF so the
            #   libraries can use it
            #   g1  ->  ID=g1
            #   g1.t1  ->  ID=g1.t1;Parent=g1
            #   transcript_id "g1.t1"; gene_id "g1";  ->  ID=g1.t1.cds;Parent=g1.t1
            m_gene = re.match('(g\d+)', cols[8])
            m_transcript = re.match('((g\d+).t\d+)', cols[8])
            m_CDS = re.match('transcript_id "(g\d+.t\d+)"; gene_id "g\d+";', cols[8])

            # the input can be in GTF or GFF.  We need to reformat the 9th column for the GTF entries
            if not cols[8].startswith('ID') and not cols[8].startswith('Parent'):
                if feat_type == 'gene':
                    if m_gene:
                        cols[8] = "ID={0}".format(m_gene.group(1))
                    else:
                        raise Exception("ERROR: GTF detected but gene row has bad 9th column format: {0}".format(cols[8]))
                elif feat_type == 'transcript':
                    if m_transcript:
                        cols[8] = "ID={0};Parent={1}".format(m_transcript.group(1), m_transcript.group(2))
                    else:
                        raise Exception("ERROR: GTF detected but transcript row has bad 9th column format: {0}".format(cols[8]))
                elif feat_type == 'CDS':
                    if m_CDS:
                        cols[8] = "ID={0}.cds;Parent={0}".format(m_CDS.group(1))
                    else:
                        raise Exception("ERROR: GTF detected but CDS row has bad 9th column format: {0}".format(cols[8]))

            feat_id = gff.column_9_value(cols[8], 'ID')

            ## initialize this assembly if we haven't seen it yet
            if mol_id not in assemblies:
                assemblies[mol_id] = things.Assembly(id=mol_id)

            current_assembly = assemblies[mol_id]

            if feat_type == "gene":
                gene = things.Gene(id=feat_id)
                gene.locate_on( target=current_assembly, fmin=int(cols[3]) - 1, fmax=int(cols[4]), strand=cols[6] )

            elif feat_type == "transcript":
                mRNA = things.mRNA(id=feat_id, parent=gene)
                mRNA.locate_on( target=current_assembly, fmin=int(cols[3]) - 1, fmax=int(cols[4]), strand=cols[6] )
                gene.add_mRNA(mRNA)
                mRNAs[mRNA.id] = mRNA

                if feat_id in exon_count_by_mRNA:
                    raise Exception( "ERROR: two different mRNAs found with same ID: {0}".format(feat_id) )
                else:
                    exon_count_by_mRNA[feat_id] = 0

            elif feat_type == "CDS":
                parent_id = gff.column_9_value(cols[8], 'Parent')

                ## sanity check that we've seen this parent
                if parent_id not in mRNAs:
                    raise Exception("ERROR: Found CDS column with parent ({0}) mRNA not yet in the file".format(parent_id))

                CDS = things.CDS(id=feat_id, parent=mRNAs[parent_id])
                CDS.locate_on( target=current_assembly, fmin=int(cols[3]) - 1, fmax=int(cols[4]), strand=cols[6], phase=int(cols[7]) )
                mRNA.add_CDS(CDS)

                ## exons weren't explicitly defined in the input file, so we need to derive new IDs for them
                exon_count_by_mRNA[parent_id] += 1
                exon_id = "{0}.exon{1}".format(parent_id, exon_count_by_mRNA[parent_id])

                exon = things.Exon(id=exon_id, parent=mRNAs[parent_id])
                exon.locate_on( target=current_assembly, fmin=int(cols[3]) - 1, fmax=int(cols[4]), strand=cols[6] )
                mRNA.add_exon(exon)

if __name__ == '__main__':
    main()

<commit_msg>minor: updated documentation after having tested on Augustus 3.3 output<commit_after>#!/usr/bin/env python3

import argparse
import re

from biocode import gff, things

'''
This script converts native (GTF) or GFF output (via the --gff3 option) of Augustus
into GFF3 format.

Admittedly, when you run augustus with --gff3=on you will get legal GFF3 with lots of
comments.  What I often need is GFF3 that also follows the gene model graph conventions
defined in the GFF3, which the default output does not.  There are no exon or mRNA
features, for example.

This script reads the native GFF output and makes it conform to the suggested canonical
gene model, defined here:

http://www.sequenceontology.org/gff3.shtml

WARNING: The augustus output format appears to change pretty frequently.  I've provided
an example of the expected input below, and the versions I tested with are 2.7, 3.0.1 and 3.3

WARNING: Only the gene/transcript/CDS rows are kept (and exon rows are created).  All
others are discarded.

EXPECTED INPUT example:

###
# start gene g2
FO082871        AUGUSTUS        gene    16991   18816   0.04    +       .       ID=g2
FO082871        AUGUSTUS        transcript      16991   18816   0.04    +       .       ID=g2.t1;Parent=g2
FO082871        AUGUSTUS        transcription_start_site        16991   16991   .       +       .       Parent=g2.t1
FO082871        AUGUSTUS        five_prime_utr  16991   17193   0.04    +       .       Parent=g2.t1
FO082871        AUGUSTUS        start_codon     17194   17196   .       +       0       Parent=g2.t1
FO082871        AUGUSTUS        intron  17317   17557   0.15    +       .       Parent=g2.t1
FO082871        AUGUSTUS        CDS     17194   17316   0.15    +       0       ID=g2.t1.cds;Parent=g2.t1
FO082871        AUGUSTUS        CDS     17558   17647   0.15    +       0       ID=g2.t1.cds;Parent=g2.t1
FO082871        AUGUSTUS        stop_codon      17645   17647   .       +       0       Parent=g2.t1
FO082871        AUGUSTUS        three_prime_utr 17648   18816   0.07    +       .       Parent=g2.t1
FO082871        AUGUSTUS        transcription_end_site  18816   18816   .       +       .       Parent=g2.t1
# protein sequence = [MQDNYTDGQPGIHRAIAKLEAIINEVDQQLYDYLLERQIHLSPPSKYWEISDIDELVAKAYVCFYAGLCC]
# Evidence for and against this transcript:
# % of transcript supported by hints (any source): 80
# CDS exons: 2/2
#      W:   2
# CDS introns: 0/1
# 5'UTR exons and introns: 1/1
#      W:   1
# 3'UTR exons and introns: 1/1
#      W:   1
# hint groups fully obeyed: 148
#      W: 148
# incompatible hint groups: 11
#      W:  11
# end gene g2

'''

def main():
    parser = argparse.ArgumentParser( description='Convert native (GTF) or GFF output from Augustus into GFF3 format')

    ## output file to be written
    parser.add_argument('-i', '--input', type=str, required=True, help='Path to a GFF file created by Augustus' )
    parser.add_argument('-o', '--output', type=str, required=True, help='Path to an output file to be created' )
    args = parser.parse_args()

    assemblies = dict()
    current_assembly = None

    gene = None
    mRNAs = dict()
    in_sequence = False
    current_sequence = None
    current_gene_comment_lines = list()

    ## Used for tracking the exon count for each gene (for ID purposes)
    exon_count_by_mRNA = dict()

    fout = open(args.output, mode='wt', encoding='utf-8')
    fout.write("##gff-version 3\n")

    for line in open(args.input):
        if line.startswith("#"):
            current_gene_comment_lines.append(line)

            if line.startswith("# end gene "):
                ## purge the comments, then write the gene
                fout.write( "".join(current_gene_comment_lines) )
                gene.print_as(fh=fout, source='AUGUSTUS', format='gff3')

                gene = None
                mRNAs = dict()
                in_sequence = False
                current_sequence = None
                current_gene_comment_lines = list()

            elif line.startswith("# protein sequence = ["):
                pass
            elif in_sequence is True:
                # build 'current_sequence'
                pass

        else:
            cols = line.split("\t")

            if len(cols) != 9:
                continue

            mol_id = cols[0]
            feat_type = cols[2]

            if feat_type not in ['gene', 'transcript', 'CDS']:
                continue

            ## The output format is GTF by default and (mostly) GFF if the --gff option is used.
            #   If GTF is detected, let's start by transforming the 9th column into GFF so the
            #   libraries can use it
            #   g1  ->  ID=g1
            #   g1.t1  ->  ID=g1.t1;Parent=g1
            #   transcript_id "g1.t1"; gene_id "g1";  ->  ID=g1.t1.cds;Parent=g1.t1
            m_gene = re.match('(g\d+)', cols[8])
            m_transcript = re.match('((g\d+).t\d+)', cols[8])
            m_CDS = re.match('transcript_id "(g\d+.t\d+)"; gene_id "g\d+";', cols[8])

            # the input can be in GTF or GFF.  We need to reformat the 9th column for the GTF entries
            if not cols[8].startswith('ID') and not cols[8].startswith('Parent'):
                if feat_type == 'gene':
                    if m_gene:
                        cols[8] = "ID={0}".format(m_gene.group(1))
                    else:
                        raise Exception("ERROR: GTF detected but gene row has bad 9th column format: {0}".format(cols[8]))
                elif feat_type == 'transcript':
                    if m_transcript:
                        cols[8] = "ID={0};Parent={1}".format(m_transcript.group(1), m_transcript.group(2))
                    else:
                        raise Exception("ERROR: GTF detected but transcript row has bad 9th column format: {0}".format(cols[8]))
                elif feat_type == 'CDS':
                    if m_CDS:
                        cols[8] = "ID={0}.cds;Parent={0}".format(m_CDS.group(1))
                    else:
                        raise Exception("ERROR: GTF detected but CDS row has bad 9th column format: {0}".format(cols[8]))

            feat_id = gff.column_9_value(cols[8], 'ID')

            ## initialize this assembly if we haven't seen it yet
            if mol_id not in assemblies:
                assemblies[mol_id] = things.Assembly(id=mol_id)

            current_assembly = assemblies[mol_id]

            if feat_type == "gene":
                gene = things.Gene(id=feat_id)
                gene.locate_on( target=current_assembly, fmin=int(cols[3]) - 1, fmax=int(cols[4]), strand=cols[6] )

            elif feat_type == "transcript":
                mRNA = things.mRNA(id=feat_id, parent=gene)
                mRNA.locate_on( target=current_assembly, fmin=int(cols[3]) - 1, fmax=int(cols[4]), strand=cols[6] )
                gene.add_mRNA(mRNA)
                mRNAs[mRNA.id] = mRNA

                if feat_id in exon_count_by_mRNA:
                    raise Exception( "ERROR: two different mRNAs found with same ID: {0}".format(feat_id) )
                else:
                    exon_count_by_mRNA[feat_id] = 0

            elif feat_type == "CDS":
                parent_id = gff.column_9_value(cols[8], 'Parent')

                ## sanity check that we've seen this parent
                if parent_id not in mRNAs:
                    raise Exception("ERROR: Found CDS column with parent ({0}) mRNA not yet in the file".format(parent_id))

                CDS = things.CDS(id=feat_id, parent=mRNAs[parent_id])
                CDS.locate_on( target=current_assembly, fmin=int(cols[3]) - 1, fmax=int(cols[4]), strand=cols[6], phase=int(cols[7]) )
                mRNA.add_CDS(CDS)

                ## exons weren't explicitly defined in the input file, so we need to derive new IDs for them
                exon_count_by_mRNA[parent_id] += 1
                exon_id = "{0}.exon{1}".format(parent_id, exon_count_by_mRNA[parent_id])

                exon = things.Exon(id=exon_id, parent=mRNAs[parent_id])
                exon.locate_on( target=current_assembly, fmin=int(cols[3]) - 1, fmax=int(cols[4]), strand=cols[6] )
                mRNA.add_exon(exon)

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>import os
import sys
import traceback

from optparse import make_option
from subprocess import Popen, PIPE

from django.db import connections, transaction, DEFAULT_DB_ALIAS
from django.db.models import get_model
from django.conf import settings
from django.core.management import call_command
from django.core.management.base import BaseCommand, CommandError
from django.core.management.sql import emit_post_sync_signal
from django.utils.importlib import import_module

from nashvegas.models import Migration
from nashvegas.utils import get_sql_for_new_models

sys.path.append("migrations")

class MigrationError(Exception):
    pass

class Command(BaseCommand):

    option_list = BaseCommand.option_list + (
        make_option("-l", "--list", action = "store_true",
                    dest = "do_list", default = False,
                    help = "Enumerate the list of migrations to execute."),
        make_option("-e", "--execute", action = "store_true",
                    dest = "do_execute", default = False,
                    help = "Execute migrations not in versions table."),
        make_option("-c", "--create", action = "store_true",
                    dest = "do_create", default = False,
                    help = "Generates sql for models that are installed but not in your database."),
        make_option("-s", "--seed", action = "store_true",
                    dest = "do_seed", default = False,
                    help = "Seed nashvegas with migrations that have previously been applied in another manner."),
        make_option("--database", action="store", dest="database",
                    default=DEFAULT_DB_ALIAS, help="Nominates a database to synchronize. "
            "Defaults to the \"default\" database."),
        make_option("-p", "--path", dest = "path",
            default = os.path.join(
                os.path.dirname(
                    os.path.normpath(
                        os.sys.modules[settings.SETTINGS_MODULE].__file__
                    )
                ), "migrations"
            ),
            help="The path to the database migration scripts."))
    help = "Upgrade database."

    def _filter_down(self, stop_at=None):

        if stop_at is None:
            stop_at = float("inf")

        applied = []
        to_execute = []
        scripts_in_directory = []

        try:
            already_applied = Migration.objects.all().order_by("migration_label")

            for x in already_applied:
                applied.append(x.migration_label)

            in_directory = os.listdir(self.path)
            in_directory = [migration for migration in in_directory if
                            not migration.startswith(".")]
            in_directory.sort()
            applied.sort()

            for script in in_directory:
                name, ext = os.path.splitext(script)
                try:
                    number = int(name.split("_")[0])
                except ValueError:
                    raise MigrationError("Invalid migration file prefix (must begin with a number)")
                if ext in [".sql", ".py"]:
                    scripts_in_directory.append((number, script))

            for number, script in scripts_in_directory:
                if script not in applied and number <= stop_at:
                    to_execute.append(script)
        except OSError, e:
            print str(e)

        return to_execute

    def _get_rev(self, fpath):
        """
        Get an SCM verion number. Try svn and git.
        """
        rev = None

        try:
            cmd = ["git", "log", "-n1", "--pretty=format:\"%h\"", fpath]
            rev = Popen(cmd, stdout=PIPE, stderr=PIPE).communicate()[0]
        except:
            pass

        if not rev:
            try:
                cmd = ["svn", "info", fpath]
                svninfo = Popen(cmd, stdout=PIPE, stderr=PIPE).stdout.readlines()
                for info in svninfo:
                    tokens = info.split(":")
                    if tokens[0].strip() == "Last Changed Rev":
                        rev = tokens[1].strip()
            except:
                pass

        return rev

    def init_nashvegas(self):
        # Copied from line 35 of django.core.management.commands.syncdb
        # Import the 'management' module within each installed app, to register
        # dispatcher events.
        for app_name in settings.INSTALLED_APPS:
            try:
                import_module('.management', app_name)
            except ImportError, exc:
                # This is slightly hackish. We want to ignore ImportErrors
                # if the "management" module itself is missing -- but we don't
                # want to ignore the exception if the management module exists
                # but raises an ImportError for some reason. The only way we
                # can do this is to check the text of the exception. Note that
                # we're a bit broad in how we check the text, because different
                # Python implementations may not use the same text.
                # CPython uses the text "No module named management"
                # PyPy uses "No module named myproject.myapp.management"
                msg = exc.args[0]
                if not msg.startswith('No module named') or 'management' not in msg:
                    raise

        # @@@ make cleaner / check explicitly for model instead of looping over and doing string comparisons
        connection = connections[self.db]
        cursor = connection.cursor()
        all_new = get_sql_for_new_models()
        for s in all_new:
            if "nashvegas_migration" in s:
                cursor.execute(s)
                transaction.commit_unless_managed(using=self.db)
                return

    def create_migrations(self):
        statements = get_sql_for_new_models(self.args)
        if len(statements) > 0:
            for s in statements:
                print s

    @transaction.commit_manually
    def execute_migrations(self, show_traceback=False):
        migrations = self._filter_down()

        if not len(migrations):
            sys.stdout.write("There are no migrations to apply.\n")

        created_models = set()

        try:
            for migration in migrations:
                migration_path = os.path.join(self.path, migration)
                fp = open(migration_path, "rb")
                lines = fp.readlines()
                fp.close()
                content = "".join(lines)

                if migration_path.endswith(".sql"):
                    to_execute = "".join(
                        [l for l in lines if not l.startswith("### New Model: ")]
                    )
                    connection = connections[self.db]
                    cursor = connection.cursor()

                    sys.stdout.write("Executing %s... " % migration)

                    try:
                        cursor.execute(to_execute)
                    except Exception:
                        sys.stdout.write("failed\n")
                        if show_traceback:
                            traceback.print_exc()
                        raise MigrationError()
                    else:
                        sys.stdout.write("success\n")

                    for l in lines:
                        if l.startswith("### New Model: "):
                            created_models.add(
                                get_model(
                                    *l.replace("### New Model: ", "").strip().split(".")
                                )
                            )
                elif migration_path.endswith(".py"):
                    sys.stdout.write("Executing %s... " % migration)

                    module = {}
                    execfile(migration_path, {}, module)

                    if "migrate" in module and callable(module["migrate"]):
                        try:
                            module["migrate"]()
                        except Exception:
                            sys.stdout.write("failed\n")
                            if show_traceback:
                                traceback.print_exc()
                            raise MigrationError()
                        else:
                            sys.stdout.write("success\n")

                Migration.objects.create(
                    migration_label=migration,
                    content=content,
                    scm_version=self._get_rev(migration_path)
                )
        except Exception:
            transaction.rollback(using=self.db)
            sys.stdout.write("Rolled back all migrations.\n")
            sys.exit(1)
        else:
            sys.stdout.write("Emitting post sync signal.\n")
            emit_post_sync_signal(
                created_models,
                self.verbosity,
                self.interactive,
                self.db
            )
            sys.stdout.write("Running loaddata for initial_data fixtures.\n")
            call_command(
                "loaddata",
                "initial_data",
                verbosity=self.verbosity,
                database=self.db
            )
            transaction.commit(using=self.db)

    def seed_migrations(self, stop_at=None):
        # @@@ the command-line interface needs to be re-thinked
        try:
            stop_at = int(self.args[0])
        except ValueError:
            raise CommandError("Invalid --seed migration")
        migrations = [os.path.join(self.path, m) for m in self._filter_down(stop_at=stop_at)]
        for migration in migrations:
            m, created = Migration.objects.get_or_create(
                migration_label=os.path.basename(migration),
                content=open(migration, "rb").read()
            )
            if created:
                # this might have been executed prior to committing
                m.scm_version = self._get_rev(migration)
                m.save()
                print m.migration_label, "has been seeded"
            else:
                print m.migration_label, "was already applied."

    def list_migrations(self):
        migrations = self._filter_down()
        if len(migrations) == 0:
            print "There are no migrations to apply."
            return

        print "Migrations to Apply:"
        for script in migrations:
            print "\t%s" % script

    def handle(self, *args, **options):
        """
        Upgrades the database.

        Executes SQL scripts that haven't already been applied to the
        database.
        """
        self.do_list = options.get("do_list")
        self.do_execute = options.get("do_execute")
        self.do_create = options.get("do_create")
        self.do_seed = options.get("do_seed")
        self.args = args

        self.path = options.get("path")
        self.verbosity = int(options.get("verbosity", 1))
        self.interactive = options.get("interactive")
        self.db = options.get("database", DEFAULT_DB_ALIAS)

        self.init_nashvegas()

        if self.do_create:
            self.create_migrations()

        if self.do_execute:
            self.execute_migrations(show_traceback=True)

        if self.do_list:
            self.list_migrations()

        if self.do_seed:
            self.seed_migrations()

<commit_msg>moved coverage of commit/rollback to include post_sync signal allowing exceptions to be properly raised<commit_after>import os
import sys
import traceback

from optparse import make_option
from subprocess import Popen, PIPE

from django.db import connections, transaction, DEFAULT_DB_ALIAS
from django.db.models import get_model
from django.conf import settings
from django.core.management import call_command
from django.core.management.base import BaseCommand, CommandError
from django.core.management.sql import emit_post_sync_signal
from django.utils.importlib import import_module

from nashvegas.models import Migration
from nashvegas.utils import get_sql_for_new_models

sys.path.append("migrations")

class MigrationError(Exception):
    pass

class Command(BaseCommand):

    option_list = BaseCommand.option_list + (
        make_option("-l", "--list", action = "store_true",
                    dest = "do_list", default = False,
                    help = "Enumerate the list of migrations to execute."),
        make_option("-e", "--execute", action = "store_true",
                    dest = "do_execute", default = False,
                    help = "Execute migrations not in versions table."),
        make_option("-c", "--create", action = "store_true",
                    dest = "do_create", default = False,
                    help = "Generates sql for models that are installed but not in your database."),
        make_option("-s", "--seed", action = "store_true",
                    dest = "do_seed", default = False,
                    help = "Seed nashvegas with migrations that have previously been applied in another manner."),
        make_option("--database", action="store", dest="database",
                    default=DEFAULT_DB_ALIAS, help="Nominates a database to synchronize. "
            "Defaults to the \"default\" database."),
        make_option("-p", "--path", dest = "path",
            default = os.path.join(
                os.path.dirname(
                    os.path.normpath(
                        os.sys.modules[settings.SETTINGS_MODULE].__file__
                    )
                ), "migrations"
            ),
            help="The path to the database migration scripts."))
    help = "Upgrade database."

    def _filter_down(self, stop_at=None):

        if stop_at is None:
            stop_at = float("inf")

        applied = []
        to_execute = []
        scripts_in_directory = []

        try:
            already_applied = Migration.objects.all().order_by("migration_label")

            for x in already_applied:
                applied.append(x.migration_label)

            in_directory = os.listdir(self.path)
            in_directory = [migration for migration in in_directory if
                            not migration.startswith(".")]
            in_directory.sort()
            applied.sort()

            for script in in_directory:
                name, ext = os.path.splitext(script)
                try:
                    number = int(name.split("_")[0])
                except ValueError:
                    raise MigrationError("Invalid migration file prefix (must begin with a number)")
                if ext in [".sql", ".py"]:
                    scripts_in_directory.append((number, script))

            for number, script in scripts_in_directory:
                if script not in applied and number <= stop_at:
                    to_execute.append(script)
        except OSError, e:
            print str(e)

        return to_execute

    def _get_rev(self, fpath):
        """
        Get an SCM verion number. Try svn and git.
        """
        rev = None

        try:
            cmd = ["git", "log", "-n1", "--pretty=format:\"%h\"", fpath]
            rev = Popen(cmd, stdout=PIPE, stderr=PIPE).communicate()[0]
        except:
            pass

        if not rev:
            try:
                cmd = ["svn", "info", fpath]
                svninfo = Popen(cmd, stdout=PIPE, stderr=PIPE).stdout.readlines()
                for info in svninfo:
                    tokens = info.split(":")
                    if tokens[0].strip() == "Last Changed Rev":
                        rev = tokens[1].strip()
            except:
                pass

        return rev

    def init_nashvegas(self):
        # Copied from line 35 of django.core.management.commands.syncdb
        # Import the 'management' module within each installed app, to register
        # dispatcher events.
        for app_name in settings.INSTALLED_APPS:
            try:
                import_module('.management', app_name)
            except ImportError, exc:
                # This is slightly hackish. We want to ignore ImportErrors
                # if the "management" module itself is missing -- but we don't
                # want to ignore the exception if the management module exists
                # but raises an ImportError for some reason. The only way we
                # can do this is to check the text of the exception. Note that
                # we're a bit broad in how we check the text, because different
                # Python implementations may not use the same text.
                # CPython uses the text "No module named management"
                # PyPy uses "No module named myproject.myapp.management"
                msg = exc.args[0]
                if not msg.startswith('No module named') or 'management' not in msg:
                    raise

        # @@@ make cleaner / check explicitly for model instead of looping over and doing string comparisons
        connection = connections[self.db]
        cursor = connection.cursor()
        all_new = get_sql_for_new_models()
        for s in all_new:
            if "nashvegas_migration" in s:
                cursor.execute(s)
                transaction.commit_unless_managed(using=self.db)
                return

    def create_migrations(self):
        statements = get_sql_for_new_models(self.args)
        if len(statements) > 0:
            for s in statements:
                print s

    @transaction.commit_manually
    def execute_migrations(self, show_traceback=False):
        migrations = self._filter_down()

        if not len(migrations):
            sys.stdout.write("There are no migrations to apply.\n")

        created_models = set()

        try:
            for migration in migrations:
                migration_path = os.path.join(self.path, migration)
                fp = open(migration_path, "rb")
                lines = fp.readlines()
                fp.close()
                content = "".join(lines)

                if migration_path.endswith(".sql"):
                    to_execute = "".join(
                        [l for l in lines if not l.startswith("### New Model: ")]
                    )
                    connection = connections[self.db]
                    cursor = connection.cursor()

                    sys.stdout.write("Executing %s... " % migration)

                    try:
                        cursor.execute(to_execute)
                    except Exception:
                        sys.stdout.write("failed\n")
                        if show_traceback:
                            traceback.print_exc()
                        raise MigrationError()
                    else:
                        sys.stdout.write("success\n")

                    for l in lines:
                        if l.startswith("### New Model: "):
                            created_models.add(
                                get_model(
                                    *l.replace("### New Model: ", "").strip().split(".")
                                )
                            )
                elif migration_path.endswith(".py"):
                    sys.stdout.write("Executing %s... " % migration)

                    module = {}
                    execfile(migration_path, {}, module)

                    if "migrate" in module and callable(module["migrate"]):
                        try:
                            module["migrate"]()
                        except Exception:
                            sys.stdout.write("failed\n")
                            if show_traceback:
                                traceback.print_exc()
                            raise MigrationError()
                        else:
                            sys.stdout.write("success\n")

                Migration.objects.create(
                    migration_label=migration,
                    content=content,
                    scm_version=self._get_rev(migration_path)
                )
            sys.stdout.write("Emitting post sync signal.\n")
            emit_post_sync_signal(
                created_models,
                self.verbosity,
                self.interactive,
                self.db
            )
            sys.stdout.write("Running loaddata for initial_data fixtures.\n")
            call_command(
                "loaddata",
                "initial_data",
                verbosity=self.verbosity,
                database=self.db
            )
        except Exception:
            transaction.rollback(using=self.db)
            sys.stdout.write("Rolled back all migrations.\n")
            raise
        else:
            transaction.commit(using=self.db)

    def seed_migrations(self, stop_at=None):
        # @@@ the command-line interface needs to be re-thinked
        try:
            stop_at = int(self.args[0])
        except ValueError:
            raise CommandError("Invalid --seed migration")
        migrations = [os.path.join(self.path, m) for m in self._filter_down(stop_at=stop_at)]
        for migration in migrations:
            m, created = Migration.objects.get_or_create(
                migration_label=os.path.basename(migration),
                content=open(migration, "rb").read()
            )
            if created:
                # this might have been executed prior to committing
                m.scm_version = self._get_rev(migration)
                m.save()
                print m.migration_label, "has been seeded"
            else:
                print m.migration_label, "was already applied."

    def list_migrations(self):
        migrations = self._filter_down()
        if len(migrations) == 0:
            print "There are no migrations to apply."
            return

        print "Migrations to Apply:"
        for script in migrations:
            print "\t%s" % script

    def handle(self, *args, **options):
        """
        Upgrades the database.

        Executes SQL scripts that haven't already been applied to the
        database.
        """
        self.do_list = options.get("do_list")
        self.do_execute = options.get("do_execute")
        self.do_create = options.get("do_create")
        self.do_seed = options.get("do_seed")
        self.args = args

        self.path = options.get("path")
        self.verbosity = int(options.get("verbosity", 1))
        self.interactive = options.get("interactive")
        self.db = options.get("database", DEFAULT_DB_ALIAS)

        self.init_nashvegas()

        if self.do_create:
            self.create_migrations()

        if self.do_execute:
            self.execute_migrations(show_traceback=True)

        if self.do_list:
            self.list_migrations()

        if self.do_seed:
            self.seed_migrations()

<|endoftext|>


<commit_before>import contextlib

import optuna

from cupy.core import _optimize_config
from cupyx import time

def _optimize(optimize_config, target_func, suggest_func, default_best):
    assert isinstance(optimize_config, _optimize_config._OptimizationConfig)
    assert callable(target_func)
    assert callable(suggest_func)

    def objective(trial):
        args = suggest_func(trial)
        max_total_time = optimize_config.max_total_time_per_trial
        perf = time.repeat(target_func, args, max_duration=max_total_time)
        return perf.gpu_times.mean()

    study = optuna.create_study()
    study.enqueue_trial(default_best)
    study.optimize(
        objective,
        n_trials=optimize_config.max_trials,
        timeout=optimize_config.timeout)
    return study.best_trial

@contextlib.contextmanager
def optimize(*, key=None, **config_dict):
    old_context = _optimize_config.get_current_context()
    context = _optimize_config.get_new_context(key, _optimize, config_dict)
    _optimize_config.set_current_context(context)

    try:
        yield context
    finally:
        _optimize_config.set_current_context(old_context)
<commit_msg>Add documentation for cupyx.optimizing.optimize<commit_after>import contextlib

import optuna

from cupy.core import _optimize_config
from cupyx import time

def _optimize(optimize_config, target_func, suggest_func, default_best):
    assert isinstance(optimize_config, _optimize_config._OptimizationConfig)
    assert callable(target_func)
    assert callable(suggest_func)

    def objective(trial):
        args = suggest_func(trial)
        max_total_time = optimize_config.max_total_time_per_trial
        perf = time.repeat(target_func, args, max_duration=max_total_time)
        return perf.gpu_times.mean()

    study = optuna.create_study()
    study.enqueue_trial(default_best)
    study.optimize(
        objective,
        n_trials=optimize_config.max_trials,
        timeout=optimize_config.timeout)
    return study.best_trial

@contextlib.contextmanager
def optimize(*, key=None, **config_dict):
    """Context manager that optimizes kernel launch parameters.

    In this context, CuPy's routines find the best kernel launch parameter
    values (e.g., the number of threads and blocks). The found values are
    cached and reused with keys as the shapes, strides and dtypes of the
    given inputs arrays.

    Args:
        key (string or None): The cache key of optimizations.
        max_trials (int): The number of trials that defaults to 100.
        timeout (float):
            Stops study after the given number of seconds. Default is 1.
        max_total_time_per_trial (float):
            Repeats measuring the execution time of the routine for the
            given number of seconds. Default is 0.1.

    Examples
    --------
    >>> import cupy
    >>> from cupyx import optimizing
    >>>
    >>> x = cupy.arange(100)
    >>> with optimizing.optimize():
    ...     cupy.sum(x)
    ...
    array(4950)
    """
    old_context = _optimize_config.get_current_context()
    context = _optimize_config.get_new_context(key, _optimize, config_dict)
    _optimize_config.set_current_context(context)

    try:
        yield context
    finally:
        _optimize_config.set_current_context(old_context)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

'''
PxUSBm.py
   (Partition Expansion USB mount)

This module is a marriage of code to do partition expansion during the first power on (formerly expandFS.py)
 and code to monitor USB activity to mount and unmount USB sticks as they are added and removed from
 USB ports on the ConnectBox. (Testing has shown problems with using two python scripts in the rc.local file.)
'''

# Python code to expand filesystem

'''
General outline of Partition Expansion need:

The building of an image results in a single partition which is much smaller than the size of the uSD card
(typically 2 - 3 GB). It is desired that this partition be expanded to the full size of the uSD card on
which the image is placed. To do this, we will use the fdisk program to find the starting sector of the
partition. If we then delete the partition, followed by the creation of a new primary partion we can set
the start of the new partition at the starting sector of the old partition, and then set the size of the
new partition to be the remainder of the disk. During this process, two reboots will be required, one after
setting up the new partition with fdisk and one after doing the file expansion using resize2fs.

Method:

We will use the library pexpect to handle the interactions between this program and the fdisk program and
the library re to handle the regex calls to sort out the number of the starting sector. We also need to
handle the selection of whether we are in the fdisk section of the process or the resize2fs section. To do
this, we will use a second file, expand_progress.txt. Initially, that file will not exist. Upon the first
boot of the uSD with a new image, this code will be called (rc.local). At the top of this code, we will
look for the expand_progress.txt file. If it doesn't exist, we know that we are in the fdisk section, and
so that section will be run. If it completes successfully, the expand_progress file will be created
"fdisk_done" will be written and a reboot will be issued by this code. At reboot, this file will be
called and again, will test for the existence of expand_progress.txt. Upon finding the file, it will
read the contents ("fdisk_done") and if "fdisk_done" is the only entry,
will proceed to run the resize2fs section of the code. Upon successful completion of that section,
the expand_progress.txt file will be opened an written with "resize2fs_done" and a second reboot invoked. After
this reboot, this code will look for the expand_progress.txt file, find it present (fdisk stuff done),
read the contents (not empty, so resize2fs complete) and exit. Thus at each reboot from that point on,
this code will quickly test the expand_progress.txt file an finding the process complete, will exit.

'''

import pexpect
import time
import re
import os

# globals for Partion expansion
progress_file = '/usr/local/connectbox/bin/expand_progress.txt'

# globals for USB monitoring
DEBUG = 0
global total
total = 0
c=["","",""]
loc=[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1]
global mnt
mnt=[-1,-1,-1,-1,-1,-1,-1,-1,-1, -1, -1]
d=["","","","","","","","","","","","","","","","","","","","","",""]

def mountCheck():
    global mnt
    global total
    j = 0                   #mount iterator looking for unmounted devices
    b = os.popen('lsblk').read()
    while (j < total):
      if DEBUG: print("loop 1, iterate",j)
      if (mnt[j] >= 0):
        if not ('usb' + chr(ord('0')+j) in b):
          c = "umount /media/usb" + chr(ord('0')+j)
          res = os.system(c)
          if DEBUG: print("completed unmount /media/usb", chr(ord('0')+j))
          if not res:
            mnt[j] = -1
            if j>0:
               os.popen('rmdir /media/usb'+chr(ord('0')+j))
            j += 1
          else:
            if DEBUG: print("Failed to " + c)
            mnt[j] = -1
            j += 1
        else:
          #Were here with the device still mounted
          j += 1
      else:
        #were here because there was no mount device detected which is odd as its supposed to be
        if DEBUG: print("Error as device should have been mounted usb", j)
        j += 1
    i=0                       #line iterator
    j=0                       #used for finding sdx1's
    k=0                       #used for usb mounts
    loc=[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1]
    b = os.popen('lsblk').read()
    c = b.partition("\n")
# while we have lines to parse we check each line for an sdx1 device and mount it if not already
    while ((c[0] != "") and (i<10)):
      if DEBUG: print("Loop 2, iterate:",i, c[0])
      d[i] = c[0]
      e=re.search('sd[a-z]1', d[i])
      if e:
        loc[j]=i
        if not (('usb' in d[i]) or ('part /' in d[i])):
          while (k < 10) and (mnt[k] >= 0):
            if DEBUG: print("loop 3, iterate:",k)
            k += 1
          if not (os.path.exists("/media/usb"+chr(ord("0")+k))):
            res = os.system("mkdir /media/usb"+chr(ord("0")+k))
          b = "mount /dev/" + e.group() + " /media/usb" + chr(ord('0')+k)
          res = os.system(b)
          if DEBUG: print("completed mount /dev/",e.group)
          k += 1
          j += 1
        else:
          if ('usb' in d[i]):
            a = d[i].partition("usb")
            if (a[2] != "") and (a[2].isalnum()):
              l = ord(a[2])-ord("0")
              mnt[l]=i
              j+=1
              if DEBUG: print("/dev/sdx1 is already mounted as usb",chr(l+ord('0')))
            else:
              if DEBUG: print("Error parsing usb# in line", i)
              j+= 1
          else:
              if DEBUG: print("/dev/sdx1 is already mounted but not as usb", d[i])
              j+= 1
      c = c[2].partition("\n")
      i += 1
# now that we have looked at all lines in the current lsblk we need to count up the mounts
    j = -1
    i = 0
    while (i < 10):            # we will check a maximum of 10 mounts
      if DEBUG: print("loop 4, iterate:",i)
      if (mnt[i] != -1):
        i +=1
        j +=1
      else:                   # we have a hole or are done but need to validate
        k = i+1
        while (k < 10) and (mnt[i] == -1):  #checking next mount to see if it is valid
          if DEBUG: print("loop 5, iterate:",k,i)
          if (mnt[k] != -1):
            mnt[i] = mnt[k]   # move the mount into this hole and clear the other point
            j += 1
            mnt[k] = -1
            i += 1
            k += 1
            if DEBUG: print("had to move mount to new location",k," from ",i)
          else:         # we have no mount here
            k += 1
        if (k == 10):           # if we have look at all mounts then were done otherwise we will check again
          i=10
    total = j+1
    if DEBUG: print("total of devices mounted is", total)
    if DEBUG: print("located", mnt)
    return()

def do_resize2fs():
	# find the filesystem name ... like "/dev/mmcblk0p1"
    out = pexpect.run('df -h')
    p = re.compile('/dev/mmcblk[0-9]p[0-9]+')
    out = out.decode('utf-8')
    m = p.search(out)
    FS = m.group()

    # we have the FS string, so build the command and run it
    cmd = 'resize2fs ' + FS
    out = pexpect.run(cmd, timeout=600)  # 10 minutes should be enough for the resize
    out = out.decode('utf-8')
    if "blocks long" in out:
        print("resize2fs complete... now reboot")
        f = open(progress_file, "w")
        f.write("resize2fs_done")
        f.close()
        os.system('reboot')

def do_fdisk():

    child = pexpect.spawn('fdisk /dev/mmcblk0', timeout = 10)
    try:
	    i = child.expect(['Command (m for help)*', 'No such file or directory']) # the match is looking for the LAST thing that came up so we need the *
    except:
	    print("Exception thrown")
	    print("debug info:")
	    print (str(child))

	# the value of i is the reference to which of the [] arguments was found
    if i==1:
        print("There is no /dev/mmcblk0 partition... exiting")
        child.kill(0)
    if i==0:
        print("Found it!")
	# continuing

	# "p" get partition info and search out the starting sector of mmcblk0p1
    child.sendline('p')
    i = child.expect('Command (m for help)*')
	# the child.before contains all that came BEFORE we found the expected text
	#print (child.before)
    response = child.before

	# change from binary to string
    respString = response.decode('utf-8')

    p = re.compile('mmcblk[0-9]p[0-9]\s*[0-9]+')		# create a regexp to get close to the sector info
    m = p.search(respString)		# should find "mmcblk0p1   8192" or similar, saving as object m
    match = m.group()				# get the text of the find
    p = re.compile('\s[0-9]+')		# a new regex to find just the number from the match above
    m = p.search(match)
    startSector = m.group()
    print("starting sector = ", startSector )

	# "d" for delete the partition
    child.sendline('d')
    i = child.expect('Command (m for help)*')
#	print("after delete ",child.before)

	# "n" for new partition
    child.sendline('n')
    i = child.expect('(default p):*')
#	print("after new ",child.before)

	# "p" for primary partition
    child.sendline('p')
    i = child.expect('default 1*')
#	print ("after p ", child.before)

	# "1" for partition number 1
    child.sendline('1')
    i = child.expect('default 2048*')
#	print (child.before)

	# send the startSector number
    child.sendline(startSector)
    child.expect('Last sector*')
#	print("At last sector... the after is: ", child.after)

	# take default for last sector
    child.sendline('')
    i = child.expect('signature*')

	# "N" don't remove the signature
    child.sendline('N')
    i = child.expect('Command (m for help)*')

	# "w" for write and exit
    child.sendline('w')
    i = child.expect('Syncing disks*')

    print("exiting the fdisk program... now reboot")
    f = open(progress_file, "w")
    f.write("fdisk_done")
    f.close()
    os.system('reboot')

if __name__ == "__main__":
# First handle the partition expansion

    # Sort out how far we are in the partition expansion process
    file_exists = os.path.exists(progress_file)
    if file_exists == False:
        do_fdisk()             # this ends in reboot() so won't return

    else:
        f = open(progress_file, "r")
        progress = f.read()
        f.close()
        if "resize2fs_done" not in progress:
            do_resize2fs()     # this ends in reboot() so won't return

# Once partition expansion is complete, handle the ongoing monitor of USBs

    while True:
        loop_time = 3
        mountCheck()
        time.sleep(loop_time)

<commit_msg>Update usr_local_connectbox_bin_PxUSBm.py<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

'''
PxUSBm.py
   (Partition Expansion USB mount)

This module is a marriage of code to do partition expansion during the first power on (formerly expandFS.py)
 and code to monitor USB activity to mount and unmount USB sticks as they are added and removed from
 USB ports on the ConnectBox. (Testing has shown problems with using two python scripts in the rc.local file.)
'''

# Python code to expand filesystem

'''
General outline of Partition Expansion need:

The building of an image results in a single partition which is much smaller than the size of the uSD card
(typically 2 - 3 GB). It is desired that this partition be expanded to the full size of the uSD card on
which the image is placed. To do this, we will use the fdisk program to find the starting sector of the
partition. If we then delete the partition, followed by the creation of a new primary partion we can set
the start of the new partition at the starting sector of the old partition, and then set the size of the
new partition to be the remainder of the disk. During this process, two reboots will be required, one after
setting up the new partition with fdisk and one after doing the file expansion using resize2fs.

Method:

We will use the library pexpect to handle the interactions between this program and the fdisk program and
the library re to handle the regex calls to sort out the number of the starting sector. We also need to
handle the selection of whether we are in the fdisk section of the process or the resize2fs section. To do
this, we will use a second file, expand_progress.txt. Initially, that file will not exist. Upon the first
boot of the uSD with a new image, this code will be called (rc.local). At the top of this code, we will
look for the expand_progress.txt file. If it doesn't exist, we know that we are in the fdisk section, and
so that section will be run. If it completes successfully, the expand_progress file will be created
"fdisk_done" will be written and a reboot will be issued by this code. At reboot, this file will be
called and again, will test for the existence of expand_progress.txt. Upon finding the file, it will
read the contents ("fdisk_done") and if "fdisk_done" is the only entry,
will proceed to run the resize2fs section of the code. Upon successful completion of that section,
the expand_progress.txt file will be opened an written with "resize2fs_done" and a second reboot invoked. After
this reboot, this code will look for the expand_progress.txt file, find it present (fdisk stuff done),
read the contents (not empty, so resize2fs complete) and exit. Thus at each reboot from that point on,
this code will quickly test the expand_progress.txt file an finding the process complete, will exit.

'''

import pexpect
import time
import re
import os

# globals for Partion expansion
progress_file = '/usr/local/connectbox/bin/expand_progress.txt'

# globals for USB monitoring
DEBUG = 0
global total
total = 0
c=["","",""]
loc=[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1]
# mnt[x]>=0 where x is the usb port of the mount and mnt[x] is the line of the /dev/sdx1
global mnt
mnt=[-1,-1,-1,-1,-1,-1,-1,-1,-1, -1, -1]
d=["","","","","","","","","","","","","","","","","","","","","",""]

def mountCheck():
    global mnt
    global total
    j = 0                   #mount iterator looking for unmounted devices
    b = os.popen('lsblk').read()
    while (j < total):
      if DEBUG: print("loop 1, iterate",j)
      if (mnt[j] >= 0):
        if not ('usb' + chr(ord('0')+j) in b):
          c = "umount /media/usb" + chr(ord('0')+j)
          res = os.system(c)
          if DEBUG: print("completed unmount /media/usb", chr(ord('0')+j))
          if not res:
            mnt[j] = -1
            if j>0:
               os.popen('rmdir /media/usb'+chr(ord('0')+j))
            j += 1
          else:
            if DEBUG: print("Failed to " + c)
            mnt[j] = -1
            j += 1
        else:
          #Were here with the device still mounted
          j += 1
      else:
        #were here because there was no mount device detected
        if DEBUG: print("device not mounted usb", j)
        j += 1
    i=0                       #line iterator
    j=0                       #used for finding sdx1's
    k=0                       #used for usb mounts
    loc=[-1,-1,-1,-1,-1,-1,-1,-1,-1,-1]
    b = os.popen('lsblk').read()
    c = b.partition("\n")
# while we have lines to parse we check each line for an sdx1 device and mount it if not already
    while ((c[0] != "") and (i<10)):
      if DEBUG: print("Loop 2, iterate:",i, c[0])
      d[i] = c[0]
      e=re.search('sd[a-z]1', d[i])
      if e:
        loc[j]=i
        if not (('usb' in d[i]) or ('part /' in d[i])):     #True if were not mounted but should be
          while (k < 10) and (mnt[k] >= 0):                 #Find an empty usbX to mount to
            if DEBUG: print("loop 3, iterate:",k)
            k += 1
          if not (os.path.exists("/media/usb"+chr(ord("0")+k))):  #if the /mount/usbx isn't there create it
            res = os.system("mkdir /media/usb"+chr(ord("0")+k))
          b = "mount /dev/" + e.group() + " /media/usb" + chr(ord('0')+k+ "-o noatime,nodev,nosuid,sync,iocharset=utf8")
          res = os.system(b)
          if DEBUG: print("completed mount /dev/",e.group)
          mnt[k]=i
          k += 1
          j += 1
        else:
          if ('usb' in d[i]):
            a = d[i].partition("usb")
            if (a[2] != "") and (a[2].isalnum()):
              l = ord(a[2])-ord("0")
              mnt[l]=i
              j+=1
              if DEBUG: print("/dev/sdx1 is already mounted as usb",chr(l+ord('0')))
            else:
              if DEBUG: print("Error parsing usb# in line", i)
              j+= 1
          else:
              if DEBUG: print("/dev/sdx1 is already mounted but not as usb", d[i])
              j+= 1
      c = c[2].partition("\n")
      i += 1
# now that we have looked at all lines in the current lsblk we need to count up the mounts
    j = -1
    i = 0
    while (i < 10):            # we will check a maximum of 10 mounts
      if DEBUG: print("loop 4, iterate:",i)
      if (mnt[i] != -1):
        i +=1
        j +=1
      else:                   # we have a hole or are done but need to validate
        k = i+1
        while (k < 10) and (mnt[i] == -1):  #checking next mount to see if it is valid
          if DEBUG: print("loop 5, iterate:",k,i)
          if (mnt[k] != -1):
            mnt[i] = mnt[k]   # move the mount into this hole and clear the other point
            j += 1
            mnt[k] = -1
            i += 1
            k += 1
            if DEBUG: print("had to move mount to new location",k," from ",i)
          else:     # we have no mount here
            k += 1
        if (k == 10):           # if we have look at all mounts then were done otherwise we will check again
          i=10
    total = j+1
    if DEBUG: print("total of devices mounted is", total)
    if DEBUG: print("located", mnt)
    return()

def do_resize2fs():
	# find the filesystem name ... like "/dev/mmcblk0p1"
    out = pexpect.run('df -h')
    p = re.compile('/dev/mmcblk[0-9]p[0-9]+')
    out = out.decode('utf-8')
    m = p.search(out)
    FS = m.group()

    # we have the FS string, so build the command and run it
    cmd = 'resize2fs ' + FS
    out = pexpect.run(cmd, timeout=600)  # 10 minutes should be enough for the resize
    out = out.decode('utf-8')
    if "blocks long" in out:
        print("resize2fs complete... now reboot")
        f = open(progress_file, "w")
        f.write("resize2fs_done")
        f.close()
        os.system('reboot')

def do_fdisk():

    child = pexpect.spawn('fdisk /dev/mmcblk0', timeout = 10)
    try:
	    i = child.expect(['Command (m for help)*', 'No such file or directory']) # the match is looking for the LAST thing that came up so we need the *
    except:
	    print("Exception thrown")
	    print("debug info:")
	    print (str(child))

	# the value of i is the reference to which of the [] arguments was found
    if i==1:
        print("There is no /dev/mmcblk0 partition... exiting")
        child.kill(0)
    if i==0:
        print("Found it!")
	# continuing

	# "p" get partition info and search out the starting sector of mmcblk0p1
    child.sendline('p')
    i = child.expect('Command (m for help)*')
	# the child.before contains all that came BEFORE we found the expected text
	#print (child.before)
    response = child.before

	# change from binary to string
    respString = response.decode('utf-8')

    p = re.compile('mmcblk[0-9]p[0-9]\s*[0-9]+')		# create a regexp to get close to the sector info
    m = p.search(respString)		# should find "mmcblk0p1   8192" or similar, saving as object m
    match = m.group()				# get the text of the find
    p = re.compile('\s[0-9]+')		# a new regex to find just the number from the match above
    m = p.search(match)
    startSector = m.group()
    print("starting sector = ", startSector )

	# "d" for delete the partition
    child.sendline('d')
    i = child.expect('Command (m for help)*')
#	print("after delete ",child.before)

	# "n" for new partition
    child.sendline('n')
    i = child.expect('(default p):*')
#	print("after new ",child.before)

	# "p" for primary partition
    child.sendline('p')
    i = child.expect('default 1*')
#	print ("after p ", child.before)

	# "1" for partition number 1
    child.sendline('1')
    i = child.expect('default 2048*')
#	print (child.before)

	# send the startSector number
    child.sendline(startSector)
    child.expect('Last sector*')
#	print("At last sector... the after is: ", child.after)

	# take default for last sector
    child.sendline('')
    i = child.expect('signature*')

	# "N" don't remove the signature
    child.sendline('N')
    i = child.expect('Command (m for help)*')

	# "w" for write and exit
    child.sendline('w')
    i = child.expect('Syncing disks*')

    print("exiting the fdisk program... now reboot")
    f = open(progress_file, "w")
    f.write("fdisk_done")
    f.close()
    os.system('reboot')

if __name__ == "__main__":
# First handle the partition expansion

    # Sort out how far we are in the partition expansion process
    file_exists = os.path.exists(progress_file)
    if file_exists == False:
        do_fdisk()             # this ends in reboot() so won't return

    else:
        f = open(progress_file, "r")
        progress = f.read()
        f.close()
        if "resize2fs_done" not in progress:
            do_resize2fs()     # this ends in reboot() so won't return

# Once partition expansion is complete, handle the ongoing monitor of USBs

    while True:
        loop_time = 3
        mountCheck()
        time.sleep(loop_time)

<|endoftext|>