<commit_before>#!/usr/bin/env python
# vim: set ts=4 sw=4 et sts=4 ai:

"""
Tool to figure find the USB device that a board is available at.

This is a complicated process as the FX2 is software configurable and hence
could end up under many different VID:PID names based on what firmware is
currently loaded onto it.
"""

import os
from collections import namedtuple
import logging

PathBase = namedtuple('PathBase', ['bus', 'address'])
class Path(PathBase):
    def __init__(self, *args, **kw):
        PathBase.__init__(self, *args, **kw)
        assert os.path.exists(self.path), "%r %r" % (self.path, self)

    @property
    def path(self):
        return '/dev/bus/usb/%03i/%03i' % (self.bus, self.address)

    def __str__(self):
        return self.path

    def __cmp__(self, other):
        if isinstance(other, Path):
            return cmp(self.path, other.path)
        return cmp(self.path, other)

Device = namedtuple('Device', ['path', 'vid', 'pid', 'serial'])
Device.__str__ = lambda self: "Device(%04x:%04x %s)" % (self.vid, self.pid, [self.path, repr(self.serial)][bool(self.serial)])
#Device.__cmp__ = lambda a, b: cmp(a.path, b.path)

def find_usb_devices_libusb():
    class LibDevice(Device):
        def inuse(self, dev=None):
            try:
                if dev is None:
                    dev = usb.core.find(bus=self.path.bus, address=self.path.address)

                #config = dev.get_active_configuration()
                active = False
                for config in dev:
                    for i, inf in enumerate(config):
                        inf_active = dev.is_kernel_driver_active(inf.bInterfaceNumber)
                        active = active or inf_active
                return active
            except usb.core.USBError:
                return None

        def detach(self):
            # Detach any driver currently attached.
            dev = usb.core.find(bus=self.path.bus, address=self.path.address)

            if not self.inuse(dev):
                return True

            config = dev.get_active_configuration()
            for inf in config:
                if dev.is_kernel_driver_active(inf.bInterfaceNumber):
                    dev.detach_kernel_driver(inf.bInterfaceNumber)

    import usb
    if usb.__file__.endswith('.so'):
        logging.warning("Your python usb module is old.")
    import usb.util

    busses = usb.busses()

    devobjs = []
    for dev in usb.core.find(find_all=True):
        serial = None
        if dev.iSerialNumber > 0:
            try:
                serial = dev.serial_number
            except usb.USBError:
                pass

        devobjs.append(LibDevice(vid=dev.idVendor, pid=dev.idProduct, serial=serial, path=Path(bus=dev.bus, address=dev.address)))
    return devobjs

def find_usb_devices_lsusb():
    import re
    import subprocess

    # 'Bus 002 Device 002: ID 8087:0024 Intel Corp. Integrated Rate Matching Hub'
    lsusb_device_regex = re.compile(
        "Bus (?P<bus>[0-9]+) Device (?P<address>[0-9]+):"
        " ID (?P<vid>[0-9a-f]+):(?P<pid>[0-9a-f]+)")

    class LsusbDevice(Device):
        def __new__(cls, *args, **kw):
            return Device.__new__(cls, *args, serial=None, **kw)

        @property
        def serial(self):
            # Get the serial number from the device and cache it.
            if not hasattr(self, "_serial"):
                serialpath = os.path.join(self.syspaths[0], "serial")
                if not os.path.exists(serialpath):
                    self._serial = None
                else:
                    self._serial = open(serialpath, "r").read().strip()
            return self._serial

        @property
        def syspaths(self):
            if not hasattr(self, "_syspaths"):
                self._syspaths = find_sys(self.path)
            return self._syspaths

        def inuse(self):
            return bool(self.drivers())

        def drivers(self):
            drivers = {}
            for path in self.syspaths[1:]:
                driver_path = os.path.join(path, "driver")
                if os.path.exists(driver_path):
                    drivers[path] = os.readlink(driver_path)
            return tuple(set(d.split('/')[-1] for d in drivers.values()))

        def detach(self):
            for path in self.syspaths[1:]:
                driver_path = os.path.join(path, "driver")
                if os.path.exists(driver_path):
                    unbind_path = os.path.join(driver_path, "unbind")
                    assert os.path.exists(unbind_path), unbind_path
                    interface = os.path.split(path)[-1]
                    open(unbind_path, "w").write(interface)

    devobjs = []
    output = subprocess.check_output('lsusb')
    for line in output.splitlines():
        bits = lsusb_device_regex.match(line)
        assert bits, repr(line)

        vid = int(bits.group('vid'), base=16)
        pid = int(bits.group('pid'), base=16)
        bus = int(bits.group('bus'), base=10)
        address = int(bits.group('address'), base=10)
        devobjs.append(LsusbDevice(
            vid=vid, pid=pid, path=Path(bus=bus, address=address),
            ))

    return devobjs

SYS_ROOT = '/sys/bus/usb/devices'

def get_path_from_sysdir(dirpath):
    buspath = os.path.join(dirpath, 'busnum')
    devpath = os.path.join(dirpath, 'devnum')
    if not os.path.exists(buspath):
        logging.info("Skipping %s (no busnum)", dirname)
        return None
    if not os.path.exists(devpath):
        logging.info("Skipping %s (no devnum)", dirname)
        return None

    busnum = int(open(buspath, 'r').read().strip())
    devnum = int(open(devpath, 'r').read().strip())

    return Path(bus=busnum, address=devnum)

def create_sys_mapping():
    # 1-1.3.1  --> (Device)    bus-port.port.port
    # 1-0:1.0  --> (Interface) bus-port.port.port:config.interface
    # usb1     --> bus<number>

    devices = {}
    interfaces = {}

    dirs = list(sorted(os.listdir(SYS_ROOT)))
    for dirname in dirs:
        dirpath = os.path.join(SYS_ROOT, dirname)
        if ":" in dirname:
            continue
        path = get_path_from_sysdir(dirpath)
        assert path
        devices[dirpath] = path
        assert path not in interfaces
        interfaces[path] = [dirpath]

    for dirname in dirs:
        dirpath = os.path.join(SYS_ROOT, dirname)
        if ":" not in dirname:
            continue

        device, interface = dirname.split(':')
        if device.endswith('-0'):
            device = "usb%s" % (device[:-2])

        devpath = os.path.join(SYS_ROOT, device)
        assert os.path.exists(devpath)
        assert devpath in devices

        interfaces[devices[devpath]].append(dirpath)

    return interfaces

def find_sys(path, mapping={}):
    if not mapping:
        mapping.update(create_sys_mapping())
    return mapping[path]

def test_libusb_and_lsusb_equal():
    libusb_devices = find_usb_devices_libusb()
    lsusb_devices = find_usb_devices_lsusb()
    for libobj, lsobj in zip(sorted(libusb_devices), sorted(lsusb_devices)):
        print "%s -- lib: %-40s ls: %-40s -- %-40s  drivers: %s" % (libobj.path, libobj, lsobj, find_sys(libobj.path)[0], lsobj.drivers())
        assert libobj.vid == lsobj.vid, "%r == %r" % (libobj.vid, lsobj.vid)
        assert libobj.pid == lsobj.pid, "%r == %r" % (libobj.pid, lsobj.pid)
        if libobj.serial:
            assert libobj.serial == lsobj.serial, "%r == %r" % (libobj.serial, lsobj.serial)
        assert libobj.path == lsobj.path, "%r == %r" % (libobj.path, lsobj.path)

        lsobj_inuse = lsobj.inuse()
        libobj_inuse = libobj.inuse()
        if libobj_inuse is not None:
            assert libobj_inuse == lsobj_inuse, "%r == %r" % (libobj_inuse, lsobj_inuse)

test_libusb_and_lsusb_equal()

BOARD_TYPES = ['opsis', 'atlys']
BOARD_NAMES = {
    'atlys': "Digilent Atlys",
    'opsis': "Numato Opsis",
    }
BOARD_STATES = ['unconfigured', 'jtag', 'operational']

USBJTAG_MAPPING = {
    'hw_nexys': 'atlys',
    'hw_opsis': 'opsis',
    }
USBJTAG_RMAPPING = {v:k for k,v in USBJTAG_MAPPING.items()}
print USBJTAG_RMAPPING

Board = namedtuple("Board", ["dev", "type", "state"])

import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--no-modify', help="Don't modify the state of the system in any way.")

parser.add_argument('--mac', help='Find board with the given MAC address.')
parser.add_argument('--dna', help='Find board with the given Device DNA.')
parser.add_argument('--position', help="""Find board using a given position in the USB structure.
Example:
 1-2.3 - Bus 1, Port 2 (which is a hub), Port 3
 5-6.7.8 - Bus 5, Port 2 (which is a hub), Port 7 (which is a hub), Port 8 (which is a hub)

While this *should* be static across reboots, but sadly on some machines it isn't :(
""")

parser.add_argument('--get-usbfs', help='Return the /dev/bus/usb path for a device.')
parser.add_argument('--get-sysfs', help='Return the /sys/bus/usb/devices path for a device.')

parser.add_argument('--get-state', help='Return the state the device is in. Possible states are: %r' % BOARD_STATES)
parser.add_argument('--get-video-device', help='Get the primary video device path.')
parser.add_argument('--get-serial-device', help='Get the serial device path.')

parser.add_argument('--use-hardware-serial', help='Use the hardware serial port on the Atlys board.')

parser.add_argument('--load-firmware', help='Load the firmware file onto the device.')

boards = []
for device in find_usb_devices_lsusb():
    # Digilent Atlys board with stock "Adept" firmware
    # Bus 003 Device 019: ID 1443:0007 Digilent Development board JTAG
    if device.vid == 0x1443 and device.pid == 0x0007:
        boards.append(Board(dev=device, type="atlys", state="unconfigured"))

    # The Numato Opsis will boot in the following mode when the EEPROM is not set up correctly.
    # Bus 003 Device 091: ID 04b4:8613 Cypress Semiconductor Corp. CY7C68013 EZ-USB FX2 USB 2.0 Development Kit
    elif device.vid == 0x04b4 and device.pid == 0x8613:
        boards.append(Board(dev=device, type="opsis", state="unconfigured"))

    # Boards loaded with the ixo-usb-jtag firmware from mithro's repo
    # https://github.com/mithro/ixo-usb-jtag
    # Bus 003 Device 090: ID 16c0:06ad Van Ooijen Technische Informatica
    elif device.vid == 0x16c0 and device.pid == 0x06ad:
        if device.serial not in USBJTAG_MAPPING:
            logging.warn("Unknown usb-jtag device!")
            continue
        boards.append(Board(dev=device, type=USBJTAG_MAPPING[device.serial], state="usbjtag"))

for board in boards:
    print "%s as %s at %s" % (
        BOARD_NAMES[board.type],
        board.state,
        board.dev.path,
        )
    if board.dev.inuse():
        print " Board is currently used by drivers %s" % (board.dev.drivers(),)
        board.dev.detach()

    if board.state == "unconfigured":
        print " Configure with 'fxload -t fx2lp -D %s -I %s'" % (
            board.dev.path,
            "%s.hex" % USBJTAG_RMAPPING[board.type],
            )

"""

# Exar USB-UART device on the Digilent Atlys board
# Bus 003 Device 018: ID 04e2:1410 Exar Corp.
# Driver at git+ssh://github.com/mithro/exar-uart-driver.git
if dev.driver() != "vizzini":
  # Need to install driver...
  pass

  - Digilent Atlys board loaded with Nero-USB JTAG firmware
  - Digilent Atlys board loaded with MakeStuff firmware
  - Digilent Atlys board loaded with HDMI2USB "unconfigured" firmware
  - Digilent Atlys board loaded with HDMI2USB "configured" firmware

  - Unconfigured Cypress FX2, could be a Numato Opsis

  - Numato Opsis board with no FX2 firmware loaded
  - Numato Opsis board loaded with Nero-USB JTAG firmware
  - Numato Opsis board loaded with MakeStuff firmware
  - Numato Opsis board loaded with HDMI2USB firmware

"""
<commit_msg>Adding a serial state too.<commit_after>#!/usr/bin/env python
# vim: set ts=4 sw=4 et sts=4 ai:

"""
Tool to figure find the USB device that a board is available at.

This is a complicated process as the FX2 is software configurable and hence
could end up under many different VID:PID names based on what firmware is
currently loaded onto it.
"""

import os
from collections import namedtuple
import logging

PathBase = namedtuple('PathBase', ['bus', 'address'])
class Path(PathBase):
    def __init__(self, *args, **kw):
        PathBase.__init__(self, *args, **kw)
        assert os.path.exists(self.path), "%r %r" % (self.path, self)

    @property
    def path(self):
        return '/dev/bus/usb/%03i/%03i' % (self.bus, self.address)

    def __str__(self):
        return self.path

    def __cmp__(self, other):
        if isinstance(other, Path):
            return cmp(self.path, other.path)
        return cmp(self.path, other)

Device = namedtuple('Device', ['path', 'vid', 'pid', 'serial'])
Device.__str__ = lambda self: "Device(%04x:%04x %s)" % (self.vid, self.pid, [self.path, repr(self.serial)][bool(self.serial)])
#Device.__cmp__ = lambda a, b: cmp(a.path, b.path)

def find_usb_devices_libusb():
    class LibDevice(Device):
        def inuse(self, dev=None):
            try:
                if dev is None:
                    dev = usb.core.find(bus=self.path.bus, address=self.path.address)

                #config = dev.get_active_configuration()
                active = False
                for config in dev:
                    for i, inf in enumerate(config):
                        inf_active = dev.is_kernel_driver_active(inf.bInterfaceNumber)
                        active = active or inf_active
                return active
            except usb.core.USBError:
                return None

        def detach(self):
            # Detach any driver currently attached.
            dev = usb.core.find(bus=self.path.bus, address=self.path.address)

            if not self.inuse(dev):
                return True

            config = dev.get_active_configuration()
            for inf in config:
                if dev.is_kernel_driver_active(inf.bInterfaceNumber):
                    dev.detach_kernel_driver(inf.bInterfaceNumber)

    import usb
    if usb.__file__.endswith('.so'):
        logging.warning("Your python usb module is old.")
    import usb.util

    busses = usb.busses()

    devobjs = []
    for dev in usb.core.find(find_all=True):
        serial = None
        if dev.iSerialNumber > 0:
            try:
                serial = dev.serial_number
            except usb.USBError:
                pass

        devobjs.append(LibDevice(vid=dev.idVendor, pid=dev.idProduct, serial=serial, path=Path(bus=dev.bus, address=dev.address)))
    return devobjs

def find_usb_devices_lsusb():
    import re
    import subprocess

    # 'Bus 002 Device 002: ID 8087:0024 Intel Corp. Integrated Rate Matching Hub'
    lsusb_device_regex = re.compile(
        "Bus (?P<bus>[0-9]+) Device (?P<address>[0-9]+):"
        " ID (?P<vid>[0-9a-f]+):(?P<pid>[0-9a-f]+)")

    class LsusbDevice(Device):
        def __new__(cls, *args, **kw):
            return Device.__new__(cls, *args, serial=None, **kw)

        @property
        def serial(self):
            # Get the serial number from the device and cache it.
            if not hasattr(self, "_serial"):
                serialpath = os.path.join(self.syspaths[0], "serial")
                if not os.path.exists(serialpath):
                    self._serial = None
                else:
                    self._serial = open(serialpath, "r").read().strip()
            return self._serial

        @property
        def syspaths(self):
            if not hasattr(self, "_syspaths"):
                self._syspaths = find_sys(self.path)
            return self._syspaths

        def inuse(self):
            return bool(self.drivers())

        def drivers(self):
            drivers = {}
            for path in self.syspaths[1:]:
                driver_path = os.path.join(path, "driver")
                if os.path.exists(driver_path):
                    drivers[path] = os.readlink(driver_path)
            return tuple(set(d.split('/')[-1] for d in drivers.values()))

        def detach(self):
            for path in self.syspaths[1:]:
                driver_path = os.path.join(path, "driver")
                if os.path.exists(driver_path):
                    unbind_path = os.path.join(driver_path, "unbind")
                    assert os.path.exists(unbind_path), unbind_path
                    interface = os.path.split(path)[-1]
                    open(unbind_path, "w").write(interface)

    devobjs = []
    output = subprocess.check_output('lsusb')
    for line in output.splitlines():
        bits = lsusb_device_regex.match(line)
        assert bits, repr(line)

        vid = int(bits.group('vid'), base=16)
        pid = int(bits.group('pid'), base=16)
        bus = int(bits.group('bus'), base=10)
        address = int(bits.group('address'), base=10)
        devobjs.append(LsusbDevice(
            vid=vid, pid=pid, path=Path(bus=bus, address=address),
            ))

    return devobjs

SYS_ROOT = '/sys/bus/usb/devices'

def get_path_from_sysdir(dirpath):
    buspath = os.path.join(dirpath, 'busnum')
    devpath = os.path.join(dirpath, 'devnum')
    if not os.path.exists(buspath):
        logging.info("Skipping %s (no busnum)", dirname)
        return None
    if not os.path.exists(devpath):
        logging.info("Skipping %s (no devnum)", dirname)
        return None

    busnum = int(open(buspath, 'r').read().strip())
    devnum = int(open(devpath, 'r').read().strip())

    return Path(bus=busnum, address=devnum)

def create_sys_mapping():
    # 1-1.3.1  --> (Device)    bus-port.port.port
    # 1-0:1.0  --> (Interface) bus-port.port.port:config.interface
    # usb1     --> bus<number>

    devices = {}
    interfaces = {}

    dirs = list(sorted(os.listdir(SYS_ROOT)))
    for dirname in dirs:
        dirpath = os.path.join(SYS_ROOT, dirname)
        if ":" in dirname:
            continue
        path = get_path_from_sysdir(dirpath)
        assert path
        devices[dirpath] = path
        assert path not in interfaces
        interfaces[path] = [dirpath]

    for dirname in dirs:
        dirpath = os.path.join(SYS_ROOT, dirname)
        if ":" not in dirname:
            continue

        device, interface = dirname.split(':')
        if device.endswith('-0'):
            device = "usb%s" % (device[:-2])

        devpath = os.path.join(SYS_ROOT, device)
        assert os.path.exists(devpath)
        assert devpath in devices

        interfaces[devices[devpath]].append(dirpath)

    return interfaces

def find_sys(path, mapping={}):
    if not mapping:
        mapping.update(create_sys_mapping())
    return mapping[path]

def test_libusb_and_lsusb_equal():
    libusb_devices = find_usb_devices_libusb()
    lsusb_devices = find_usb_devices_lsusb()
    for libobj, lsobj in zip(sorted(libusb_devices), sorted(lsusb_devices)):
        print "%s -- lib: %-40s ls: %-40s -- %-40s  drivers: %s" % (libobj.path, libobj, lsobj, find_sys(libobj.path)[0], lsobj.drivers())
        assert libobj.vid == lsobj.vid, "%r == %r" % (libobj.vid, lsobj.vid)
        assert libobj.pid == lsobj.pid, "%r == %r" % (libobj.pid, lsobj.pid)
        if libobj.serial:
            assert libobj.serial == lsobj.serial, "%r == %r" % (libobj.serial, lsobj.serial)
        assert libobj.path == lsobj.path, "%r == %r" % (libobj.path, lsobj.path)

        lsobj_inuse = lsobj.inuse()
        libobj_inuse = libobj.inuse()
        if libobj_inuse is not None:
            assert libobj_inuse == lsobj_inuse, "%r == %r" % (libobj_inuse, lsobj_inuse)

test_libusb_and_lsusb_equal()

BOARD_TYPES = ['opsis', 'atlys']
BOARD_NAMES = {
    'atlys': "Digilent Atlys",
    'opsis': "Numato Opsis",
    }
BOARD_STATES = ['unconfigured', 'jtag', 'serial', 'operational']

USBJTAG_MAPPING = {
    'hw_nexys': 'atlys',
    'hw_opsis': 'opsis',
    }
USBJTAG_RMAPPING = {v:k for k,v in USBJTAG_MAPPING.items()}

Board = namedtuple("Board", ["dev", "type", "state"])

import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--no-modify', help="Don't modify the state of the system in any way.")

parser.add_argument('--mac', help='Find board with the given MAC address.')
parser.add_argument('--dna', help='Find board with the given Device DNA.')
parser.add_argument('--position', help="""Find board using a given position in the USB structure.
Example:
 1-2.3 - Bus 1, Port 2 (which is a hub), Port 3
 5-6.7.8 - Bus 5, Port 2 (which is a hub), Port 7 (which is a hub), Port 8 (which is a hub)

While this *should* be static across reboots, but sadly on some machines it isn't :(
""")

parser.add_argument('--get-usbfs', help='Return the /dev/bus/usb path for a device.')
parser.add_argument('--get-sysfs', help='Return the /sys/bus/usb/devices path for a device.')

parser.add_argument('--get-state', help='Return the state the device is in. Possible states are: %r' % BOARD_STATES)
parser.add_argument('--get-video-device', help='Get the primary video device path.')
parser.add_argument('--get-serial-device', help='Get the serial device path.')

parser.add_argument('--use-hardware-serial', help='Use the hardware serial port on the Atlys board.')

parser.add_argument('--load-firmware', help='Load the firmware file onto the device.')

boards = []
for device in find_usb_devices_lsusb():
    # Digilent Atlys board with stock "Adept" firmware
    # Bus 003 Device 019: ID 1443:0007 Digilent Development board JTAG
    if device.vid == 0x1443 and device.pid == 0x0007:
        boards.append(Board(dev=device, type="atlys", state="unconfigured"))

    # The Numato Opsis will boot in the following mode when the EEPROM is not set up correctly.
    # Bus 003 Device 091: ID 04b4:8613 Cypress Semiconductor Corp. CY7C68013 EZ-USB FX2 USB 2.0 Development Kit
    elif device.vid == 0x04b4 and device.pid == 0x8613:
        boards.append(Board(dev=device, type="opsis", state="unconfigured"))

    # Boards loaded with the ixo-usb-jtag firmware from mithro's repo
    # https://github.com/mithro/ixo-usb-jtag
    # Bus 003 Device 090: ID 16c0:06ad Van Ooijen Technische Informatica
    elif device.vid == 0x16c0 and device.pid == 0x06ad:
        if device.serial not in USBJTAG_MAPPING:
            logging.warn("Unknown usb-jtag device!")
            continue
        boards.append(Board(dev=device, type=USBJTAG_MAPPING[device.serial], state="usbjtag"))

for board in boards:
    print "%s as %s at %s" % (
        BOARD_NAMES[board.type],
        board.state,
        board.dev.path,
        )
    if board.dev.inuse():
        print " Board is currently used by drivers %s" % (board.dev.drivers(),)
        board.dev.detach()

    if board.state == "unconfigured":
        print " Configure with 'fxload -t fx2lp -D %s -I %s'" % (
            board.dev.path,
            "%s.hex" % USBJTAG_RMAPPING[board.type],
            )

"""

# Exar USB-UART device on the Digilent Atlys board
# Bus 003 Device 018: ID 04e2:1410 Exar Corp.
# Driver at git+ssh://github.com/mithro/exar-uart-driver.git
if dev.driver() != "vizzini":
  # Need to install driver...
  pass

  - Digilent Atlys board loaded with Nero-USB JTAG firmware
  - Digilent Atlys board loaded with MakeStuff firmware
  - Digilent Atlys board loaded with HDMI2USB "unconfigured" firmware
  - Digilent Atlys board loaded with HDMI2USB "configured" firmware

  - Unconfigured Cypress FX2, could be a Numato Opsis

  - Numato Opsis board with no FX2 firmware loaded
  - Numato Opsis board loaded with Nero-USB JTAG firmware
  - Numato Opsis board loaded with MakeStuff firmware
  - Numato Opsis board loaded with HDMI2USB firmware

"""
<|endoftext|>


<commit_before>from prflask import db, app
from sqlalchemy.orm import relationship
from datetime import datetime
import flask.ext.restless

class Request(db.Model):
# The public records request
	__tablename__ = 'request'
	id = db.Column(db.Integer, primary_key =True)
	date_created = db.Column(db.DateTime)
	status_updated = db.Column(db.DateTime)
	text = db.Column(db.String(), unique=True) # The actual request text.
	subscribers = relationship("Subscriber") # The list of subscribers following this request.
	owners = relationship("Owner") # The list of city staff ever assigned to the request.
	current_owner = db.Column(db.Integer) # The Owner ID for the city staff that currently 'owns' the request.
	records = relationship("Record") # The list of records that have been uploaded for this request.
	notes = relationship("Note") # The list of notes appended to this request.
	status = db.Column(db.String(400)) # The status of the request (open, closed, etc.)
	def __init__(self, text):
		self.text = text
		self.date_created = datetime.now()
	def __repr__(self):
		return self.text

class Owner(db.Model):
# A member of city staff assigned to a particular request, that may or may not upload records towards that request.
	__tablename__ = 'owner'
	id = db.Column(db.Integer, primary_key =True)
	alias = db.Column(db.String(100))
	email = db.Column(db.String(100))
	date_created = db.Column(db.DateTime)
	records = relationship("Record") # All records that have been uploaded
	request_id = db.Column(db.Integer, db.ForeignKey('request.id'))
	reason = db.Column(db.String()) # Reason they were assigned
	def __init__(self, request_id, email, reason= None, alias = None):
		self.reason = reason
		self.alias = alias
		self.email = email
		self.request_id = request_id
		self.date_created = datetime.now()
	def __repr__(self):
		return self.alias

class Subscriber(db.Model):
# A person subscribed to a request, who may or may not have created the request, and may or may not own a part of the request.
	__tablename__ = 'subscriber'
	id = db.Column(db.Integer, primary_key = True)
	email = db.Column(db.String(100))
	alias = db.Column(db.String(100))
	date_created = db.Column(db.DateTime)
	request_id = db.Column(db.Integer, db.ForeignKey('request.id'))
	creator = db.Boolean() # True if they created the initial request.
	owner_id = db.Column(db.Integer, db.ForeignKey('owner.id')) # Not null if responsible for fulfilling a part of the request
 	def __init__(self, request_id, email, alias = None):
 		self.email = email
 		self.alias = alias
		self.request_id = request_id
		self.date_created = datetime.now()
	def __repr__(self):
		return self.alias

class Record(db.Model):
# A record that has been uploaded for a request.
	__tablename__ = 'record'
	id = db.Column(db.Integer, primary_key = True)
	date_created = db.Column(db.DateTime)
	owner_id = db.Column(db.Integer, db.ForeignKey('owner.id')) # The city staff who uploaded the record
	doc_id = db.Column(db.Integer) # The document ID. Currently using Scribd API to upload documents.
	request_id = db.Column(db.Integer, db.ForeignKey('request.id')) # The request this record was uploaded for
	description = db.Column(db.String(400)) # A short description of what the record is.
	filename = db.Column(db.String(400)) # The original name of the file being uploaded.
	url = db.Column(db.String()) # Where it exists on the internet.
	def __init__(self, request_id, owner_id, url = None, doc_id = None, description = None):
		self.doc_id = doc_id
		self.request_id = request_id
		self.owner_id = owner_id
		self.date_created = datetime.now()
		self.description = description
		self.url = url
	def __repr__(self):
		return self.records

class Note(db.Model):
# A note on a request.
	__tablename__ = 'note'
	id = db.Column(db.Integer, primary_key = True)
	date_created = db.Column(db.DateTime)
	text = db.Column(db.String())
	request_id = db.Column(db.Integer, db.ForeignKey('request.id')) # The request it belongs to.
	subscriber_id = db.Column(db.Integer, db.ForeignKey('subscriber.id')) # The subscriber who wrote the note
	owner_id = db.Column(db.Integer, db.ForeignKey('owner.id'))
	def __init__(self, request_id, text, subscriber_id = None, owner_id = None):
		self.text = text
		self.request_id = request_id
		self.subscriber_id = subscriber_id
		self.owner_id = owner_id
		self.date_created = datetime.now()
	def __repr__(self):
		return self.text

# Create API
manager = flask.ext.restless.APIManager(app, flask_sqlalchemy_db=db)
manager.create_api(Request, methods=['GET'], exclude_columns=['subscribers'])
manager.create_api(Note, methods=['GET'])
manager.create_api(Record, methods=['GET'])
<commit_msg>Adding user<commit_after>from prflask import db, app
from sqlalchemy.orm import relationship
from datetime import datetime
import flask.ext.restless

class User(db.Model):
	id = db.Column(db.Integer, primary_key = True)
	alias = db.Column(db.String(100))
	email = db.Column(db.String(100), unique=True)
	date_created = db.Column(db.DateTime)
	owners = db.relationship("Owner")
	subscribers = db.relationship("Subscriber")
	def __init__(self, email, alias = None):
		self.email = email
		self.alias = alias
		self.date_created = datetime.now()
	def __repr__(self):
		return self.email

class Request(db.Model):
# The public records request
	__tablename__ = 'request'
	id = db.Column(db.Integer, primary_key =True)
	date_created = db.Column(db.DateTime)
	status_updated = db.Column(db.DateTime)
	text = db.Column(db.String(), unique=True) # The actual request text.
	subscribers = relationship("Subscriber") # The list of subscribers following this request.
	owners = relationship("Owner") # The list of city staff ever assigned to the request.
	current_owner = db.Column(db.Integer) # The Owner ID for the city staff that currently 'owns' the request.
	records = relationship("Record") # The list of records that have been uploaded for this request.
	notes = relationship("Note") # The list of notes appended to this request.
	status = db.Column(db.String(400)) # The status of the request (open, closed, etc.)
	def __init__(self, text):
		self.text = text
		self.date_created = datetime.now()
	def __repr__(self):
		return self.text

class Owner(db.Model):
# A member of city staff assigned to a particular request, that may or may not upload records towards that request.
	__tablename__ = 'owner'
	id = db.Column(db.Integer, primary_key =True)
	user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
	records = relationship("Record") # All records that have been uploaded
	request_id = db.Column(db.Integer, db.ForeignKey('request.id'))
	reason = db.Column(db.String()) # Reason they were assigned
	date_created = db.Column(db.DateTime)
	def __init__(self, request_id, user_id, reason= None):
		self.reason = reason
		self.user_id = user_id
		self.request_id = request_id
		self.date_created = datetime.now()
	def __repr__(self):
		return self.user_id

class Subscriber(db.Model):
# A person subscribed to a request, who may or may not have created the request, and may or may not own a part of the request.
	__tablename__ = 'subscriber'
	id = db.Column(db.Integer, primary_key = True)
	user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
	request_id = db.Column(db.Integer, db.ForeignKey('request.id'))
	creator = db.Boolean() # True if they created the initial request.
	date_created = db.Column(db.DateTime)
	owner_id = db.Column(db.Integer, db.ForeignKey('owner.id')) # Not null if responsible for fulfilling a part of the request
 	def __init__(self, request_id, user_id):
 		self.user_id = user_id
		self.request_id = request_id
		self.date_created = datetime.now()
	def __repr__(self):
		return self.user_id

class Record(db.Model):
# A record that has been uploaded for a request.
	__tablename__ = 'record'
	id = db.Column(db.Integer, primary_key = True)
	date_created = db.Column(db.DateTime)
	owner_id = db.Column(db.Integer, db.ForeignKey('owner.id')) # The city staff who uploaded the record
	doc_id = db.Column(db.Integer) # The document ID. Currently using Scribd API to upload documents.
	request_id = db.Column(db.Integer, db.ForeignKey('request.id')) # The request this record was uploaded for
	description = db.Column(db.String(400)) # A short description of what the record is.
	filename = db.Column(db.String(400)) # The original name of the file being uploaded.
	url = db.Column(db.String()) # Where it exists on the internet.
	def __init__(self, request_id, owner_id, url = None, doc_id = None, description = None):
		self.doc_id = doc_id
		self.request_id = request_id
		self.owner_id = owner_id
		self.date_created = datetime.now()
		self.description = description
		self.url = url
	def __repr__(self):
		return self.records

class Note(db.Model):
# A note on a request.
	__tablename__ = 'note'
	id = db.Column(db.Integer, primary_key = True)
	date_created = db.Column(db.DateTime)
	text = db.Column(db.String())
	request_id = db.Column(db.Integer, db.ForeignKey('request.id')) # The request it belongs to.
	subscriber_id = db.Column(db.Integer, db.ForeignKey('subscriber.id')) # The subscriber who wrote the note
	owner_id = db.Column(db.Integer, db.ForeignKey('owner.id'))
	def __init__(self, request_id, text, subscriber_id = None, owner_id = None):
		self.text = text
		self.request_id = request_id
		self.subscriber_id = subscriber_id
		self.owner_id = owner_id
		self.date_created = datetime.now()
	def __repr__(self):
		return self.text

# Create API
manager = flask.ext.restless.APIManager(app, flask_sqlalchemy_db=db)
manager.create_api(Request, methods=['GET'], exclude_columns=['subscribers'])
manager.create_api(Note, methods=['GET'])
manager.create_api(Record, methods=['GET'])
manager.create_api(User, methods=['GET'])
<|endoftext|>


<commit_before>"""Common Module contains code shared by robotics and machine vision toolboxes"""
import numpy as np
from . import test_args

def ishomog(tr, dim, rtest=''):
    """ISHOMOG Test if SE(3) homogeneous transformation matrix.
    ISHOMOG(T) is true if the argument T is of dimension 4x4 or 4x4xN, else false.
    ISHOMOG(T, 'valid') as above, but also checks the validity of the rotation sub-matrix.
    See Also: isrot, ishomog2, isvec"""

    assert dim == [3, 3] or dim == [4, 4]
    if rtest == 'valid':
        is_valid = lambda matrix: abs(np.linalg.det(matrix) - 1) < np.spacing(1)
    flag = True
    if test_args.is_mat_list(tr):
        for matrix in tr:
            if not(matrix.shape[0] == dim[0] and matrix.shape[1] == dim[0]):
                flag = False
        # if rtest = 'valid'
        if flag and rtest == 'valid':
            flag = is_valid(tr[0]) # As in matlab code only first matrix is passed for validity test
            # TODO-Do we need to test all matrices in list for validity of rotation submatrix
    elif isinstance(tr, np.matrix):
        if tr.shape[0] == dim[0] and tr.shape[1] == dim[0]:
            if flag and rtest == 'valid':
                flag = is_valid(tr)
        else:
            flag = False
    else:
        raise ValueError('Invalid data type passed to common.ishomog()')
    return flag
<commit_msg>task 1.3.2 Updated common.py<commit_after>"""Common Module contains code shared by robotics and machine vision toolboxes"""
import numpy as np
from . import test_args

def ishomog(tr, dim, rtest=''):
    """ISHOMOG Test if SE(3) homogeneous transformation matrix.
    ISHOMOG(T) is true if the argument T is of dimension 4x4 or 4x4xN, else false.
    ISHOMOG(T, 'valid') as above, but also checks the validity of the rotation sub-matrix.
    See Also: isrot, ishomog2, isvec"""

    assert dim == [3, 3] or dim == [4, 4]
    if rtest == 'valid':
        is_valid = lambda matrix: abs(np.linalg.det(matrix) - 1) < np.spacing(1)
    flag = True
    if test_args.is_mat_list(tr):
        for matrix in tr:
            if not(matrix.shape[0] == dim[0] and matrix.shape[1] == dim[0]):
                flag = False
        # if rtest = 'valid'
        if flag and rtest == 'valid':
            flag = is_valid(tr[0]) # As in matlab code only first matrix is passed for validity test
            # TODO-Do we need to test all matrices in list for validity of rotation submatrix
    elif isinstance(tr, np.matrix):
        if tr.shape[0] == dim[0] and tr.shape[1] == dim[0]:
            if flag and rtest == 'valid':
                flag = is_valid(tr)
        else:
            flag = False
    else:
        raise ValueError('Invalid data type passed to common.ishomog()')
    return flag

def isvec(v, l = 3):
    '''
    ISVEC Test if vector
    '''
    d = v.shape
    h = d.size == 2 and min(v.shape) == 1 and v.size == l

    return h
<|endoftext|>


<commit_before>import re
import requests
import urllib

from smartbot import utils
from smartbot.formatting import *

class Plugin:
    def __init__(self, key):
        self.key = key

    def on_message(self, bot, msg, reply):
        match = re.findall(r"https?://(?:www\.)?youtube\.com/watch\?v=([^\s]+)", msg["message"], re.IGNORECASE)
        for i, id in enumerate(match):
            url = "https://www.googleapis.com/youtube/v3/videos?key={}&part=contentDetails%2Csnippet%2Cstatistics&id={}".format(
                urllib.parse.quote(self.key),
                urllib.parse.quote(id)
            )
            headers = {"User-Agent": "SmartBot"}
            payload = {
                "key": self.key,
                "id": id,
                "part": ",".join(["contentDetails", "snippet", "statistics"]),
            }

            res = requests.get(url, headers=headers).json()
            if res["items"]:
                video = res["items"][0]
                thumbnail = video["snippet"]["thumbnails"]["default"]["url"]
                duration = video["contentDetails"]["duration"]
                views = video["statistics"]["viewCount"]
                likes = video["statistics"]["likeCount"]
                dislikes = video["statistics"]["dislikeCount"]
                reply("{}: {} | {} {} {} | {}".format(bot.format("[{}]".format(i), Style.bold),
                                                      duration,
                                                      views,
                                                      bot.format(likes, Colour.fg_green),
                                                      bot.format(dislikes, Colour.fg_red),
                                                      bot.format(thumbnail, Colour.fg_grey)))

    def on_help(self):
        return "Sends information about YouTube URLs posted."
<commit_msg>Add support for shortened youtube URLs<commit_after>import re
import requests
import urllib

from smartbot import utils
from smartbot.formatting import *

class Plugin:
    def __init__(self, key):
        self.key = key

    def on_message(self, bot, msg, reply):
        match = re.findall(r"(?:https?://)?(?:www\.)?youtu\.?be(?:\.com)?/(?:watch\?v=)?([^\s]+)", msg["message"], re.IGNORECASE)
        for i, id in enumerate(match):
            url = "https://www.googleapis.com/youtube/v3/videos?key={}&part=contentDetails%2Csnippet%2Cstatistics&id={}".format(
                urllib.parse.quote(self.key),
                urllib.parse.quote(id)
            )
            headers = {"User-Agent": "SmartBot"}
            payload = {
                "key": self.key,
                "id": id,
                "part": ",".join(["contentDetails", "snippet", "statistics"]),
            }

            res = requests.get(url, headers=headers).json()
            if res["items"]:
                video = res["items"][0]
                thumbnail = video["snippet"]["thumbnails"]["default"]["url"]
                duration = video["contentDetails"]["duration"]
                views = video["statistics"]["viewCount"]
                likes = video["statistics"]["likeCount"]
                dislikes = video["statistics"]["dislikeCount"]
                reply("{}: {} | {} {} {} | {}".format(bot.format("[{}]".format(i), Style.bold),
                                                      duration,
                                                      views,
                                                      bot.format(likes, Colour.fg_green),
                                                      bot.format(dislikes, Colour.fg_red),
                                                      bot.format(thumbnail, Colour.fg_grey)))

    def on_help(self):
        return "Sends information about YouTube URLs posted."
<|endoftext|>


<commit_before># coding: utf-8
import re
import uuid
import graphviz as gv

from .graphviz_style import *

# index of subgraph
CLUSTER_INDEX = 0

def tf_digraph(name=None, name_scope=None, style=True):
    """
    Return graphviz.dot.Digraph with TensorBoard-like style.
    @param  name
    @param  name_scope
    @param  style
    @return graphviz.dot.Digraph object
    """
    digraph = gv.Digraph(name=name)
    if name_scope:
        digraph.graph_attr['label'] = name_scope
    if style is False: return digraph

    if name_scope:
        digraph.graph_attr.update(name_scope_graph_pref)
    else:
        digraph.graph_attr.update(non_name_scope_graph_pref)
    digraph.graph_attr.update(graph_pref)
    digraph.node_attr.update(node_pref)
    digraph.edge_attr.update(edge_pref)
    return digraph

def nested_dict(dict_, keys, val):
    """
    Assign value to dictionary.
    @param  dict_
    @param  keys
    @param  val
    @return dictionary
    """
    cloned = dict_.copy()
    if len(keys) == 1:
        cloned[keys[0]] = val
        return cloned
    dd = cloned[keys[0]]
    for k in keys[1:len(keys)-1]:
        dd = dd[k]
    last_key = keys[len(keys)-1]
    dd[last_key] = val
    return cloned

def node_abs_paths(node):
    """
    Return absolute node path name.
    @param  node
    @return string
    """
    node_names = node.name.split('/')
    return ['/'.join(node_names[0:i+1]) for i in range(len(node_names))]

def node_table(tfgraph, depth=1):
    """
    Return dictionary of node.
    @param  tfgraph
    @param  depth
    @return dictionary
    """
    table = {}
    max_depth = depth
    ops = tfgraph.get_operations()
    for depth_i in range(max_depth):
        for op in ops:
            abs_paths = node_abs_paths(op)
            if depth_i >= len(abs_paths): continue
            ps = abs_paths[:depth_i+1]
            if len(ps) == 1:
                key = '/'.join(abs_paths[0:depth_i+1])
                if not key in table: table[key] = {}
            else:
                table = nested_dict(table, ps, {})
    return table

def node_shape(tfnode, depth=1):
    """
    Return node and the children.
    @param  tfnode
    @param  depth
    @return string, list
    """
    outpt_name = tfnode.name
    if len(outpt_name.split('/')) < depth: return None
    on = '/'.join(outpt_name.split('/')[:depth]) # output node
    result = re.match(r"(.*):\d*$", on)
    if not result: return None
    on = result.groups()[0]
    if tfnode.shape.ndims is None:
        return on, []
    else:
        return on, tfnode.shape.as_list()

def node_input_table(tfgraph, depth=1):
    """
    Return table of operations
    @param  tfgraph
    @param  depth
    @return dictionary, table of operations
    """
    table = {}
    inpt_op_table = {}
    inpt_op_shape_table = {}
    for op in tfgraph.get_operations():
        op_name = op.name.split('/')[0:depth]
        opn = '/'.join(op_name)
        if not opn in inpt_op_table:
            inpt_op_table[opn] = []
        inpt_op_list = ['/'.join(inpt_op.split('/')[0:depth]) for inpt_op in op.node_def.input]
        inpt_op_table[opn].append(inpt_op_list)
        for output in op.outputs:
            for i in range(depth):
                shape = node_shape(output, depth=i+1)
                if shape: inpt_op_shape_table[shape[0]] = shape[1]
    for opn in inpt_op_table.keys():
        t_l = []
        for ll in inpt_op_table[opn]:
            list.extend(t_l, ll)
        table[opn] = list(set(t_l))
    return table, inpt_op_shape_table

def add_nodes(node_table, name=None, name_scope=None, style=True):
    """
    Add TensorFlow graph's nodes to graphviz.dot.Digraph.
    @param  node_table
    @param  name
    @param  name_scope
    @param  style
    @return graphviz.dot.Digraph object
    """
    global CLUSTER_INDEX
    if name:
        digraph = tf_digraph(name=name, name_scope=name_scope, style=style)
    else:
        digraph = tf_digraph(name=str(uuid.uuid4().get_hex().upper()[0:6]), name_scope=name_scope, style=style)
    graphs = []
    for key, value in node_table.items():
        if len(value) > 0:
            sg = add_nodes(value, name='cluster_%i' % CLUSTER_INDEX, name_scope=key.split('/')[-1], style=style)
            sg.node(key, key.split('/')[-1])
            CLUSTER_INDEX += 1
            graphs.append(sg)
        else:
            digraph.node(key, key.split('/')[-1])
    for tg in graphs:
        digraph.subgraph(tg)
    return digraph

def edge_label(shape):
    """
    Returen texts of graph's edges.
    @param  shape
    @return
    """
    if len(shape) == 0: return ''
    if shape[0] is None: label = "?"
    else: label = "%i" % shape[0]
    for s in shape[1:]:
        if s is None: label += "×?"
        else: label += u"×%i" % s
    return label

def add_edges(digraph, node_inpt_table, node_inpt_shape_table):
    """
    Add TensorFlow graph's edges to graphviz.dot.Digraph.
    @param  dirgraph
    @param  node_inpt_table
    @param  node_inpt_shape_table
    @return  graphviz.dot.Digraph
    """
    for node, node_inputs in node_inpt_table.items():
        if re.match(r"\^", node): continue
        for ni in node_inputs:
            if ni == node: continue
            if re.match(r"\^", ni): continue
            if not ni in node_inpt_shape_table:
                digraph.edge(ni, node)
            else:
                shape = node_inpt_shape_table[ni]
                digraph.edge(ni, node, label=edge_label(shape))
    return digraph

def board(tfgraph, depth=1, name='G', style=True):
    """
    Return graphviz.dot.Digraph object with TensorFlow's Graphs.
    @param  depth
    @param  name
    @param  style
    @return  graphviz.dot.Digraph
    """
    global CLUSTER_INDEX
    CLUSTER_INDEX = 0
    _node_table = node_table(tfgraph, depth=depth)
    _node_inpt_table, _node_inpt_shape_table = node_input_table(tfgraph, depth=depth)
    digraph = add_nodes(_node_table, name=name, style=style)
    digraph = add_edges(digraph, _node_inpt_table, _node_inpt_shape_table)
    return digraph
<commit_msg>added name_regex filter to allow showing only part of the graph<commit_after># coding: utf-8
import re
import uuid
import graphviz as gv

from .graphviz_style import *

# index of subgraph
CLUSTER_INDEX = 0

def tf_digraph(name=None, name_scope=None, style=True):
    """
    Return graphviz.dot.Digraph with TensorBoard-like style.
    @param  name
    @param  name_scope
    @param  style
    @return graphviz.dot.Digraph object
    """
    digraph = gv.Digraph(name=name)
    if name_scope:
        digraph.graph_attr['label'] = name_scope
    if style is False: return digraph

    if name_scope:
        digraph.graph_attr.update(name_scope_graph_pref)
    else:
        digraph.graph_attr.update(non_name_scope_graph_pref)
    digraph.graph_attr.update(graph_pref)
    digraph.node_attr.update(node_pref)
    digraph.edge_attr.update(edge_pref)
    return digraph

def nested_dict(dict_, keys, val):
    """
    Assign value to dictionary.
    @param  dict_
    @param  keys
    @param  val
    @return dictionary
    """
    cloned = dict_.copy()
    if len(keys) == 1:
        cloned[keys[0]] = val
        return cloned
    dd = cloned[keys[0]]
    for k in keys[1:len(keys)-1]:
        dd = dd[k]
    last_key = keys[len(keys)-1]
    dd[last_key] = val
    return cloned

def node_abs_paths(node):
    """
    Return absolute node path name.
    @param  node
    @return string
    """
    node_names = node.name.split('/')
    return ['/'.join(node_names[0:i+1]) for i in range(len(node_names))]

def node_table(tfgraph, depth=1, name_regex=''):
    """
    Return dictionary of node.
    @param  tfgraph
    @param  depth
    @param  name_regex
    @return dictionary
    """
    table = {}
    max_depth = depth
    ops = tfgraph.get_operations()
    for depth_i in range(max_depth):
        for op in ops:
            abs_paths = node_abs_paths(op)
            if depth_i >= len(abs_paths): continue
            if name_regex and not re.match(name_regex, op.name): continue
            ps = abs_paths[:depth_i+1]
            if len(ps) == 1:
                key = '/'.join(abs_paths[0:depth_i+1])
                if not key in table: table[key] = {}
            else:
                table = nested_dict(table, ps, {})
    return table

def node_shape(tfnode, depth=1):
    """
    Return node and the children.
    @param  tfnode
    @param  depth
    @return string, list
    """
    outpt_name = tfnode.name
    if len(outpt_name.split('/')) < depth: return None
    on = '/'.join(outpt_name.split('/')[:depth]) # output node
    result = re.match(r"(.*):\d*$", on)
    if not result: return None
    on = result.groups()[0]
    if tfnode.shape.ndims is None:
        return on, []
    else:
        return on, tfnode.shape.as_list()

def node_input_table(tfgraph, depth=1, name_regex=''):
    """
    Return table of operations
    @param  tfgraph
    @param  depth
    @param  name_regex
    @return dictionary, table of operations
    """
    table = {}
    inpt_op_table = {}
    inpt_op_shape_table = {}
    for op in tfgraph.get_operations():
        if name_regex and not re.match(name_regex, op.name): continue
        op_name = op.name.split('/')[0:depth]
        opn = '/'.join(op_name)
        if not opn in inpt_op_table:
            inpt_op_table[opn] = []
        inpt_op_list = ['/'.join(inpt_op.split('/')[0:depth]) \
            for inpt_op in op.node_def.input if not name_regex or re.match(name_regex, inpt_op)]
        inpt_op_table[opn].append(inpt_op_list)
        for output in op.outputs:
            for i in range(depth):
                shape = node_shape(output, depth=i+1)
                if shape: inpt_op_shape_table[shape[0]] = shape[1]
    for opn in inpt_op_table.keys():
        t_l = []
        for ll in inpt_op_table[opn]:
            list.extend(t_l, ll)
        table[opn] = list(set(t_l))
    return table, inpt_op_shape_table

def add_nodes(node_table, name=None, name_scope=None, style=True):
    """
    Add TensorFlow graph's nodes to graphviz.dot.Digraph.
    @param  node_table
    @param  name
    @param  name_scope
    @param  style
    @return graphviz.dot.Digraph object
    """
    global CLUSTER_INDEX
    if name:
        digraph = tf_digraph(name=name, name_scope=name_scope, style=style)
    else:
        digraph = tf_digraph(name=str(uuid.uuid4().get_hex().upper()[0:6]), name_scope=name_scope, style=style)
    graphs = []
    for key, value in node_table.items():
        if len(value) > 0:
            sg = add_nodes(value, name='cluster_%i' % CLUSTER_INDEX, name_scope=key.split('/')[-1], style=style)
            sg.node(key, key.split('/')[-1])
            CLUSTER_INDEX += 1
            graphs.append(sg)
        else:
            digraph.node(key, key.split('/')[-1])
    for tg in graphs:
        digraph.subgraph(tg)
    return digraph

def edge_label(shape):
    """
    Returen texts of graph's edges.
    @param  shape
    @return
    """
    if len(shape) == 0: return ''
    if shape[0] is None: label = "?"
    else: label = "%i" % shape[0]
    for s in shape[1:]:
        if s is None: label += "×?"
        else: label += u"×%i" % s
    return label

def add_edges(digraph, node_inpt_table, node_inpt_shape_table):
    """
    Add TensorFlow graph's edges to graphviz.dot.Digraph.
    @param  dirgraph
    @param  node_inpt_table
    @param  node_inpt_shape_table
    @return  graphviz.dot.Digraph
    """
    for node, node_inputs in node_inpt_table.items():
        if re.match(r"\^", node): continue
        for ni in node_inputs:
            if ni == node: continue
            if re.match(r"\^", ni): continue
            if not ni in node_inpt_shape_table:
                digraph.edge(ni, node)
            else:
                shape = node_inpt_shape_table[ni]
                digraph.edge(ni, node, label=edge_label(shape))
    return digraph

def board(tfgraph, depth=1, name='G', style=True, name_regex=''):
    """
    Return graphviz.dot.Digraph object with TensorFlow's Graphs.
    @param  depth
    @param  name
    @param  style
    @param  name_regex
    @return  graphviz.dot.Digraph
    """
    global CLUSTER_INDEX
    CLUSTER_INDEX = 0
    _node_table = node_table(tfgraph, depth=depth, name_regex=name_regex)
    _node_inpt_table, _node_inpt_shape_table = node_input_table(tfgraph, depth=depth, name_regex=name_regex)
    digraph = add_nodes(_node_table, name=name, style=style)
    digraph = add_edges(digraph, _node_inpt_table, _node_inpt_shape_table)
    return digraph
<|endoftext|>


<commit_before>"""
:py:class:`Model` is an abstract class representing
an AllenNLP model.
"""

import logging
import os
from typing import Dict, Union, List, Set

import numpy
import torch

from allennlp.common.checks import ConfigurationError
from allennlp.common.params import Params
from allennlp.common.registrable import Registrable
from allennlp.data import Instance, Vocabulary
from allennlp.data.dataset import Batch
from allennlp.nn import util
from allennlp.nn.regularizers import RegularizerApplicator

logger = logging.getLogger(__name__)  # pylint: disable=invalid-name

# When training a model, many sets of weights are saved. By default we want to
# save/load this set of weights.
_DEFAULT_WEIGHTS = "best.th"

class Model(torch.nn.Module, Registrable):
    """
    This abstract class represents a model to be trained. Rather than relying completely
    on the Pytorch Module, we modify the output spec of ``forward`` to be a dictionary.

    Models built using this API are still compatible with other pytorch models and can
    be used naturally as modules within other models - outputs are dictionaries, which
    can be unpacked and passed into other layers. One caveat to this is that if you
    wish to use an AllenNLP model inside a Container (such as nn.Sequential), you must
    interleave the models with a wrapper module which unpacks the dictionary into
    a list of tensors.

    In order for your model to be trained using the :class:`~allennlp.training.Trainer`
    api, the output dictionary of your Model must include a "loss" key, which will be
    optimised during the training process.

    Finally, you can optionally implement :func:`Model.get_metrics` in order to make use
    of early stopping and best-model serialization based on a validation metric in
    :class:`~allennlp.training.Trainer`. Metrics that begin with "_" will not be logged
    to the progress bar by :class:`~allennlp.training.Trainer`.
    """
    _warn_for_unseparable_batches: Set[str] = set()

    def __init__(self,
                 vocab: Vocabulary,
                 regularizer: RegularizerApplicator = None) -> None:
        super().__init__()
        self.vocab = vocab
        self._regularizer = regularizer

    def get_regularization_penalty(self) -> Union[float, torch.Tensor]:
        """
        Computes the regularization penalty for the model.
        Returns 0 if the model was not configured to use regularization.
        """
        if self._regularizer is None:
            return 0.0
        else:
            return self._regularizer(self)

    def get_parameters_for_histogram_tensorboard_logging( # pylint: disable=invalid-name
            self) -> List[str]:
        """
        Returns the name of model parameters used for logging histograms to tensorboard.
        """
        return [name for name, _ in self.named_parameters()]

    def forward(self, *inputs) -> Dict[str, torch.Tensor]:  # pylint: disable=arguments-differ
        """
        Defines the forward pass of the model. In addition, to facilitate easy training,
        this method is designed to compute a loss function defined by a user.

        The input is comprised of everything required to perform a
        training update, `including` labels - you define the signature here!
        It is down to the user to ensure that inference can be performed
        without the presence of these labels. Hence, any inputs not available at
        inference time should only be used inside a conditional block.

        The intended sketch of this method is as follows::

            def forward(self, input1, input2, targets=None):
                ....
                ....
                output1 = self.layer1(input1)
                output2 = self.layer2(input2)
                output_dict = {"output1": output1, "output2": output2}
                if targets is not None:
                    # Function returning a scalar torch.Tensor, defined by the user.
                    loss = self._compute_loss(output1, output2, targets)
                    output_dict["loss"] = loss
                return output_dict

        Parameters
        ----------
        inputs:
            Tensors comprising everything needed to perform a training update, `including` labels,
            which should be optional (i.e have a default value of ``None``).  At inference time,
            simply pass the relevant inputs, not including the labels.

        Returns
        -------
        output_dict: ``Dict[str, torch.Tensor]``
            The outputs from the model. In order to train a model using the
            :class:`~allennlp.training.Trainer` api, you must provide a "loss" key pointing to a
            scalar ``torch.Tensor`` representing the loss to be optimized.
        """
        raise NotImplementedError

    def forward_on_instance(self, instance: Instance) -> Dict[str, numpy.ndarray]:
        """
        Takes an :class:`~allennlp.data.instance.Instance`, which typically has raw text in it,
        converts that text into arrays using this model's :class:`Vocabulary`, passes those arrays
        through :func:`self.forward()` and :func:`self.decode()` (which by default does nothing)
        and returns the result.  Before returning the result, we convert any
        ``torch.Tensors`` into numpy arrays and remove the batch dimension.
        """
        return self.forward_on_instances([instance])[0]

    def forward_on_instances(self,
                             instances: List[Instance]) -> List[Dict[str, numpy.ndarray]]:
        """
        Takes a list of  :class:`~allennlp.data.instance.Instance`s, converts that text into
        arrays using this model's :class:`Vocabulary`, passes those arrays through
        :func:`self.forward()` and :func:`self.decode()` (which by default does nothing)
        and returns the result.  Before returning the result, we convert any
        ``torch.Tensors`` into numpy arrays and separate the
        batched output into a list of individual dicts per instance. Note that typically
        this will be faster on a GPU (and conditionally, on a CPU) than repeated calls to
        :func:`forward_on_instance`.

        Parameters
        ----------
        instances : List[Instance], required
            The instances to run the model on.
        cuda_device : int, required
            The GPU device to use.  -1 means use the CPU.

        Returns
        -------
        A list of the models output for each instance.
        """
        batch_size = len(instances)
        with torch.no_grad():
            cuda_device = self._get_prediction_device()
            dataset = Batch(instances)
            dataset.index_instances(self.vocab)
            model_input = dataset.as_tensor_dict(cuda_device=cuda_device)
            outputs = self.decode(self(**model_input))

            instance_separated_output: List[Dict[str, numpy.ndarray]] = [{} for _ in dataset.instances]
            for name, output in list(outputs.items()):
                if isinstance(output, torch.Tensor):
                    # NOTE(markn): This is a hack because 0-dim pytorch tensors are not iterable.
                    # This occurs with batch size 1, because we still want to include the loss in that case.
                    if output.dim() == 0:
                        output = output.unsqueeze(0)

                    if output.size(0) != batch_size:
                        self._maybe_warn_for_unseparable_batches(name)
                        continue
                    output = output.detach().cpu().numpy()
                elif len(output) != batch_size:
                    self._maybe_warn_for_unseparable_batches(name)
                    continue
                outputs[name] = output
                for instance_output, batch_element in zip(instance_separated_output, output):
                    instance_output[name] = batch_element
            return instance_separated_output

    def decode(self, output_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
        """
        Takes the result of :func:`forward` and runs inference / decoding / whatever
        post-processing you need to do your model.  The intent is that ``model.forward()`` should
        produce potentials or probabilities, and then ``model.decode()`` can take those results and
        run some kind of beam search or constrained inference or whatever is necessary.  This does
        not handle all possible decoding use cases, but it at least handles simple kinds of
        decoding.

        This method `modifies` the input dictionary, and also `returns` the same dictionary.

        By default in the base class we do nothing.  If your model has some special decoding step,
        override this method.
        """
        # pylint: disable=no-self-use
        return output_dict

    def get_metrics(self, reset: bool = False) -> Dict[str, float]:
        """
        Returns a dictionary of metrics. This method will be called by
        :class:`allennlp.training.Trainer` in order to compute and use model metrics for early
        stopping and model serialization.  We return an empty dictionary here rather than raising
        as it is not required to implement metrics for a new model.  A boolean `reset` parameter is
        passed, as frequently a metric accumulator will have some state which should be reset
        between epochs. This is also compatible with :class:`~allennlp.training.Metric`s. Metrics
        should be populated during the call to ``forward``, with the
        :class:`~allennlp.training.Metric` handling the accumulation of the metric until this
        method is called.
        """
        # pylint: disable=unused-argument,no-self-use
        return {}

    def _get_prediction_device(self) -> int:
        """
        This method checks the device of the model parameters to determine the cuda_device
        this model should be run on for predictions.  If there are no parameters, it returns -1.

        Returns
        -------
        The cuda device this model should run on for predictions.
        """
        devices = {util.get_device_of(param) for param in self.parameters()}

        if len(devices) > 1:
            devices_string = ", ".join(str(x) for x in devices)
            raise ConfigurationError(f"Parameters have mismatching cuda_devices: {devices_string}")
        elif len(devices) == 1:
            return devices.pop()
        else:
            return -1

    def _maybe_warn_for_unseparable_batches(self, output_key: str):
        """
        This method warns once if a user implements a model which returns a dictionary with
        values which we are unable to split back up into elements of the batch. This is controlled
        by a class attribute ``_warn_for_unseperable_batches`` because it would be extremely verbose
        otherwise.
        """
        if  output_key in self._warn_for_unseparable_batches:
            logger.warning(f"Encountered the {output_key} key in the model's return dictionary which "
                           "couldn't be split by the batch size. Key will be ignored.")
            # We only want to warn once for this key,
            # so we set this to false so we don't warn again.
            self._warn_for_unseparable_batches.add(output_key)

    @classmethod
    def _load(cls,
              config: Params,
              serialization_dir: str,
              weights_file: str = None,
              cuda_device: int = -1) -> 'Model':
        """
        Instantiates an already-trained model, based on the experiment
        configuration and some optional overrides.
        """
        weights_file = weights_file or os.path.join(serialization_dir, _DEFAULT_WEIGHTS)

        # Load vocabulary from file
        vocab_dir = os.path.join(serialization_dir, 'vocabulary')
        vocab = Vocabulary.from_files(vocab_dir)

        model_params = config.get('model')

        # The experiment config tells us how to _train_ a model, including where to get pre-trained
        # embeddings from.  We're now _loading_ the model, so those embeddings will already be
        # stored in our weights.  We don't need any pretrained weight file anymore, and we don't
        # want the code to look for it, so we remove it from the parameters here.
        remove_pretrained_embedding_params(model_params)
        model = Model.from_params(vocab=vocab, params=model_params)
        model_state = torch.load(weights_file, map_location=util.device_mapping(cuda_device))
        model.load_state_dict(model_state)

        # Force model to cpu or gpu, as appropriate, to make sure that the embeddings are
        # in sync with the weights
        if cuda_device >= 0:
            model.cuda(cuda_device)
        else:
            model.cpu()

        return model

    @classmethod
    def load(cls,
             config: Params,
             serialization_dir: str,
             weights_file: str = None,
             cuda_device: int = -1) -> 'Model':
        """
        Instantiates an already-trained model, based on the experiment
        configuration and some optional overrides.

        Parameters
        ----------
        config: Params
            The configuration that was used to train the model. It should definitely
            have a `model` section, and should probably have a `trainer` section
            as well.
        serialization_dir: str = None
            The directory containing the serialized weights, parameters, and vocabulary
            of the model.
        weights_file: str = None
            By default we load the weights from `best.th` in the serialization
            directory, but you can override that value here.
        cuda_device: int = -1
            By default we load the model on the CPU, but if you want to load it
            for GPU usage you can specify the id of your GPU here

        Returns
        -------
        model: Model
            The model specified in the configuration, loaded with the serialized
            vocabulary and the trained weights.
        """

        # Peak at the class of the model.
        model_type = config["model"]["type"]

        # Load using an overridable _load method.
        # This allows subclasses of Model to override _load.
        # pylint: disable=protected-access
        return cls.by_name(model_type)._load(config, serialization_dir, weights_file, cuda_device)

def remove_pretrained_embedding_params(params: Params):
    keys = params.keys()
    if 'pretrained_file' in keys:
        del params['pretrained_file']
    for value in params.values():
        if isinstance(value, Params):
            remove_pretrained_embedding_params(value)
<commit_msg>Show warning before ignoring key with unseparable batches from model.forward. (#1520)<commit_after>"""
:py:class:`Model` is an abstract class representing
an AllenNLP model.
"""

import logging
import os
from typing import Dict, Union, List, Set

import numpy
import torch

from allennlp.common.checks import ConfigurationError
from allennlp.common.params import Params
from allennlp.common.registrable import Registrable
from allennlp.data import Instance, Vocabulary
from allennlp.data.dataset import Batch
from allennlp.nn import util
from allennlp.nn.regularizers import RegularizerApplicator

logger = logging.getLogger(__name__)  # pylint: disable=invalid-name

# When training a model, many sets of weights are saved. By default we want to
# save/load this set of weights.
_DEFAULT_WEIGHTS = "best.th"

class Model(torch.nn.Module, Registrable):
    """
    This abstract class represents a model to be trained. Rather than relying completely
    on the Pytorch Module, we modify the output spec of ``forward`` to be a dictionary.

    Models built using this API are still compatible with other pytorch models and can
    be used naturally as modules within other models - outputs are dictionaries, which
    can be unpacked and passed into other layers. One caveat to this is that if you
    wish to use an AllenNLP model inside a Container (such as nn.Sequential), you must
    interleave the models with a wrapper module which unpacks the dictionary into
    a list of tensors.

    In order for your model to be trained using the :class:`~allennlp.training.Trainer`
    api, the output dictionary of your Model must include a "loss" key, which will be
    optimised during the training process.

    Finally, you can optionally implement :func:`Model.get_metrics` in order to make use
    of early stopping and best-model serialization based on a validation metric in
    :class:`~allennlp.training.Trainer`. Metrics that begin with "_" will not be logged
    to the progress bar by :class:`~allennlp.training.Trainer`.
    """
    _warn_for_unseparable_batches: Set[str] = set()

    def __init__(self,
                 vocab: Vocabulary,
                 regularizer: RegularizerApplicator = None) -> None:
        super().__init__()
        self.vocab = vocab
        self._regularizer = regularizer

    def get_regularization_penalty(self) -> Union[float, torch.Tensor]:
        """
        Computes the regularization penalty for the model.
        Returns 0 if the model was not configured to use regularization.
        """
        if self._regularizer is None:
            return 0.0
        else:
            return self._regularizer(self)

    def get_parameters_for_histogram_tensorboard_logging( # pylint: disable=invalid-name
            self) -> List[str]:
        """
        Returns the name of model parameters used for logging histograms to tensorboard.
        """
        return [name for name, _ in self.named_parameters()]

    def forward(self, *inputs) -> Dict[str, torch.Tensor]:  # pylint: disable=arguments-differ
        """
        Defines the forward pass of the model. In addition, to facilitate easy training,
        this method is designed to compute a loss function defined by a user.

        The input is comprised of everything required to perform a
        training update, `including` labels - you define the signature here!
        It is down to the user to ensure that inference can be performed
        without the presence of these labels. Hence, any inputs not available at
        inference time should only be used inside a conditional block.

        The intended sketch of this method is as follows::

            def forward(self, input1, input2, targets=None):
                ....
                ....
                output1 = self.layer1(input1)
                output2 = self.layer2(input2)
                output_dict = {"output1": output1, "output2": output2}
                if targets is not None:
                    # Function returning a scalar torch.Tensor, defined by the user.
                    loss = self._compute_loss(output1, output2, targets)
                    output_dict["loss"] = loss
                return output_dict

        Parameters
        ----------
        inputs:
            Tensors comprising everything needed to perform a training update, `including` labels,
            which should be optional (i.e have a default value of ``None``).  At inference time,
            simply pass the relevant inputs, not including the labels.

        Returns
        -------
        output_dict: ``Dict[str, torch.Tensor]``
            The outputs from the model. In order to train a model using the
            :class:`~allennlp.training.Trainer` api, you must provide a "loss" key pointing to a
            scalar ``torch.Tensor`` representing the loss to be optimized.
        """
        raise NotImplementedError

    def forward_on_instance(self, instance: Instance) -> Dict[str, numpy.ndarray]:
        """
        Takes an :class:`~allennlp.data.instance.Instance`, which typically has raw text in it,
        converts that text into arrays using this model's :class:`Vocabulary`, passes those arrays
        through :func:`self.forward()` and :func:`self.decode()` (which by default does nothing)
        and returns the result.  Before returning the result, we convert any
        ``torch.Tensors`` into numpy arrays and remove the batch dimension.
        """
        return self.forward_on_instances([instance])[0]

    def forward_on_instances(self,
                             instances: List[Instance]) -> List[Dict[str, numpy.ndarray]]:
        """
        Takes a list of  :class:`~allennlp.data.instance.Instance`s, converts that text into
        arrays using this model's :class:`Vocabulary`, passes those arrays through
        :func:`self.forward()` and :func:`self.decode()` (which by default does nothing)
        and returns the result.  Before returning the result, we convert any
        ``torch.Tensors`` into numpy arrays and separate the
        batched output into a list of individual dicts per instance. Note that typically
        this will be faster on a GPU (and conditionally, on a CPU) than repeated calls to
        :func:`forward_on_instance`.

        Parameters
        ----------
        instances : List[Instance], required
            The instances to run the model on.
        cuda_device : int, required
            The GPU device to use.  -1 means use the CPU.

        Returns
        -------
        A list of the models output for each instance.
        """
        batch_size = len(instances)
        with torch.no_grad():
            cuda_device = self._get_prediction_device()
            dataset = Batch(instances)
            dataset.index_instances(self.vocab)
            model_input = dataset.as_tensor_dict(cuda_device=cuda_device)
            outputs = self.decode(self(**model_input))

            instance_separated_output: List[Dict[str, numpy.ndarray]] = [{} for _ in dataset.instances]
            for name, output in list(outputs.items()):
                if isinstance(output, torch.Tensor):
                    # NOTE(markn): This is a hack because 0-dim pytorch tensors are not iterable.
                    # This occurs with batch size 1, because we still want to include the loss in that case.
                    if output.dim() == 0:
                        output = output.unsqueeze(0)

                    if output.size(0) != batch_size:
                        self._maybe_warn_for_unseparable_batches(name)
                        continue
                    output = output.detach().cpu().numpy()
                elif len(output) != batch_size:
                    self._maybe_warn_for_unseparable_batches(name)
                    continue
                outputs[name] = output
                for instance_output, batch_element in zip(instance_separated_output, output):
                    instance_output[name] = batch_element
            return instance_separated_output

    def decode(self, output_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
        """
        Takes the result of :func:`forward` and runs inference / decoding / whatever
        post-processing you need to do your model.  The intent is that ``model.forward()`` should
        produce potentials or probabilities, and then ``model.decode()`` can take those results and
        run some kind of beam search or constrained inference or whatever is necessary.  This does
        not handle all possible decoding use cases, but it at least handles simple kinds of
        decoding.

        This method `modifies` the input dictionary, and also `returns` the same dictionary.

        By default in the base class we do nothing.  If your model has some special decoding step,
        override this method.
        """
        # pylint: disable=no-self-use
        return output_dict

    def get_metrics(self, reset: bool = False) -> Dict[str, float]:
        """
        Returns a dictionary of metrics. This method will be called by
        :class:`allennlp.training.Trainer` in order to compute and use model metrics for early
        stopping and model serialization.  We return an empty dictionary here rather than raising
        as it is not required to implement metrics for a new model.  A boolean `reset` parameter is
        passed, as frequently a metric accumulator will have some state which should be reset
        between epochs. This is also compatible with :class:`~allennlp.training.Metric`s. Metrics
        should be populated during the call to ``forward``, with the
        :class:`~allennlp.training.Metric` handling the accumulation of the metric until this
        method is called.
        """
        # pylint: disable=unused-argument,no-self-use
        return {}

    def _get_prediction_device(self) -> int:
        """
        This method checks the device of the model parameters to determine the cuda_device
        this model should be run on for predictions.  If there are no parameters, it returns -1.

        Returns
        -------
        The cuda device this model should run on for predictions.
        """
        devices = {util.get_device_of(param) for param in self.parameters()}

        if len(devices) > 1:
            devices_string = ", ".join(str(x) for x in devices)
            raise ConfigurationError(f"Parameters have mismatching cuda_devices: {devices_string}")
        elif len(devices) == 1:
            return devices.pop()
        else:
            return -1

    def _maybe_warn_for_unseparable_batches(self, output_key: str):
        """
        This method warns once if a user implements a model which returns a dictionary with
        values which we are unable to split back up into elements of the batch. This is controlled
        by a class attribute ``_warn_for_unseperable_batches`` because it would be extremely verbose
        otherwise.
        """
        if  output_key not in self._warn_for_unseparable_batches:
            logger.warning(f"Encountered the {output_key} key in the model's return dictionary which "
                           "couldn't be split by the batch size. Key will be ignored.")
            # We only want to warn once for this key,
            # so we set this to false so we don't warn again.
            self._warn_for_unseparable_batches.add(output_key)

    @classmethod
    def _load(cls,
              config: Params,
              serialization_dir: str,
              weights_file: str = None,
              cuda_device: int = -1) -> 'Model':
        """
        Instantiates an already-trained model, based on the experiment
        configuration and some optional overrides.
        """
        weights_file = weights_file or os.path.join(serialization_dir, _DEFAULT_WEIGHTS)

        # Load vocabulary from file
        vocab_dir = os.path.join(serialization_dir, 'vocabulary')
        vocab = Vocabulary.from_files(vocab_dir)

        model_params = config.get('model')

        # The experiment config tells us how to _train_ a model, including where to get pre-trained
        # embeddings from.  We're now _loading_ the model, so those embeddings will already be
        # stored in our weights.  We don't need any pretrained weight file anymore, and we don't
        # want the code to look for it, so we remove it from the parameters here.
        remove_pretrained_embedding_params(model_params)
        model = Model.from_params(vocab=vocab, params=model_params)
        model_state = torch.load(weights_file, map_location=util.device_mapping(cuda_device))
        model.load_state_dict(model_state)

        # Force model to cpu or gpu, as appropriate, to make sure that the embeddings are
        # in sync with the weights
        if cuda_device >= 0:
            model.cuda(cuda_device)
        else:
            model.cpu()

        return model

    @classmethod
    def load(cls,
             config: Params,
             serialization_dir: str,
             weights_file: str = None,
             cuda_device: int = -1) -> 'Model':
        """
        Instantiates an already-trained model, based on the experiment
        configuration and some optional overrides.

        Parameters
        ----------
        config: Params
            The configuration that was used to train the model. It should definitely
            have a `model` section, and should probably have a `trainer` section
            as well.
        serialization_dir: str = None
            The directory containing the serialized weights, parameters, and vocabulary
            of the model.
        weights_file: str = None
            By default we load the weights from `best.th` in the serialization
            directory, but you can override that value here.
        cuda_device: int = -1
            By default we load the model on the CPU, but if you want to load it
            for GPU usage you can specify the id of your GPU here

        Returns
        -------
        model: Model
            The model specified in the configuration, loaded with the serialized
            vocabulary and the trained weights.
        """

        # Peak at the class of the model.
        model_type = config["model"]["type"]

        # Load using an overridable _load method.
        # This allows subclasses of Model to override _load.
        # pylint: disable=protected-access
        return cls.by_name(model_type)._load(config, serialization_dir, weights_file, cuda_device)

def remove_pretrained_embedding_params(params: Params):
    keys = params.keys()
    if 'pretrained_file' in keys:
        del params['pretrained_file']
    for value in params.values():
        if isinstance(value, Params):
            remove_pretrained_embedding_params(value)
<|endoftext|>


<commit_before>#
# Copyright (C) 2008 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import re
import sys

from command import InteractiveCommand
from editor import Editor
from error import UploadError

def _die(fmt, *args):
  msg = fmt % args
  print >>sys.stderr, 'error: %s' % msg
  sys.exit(1)

def _SplitEmails(values):
  result = []
  for str in values:
    result.extend([s.strip() for s in str.split(',')])
  return result

class Upload(InteractiveCommand):
  common = True
  helpSummary = "Upload changes for code review"
  helpUsage="""
%prog [--re --cc] {[<project>]... | --replace <project>}
"""
  helpDescription = """
The '%prog' command is used to send changes to the Gerrit code
review system.  It searches for changes in local projects that do
not yet exist in the corresponding remote repository.  If multiple
changes are found, '%prog' opens an editor to allow the
user to choose which change to upload.  After a successful upload,
repo prints the URL for the change in the Gerrit code review system.

'%prog' searches for uploadable changes in all projects listed
at the command line.  Projects can be specified either by name, or
by a relative or absolute path to the project's local directory. If
no projects are specified, '%prog' will search for uploadable
changes in all projects listed in the manifest.

If the --reviewers or --cc options are passed, those emails are
added to the respective list of users, and emails are sent to any
new users.  Users passed to --reviewers must be already registered
with the code review system, or the upload will fail.
"""

  def _Options(self, p):
    p.add_option('--replace',
                 dest='replace', action='store_true',
                 help='Upload replacement patchesets from this branch')
    p.add_option('--re', '--reviewers',
                 type='string',  action='append', dest='reviewers',
                 help='Request reviews from these people.')
    p.add_option('--cc',
                 type='string',  action='append', dest='cc',
                 help='Also send email to these email addresses.')

  def _SingleBranch(self, branch, people):
    project = branch.project
    name = branch.name
    date = branch.date
    list = branch.commits

    print 'Upload project %s/:' % project.relpath
    print '  branch %s (%2d commit%s, %s):' % (
                  name,
                  len(list),
                  len(list) != 1 and 's' or '',
                  date)
    for commit in list:
      print '         %s' % commit

    sys.stdout.write('(y/n)? ')
    answer = sys.stdin.readline().strip()
    if answer in ('y', 'Y', 'yes', '1', 'true', 't'):
      self._UploadAndReport([branch], people)
    else:
      _die("upload aborted by user")

  def _MultipleBranches(self, pending, people):
    projects = {}
    branches = {}

    script = []
    script.append('# Uncomment the branches to upload:')
    for project, avail in pending:
      script.append('#')
      script.append('# project %s/:' % project.relpath)

      b = {}
      for branch in avail:
        name = branch.name
        date = branch.date
        list = branch.commits

        if b:
          script.append('#')
        script.append('#  branch %s (%2d commit%s, %s):' % (
                      name,
                      len(list),
                      len(list) != 1 and 's' or '',
                      date))
        for commit in list:
          script.append('#         %s' % commit)
        b[name] = branch

      projects[project.relpath] = project
      branches[project.name] = b
    script.append('')

    script = Editor.EditString("\n".join(script)).split("\n")

    project_re = re.compile(r'^#?\s*project\s*([^\s]+)/:$')
    branch_re = re.compile(r'^\s*branch\s*([^\s(]+)\s*\(.*')

    project = None
    todo = []

    for line in script:
      m = project_re.match(line)
      if m:
        name = m.group(1)
        project = projects.get(name)
        if not project:
          _die('project %s not available for upload', name)
        continue

      m = branch_re.match(line)
      if m:
        name = m.group(1)
        if not project:
          _die('project for branch %s not in script', name)
        branch = branches[project.name].get(name)
        if not branch:
          _die('branch %s not in %s', name, project.relpath)
        todo.append(branch)
    if not todo:
      _die("nothing uncommented for upload")
    self._UploadAndReport(todo, people)

  def _ReplaceBranch(self, project, people):
    branch = project.CurrentBranch
    if not branch:
      print >>sys.stdout, "no branches ready for upload"
      return
    branch = project.GetUploadableBranch(branch)
    if not branch:
      print >>sys.stdout, "no branches ready for upload"
      return

    script = []
    script.append('# Replacing from branch %s' % branch.name)
    for commit in branch.commits:
      script.append('[      ] %s' % commit)
    script.append('')
    script.append('# Insert change numbers in the brackets to add a new patch set.')
    script.append('# To create a new change record, leave the brackets empty.')

    script = Editor.EditString("\n".join(script)).split("\n")

    change_re = re.compile(r'^\[\s*(\d{1,})\s*\]\s*([0-9a-f]{1,}) .*$')
    to_replace = dict()
    full_hashes = branch.unabbrev_commits

    for line in script:
      m = change_re.match(line)
      if m:
        f = m.group(2)
        try:
          f = full_hashes[f]
        except KeyError:
          print 'fh = %s' % full_hashes
          print >>sys.stderr, "error: commit %s not found" % f
          sys.exit(1)
        to_replace[m.group(1)] = f

    if not to_replace:
      print >>sys.stderr, "error: no replacements specified"
      print >>sys.stderr, "       use 'repo upload' without --replace"
      sys.exit(1)

    branch.replace_changes = to_replace
    self._UploadAndReport([branch], people)

  def _UploadAndReport(self, todo, people):
    have_errors = False
    for branch in todo:
      try:
        branch.UploadForReview(people)
        branch.uploaded = True
      except UploadError, e:
        branch.error = e
        branch.uploaded = False
        have_errors = True

    print >>sys.stderr, ''
    print >>sys.stderr, '--------------------------------------------'

    if have_errors:
      for branch in todo:
        if not branch.uploaded:
          print >>sys.stderr, '[FAILED] %-15s %-15s  (%s)' % (
                 branch.project.relpath + '/', \
                 branch.name, \
                 branch.error)
      print >>sys.stderr, ''

    for branch in todo:
        if branch.uploaded:
          print >>sys.stderr, '[OK    ] %-15s %s' % (
                 branch.project.relpath + '/',
                 branch.name)
          print >>sys.stderr, '%s' % branch.tip_url
          print >>sys.stderr, '(as %s)' % branch.owner_email
          print >>sys.stderr, ''

    if have_errors:
      sys.exit(1)

  def Execute(self, opt, args):
    project_list = self.GetProjects(args)
    pending = []
    reviewers = []
    cc = []

    if opt.reviewers:
      reviewers = _SplitEmails(opt.reviewers)
    if opt.cc:
      cc = _SplitEmails(opt.cc)
    people = (reviewers,cc)

    if opt.replace:
      if len(project_list) != 1:
        print >>sys.stderr, \
              'error: --replace requires exactly one project'
        sys.exit(1)
      self._ReplaceBranch(project_list[0], people)
      return

    for project in project_list:
      avail = project.GetUploadableBranches()
      if avail:
        pending.append((project, avail))

    if not pending:
      print >>sys.stdout, "no branches ready for upload"
    elif len(pending) == 1 and len(pending[0][1]) == 1:
      self._SingleBranch(pending[0][1][0], people)
    else:
      self._MultipleBranches(pending, people)
<commit_msg>Don't accept multiple commits for the same change in upload --replace<commit_after>#
# Copyright (C) 2008 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import re
import sys

from command import InteractiveCommand
from editor import Editor
from error import UploadError

def _die(fmt, *args):
  msg = fmt % args
  print >>sys.stderr, 'error: %s' % msg
  sys.exit(1)

def _SplitEmails(values):
  result = []
  for str in values:
    result.extend([s.strip() for s in str.split(',')])
  return result

class Upload(InteractiveCommand):
  common = True
  helpSummary = "Upload changes for code review"
  helpUsage="""
%prog [--re --cc] {[<project>]... | --replace <project>}
"""
  helpDescription = """
The '%prog' command is used to send changes to the Gerrit code
review system.  It searches for changes in local projects that do
not yet exist in the corresponding remote repository.  If multiple
changes are found, '%prog' opens an editor to allow the
user to choose which change to upload.  After a successful upload,
repo prints the URL for the change in the Gerrit code review system.

'%prog' searches for uploadable changes in all projects listed
at the command line.  Projects can be specified either by name, or
by a relative or absolute path to the project's local directory. If
no projects are specified, '%prog' will search for uploadable
changes in all projects listed in the manifest.

If the --reviewers or --cc options are passed, those emails are
added to the respective list of users, and emails are sent to any
new users.  Users passed to --reviewers must be already registered
with the code review system, or the upload will fail.
"""

  def _Options(self, p):
    p.add_option('--replace',
                 dest='replace', action='store_true',
                 help='Upload replacement patchesets from this branch')
    p.add_option('--re', '--reviewers',
                 type='string',  action='append', dest='reviewers',
                 help='Request reviews from these people.')
    p.add_option('--cc',
                 type='string',  action='append', dest='cc',
                 help='Also send email to these email addresses.')

  def _SingleBranch(self, branch, people):
    project = branch.project
    name = branch.name
    date = branch.date
    list = branch.commits

    print 'Upload project %s/:' % project.relpath
    print '  branch %s (%2d commit%s, %s):' % (
                  name,
                  len(list),
                  len(list) != 1 and 's' or '',
                  date)
    for commit in list:
      print '         %s' % commit

    sys.stdout.write('(y/n)? ')
    answer = sys.stdin.readline().strip()
    if answer in ('y', 'Y', 'yes', '1', 'true', 't'):
      self._UploadAndReport([branch], people)
    else:
      _die("upload aborted by user")

  def _MultipleBranches(self, pending, people):
    projects = {}
    branches = {}

    script = []
    script.append('# Uncomment the branches to upload:')
    for project, avail in pending:
      script.append('#')
      script.append('# project %s/:' % project.relpath)

      b = {}
      for branch in avail:
        name = branch.name
        date = branch.date
        list = branch.commits

        if b:
          script.append('#')
        script.append('#  branch %s (%2d commit%s, %s):' % (
                      name,
                      len(list),
                      len(list) != 1 and 's' or '',
                      date))
        for commit in list:
          script.append('#         %s' % commit)
        b[name] = branch

      projects[project.relpath] = project
      branches[project.name] = b
    script.append('')

    script = Editor.EditString("\n".join(script)).split("\n")

    project_re = re.compile(r'^#?\s*project\s*([^\s]+)/:$')
    branch_re = re.compile(r'^\s*branch\s*([^\s(]+)\s*\(.*')

    project = None
    todo = []

    for line in script:
      m = project_re.match(line)
      if m:
        name = m.group(1)
        project = projects.get(name)
        if not project:
          _die('project %s not available for upload', name)
        continue

      m = branch_re.match(line)
      if m:
        name = m.group(1)
        if not project:
          _die('project for branch %s not in script', name)
        branch = branches[project.name].get(name)
        if not branch:
          _die('branch %s not in %s', name, project.relpath)
        todo.append(branch)
    if not todo:
      _die("nothing uncommented for upload")
    self._UploadAndReport(todo, people)

  def _ReplaceBranch(self, project, people):
    branch = project.CurrentBranch
    if not branch:
      print >>sys.stdout, "no branches ready for upload"
      return
    branch = project.GetUploadableBranch(branch)
    if not branch:
      print >>sys.stdout, "no branches ready for upload"
      return

    script = []
    script.append('# Replacing from branch %s' % branch.name)
    for commit in branch.commits:
      script.append('[      ] %s' % commit)
    script.append('')
    script.append('# Insert change numbers in the brackets to add a new patch set.')
    script.append('# To create a new change record, leave the brackets empty.')

    script = Editor.EditString("\n".join(script)).split("\n")

    change_re = re.compile(r'^\[\s*(\d{1,})\s*\]\s*([0-9a-f]{1,}) .*$')
    to_replace = dict()
    full_hashes = branch.unabbrev_commits

    for line in script:
      m = change_re.match(line)
      if m:
        c = m.group(1)
        f = m.group(2)
        try:
          f = full_hashes[f]
        except KeyError:
          print 'fh = %s' % full_hashes
          print >>sys.stderr, "error: commit %s not found" % f
          sys.exit(1)
        if c in to_replace:
          print >>sys.stderr,\
            "error: change %s cannot accept multiple commits" % c
          sys.exit(1)
        to_replace[c] = f

    if not to_replace:
      print >>sys.stderr, "error: no replacements specified"
      print >>sys.stderr, "       use 'repo upload' without --replace"
      sys.exit(1)

    branch.replace_changes = to_replace
    self._UploadAndReport([branch], people)

  def _UploadAndReport(self, todo, people):
    have_errors = False
    for branch in todo:
      try:
        branch.UploadForReview(people)
        branch.uploaded = True
      except UploadError, e:
        branch.error = e
        branch.uploaded = False
        have_errors = True

    print >>sys.stderr, ''
    print >>sys.stderr, '--------------------------------------------'

    if have_errors:
      for branch in todo:
        if not branch.uploaded:
          print >>sys.stderr, '[FAILED] %-15s %-15s  (%s)' % (
                 branch.project.relpath + '/', \
                 branch.name, \
                 branch.error)
      print >>sys.stderr, ''

    for branch in todo:
        if branch.uploaded:
          print >>sys.stderr, '[OK    ] %-15s %s' % (
                 branch.project.relpath + '/',
                 branch.name)
          print >>sys.stderr, '%s' % branch.tip_url
          print >>sys.stderr, '(as %s)' % branch.owner_email
          print >>sys.stderr, ''

    if have_errors:
      sys.exit(1)

  def Execute(self, opt, args):
    project_list = self.GetProjects(args)
    pending = []
    reviewers = []
    cc = []

    if opt.reviewers:
      reviewers = _SplitEmails(opt.reviewers)
    if opt.cc:
      cc = _SplitEmails(opt.cc)
    people = (reviewers,cc)

    if opt.replace:
      if len(project_list) != 1:
        print >>sys.stderr, \
              'error: --replace requires exactly one project'
        sys.exit(1)
      self._ReplaceBranch(project_list[0], people)
      return

    for project in project_list:
      avail = project.GetUploadableBranches()
      if avail:
        pending.append((project, avail))

    if not pending:
      print >>sys.stdout, "no branches ready for upload"
    elif len(pending) == 1 and len(pending[0][1]) == 1:
      self._SingleBranch(pending[0][1][0], people)
    else:
      self._MultipleBranches(pending, people)
<|endoftext|>