<commit_before>#!/usr/bin/env python

"""
Arduino API Library is an API layer to quickly prototype ideas using the Arduino.
Copyright (C) 2009  Akash Xavier <akashmanohar@gmail.com>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

import serial, time

class Arduino:

    __OUTPUT_PINS = -1

    def __init__(self, port, baudrate=115200):
        self.serial = serial.Serial(port, baudrate)

    def __str__(self):
        return "Arduino is on port %s at %d baudrate" %(self.serial.port, self.serial.baudrate)

    def output(self, pinArray):
        self.__sendData(len(pinArray))

        if(isinstance(pinArray, list) or isinstance(pinArray, tuple)):
            self.__OUTPUT_PINS = pinArray
            for each_pin in pinArray:
                self.__sendPin(each_pin)
        return True

    def setLow(self, pin):
        self.__sendData('0')
        self.__sendPin(pin)
        return True

    def setHigh(self, pin):
        self.__sendData('1')
        self.__sendPin(pin)
        return True

    def getState(self, pin):
        self.__sendData('2')
        self.__sendPin(pin)
        return self.__formatPinState(self.__getData())

    def analogWrite(self, pin, value):
        self.__sendData('3')
        hex_value = hex(value)[2:]
        if(len(hex_value)==1):
            self.__sendData('0')
        else:
            self.__sendData(hex_value[0])
        self.__sendData(hex_value[1])
        return True

    def analogRead(self, pin, value):
        self.__sendData('4')
        return self.__getData()

    def turnOff(self):
        for each_pin in self.__OUTPUT_PINS:
            self.setLow(each_pin)
        return True

    def __sendPin(self, pin):
        pin_in_char = chr(pin+48)
        self.__sendData(pin_in_char)

    def __sendData(self, serial_data):
        while(self.__getData()!="what"):
            pass
        self.serial.write(str(serial_data))

    def __getData(self):
        return self.serial.readline().replace("\r\n","")

    def __formatPinState(self, pinValue):
        if pinValue=='1':
            return True
        else:
            return False

    def close(self):
        self.serial.close()
        return True

    """
    def __del__(self):
        #close serial connection once program ends
        #this fixes the problem of port getting locked or unrecoverable in some linux systems
        self.serial.close()
    """

<commit_msg>fixed analogRead with jbuckner's solution<commit_after>#!/usr/bin/env python

"""
Arduino API Library is an API layer to quickly prototype ideas using the Arduino.
Copyright (C) 2009  Akash Xavier <akashmanohar@gmail.com>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

import serial, time

class Arduino:

    __OUTPUT_PINS = -1

    def __init__(self, port, baudrate=115200):
        self.serial = serial.Serial(port, baudrate)

    def __str__(self):
        return "Arduino is on port %s at %d baudrate" %(self.serial.port, self.serial.baudrate)

    def output(self, pinArray):
        self.__sendData(len(pinArray))

        if(isinstance(pinArray, list) or isinstance(pinArray, tuple)):
            self.__OUTPUT_PINS = pinArray
            for each_pin in pinArray:
                self.__sendPin(each_pin)
        return True

    def setLow(self, pin):
        self.__sendData('0')
        self.__sendPin(pin)
        return True

    def setHigh(self, pin):
        self.__sendData('1')
        self.__sendPin(pin)
        return True

    def getState(self, pin):
        self.__sendData('2')
        self.__sendPin(pin)
        return self.__formatPinState(self.__getData())

    def analogWrite(self, pin, value):
        self.__sendData('3')
        hex_value = hex(value)[2:]
        if(len(hex_value)==1):
            self.__sendData('0')
        else:
            self.__sendData(hex_value[0])
        self.__sendData(hex_value[1])
        return True

    def analogRead(self, pin):
        self.__sendData('4')
        self.__sendPin(pin)
        return self.__getData()

    def turnOff(self):
        for each_pin in self.__OUTPUT_PINS:
            self.setLow(each_pin)
        return True

    def __sendPin(self, pin):
        pin_in_char = chr(pin+48)
        self.__sendData(pin_in_char)

    def __sendData(self, serial_data):
        while(self.__getData()!="what"):
            pass
        self.serial.write(str(serial_data))

    def __getData(self):
        return self.serial.readline().replace("\r\n","")

    def __formatPinState(self, pinValue):
        if pinValue=='1':
            return True
        else:
            return False

    def close(self):
        self.serial.close()
        return True

    """
    def __del__(self):
        #close serial connection once program ends
        #this fixes the problem of port getting locked or unrecoverable in some linux systems
        self.serial.close()
    """

<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

#  ================ Start Argument Parsing ==============================

"""arm_now.
Usage:
  arm_now list [--all]
  arm_now start [<arch>] [--clean] [-s|--sync] [--offline]
                [--autostart=<script>] [--add-qemu-options=<options>]
                [--real-source] [--redir=<port>]...
  arm_now clean
  arm_now resize <new_size> [--correct]
  arm_now install [<arch>] [--clean] [--real-source]
  arm_now show
  arm_now offline
  arm_now -h | --help
  arm_now --version
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Commands:
  list          List all available images for all cpu.
  start         Start a vm with a <arch> cpu. (default: armv5-eabi)
  resize        Resize the current rootfs. (example: arm_now resize 1G, or +1G)
  clean         Delete the current rootfs.
  install       Download, install and config a rootfs for the given <arch>. (default: armv5-eabi)
  show          Show informations about the rootfs.
  offline       Download all rootfs and kernel so arm_now can be fully runned offline.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Options:
  --sync                        Synchronize the current directory with the vm home.
  --redir protocol:host::guest  Redirect the host port to the guest (example: --redir tcp:8000::80)
  --clean                       Clean the current image before starting.
  --add-qemu-options=<options>  Add options to qemu-system-<arch>.
                     (example: --add-qemu-options="-sandbox on" to Enable seccomp mode 2 system call filter )
  --autostart=<script>          At startup <script> is uploaded and executed inside the vm.
  --syncpath=<path>             Synchronize the <path> directory with the vm home.
  --syncroot=<path>             Synchronize the <path> directory with the vm root.
                            (Only if you need to modify the linux vm config)
  --offline                     Start with zero internet request.
  --correct                     Correct the filesystem after resize.
  -h --help                     Show this screen.
  --version                     Show version.

Defaults:
  arch          Defaults to armv5-eabi.
"""

import os
import sys
import re
from multiprocessing import Pool
import contextlib
from pathlib import Path
from subprocess import check_call

# Exall is an exception manager based on decorator/context/callback
# Check it out: https://github.com/nongiach/exall
from exall import exall, ignore
from docopt import docopt

from .utils import pgreen, pred, porange, maybe_you_meant, which
from .filesystem import Filesystem
from .config import Config, qemu_options, install_opkg
from . import options
from .download import download_image, scrawl_kernel, download, indexof_parse

def main():
    """ Call the function according to the asked command
        Read above to acknowledge supported commands
    """
    check_dependencies_or_exit()
    a = docopt(__doc__, version='arm_now 1.24')
    if not a["<arch>"] and os.path.isfile(Config.ARCH):
        with open(Config.ARCH) as F:
            a["<arch>"] = F.read()
    if a["list"]:
        do_list(a["--all"])
    elif a["start"]:
        do_start(a["<arch>"] or "armv5-eabi",
                 a["--clean"], a["--sync"] or a["-s"],
                 a["--offline"], a["--redir"],
                 ' '.join(a["--add-qemu-options"]),
                 a["--autostart"], a["--real-source"])
    elif a["clean"]:
        options.clean(Config)
    elif a["resize"]:
        do_resize(a["<new_size>"], a["--correct"])
    elif a["install"]:
        do_install(a["<arch>"] or "armv5-eabi", a["--clean"], a["--real-source"])
    elif a["show"]:
        do_show()
    elif a["offline"]:
        do_offline()

#  ================ End Argument Parsing ==============================

def do_start(arch, clean, sync, offline, redir, add_qemu_options, autostart, real_source):
    """Setup and start a virtualmachine using qemu.

    :param arch: The cpu architecture that will be started.
    :param redir: Redirect a host port to the guest.
    :param offline: skip the checks for new images.
    :param clean: Clean filesystem before starting.
    :param sync: Sync le current directory with the guest.
    """
    add_qemu_options += " " + convert_redir_to_qemu_args(redir)
    do_install(arch, clean, real_source)
    fs = Filesystem(Config.ROOTFS)
    config_filesystem(Config.ROOTFS, arch, real_source)
    if sync:
        options.sync_upload(Config.ROOTFS, src=".", dest="/root")
    options.autostart(Config.ROOTFS, autostart)
    run_qemu(arch, Config.KERNEL, Config.DTB, Config.ROOTFS, add_qemu_options)
    fs.check()
    if sync:
        options.sync_download(Config.ROOTFS, "/root.tar", ".")

def run_qemu(arch, kernel, dtb, rootfs, add_qemu_options):
    dtb = "" if not os.path.exists(dtb) else "-dtb {}".format(dtb)
    options = qemu_options[arch][1].format(arch=arch, kernel=kernel, rootfs=rootfs, dtb=dtb)
    arch = qemu_options[arch][0]
    print("Starting qemu-system-{}".format(arch))
    qemu_config = "-serial stdio -monitor null {add_qemu_options}".format(add_qemu_options=add_qemu_options)
    cmd = """stty intr ^]
       export QEMU_AUDIO_DRV="none"
       qemu-system-{arch} {options} \
               -m 256M \
               -nographic \
               {qemu_config} \
               {dtb} \
               -no-reboot
       stty intr ^c
    """.format(arch=arch, qemu_config=qemu_config, options=options, dtb=dtb)
    pgreen(cmd)
    os.system(cmd)

def is_already_created(arch):
    """ if the current kernel and rootfs is not the same arch then delete them """
    if not os.path.exists(Config.DIR + "/arch"):
        return False
    with open(Config.DIR + "/arch", "r") as F:
        old_arch = F.read()
    if old_arch == arch:
        return True
    response = input("(use --clean next time) An {} image exists, delete ? (y/n) ".format(old_arch))
    if not response.startswith("y"):
        sys.exit(1)
    options.clean(Config)
    return False

def do_install(arch, clean, real_source):
    """ download and setup filesystem and kernel
    """
    if clean:
        options.clean(Config)
    if arch not in qemu_options:
        pred("ERROR: I don't know this arch='{}' yet".format(arch), file=sys.stderr)
        porange("maybe you meant: {}".format(maybe_you_meant(arch,
                                                             qemu_options.keys()
                                                             ) or qemu_options.keys()), file=sys.stderr)
        sys.exit(1)
    if is_already_created(arch):
        porange("WARNING: {} already exists, use --clean to restart with a fresh filesystem".format(Config.DIR))
        return
    with contextlib.suppress(FileExistsError):
        os.mkdir(Config.DIR)
    download_image(arch, dest=Config.DIR, real_source=real_source)
    pgreen("[+] Installed")

# def do_install_real_source(arch, clean=False):
#     """ download and setup filesystem and kernel
#     """
#     if clean: options.clean(Config)
#     if arch not in qemu_options:
#         pred("ERROR: I don't know this arch='{}' yet".format(arch), file=sys.stderr)
#         porange("maybe you meant: {}".format(maybe_you_meant(arch,
#                                                              qemu_options.keys()
#                                                              ) or qemu_options.keys()), file=sys.stderr)
#         sys.exit(1)
#     kernel, dtb, rootfs = scrawl_kernel(arch)
#     if kernel is None or rootfs is None:
#         pred("ERROR: couldn't download files for this arch", file=sys.stderr)
#         sys.exit(1)
#     if is_already_created(arch):
#         porange("WARNING: {} already exists, use --clean to restart with a fresh filesystem".format(Config.DIR))
#         return
#     with contextlib.suppress(FileExistsError):
#         os.mkdir(Config.DIR)
#     download(kernel, Config.KERNEL, Config.DOWNLOAD_CACHE_DIR)
#     if dtb: download(dtb, Config.DTB, Config.DOWNLOAD_CACHE_DIR)
#     download(rootfs, Config.ROOTFS, Config.DOWNLOAD_CACHE_DIR)
#     with open(Config.DIR + "/arch", "w") as F:
#         F.write(arch)
#     pgreen("[+] Installed")

def config_filesystem(rootfs, arch, real_source):
    fs = Filesystem(rootfs)
    fs.rm('/etc/init.d/S40network')
    fs.rm('/etc/init.d/S90tests')
    fs.rm('/etc/issue')
    fs.create("/etc/issue",
              "Welcome to arm_now\n")
    fs.create("/etc/init.d/S95_how_to_kill_qemu",
              'echo -e "\033[0;31mpress ctrl+] to kill qemu\033[0m"\n',
              right=555)
    fs.create("/etc/init.d/S40_network", """
IFACE=$(ip a | grep -o ':.*: ' | grep -v ': lo: ' | grep -o '[^ :@]*' | head -n 1)
ifconfig "$IFACE" 10.0.2.15
route add default gw 10.0.2.2
echo 'nameserver 10.0.2.3' >> /etc/resolv.conf
""", right=555)
    if arch in install_opkg and real_source:
        fs.create("/root/install_pkg_manager.sh", """
{install_opkg}
opkg update
echo -e '\n\n now you can $ opkg install gdb'
rm /root/install_pkg_manager.sh
""".format(install_opkg=install_opkg[arch]), right=555)
        fs.create("/etc/profile.d/opkg_path.sh", """
export PATH=$PATH:/opt/bin:/opt/sbin
                """, right=555)
    fs.sed('s/init.d\/S/init.d\/K/g', '/etc/init.d/rcK', right=755)

def check_dependencies_or_exit():
    dependencies = [
            which("e2cp",
                ubuntu="apt-get install e2tools",
                arch="yaourt -S e2tools",
                darwin="brew install e2tools gettext e2fsprogs\nbrew unlink e2fsprogs && brew link e2fsprogs -f"),
            which("qemu-system-arm",
                  ubuntu="apt-get install qemu",
                  kali="apt-get install qemu-system",
                  arch="pacman -S qemu-arch-extra",
                  darwin="brew install qemu"),
            which("unzip",
                ubuntu="apt-get install unzip",
                arch="pacman -S unzip",
                darwin="brew install unzip")
            ]
    if not all(dependencies):
        print("requirements missing, plz install them", file=sys.stderr)
        sys.exit(1)

re_redir = re.compile(r"(tcp|udp):\d+:\d+")

def convert_redir_to_qemu_args(redir):
    args = []

    for r in redir:
        if not re_redir.match(r):
            pred("ERROR: Invalid argument: --redir {}".format(r))
            print("example:")
            print("\tredirect tcp host 8000 to guest 80: --redir tcp:8000:80")
            print("\tredirect udp host 4444 to guest 44: --redir udp:4444:44")
            sys.exit(1)

        args.append("-nic user,hostfwd={}::{}-:{} ".format(*r.split(":")))

    return "".join(args)

def do_resize(size, correct):
    """ Resize filesystem.

        Example: arm_now resize +10G
                 Resizing to 10 GigaBytes
    """
    fs = Filesystem(Config.ROOTFS)
    fs.resize(size)
    if correct:
        fs.correct()

def test_arch(arch):
    arch = arch[:-1]
    kernel, dtb, rootfs = scrawl_kernel(arch)
    if kernel and rootfs:
        print("{}: OK".format(arch))

def do_list(all=False):
    """ List all compactible cpu architecture
    """
    if not all:
        print('\n'.join(qemu_options.keys()))
    else:
        url = "https://toolchains.bootlin.com/downloads/releases/toolchains/"
        all_arch = indexof_parse(url)
        p = Pool(10)
        p.map(test_arch, all_arch)

def do_show():
    if not os.path.isfile(Config.ARCH) or not os.path.isfile(Config.ROOTFS):
        pred("File missing")
        return
    with open(Config.ARCH) as F:
        arch = F.read()
    print(" Info ".center(80, "~"))
    size = os.path.getsize(Config.ROOTFS)
    pgreen("arch         = {}".format(arch))
    pgreen("rootfs size  = {}M".format(size // (1024 * 1024)))
    Filesystem(Config.ROOTFS).ls("/root")
    print("~" * 80)

@exall(os.makedirs, FileExistsError, ignore)
def do_offline():
    URL = "https://github.com/nongiach/arm_now_templates/archive/master.zip"
    templates = str(Path.home()) + "/.config/arm_now/templates/"
    master_zip = str(Path.home()) + "/.config/arm_now/templates/master.zip"
    os.makedirs(templates)
    # download_from_github(arch)
    download(URL, master_zip, Config.DOWNLOAD_CACHE_DIR)
    os.chdir(templates)
    check_call("unzip master.zip", shell=True)
    check_call("mv arm_now_templates-master/* .", shell=True)
    check_call("rm -rf arm_now_templates-master/ README.md master.zip", shell=True)

if __name__ == "__main__":
    main()
<commit_msg>Update help<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

#  ================ Start Argument Parsing ==============================

"""arm_now.
Usage:
  arm_now list [--all]
  arm_now start [<arch>] [--clean] [-s|--sync] [--offline]
                [--autostart=<script>] [--add-qemu-options=<options>]
                [--real-source] [--redir=<port>]...
  arm_now clean
  arm_now resize <new_size> [--correct]
  arm_now install [<arch>] [--clean] [--real-source]
  arm_now show
  arm_now offline
  arm_now -h | --help
  arm_now --version
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Commands:
  list          List all available images for all cpu.
  start         Start a vm with a <arch> cpu. (default: armv5-eabi)
  resize        Resize the current rootfs. (example: arm_now resize 1G, or +1G)
  clean         Delete the current rootfs.
  install       Download, install and config a rootfs for the given <arch>. (default: armv5-eabi)
  show          Show informations about the rootfs.
  offline       Download all rootfs and kernel so arm_now can be fully runned offline.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Options:
  --sync                        Synchronize the current directory with the vm home.
  --redir protocol:host:guest  Redirect the host port to the guest (example: --redir tcp:8000:80)
  --clean                       Clean the current image before starting.
  --add-qemu-options=<options>  Add options to qemu-system-<arch>.
                     (example: --add-qemu-options="-sandbox on" to Enable seccomp mode 2 system call filter )
  --autostart=<script>          At startup <script> is uploaded and executed inside the vm.
  --syncpath=<path>             Synchronize the <path> directory with the vm home.
  --syncroot=<path>             Synchronize the <path> directory with the vm root.
                            (Only if you need to modify the linux vm config)
  --offline                     Start with zero internet request.
  --correct                     Correct the filesystem after resize.
  -h --help                     Show this screen.
  --version                     Show version.

Defaults:
  arch          Defaults to armv5-eabi.
"""

import os
import sys
import re
from multiprocessing import Pool
import contextlib
from pathlib import Path
from subprocess import check_call

# Exall is an exception manager based on decorator/context/callback
# Check it out: https://github.com/nongiach/exall
from exall import exall, ignore
from docopt import docopt

from .utils import pgreen, pred, porange, maybe_you_meant, which
from .filesystem import Filesystem
from .config import Config, qemu_options, install_opkg
from . import options
from .download import download_image, scrawl_kernel, download, indexof_parse

def main():
    """ Call the function according to the asked command
        Read above to acknowledge supported commands
    """
    check_dependencies_or_exit()
    a = docopt(__doc__, version='arm_now 1.24')
    if not a["<arch>"] and os.path.isfile(Config.ARCH):
        with open(Config.ARCH) as F:
            a["<arch>"] = F.read()
    if a["list"]:
        do_list(a["--all"])
    elif a["start"]:
        do_start(a["<arch>"] or "armv5-eabi",
                 a["--clean"], a["--sync"] or a["-s"],
                 a["--offline"], a["--redir"],
                 ' '.join(a["--add-qemu-options"]),
                 a["--autostart"], a["--real-source"])
    elif a["clean"]:
        options.clean(Config)
    elif a["resize"]:
        do_resize(a["<new_size>"], a["--correct"])
    elif a["install"]:
        do_install(a["<arch>"] or "armv5-eabi", a["--clean"], a["--real-source"])
    elif a["show"]:
        do_show()
    elif a["offline"]:
        do_offline()

#  ================ End Argument Parsing ==============================

def do_start(arch, clean, sync, offline, redir, add_qemu_options, autostart, real_source):
    """Setup and start a virtualmachine using qemu.

    :param arch: The cpu architecture that will be started.
    :param redir: Redirect a host port to the guest.
    :param offline: skip the checks for new images.
    :param clean: Clean filesystem before starting.
    :param sync: Sync le current directory with the guest.
    """
    add_qemu_options += " " + convert_redir_to_qemu_args(redir)
    do_install(arch, clean, real_source)
    fs = Filesystem(Config.ROOTFS)
    config_filesystem(Config.ROOTFS, arch, real_source)
    if sync:
        options.sync_upload(Config.ROOTFS, src=".", dest="/root")
    options.autostart(Config.ROOTFS, autostart)
    run_qemu(arch, Config.KERNEL, Config.DTB, Config.ROOTFS, add_qemu_options)
    fs.check()
    if sync:
        options.sync_download(Config.ROOTFS, "/root.tar", ".")

def run_qemu(arch, kernel, dtb, rootfs, add_qemu_options):
    dtb = "" if not os.path.exists(dtb) else "-dtb {}".format(dtb)
    options = qemu_options[arch][1].format(arch=arch, kernel=kernel, rootfs=rootfs, dtb=dtb)
    arch = qemu_options[arch][0]
    print("Starting qemu-system-{}".format(arch))
    qemu_config = "-serial stdio -monitor null {add_qemu_options}".format(add_qemu_options=add_qemu_options)
    cmd = """stty intr ^]
       export QEMU_AUDIO_DRV="none"
       qemu-system-{arch} {options} \
               -m 256M \
               -nographic \
               {qemu_config} \
               {dtb} \
               -no-reboot
       stty intr ^c
    """.format(arch=arch, qemu_config=qemu_config, options=options, dtb=dtb)
    pgreen(cmd)
    os.system(cmd)

def is_already_created(arch):
    """ if the current kernel and rootfs is not the same arch then delete them """
    if not os.path.exists(Config.DIR + "/arch"):
        return False
    with open(Config.DIR + "/arch", "r") as F:
        old_arch = F.read()
    if old_arch == arch:
        return True
    response = input("(use --clean next time) An {} image exists, delete ? (y/n) ".format(old_arch))
    if not response.startswith("y"):
        sys.exit(1)
    options.clean(Config)
    return False

def do_install(arch, clean, real_source):
    """ download and setup filesystem and kernel
    """
    if clean:
        options.clean(Config)
    if arch not in qemu_options:
        pred("ERROR: I don't know this arch='{}' yet".format(arch), file=sys.stderr)
        porange("maybe you meant: {}".format(maybe_you_meant(arch,
                                                             qemu_options.keys()
                                                             ) or qemu_options.keys()), file=sys.stderr)
        sys.exit(1)
    if is_already_created(arch):
        porange("WARNING: {} already exists, use --clean to restart with a fresh filesystem".format(Config.DIR))
        return
    with contextlib.suppress(FileExistsError):
        os.mkdir(Config.DIR)
    download_image(arch, dest=Config.DIR, real_source=real_source)
    pgreen("[+] Installed")

# def do_install_real_source(arch, clean=False):
#     """ download and setup filesystem and kernel
#     """
#     if clean: options.clean(Config)
#     if arch not in qemu_options:
#         pred("ERROR: I don't know this arch='{}' yet".format(arch), file=sys.stderr)
#         porange("maybe you meant: {}".format(maybe_you_meant(arch,
#                                                              qemu_options.keys()
#                                                              ) or qemu_options.keys()), file=sys.stderr)
#         sys.exit(1)
#     kernel, dtb, rootfs = scrawl_kernel(arch)
#     if kernel is None or rootfs is None:
#         pred("ERROR: couldn't download files for this arch", file=sys.stderr)
#         sys.exit(1)
#     if is_already_created(arch):
#         porange("WARNING: {} already exists, use --clean to restart with a fresh filesystem".format(Config.DIR))
#         return
#     with contextlib.suppress(FileExistsError):
#         os.mkdir(Config.DIR)
#     download(kernel, Config.KERNEL, Config.DOWNLOAD_CACHE_DIR)
#     if dtb: download(dtb, Config.DTB, Config.DOWNLOAD_CACHE_DIR)
#     download(rootfs, Config.ROOTFS, Config.DOWNLOAD_CACHE_DIR)
#     with open(Config.DIR + "/arch", "w") as F:
#         F.write(arch)
#     pgreen("[+] Installed")

def config_filesystem(rootfs, arch, real_source):
    fs = Filesystem(rootfs)
    fs.rm('/etc/init.d/S40network')
    fs.rm('/etc/init.d/S90tests')
    fs.rm('/etc/issue')
    fs.create("/etc/issue",
              "Welcome to arm_now\n")
    fs.create("/etc/init.d/S95_how_to_kill_qemu",
              'echo -e "\033[0;31mpress ctrl+] to kill qemu\033[0m"\n',
              right=555)
    fs.create("/etc/init.d/S40_network", """
IFACE=$(ip a | grep -o ':.*: ' | grep -v ': lo: ' | grep -o '[^ :@]*' | head -n 1)
ifconfig "$IFACE" 10.0.2.15
route add default gw 10.0.2.2
echo 'nameserver 10.0.2.3' >> /etc/resolv.conf
""", right=555)
    if arch in install_opkg and real_source:
        fs.create("/root/install_pkg_manager.sh", """
{install_opkg}
opkg update
echo -e '\n\n now you can $ opkg install gdb'
rm /root/install_pkg_manager.sh
""".format(install_opkg=install_opkg[arch]), right=555)
        fs.create("/etc/profile.d/opkg_path.sh", """
export PATH=$PATH:/opt/bin:/opt/sbin
                """, right=555)
    fs.sed('s/init.d\/S/init.d\/K/g', '/etc/init.d/rcK', right=755)

def check_dependencies_or_exit():
    dependencies = [
            which("e2cp",
                ubuntu="apt-get install e2tools",
                arch="yaourt -S e2tools",
                darwin="brew install e2tools gettext e2fsprogs\nbrew unlink e2fsprogs && brew link e2fsprogs -f"),
            which("qemu-system-arm",
                  ubuntu="apt-get install qemu",
                  kali="apt-get install qemu-system",
                  arch="pacman -S qemu-arch-extra",
                  darwin="brew install qemu"),
            which("unzip",
                ubuntu="apt-get install unzip",
                arch="pacman -S unzip",
                darwin="brew install unzip")
            ]
    if not all(dependencies):
        print("requirements missing, plz install them", file=sys.stderr)
        sys.exit(1)

re_redir = re.compile(r"(tcp|udp):\d+:\d+")

def convert_redir_to_qemu_args(redir):
    args = []

    for r in redir:
        if not re_redir.match(r):
            pred("ERROR: Invalid argument: --redir {}".format(r))
            print("example:")
            print("\tredirect tcp host 8000 to guest 80: --redir tcp:8000:80")
            print("\tredirect udp host 4444 to guest 44: --redir udp:4444:44")
            sys.exit(1)

        args.append("-nic user,hostfwd={}::{}-:{} ".format(*r.split(":")))

    return "".join(args)

def do_resize(size, correct):
    """ Resize filesystem.

        Example: arm_now resize +10G
                 Resizing to 10 GigaBytes
    """
    fs = Filesystem(Config.ROOTFS)
    fs.resize(size)
    if correct:
        fs.correct()

def test_arch(arch):
    arch = arch[:-1]
    kernel, dtb, rootfs = scrawl_kernel(arch)
    if kernel and rootfs:
        print("{}: OK".format(arch))

def do_list(all=False):
    """ List all compactible cpu architecture
    """
    if not all:
        print('\n'.join(qemu_options.keys()))
    else:
        url = "https://toolchains.bootlin.com/downloads/releases/toolchains/"
        all_arch = indexof_parse(url)
        p = Pool(10)
        p.map(test_arch, all_arch)

def do_show():
    if not os.path.isfile(Config.ARCH) or not os.path.isfile(Config.ROOTFS):
        pred("File missing")
        return
    with open(Config.ARCH) as F:
        arch = F.read()
    print(" Info ".center(80, "~"))
    size = os.path.getsize(Config.ROOTFS)
    pgreen("arch         = {}".format(arch))
    pgreen("rootfs size  = {}M".format(size // (1024 * 1024)))
    Filesystem(Config.ROOTFS).ls("/root")
    print("~" * 80)

@exall(os.makedirs, FileExistsError, ignore)
def do_offline():
    URL = "https://github.com/nongiach/arm_now_templates/archive/master.zip"
    templates = str(Path.home()) + "/.config/arm_now/templates/"
    master_zip = str(Path.home()) + "/.config/arm_now/templates/master.zip"
    os.makedirs(templates)
    # download_from_github(arch)
    download(URL, master_zip, Config.DOWNLOAD_CACHE_DIR)
    os.chdir(templates)
    check_call("unzip master.zip", shell=True)
    check_call("mv arm_now_templates-master/* .", shell=True)
    check_call("rm -rf arm_now_templates-master/ README.md master.zip", shell=True)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests of utilities supporting export to SavedModel."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import sys
import tempfile
import time

# pylint: disable=g-import-not-at-top

# TODO(jart): #6568 Remove this hack that makes dlopen() not crash.
if hasattr(sys, "getdlopenflags") and hasattr(sys, "setdlopenflags"):
  import ctypes
  sys.setdlopenflags(sys.getdlopenflags() | ctypes.RTLD_GLOBAL)

from tensorflow.contrib.learn.python.learn import export_strategy as export_strategy_lib
from tensorflow.contrib.learn.python.learn.estimators import constants
from tensorflow.contrib.learn.python.learn.estimators import model_fn
from tensorflow.contrib.learn.python.learn.utils import input_fn_utils
from tensorflow.contrib.learn.python.learn.utils import saved_model_export_utils
from tensorflow.core.framework import tensor_shape_pb2
from tensorflow.core.framework import types_pb2
from tensorflow.core.protobuf import meta_graph_pb2
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.ops import array_ops
from tensorflow.python.platform import gfile
from tensorflow.python.platform import test
from tensorflow.python.saved_model import signature_constants
from tensorflow.python.saved_model import signature_def_utils

class SavedModelExportUtilsTest(test.TestCase):

  def test_build_standardized_signature_def(self):
    input_tensors = {
        "input-1":
            array_ops.placeholder(
                dtypes.float32, 1, name="input-tensor-1")
    }
    output_tensors = {
        "output-1":
            array_ops.placeholder(
                dtypes.float32, 1, name="output-tensor-1")
    }
    problem_type = constants.ProblemType.LINEAR_REGRESSION
    regression_signature_def = (
        saved_model_export_utils.build_standardized_signature_def(
            input_tensors, output_tensors, problem_type))
    expected_regression_signature_def = meta_graph_pb2.SignatureDef()
    shape = tensor_shape_pb2.TensorShapeProto(
        dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)])
    dtype = types_pb2.DataType.Value("DT_FLOAT")
    expected_regression_signature_def.inputs[
        signature_constants.REGRESS_INPUTS].CopyFrom(
            meta_graph_pb2.TensorInfo(
                name="input-tensor-1:0", dtype=dtype, tensor_shape=shape))
    expected_regression_signature_def.outputs[
        signature_constants.REGRESS_OUTPUTS].CopyFrom(
            meta_graph_pb2.TensorInfo(
                name="output-tensor-1:0", dtype=dtype, tensor_shape=shape))

    expected_regression_signature_def.method_name = (
        signature_constants.REGRESS_METHOD_NAME)
    self.assertEqual(regression_signature_def,
                     expected_regression_signature_def)

  def test_get_input_alternatives(self):
    input_ops = input_fn_utils.InputFnOps("bogus features dict", None,
                                          "bogus default input dict")

    input_alternatives, _ = saved_model_export_utils.get_input_alternatives(
        input_ops)
    self.assertEqual(input_alternatives[
        saved_model_export_utils.DEFAULT_INPUT_ALTERNATIVE_KEY],
                     "bogus default input dict")
    # self.assertEqual(input_alternatives[
    #     saved_model_export_utils.FEATURES_INPUT_ALTERNATIVE_KEY],
    #                  "bogus features dict")

  def test_get_output_alternatives_explicit(self):
    provided_output_alternatives = {
        "head-1": (constants.ProblemType.LINEAR_REGRESSION,
                   "bogus output dict"),
        "head-2": (constants.ProblemType.CLASSIFICATION, "bogus output dict 2"),
        "head-3": (constants.ProblemType.UNSPECIFIED, "bogus output dict 3"),
    }
    model_fn_ops = model_fn.ModelFnOps(
        model_fn.ModeKeys.INFER,
        predictions={"some_output": "bogus_tensor"},
        output_alternatives=provided_output_alternatives)
    output_alternatives, _ = saved_model_export_utils.get_output_alternatives(
        model_fn_ops, "head-1")

    self.assertEqual(provided_output_alternatives, output_alternatives)

  def test_get_output_alternatives_implicit(self):
    prediction_tensor = constant_op.constant(["bogus"])
    model_fn_ops = model_fn.ModelFnOps(
        model_fn.ModeKeys.INFER,
        predictions={"some_output": prediction_tensor},
        output_alternatives=None)

    output_alternatives, _ = saved_model_export_utils.get_output_alternatives(
        model_fn_ops, "some_output")
    self.assertEqual({
        "default_output_alternative": (constants.ProblemType.UNSPECIFIED, {
            "some_output": prediction_tensor
        })
    }, output_alternatives)

  def test_get_output_alternatives_implicit_single(self):
    prediction_tensor = constant_op.constant(["bogus"])
    model_fn_ops = model_fn.ModelFnOps(
        model_fn.ModeKeys.INFER,
        predictions=prediction_tensor,
        output_alternatives=None)

    output_alternatives, _ = saved_model_export_utils.get_output_alternatives(
        model_fn_ops)
    self.assertEqual({
        "default_output_alternative": (constants.ProblemType.UNSPECIFIED, {
            "output": prediction_tensor
        })
    }, output_alternatives)

  def test_build_all_signature_defs(self):
    input_features = constant_op.constant(["10"])
    input_example = constant_op.constant(["11"])
    input_ops = input_fn_utils.InputFnOps({
        "features": input_features
    }, None, {"default input": input_example})
    input_alternatives, _ = (
        saved_model_export_utils.get_input_alternatives(input_ops))
    output_1 = constant_op.constant(["1"])
    output_2 = constant_op.constant(["2"])
    output_3 = constant_op.constant(["3"])
    provided_output_alternatives = {
        "head-1": (constants.ProblemType.LINEAR_REGRESSION, {
            "some_output_1": output_1
        }),
        "head-2": (constants.ProblemType.CLASSIFICATION, {
            "some_output_2": output_2
        }),
        "head-3": (constants.ProblemType.UNSPECIFIED, {
            "some_output_3": output_3
        }),
    }
    model_fn_ops = model_fn.ModelFnOps(
        model_fn.ModeKeys.INFER,
        predictions={"some_output": constant_op.constant(["4"])},
        output_alternatives=provided_output_alternatives)
    output_alternatives, _ = (saved_model_export_utils.get_output_alternatives(
        model_fn_ops, "head-1"))

    signature_defs = saved_model_export_utils.build_all_signature_defs(
        input_alternatives, output_alternatives, "head-1")

    expected_signature_defs = {
        "serving_default":
            signature_def_utils.regression_signature_def(input_example,
                                                         output_1),
        "default_input_alternative:head-1":
            signature_def_utils.regression_signature_def(input_example,
                                                         output_1),
        "default_input_alternative:head-2":
            signature_def_utils.classification_signature_def(input_example,
                                                             output_2, None),
        "default_input_alternative:head-3":
            signature_def_utils.predict_signature_def({
                "input": input_example
            }, {"output": output_3}),
        # "features_input_alternative:head-1":
        #     signature_def_utils.regression_signature_def(input_features,
        #                                                  output_1),
        # "features_input_alternative:head-2":
        #     signature_def_utils.classification_signature_def(input_features,
        #                                                      output_2, None),
        # "features_input_alternative:head-3":
        #     signature_def_utils.predict_signature_def({
        #         "input": input_features
        #     }, {"output": output_3}),
    }

    self.assertDictEqual(expected_signature_defs, signature_defs)

  def test_build_all_signature_defs_legacy_input_fn_not_supported(self):
    """Tests that legacy input_fn returning (features, labels) raises error.

    serving_input_fn must return InputFnOps including a default input
    alternative.
    """
    input_features = constant_op.constant(["10"])
    input_ops = ({"features": input_features}, None)
    input_alternatives, _ = (
        saved_model_export_utils.get_input_alternatives(input_ops))
    output_1 = constant_op.constant(["1"])
    output_2 = constant_op.constant(["2"])
    output_3 = constant_op.constant(["3"])
    provided_output_alternatives = {
        "head-1": (constants.ProblemType.LINEAR_REGRESSION, {
            "some_output_1": output_1
        }),
        "head-2": (constants.ProblemType.CLASSIFICATION, {
            "some_output_2": output_2
        }),
        "head-3": (constants.ProblemType.UNSPECIFIED, {
            "some_output_3": output_3
        }),
    }
    model_fn_ops = model_fn.ModelFnOps(
        model_fn.ModeKeys.INFER,
        predictions={"some_output": constant_op.constant(["4"])},
        output_alternatives=provided_output_alternatives)
    output_alternatives, _ = (saved_model_export_utils.get_output_alternatives(
        model_fn_ops, "head-1"))

    with self.assertRaisesRegexp(
        ValueError, "A default input_alternative must be provided"):
      saved_model_export_utils.build_all_signature_defs(
          input_alternatives, output_alternatives, "head-1")

  def test_get_timestamped_export_dir(self):
    export_dir_base = tempfile.mkdtemp() + "export/"
    export_dir_1 = saved_model_export_utils.get_timestamped_export_dir(
        export_dir_base)
    time.sleep(1)
    export_dir_2 = saved_model_export_utils.get_timestamped_export_dir(
        export_dir_base)
    time.sleep(1)
    export_dir_3 = saved_model_export_utils.get_timestamped_export_dir(
        export_dir_base)

    # Export directories should be named using a timestamp that is seconds
    # since epoch.  Such a timestamp is 10 digits long.
    time_1 = os.path.basename(export_dir_1)
    self.assertEqual(10, len(time_1))
    time_2 = os.path.basename(export_dir_2)
    self.assertEqual(10, len(time_2))
    time_3 = os.path.basename(export_dir_3)
    self.assertEqual(10, len(time_3))

    self.assertTrue(int(time_1) < int(time_2))
    self.assertTrue(int(time_2) < int(time_3))

  def test_garbage_collect_exports(self):
    export_dir_base = tempfile.mkdtemp() + "export/"
    gfile.MkDir(export_dir_base)
    export_dir_1 = _create_test_export_dir(export_dir_base)
    export_dir_2 = _create_test_export_dir(export_dir_base)
    export_dir_3 = _create_test_export_dir(export_dir_base)
    export_dir_4 = _create_test_export_dir(export_dir_base)

    self.assertTrue(gfile.Exists(export_dir_1))
    self.assertTrue(gfile.Exists(export_dir_2))
    self.assertTrue(gfile.Exists(export_dir_3))
    self.assertTrue(gfile.Exists(export_dir_4))

    # Garbage collect all but the most recent 2 exports,
    # where recency is determined based on the timestamp directory names.
    saved_model_export_utils.garbage_collect_exports(export_dir_base, 2)

    self.assertFalse(gfile.Exists(export_dir_1))
    self.assertFalse(gfile.Exists(export_dir_2))
    self.assertTrue(gfile.Exists(export_dir_3))
    self.assertTrue(gfile.Exists(export_dir_4))

  def test_get_most_recent_export(self):
    export_dir_base = tempfile.mkdtemp() + "export/"
    gfile.MkDir(export_dir_base)
    _create_test_export_dir(export_dir_base)
    _create_test_export_dir(export_dir_base)
    _create_test_export_dir(export_dir_base)
    export_dir_4 = _create_test_export_dir(export_dir_base)

    (most_recent_export_dir, most_recent_export_version) = (
        saved_model_export_utils.get_most_recent_export(export_dir_base))

    self.assertEqual(export_dir_4, most_recent_export_dir)
    self.assertEqual(export_dir_4,
                     os.path.join(export_dir_base,
                                  str(most_recent_export_version)))

  def test_make_export_strategy(self):
    """Only tests that an ExportStrategy instance is created."""
    def _serving_input_fn():
      return array_ops.constant([1]), None
    export_strategy = saved_model_export_utils.make_export_strategy(
        serving_input_fn=_serving_input_fn,
        default_output_alternative_key="default",
        assets_extra={"from/path": "to/path"},
        as_text=False,
        exports_to_keep=5)
    self.assertTrue(
        isinstance(export_strategy, export_strategy_lib.ExportStrategy))

def _create_test_export_dir(export_dir_base):
  export_dir = saved_model_export_utils.get_timestamped_export_dir(
      export_dir_base)
  gfile.MkDir(export_dir)
  time.sleep(1)
  return export_dir

if __name__ == "__main__":
  test.main()
<commit_msg>Fix Python 3 compatibility bug in test Change: 144998717<commit_after># Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests of utilities supporting export to SavedModel."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import sys
import tempfile
import time

# pylint: disable=g-import-not-at-top

# TODO(jart): #6568 Remove this hack that makes dlopen() not crash.
if hasattr(sys, "getdlopenflags") and hasattr(sys, "setdlopenflags"):
  import ctypes
  sys.setdlopenflags(sys.getdlopenflags() | ctypes.RTLD_GLOBAL)

from tensorflow.contrib.learn.python.learn import export_strategy as export_strategy_lib
from tensorflow.contrib.learn.python.learn.estimators import constants
from tensorflow.contrib.learn.python.learn.estimators import model_fn
from tensorflow.contrib.learn.python.learn.utils import input_fn_utils
from tensorflow.contrib.learn.python.learn.utils import saved_model_export_utils
from tensorflow.core.framework import tensor_shape_pb2
from tensorflow.core.framework import types_pb2
from tensorflow.core.protobuf import meta_graph_pb2
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.ops import array_ops
from tensorflow.python.platform import gfile
from tensorflow.python.platform import test
from tensorflow.python.saved_model import signature_constants
from tensorflow.python.saved_model import signature_def_utils
from tensorflow.python.util import compat

class SavedModelExportUtilsTest(test.TestCase):

  def test_build_standardized_signature_def(self):
    input_tensors = {
        "input-1":
            array_ops.placeholder(
                dtypes.float32, 1, name="input-tensor-1")
    }
    output_tensors = {
        "output-1":
            array_ops.placeholder(
                dtypes.float32, 1, name="output-tensor-1")
    }
    problem_type = constants.ProblemType.LINEAR_REGRESSION
    regression_signature_def = (
        saved_model_export_utils.build_standardized_signature_def(
            input_tensors, output_tensors, problem_type))
    expected_regression_signature_def = meta_graph_pb2.SignatureDef()
    shape = tensor_shape_pb2.TensorShapeProto(
        dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)])
    dtype = types_pb2.DataType.Value("DT_FLOAT")
    expected_regression_signature_def.inputs[
        signature_constants.REGRESS_INPUTS].CopyFrom(
            meta_graph_pb2.TensorInfo(
                name="input-tensor-1:0", dtype=dtype, tensor_shape=shape))
    expected_regression_signature_def.outputs[
        signature_constants.REGRESS_OUTPUTS].CopyFrom(
            meta_graph_pb2.TensorInfo(
                name="output-tensor-1:0", dtype=dtype, tensor_shape=shape))

    expected_regression_signature_def.method_name = (
        signature_constants.REGRESS_METHOD_NAME)
    self.assertEqual(regression_signature_def,
                     expected_regression_signature_def)

  def test_get_input_alternatives(self):
    input_ops = input_fn_utils.InputFnOps("bogus features dict", None,
                                          "bogus default input dict")

    input_alternatives, _ = saved_model_export_utils.get_input_alternatives(
        input_ops)
    self.assertEqual(input_alternatives[
        saved_model_export_utils.DEFAULT_INPUT_ALTERNATIVE_KEY],
                     "bogus default input dict")
    # self.assertEqual(input_alternatives[
    #     saved_model_export_utils.FEATURES_INPUT_ALTERNATIVE_KEY],
    #                  "bogus features dict")

  def test_get_output_alternatives_explicit(self):
    provided_output_alternatives = {
        "head-1": (constants.ProblemType.LINEAR_REGRESSION,
                   "bogus output dict"),
        "head-2": (constants.ProblemType.CLASSIFICATION, "bogus output dict 2"),
        "head-3": (constants.ProblemType.UNSPECIFIED, "bogus output dict 3"),
    }
    model_fn_ops = model_fn.ModelFnOps(
        model_fn.ModeKeys.INFER,
        predictions={"some_output": "bogus_tensor"},
        output_alternatives=provided_output_alternatives)
    output_alternatives, _ = saved_model_export_utils.get_output_alternatives(
        model_fn_ops, "head-1")

    self.assertEqual(provided_output_alternatives, output_alternatives)

  def test_get_output_alternatives_implicit(self):
    prediction_tensor = constant_op.constant(["bogus"])
    model_fn_ops = model_fn.ModelFnOps(
        model_fn.ModeKeys.INFER,
        predictions={"some_output": prediction_tensor},
        output_alternatives=None)

    output_alternatives, _ = saved_model_export_utils.get_output_alternatives(
        model_fn_ops, "some_output")
    self.assertEqual({
        "default_output_alternative": (constants.ProblemType.UNSPECIFIED, {
            "some_output": prediction_tensor
        })
    }, output_alternatives)

  def test_get_output_alternatives_implicit_single(self):
    prediction_tensor = constant_op.constant(["bogus"])
    model_fn_ops = model_fn.ModelFnOps(
        model_fn.ModeKeys.INFER,
        predictions=prediction_tensor,
        output_alternatives=None)

    output_alternatives, _ = saved_model_export_utils.get_output_alternatives(
        model_fn_ops)
    self.assertEqual({
        "default_output_alternative": (constants.ProblemType.UNSPECIFIED, {
            "output": prediction_tensor
        })
    }, output_alternatives)

  def test_build_all_signature_defs(self):
    input_features = constant_op.constant(["10"])
    input_example = constant_op.constant(["11"])
    input_ops = input_fn_utils.InputFnOps({
        "features": input_features
    }, None, {"default input": input_example})
    input_alternatives, _ = (
        saved_model_export_utils.get_input_alternatives(input_ops))
    output_1 = constant_op.constant(["1"])
    output_2 = constant_op.constant(["2"])
    output_3 = constant_op.constant(["3"])
    provided_output_alternatives = {
        "head-1": (constants.ProblemType.LINEAR_REGRESSION, {
            "some_output_1": output_1
        }),
        "head-2": (constants.ProblemType.CLASSIFICATION, {
            "some_output_2": output_2
        }),
        "head-3": (constants.ProblemType.UNSPECIFIED, {
            "some_output_3": output_3
        }),
    }
    model_fn_ops = model_fn.ModelFnOps(
        model_fn.ModeKeys.INFER,
        predictions={"some_output": constant_op.constant(["4"])},
        output_alternatives=provided_output_alternatives)
    output_alternatives, _ = (saved_model_export_utils.get_output_alternatives(
        model_fn_ops, "head-1"))

    signature_defs = saved_model_export_utils.build_all_signature_defs(
        input_alternatives, output_alternatives, "head-1")

    expected_signature_defs = {
        "serving_default":
            signature_def_utils.regression_signature_def(input_example,
                                                         output_1),
        "default_input_alternative:head-1":
            signature_def_utils.regression_signature_def(input_example,
                                                         output_1),
        "default_input_alternative:head-2":
            signature_def_utils.classification_signature_def(input_example,
                                                             output_2, None),
        "default_input_alternative:head-3":
            signature_def_utils.predict_signature_def({
                "input": input_example
            }, {"output": output_3}),
        # "features_input_alternative:head-1":
        #     signature_def_utils.regression_signature_def(input_features,
        #                                                  output_1),
        # "features_input_alternative:head-2":
        #     signature_def_utils.classification_signature_def(input_features,
        #                                                      output_2, None),
        # "features_input_alternative:head-3":
        #     signature_def_utils.predict_signature_def({
        #         "input": input_features
        #     }, {"output": output_3}),
    }

    self.assertDictEqual(expected_signature_defs, signature_defs)

  def test_build_all_signature_defs_legacy_input_fn_not_supported(self):
    """Tests that legacy input_fn returning (features, labels) raises error.

    serving_input_fn must return InputFnOps including a default input
    alternative.
    """
    input_features = constant_op.constant(["10"])
    input_ops = ({"features": input_features}, None)
    input_alternatives, _ = (
        saved_model_export_utils.get_input_alternatives(input_ops))
    output_1 = constant_op.constant(["1"])
    output_2 = constant_op.constant(["2"])
    output_3 = constant_op.constant(["3"])
    provided_output_alternatives = {
        "head-1": (constants.ProblemType.LINEAR_REGRESSION, {
            "some_output_1": output_1
        }),
        "head-2": (constants.ProblemType.CLASSIFICATION, {
            "some_output_2": output_2
        }),
        "head-3": (constants.ProblemType.UNSPECIFIED, {
            "some_output_3": output_3
        }),
    }
    model_fn_ops = model_fn.ModelFnOps(
        model_fn.ModeKeys.INFER,
        predictions={"some_output": constant_op.constant(["4"])},
        output_alternatives=provided_output_alternatives)
    output_alternatives, _ = (saved_model_export_utils.get_output_alternatives(
        model_fn_ops, "head-1"))

    with self.assertRaisesRegexp(
        ValueError, "A default input_alternative must be provided"):
      saved_model_export_utils.build_all_signature_defs(
          input_alternatives, output_alternatives, "head-1")

  def test_get_timestamped_export_dir(self):
    export_dir_base = tempfile.mkdtemp() + "export/"
    export_dir_1 = saved_model_export_utils.get_timestamped_export_dir(
        export_dir_base)
    time.sleep(1)
    export_dir_2 = saved_model_export_utils.get_timestamped_export_dir(
        export_dir_base)
    time.sleep(1)
    export_dir_3 = saved_model_export_utils.get_timestamped_export_dir(
        export_dir_base)

    # Export directories should be named using a timestamp that is seconds
    # since epoch.  Such a timestamp is 10 digits long.
    time_1 = os.path.basename(export_dir_1)
    self.assertEqual(10, len(time_1))
    time_2 = os.path.basename(export_dir_2)
    self.assertEqual(10, len(time_2))
    time_3 = os.path.basename(export_dir_3)
    self.assertEqual(10, len(time_3))

    self.assertTrue(int(time_1) < int(time_2))
    self.assertTrue(int(time_2) < int(time_3))

  def test_garbage_collect_exports(self):
    export_dir_base = tempfile.mkdtemp() + "export/"
    gfile.MkDir(export_dir_base)
    export_dir_1 = _create_test_export_dir(export_dir_base)
    export_dir_2 = _create_test_export_dir(export_dir_base)
    export_dir_3 = _create_test_export_dir(export_dir_base)
    export_dir_4 = _create_test_export_dir(export_dir_base)

    self.assertTrue(gfile.Exists(export_dir_1))
    self.assertTrue(gfile.Exists(export_dir_2))
    self.assertTrue(gfile.Exists(export_dir_3))
    self.assertTrue(gfile.Exists(export_dir_4))

    # Garbage collect all but the most recent 2 exports,
    # where recency is determined based on the timestamp directory names.
    saved_model_export_utils.garbage_collect_exports(export_dir_base, 2)

    self.assertFalse(gfile.Exists(export_dir_1))
    self.assertFalse(gfile.Exists(export_dir_2))
    self.assertTrue(gfile.Exists(export_dir_3))
    self.assertTrue(gfile.Exists(export_dir_4))

  def test_get_most_recent_export(self):
    export_dir_base = tempfile.mkdtemp() + "export/"
    gfile.MkDir(export_dir_base)
    _create_test_export_dir(export_dir_base)
    _create_test_export_dir(export_dir_base)
    _create_test_export_dir(export_dir_base)
    export_dir_4 = _create_test_export_dir(export_dir_base)

    (most_recent_export_dir, most_recent_export_version) = (
        saved_model_export_utils.get_most_recent_export(export_dir_base))

    self.assertEqual(compat.as_bytes(export_dir_4),
                     compat.as_bytes(most_recent_export_dir))
    self.assertEqual(compat.as_bytes(export_dir_4),
                     os.path.join(compat.as_bytes(export_dir_base),
                                  compat.as_bytes(
                                      str(most_recent_export_version))))

  def test_make_export_strategy(self):
    """Only tests that an ExportStrategy instance is created."""
    def _serving_input_fn():
      return array_ops.constant([1]), None
    export_strategy = saved_model_export_utils.make_export_strategy(
        serving_input_fn=_serving_input_fn,
        default_output_alternative_key="default",
        assets_extra={"from/path": "to/path"},
        as_text=False,
        exports_to_keep=5)
    self.assertTrue(
        isinstance(export_strategy, export_strategy_lib.ExportStrategy))

def _create_test_export_dir(export_dir_base):
  export_dir = saved_model_export_utils.get_timestamped_export_dir(
      export_dir_base)
  gfile.MkDir(export_dir)
  time.sleep(1)
  return export_dir

if __name__ == "__main__":
  test.main()
<|endoftext|>


<commit_before>from copy import deepcopy
from collections import MutableMapping

class Cache(MutableMapping):
    """
    In-memory Cache storage. Reduces overheads on repeated requests, at the cost
    that some responses may have outdated data.

    Generally the cache will try to return copies of objects, rather than the
    objects themselves. This is to maintain a semi-immutable cache resembling
    the API as closely as possible.

    Call ```Cache.purge()``` to purge any data, any subsequent
    requests will return fresh data from Assembla.
    """

    def __init__(self, parent, schema, cache_responses=True):
        self.store = dict()
        self.schema = schema
        # Initiate the cache, using :schema
        self.clear()
        self.cache_responses = cache_responses
        self.parent = parent

    def __setitem__(self, key, value):
        self.store[key] = value
    set = __setitem__

    def __getitem__(self, item):
        """
        Return a copy of an item, rather than the actual item.

        The cache stores lists of objects, rather than returning the object
        directly, this traverses the stored list and returns deep copies of each
        object.
        """
        return map(deepcopy, self.store[item])

    def __delitem__(self, key):
        del self.store.pop[key]

    def __iter__(self):
        return iter(self.store)

    def __len__(self):
        return len(self.store)

    def has_cached(self, item):
        return True if self.store.has_key(item) and self.store[item] is not None else False

    def activate(self):
        self.cache_responses = True

    def deactivate(self):
        self.cache_responses = False

    def purge(self):
        """
        Purges the cache of any data it may have by reinstantiating itself from
        the schema
        """
        self.store.clear()
        map(lambda name: self.__setitem__(name, []), self.schema)
<commit_msg>Small cleanup in the Cache.<commit_after>from copy import deepcopy
from collections import MutableMapping

class Cache(MutableMapping):
    """
    In-memory Cache storage. Reduces overheads on repeated requests, at the cost
    that some responses may have outdated data.

    Generally the cache will try to return copies of objects, rather than the
    objects themselves. This is to maintain a semi-immutable cache resembling
    the API as closely as possible.

    Call ```Cache.purge()``` to purge any data, any subsequent
    requests will return fresh data from Assembla.
    """

    def __init__(self, parent, schema, cache_responses=True):
        self.store = dict()
        self.schema = schema
        # Initiate the cache, using :schema
        self.clear()
        self.cache_responses = cache_responses
        self.parent = parent

    def __setitem__(self, key, value):
        self.store[key] = value
    set = __setitem__

    def __getitem__(self, item):
        """
        Return a copy of an item, rather than the actual item.

        The cache stores lists of objects, rather than returning the object
        directly, this traverses the stored list and returns deep copies of each
        object.
        """
        return map(deepcopy, self.store[item])

    def __delitem__(self, key):
        del self.store.pop[key]

    def __iter__(self):
        return iter(self.store)

    def __len__(self):
        return len(self.store)

    def has_cached(self, item):
        return self.store.has_key(item) and self.store[item]

    def activate(self):
        self.cache_responses = True

    def deactivate(self):
        self.cache_responses = False

    def purge(self):
        """
        Purges the cache of any data it may have by reinstantiating itself from
        the schema
        """
        self.store.clear()
        for name in self.schema:
            self.__setitem__(name, [])
<|endoftext|>


<commit_before>##
## runner.py
## Login : David Rousselie <dax@happycoders.org>
## Started on  Thu May 17 22:02:08 2007 David Rousselie
## $Id$
##
## Copyright (C) 2007 David Rousselie
## This program is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 2 of the License, or
## (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program; if not, write to the Free Software
## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##

import logging
import os
import sys
from ConfigParser import ConfigParser
from getopt import gnu_getopt
import threading

from jcl.lang import Lang
from jcl.jabber.component import JCLComponent
import jcl.model as model
from jcl.model.account import Account, PresenceAccount, User, LegacyJID

class JCLRunner(object):

    def __init__(self, component_name, component_version):
        """
        options: list of tuples:
        - short_opt: same as getopt
        - long_opt: same as getopt
        """
        self.component_short_name = "JCL"
        self.component_name = component_name
        self.component_version = component_version
        self.config_file = "jcl.conf"
        self.server = "localhost"
        self.port = 5347
        self.secret = "secret"
        self.service_jid = "jcl.localhost"
        self.language = "en"
        self.db_url = "sqlite:///var/spool/jabber/jcl.db"
        self.pid_file = "/var/run/jabber/jcl.pid"
        self.log_stdout = False
        self.log_file = None
        self.console = False
        self.options = [("c:", "config-file=", None,
                         " FILE\t\t\t\tConfiguration file to use",
                         lambda arg: self.set_attr("config_file", arg)),
                        ("S:", "server=", "jabber",
                         " SERVER_ADDRESS\t\t\tAddress of the Jabber server",
                         lambda arg: self.set_attr("server", arg)),
                        ("P:", "port=", "jabber",
                         " PORT\t\t\t\t\tPort of the Jabber server to connect the component",
                         lambda arg: self.set_attr("port", int(arg))),
                        ("s:", "secret=", "jabber",
                         " SECRET\t\t\t\tComponent password to connect to the Jabber server",
                         lambda arg: self.set_attr("secret", arg)),
                        ("j:", "service-jid=", "jabber",
                         " JID\t\t\t\tJID of the component",
                         lambda arg: self.set_attr("service_jid", arg)),
                        ("l:", "language=", "jabber",
                         " LANG\t\t\t\tDefault Language of the component",
                         lambda arg: self.set_attr("language", arg)),
                        ("u:", "db-url=", "db",
                         " URL\t\t\t\tDatabase URL",
                         lambda arg: self.set_attr("db_url", arg)),
                        ("p:", "pid-file=", "component",
                         " FILE\t\t\t\tPath of the PID file",
                         lambda arg: self.set_attr("pid_file", arg)),
                        ("d", "debug", None,
                         "\t\t\t\t\tEnable debug traces",
                         lambda arg: self.set_attr("debug", True)),
                        ("C", "console", None,
                         "\t\t\t\t\tRun in console mode",
                         lambda arg: self.set_attr("console", True)),
                        ("o", "log-stdout", None,
                         "\t\t\t\t\tLog on stdout",
                         lambda arg: self.set_attr("log_stdout", True)),
                        ("f:", "log-file=", "component",
                         "\t\t\t\t\tLog in file",
                         lambda arg: self.set_attr("log_file", arg)),
                        ("h", "help", None,
                         "\t\t\t\t\tThis help",
                         lambda arg: self.print_help())]
        self.logger = logging.getLogger()
        self.__debug = False
        self.wait_event = threading.Event()

    def set_attr(self, attr, value):
        setattr(self, attr, value)

    def __configure_commandline_args(self, shortopts, longopts, cleanopts):
        (opts, args) = gnu_getopt(sys.argv[1:], shortopts, longopts)
        commandline_args = {}
        for (arg, value) in opts:
            clean_arg = arg.lstrip('-')
            if cleanopts.has_key(clean_arg):
                commandline_args[clean_arg] = value
        return commandline_args

    def __apply_commandline_args(self, commandline_args, cleanopts):
        for arg in commandline_args:
            value = commandline_args[arg]
            self.logger.debug("Applying argument " + arg + " = " +
                              value)
            cleanopts[arg][1](value)

    def __apply_configfile(self, commandline_args, cleanopts):
        if commandline_args.has_key("config-file"):
            self.config_file = commandline_args["config-file"]
        elif commandline_args.has_key("c"):
            self.config_file = commandline_args["c"]
        self.config = ConfigParser()
        self.logger.debug("Loading config file " + self.config_file)
        read_file = self.config.read(self.config_file)
        if read_file == []:
            self.logger.info("Creating empty config file " + self.config_file)
        else:
            for opt in cleanopts:
                if len(opt) > 1:
                    (section, set_func) = cleanopts[opt]
                    if section is not None:
                        attr = opt.replace("-", "_")
                        if self.config.has_section(section) \
                                and self.config.has_option(section, attr):
                            config_property = self.config.get(section, attr)
                            self.logger.debug("Setting " + attr + " = " +
                                              config_property +
                                              " from configuration file " +
                                              self.config_file)
                            set_func(config_property)

    def configure(self):
        """
        Apply configuration from command line and configuration file.
        command line arguments override configuration file properties.
        """
        shortopts = ""
        longopts = []
        cleanopts = {}
        for option in self.options:
            shortopts += option[0]
            longopts += [option[1]]
            cleanopts[option[0][0]] = (option[2], option[4])
            if option[1][-1:] == '=':
                cleanopts[option[1][:-1]] = (option[2], option[4])
            else:
                cleanopts[option[1]] = (option[2], option[4])

        commandline_args = self.__configure_commandline_args(shortopts,
                                                             longopts,
                                                             cleanopts)
        if commandline_args.has_key("debug") or commandline_args.has_key("d"):
            self.debug = True
            self.logger.debug("Debug activated")
        else:
            self.debug = False
        self.__apply_configfile(commandline_args, cleanopts)
        self.__apply_commandline_args(commandline_args, cleanopts)
        if self.log_stdout:
            self.logger.addHandler(logging.StreamHandler())
        if self.log_file is not None:
            self.logger.addHandler(logging.FileHandler(self.log_file))

    def _get_help(self):
        help = self.component_name + " v" + self.component_version + " help:\n"
        for option in self.options:
            if option[1][-1:] == '=':
                long_option = option[1][:-1]
            else:
                long_option = option[1]
            help += "\t-" + option[0][0] + ", --" + long_option + option[3] + "\n"
        return help

    def print_help(self):
        print self._get_help()
        sys.exit(0)

    def get_debug(self):
        return self.__debug

    def set_debug(self, value):
        self.__debug = value
        if self.__debug:
            self.logger.setLevel(logging.DEBUG)
        else:
            self.logger.setLevel(logging.CRITICAL)

    debug = property(get_debug, set_debug)

    def setup_db(self):
        Account.createTable(ifNotExists=True)
        PresenceAccount.createTable(ifNotExists=True)
        User.createTable(ifNotExists=True)
        LegacyJID.createTable(ifNotExists=True)

    def setup_pidfile(self):
        pidfile = open(self.pid_file, "w")
        pidfile.write(str(os.getpid()))
        pidfile.close()

    def _run(self, run_func):
        if self.console:
            self.run_console()
        else:
            try:
                self.setup_pidfile()
                model.db_connection_str = self.db_url
                model.db_connect()
                self.setup_db()
                model.db_disconnect()
                self.logger.debug(self.component_name + " v" +
                                  self.component_version + " is starting ...")
                restart = True
                while restart:
                    (restart, time_to_wait) = run_func()
                    self.wait_event.wait(time_to_wait)
                self.logger.debug(self.component_name + " is exiting")
            finally:
                if os.path.exists(self.pid_file):
                    os.remove(self.pid_file)

    def run(self):
        def run_func():
            component = JCLComponent(jid=self.service_jid,
                                     secret=self.secret,
                                     server=self.server,
                                     port=self.port,
                                     lang=Lang(self.language),
                                     config=self.config,
                                     config_file=self.config_file)
            component.version = self.component_version
            return component.run()
        self._run(run_func)

    def run_console(self):
        """
        Run JCL console. Used to access SQLObject connection
        """
        from IPython.Shell import IPShellEmbed
        # pre-import jcl.model.account to be used in the shell
        import jcl.model.account as account
        model.db_connection_str = self.db_url
        model.db_connect()
        self.setup_db()
        ipshell = IPShellEmbed(["-pi1", self.component_short_name + "[\\#]: "],
                               "Starting " + self.component_name + " v" \
                                   + self.component_version + " console.",
                               "Ending " + self.component_name + " v" \
                                   + self.component_version + " console.")
        ipshell()

def main():
    import sys
    reload(sys)
    sys.setdefaultencoding('utf-8')
    del sys.setdefaultencoding
    import jcl
    from jcl.runner import JCLRunner
    runner = JCLRunner("JCL", jcl.version)
    runner.configure()
    runner.run()

if __name__ == '__main__':
    main()
<commit_msg>Add custom logging formatter<commit_after>##
## runner.py
## Login : David Rousselie <dax@happycoders.org>
## Started on  Thu May 17 22:02:08 2007 David Rousselie
## $Id$
##
## Copyright (C) 2007 David Rousselie
## This program is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 2 of the License, or
## (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program; if not, write to the Free Software
## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##

import logging
import os
import sys
from ConfigParser import ConfigParser
from getopt import gnu_getopt
import threading

from jcl.lang import Lang
from jcl.jabber.component import JCLComponent
import jcl.model as model
from jcl.model.account import Account, PresenceAccount, User, LegacyJID

LOG_FORMATTER = logging.Formatter(fmt="[%(levelname)s] %(asctime)s (%(pathname)s:%(lineno)d): %(message)s")

class JCLRunner(object):

    def __init__(self, component_name, component_version):
        """
        options: list of tuples:
        - short_opt: same as getopt
        - long_opt: same as getopt
        """
        self.component_short_name = "JCL"
        self.component_name = component_name
        self.component_version = component_version
        self.config_file = "jcl.conf"
        self.server = "localhost"
        self.port = 5347
        self.secret = "secret"
        self.service_jid = "jcl.localhost"
        self.language = "en"
        self.db_url = "sqlite:///var/spool/jabber/jcl.db"
        self.pid_file = "/var/run/jabber/jcl.pid"
        self.log_stdout = False
        self.log_file = None
        self.console = False
        self.options = [("c:", "config-file=", None,
                         " FILE\t\t\t\tConfiguration file to use",
                         lambda arg: self.set_attr("config_file", arg)),
                        ("S:", "server=", "jabber",
                         " SERVER_ADDRESS\t\t\tAddress of the Jabber server",
                         lambda arg: self.set_attr("server", arg)),
                        ("P:", "port=", "jabber",
                         " PORT\t\t\t\t\tPort of the Jabber server to connect the component",
                         lambda arg: self.set_attr("port", int(arg))),
                        ("s:", "secret=", "jabber",
                         " SECRET\t\t\t\tComponent password to connect to the Jabber server",
                         lambda arg: self.set_attr("secret", arg)),
                        ("j:", "service-jid=", "jabber",
                         " JID\t\t\t\tJID of the component",
                         lambda arg: self.set_attr("service_jid", arg)),
                        ("l:", "language=", "jabber",
                         " LANG\t\t\t\tDefault Language of the component",
                         lambda arg: self.set_attr("language", arg)),
                        ("u:", "db-url=", "db",
                         " URL\t\t\t\tDatabase URL",
                         lambda arg: self.set_attr("db_url", arg)),
                        ("p:", "pid-file=", "component",
                         " FILE\t\t\t\tPath of the PID file",
                         lambda arg: self.set_attr("pid_file", arg)),
                        ("d", "debug", None,
                         "\t\t\t\t\tEnable debug traces",
                         lambda arg: self.set_attr("debug", True)),
                        ("C", "console", None,
                         "\t\t\t\t\tRun in console mode",
                         lambda arg: self.set_attr("console", True)),
                        ("o", "log-stdout", None,
                         "\t\t\t\t\tLog on stdout",
                         lambda arg: self.set_attr("log_stdout", True)),
                        ("f:", "log-file=", "component",
                         "\t\t\t\t\tLog in file",
                         lambda arg: self.set_attr("log_file", arg)),
                        ("h", "help", None,
                         "\t\t\t\t\tThis help",
                         lambda arg: self.print_help())]
        self.logger = logging.getLogger()
        self.__debug = False
        self.wait_event = threading.Event()

    def set_attr(self, attr, value):
        setattr(self, attr, value)

    def __configure_commandline_args(self, shortopts, longopts, cleanopts):
        (opts, args) = gnu_getopt(sys.argv[1:], shortopts, longopts)
        commandline_args = {}
        for (arg, value) in opts:
            clean_arg = arg.lstrip('-')
            if cleanopts.has_key(clean_arg):
                commandline_args[clean_arg] = value
        return commandline_args

    def __apply_commandline_args(self, commandline_args, cleanopts):
        for arg in commandline_args:
            value = commandline_args[arg]
            self.logger.debug("Applying argument " + arg + " = " +
                              value)
            cleanopts[arg][1](value)

    def __apply_configfile(self, commandline_args, cleanopts):
        if commandline_args.has_key("config-file"):
            self.config_file = commandline_args["config-file"]
        elif commandline_args.has_key("c"):
            self.config_file = commandline_args["c"]
        self.config = ConfigParser()
        self.logger.debug("Loading config file " + self.config_file)
        read_file = self.config.read(self.config_file)
        if read_file == []:
            self.logger.info("Creating empty config file " + self.config_file)
        else:
            for opt in cleanopts:
                if len(opt) > 1:
                    (section, set_func) = cleanopts[opt]
                    if section is not None:
                        attr = opt.replace("-", "_")
                        if self.config.has_section(section) \
                                and self.config.has_option(section, attr):
                            config_property = self.config.get(section, attr)
                            self.logger.debug("Setting " + attr + " = " +
                                              config_property +
                                              " from configuration file " +
                                              self.config_file)
                            set_func(config_property)

    def configure(self):
        """
        Apply configuration from command line and configuration file.
        command line arguments override configuration file properties.
        """
        shortopts = ""
        longopts = []
        cleanopts = {}
        for option in self.options:
            shortopts += option[0]
            longopts += [option[1]]
            cleanopts[option[0][0]] = (option[2], option[4])
            if option[1][-1:] == '=':
                cleanopts[option[1][:-1]] = (option[2], option[4])
            else:
                cleanopts[option[1]] = (option[2], option[4])

        commandline_args = self.__configure_commandline_args(shortopts,
                                                             longopts,
                                                             cleanopts)
        if commandline_args.has_key("debug") or commandline_args.has_key("d"):
            self.debug = True
            self.logger.debug("Debug activated")
        else:
            self.debug = False
        self.__apply_configfile(commandline_args, cleanopts)
        self.__apply_commandline_args(commandline_args, cleanopts)
        if self.log_stdout:
            handler = logging.StreamHandler()
            handler.setFormatter(LOG_FORMATTER)
            logging.Logger.root.addHandler(handler)
        if self.log_file is not None:
            handler = logging.FileHandler(self.log_file)
            handler.setFormatter(LOG_FORMATTER)
            logging.Logger.root.addHandler(handler)

    def _get_help(self):
        help = self.component_name + " v" + self.component_version + " help:\n"
        for option in self.options:
            if option[1][-1:] == '=':
                long_option = option[1][:-1]
            else:
                long_option = option[1]
            help += "\t-" + option[0][0] + ", --" + long_option + option[3] + "\n"
        return help

    def print_help(self):
        print self._get_help()
        sys.exit(0)

    def get_debug(self):
        return self.__debug

    def set_debug(self, value):
        self.__debug = value
        if self.__debug:
            self.logger.setLevel(logging.DEBUG)
        else:
            self.logger.setLevel(logging.CRITICAL)

    debug = property(get_debug, set_debug)

    def setup_db(self):
        Account.createTable(ifNotExists=True)
        PresenceAccount.createTable(ifNotExists=True)
        User.createTable(ifNotExists=True)
        LegacyJID.createTable(ifNotExists=True)

    def setup_pidfile(self):
        pidfile = open(self.pid_file, "w")
        pidfile.write(str(os.getpid()))
        pidfile.close()

    def _run(self, run_func):
        if self.console:
            self.run_console()
        else:
            try:
                self.setup_pidfile()
                model.db_connection_str = self.db_url
                model.db_connect()
                self.setup_db()
                model.db_disconnect()
                self.logger.debug(self.component_name + " v" +
                                  self.component_version + " is starting ...")
                restart = True
                while restart:
                    (restart, time_to_wait) = run_func()
                    self.wait_event.wait(time_to_wait)
                self.logger.debug(self.component_name + " is exiting")
            finally:
                if os.path.exists(self.pid_file):
                    os.remove(self.pid_file)

    def run(self):
        def run_func():
            component = JCLComponent(jid=self.service_jid,
                                     secret=self.secret,
                                     server=self.server,
                                     port=self.port,
                                     lang=Lang(self.language),
                                     config=self.config,
                                     config_file=self.config_file)
            component.version = self.component_version
            return component.run()
        self._run(run_func)

    def run_console(self):
        """
        Run JCL console. Used to access SQLObject connection
        """
        from IPython.Shell import IPShellEmbed
        # pre-import jcl.model.account to be used in the shell
        import jcl.model.account as account
        model.db_connection_str = self.db_url
        model.db_connect()
        self.setup_db()
        ipshell = IPShellEmbed(["-pi1", self.component_short_name + "[\\#]: "],
                               "Starting " + self.component_name + " v" \
                                   + self.component_version + " console.",
                               "Ending " + self.component_name + " v" \
                                   + self.component_version + " console.")
        ipshell()

def main():
    import sys
    reload(sys)
    sys.setdefaultencoding('utf-8')
    del sys.setdefaultencoding
    import jcl
    from jcl.runner import JCLRunner
    runner = JCLRunner("JCL", jcl.version)
    runner.configure()
    runner.run()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import sys
import cv2
import numpy as np
import pickle
from cv_pickle import CVPickle

class CVError(Exception):
    def __init__(self, message):
        self.message = message
    def __str__(self):
        return repr(self.message)

class Assistant:
    def __init__(self, test_mode=False):
        self.test_mode = test_mode

    def handle_frame(self, header, raw_data):
        if self.test_mode:
            return self.handle_test(header, raw_data)

        # PERFORM Cognitive Assistant Processing
        sys.stdout.write("TEST processing: \n%s\n" % header)
        step = header["step"]

        img_array = np.asarray(bytearray(raw_data), dtype=np.int8)
        cv_image = cv2.imdecode(img_array, -1)

        result = {}
        step = int(float(step))
        print "ste"
        print step
        if step == 0:
            # STEP 0: INITIAL CHECK
            result = self.initial_check(cv_image)
        elif step == 1:
             # STEP 1: CHECK MONITOR UPRIGHT
            print "initi setup"
            result = self.initial_setup(cv_image)
        elif step == 2:
            # STEP 2: MONITOR ON RIGHT HAND, PALM UP
        elif step == 3:
            # STEP 3: DISTANCE BTWN CUFF AND WRIST
        elif step == 4:
            # STEP 4: STRAP TIGHTENED
        elif step == 5:
            # STEP 5: ELBOW ON SURFACE
        elif step == 6:
            # STEP 6: LEFT HAND ON HEART
        elif step == 7:
            # STEP 7: CUFF ON SAME LEVEL AS LEFT HAND / START MEASUREMENT
        elif step == 8:
            # STEP 8: ALL INDICATORS ON
        elif step == 9:
            # STEP 9: STABLE READING ON SCREEN

        return result

    def handle_test(self, header, raw_data):
        sys.stdout.write("processing: \n%s\n" % header)
        step = int(float(header["step"]))

        if step == 0:
            result = {  "warning" : "Room looks good.",
                      "next_step" : 1 }

        elif step == 1:
            print "initial setup"
            result = {  "warning": "Device looks okay. Please put it on your arm.",
                          "next_step": 1 }
        elif step == 2:
            print "OK"

        return result

    def initial_check(self, img):

        hsv_image = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        avr = np.average(hsv_image)

        ## TODO: check for noise

        if avr < 55:
            return {  "warning" : "Light not good enough.",
                      "next_step" : 0,
              "debug": avr }
        else:
            return {  "warning" : "Room looks good.",
                      "next_step" : 1 }

        return { "warning" : "There is a problem with the application.",
                 "next_step" : -1 }

    def filter_matches(self, kp1, kp2, matches, ratio = 0.75):
        mkp1, mkp2 = [], []
        for m in matches:
            if len(m) == 2 and m[0].distance < m[1].distance * ratio:
                m = m[0]
                mkp1.append( kp1[m.queryIdx] )
                mkp2.append( kp2[m.trainIdx] )
        kp_pairs = zip(mkp1, mkp2)
        return kp_pairs

    def extract_cached_descriptors(self, name):

        fname = name + ".xml"

        try:
            f = open(fname, 'r')
            temp = pickle.load(f)
            kp, desc = CVPickle().unpickle_keypoints(temp)
        except:
            raise CVError("Error getting the cached descriptors")

        f.close()

        return kp, desc

    def extract_image_descriptors(self, img):
        detector = cv2.SURF(400, 5, 5)
        kp, desc = detector.detectAndCompute(img, None)
        return kp, desc

    def initial_setup(self, img):

        # default warning
        warning = "Please make sure that you are holding the device upright \
                    and to the center of your eyesight."

        try:

            img_device = cv2.imread("device.jpg", 0)
            img_scene = img
            # extract device - cached for now
            kp1, desc1 = self.extract_cached_descriptors("device.jpg")
            # extract scene
            kp2, desc2 = self.extract_image_descriptors(img_scene)

            print 'Scene - %d features' % (len(kp2))

            matcher = cv2.BFMatcher(cv2.NORM_L2)
            raw_matches = matcher.knnMatch(desc1, trainDescriptors = desc2, k = 2) #2
            kp_pairs = self.filter_matches(kp1, kp2, raw_matches)

            mkp1, mkp2 = zip(*kp_pairs)

            p1 = np.float32([kp.pt for kp in mkp1])
            p2 = np.float32([kp.pt for kp in mkp2])

            if len(kp_pairs) >= 4:
                H, status = cv2.findHomography(p1, p2, cv2.RANSAC, 5.0)
            else:
                raise CVError("Could not find Homography")

            h1, w1 = img_device.shape[:2]
            h2, w2 = img_scene.shape[:2]
            vis = np.zeros((max(h1, h2), w1+w2), np.uint8)
            vis[:h1, :w1] = img_device
            vis[:h2, w1:w1+w2] = img_scene
            vis = cv2.cvtColor(vis, cv2.COLOR_GRAY2BGR)

            corners = np.float32([[0, 0], [w1, 0], [w1, h1], [0, h1]])
            corners = np.int32( cv2.perspectiveTransform(corners.reshape(1, -1, 2), H).reshape(-1, 2) + (w1, 0) )
            """
            sample corners
            [[2284 1671]
             [2596 1165]
             [3231 1573]
             [2878 2083]]
            """
            ## check correct orientation
            x1,y1 = corners[0]
            x2,y2 = corners[1]
            x3,y3 = corners[2]
            x4,y4 = corners[3]

            if y1 > y4 or y2 > y3:
                warning = "Incorrect device orientation"
                raise CVError("Could not find Homography")
            else:
                return {  "warning": "Device looks okay. Please put it on your arm.",
                          "next_step": 1 }

        except CVError as e:
            print e
            return {  "warning": warning,
                      "next_step": 1  }

<commit_msg>added error checking for invalid step number<commit_after>import sys
import cv2
import numpy as np
import pickle
from cv_pickle import CVPickle

class CVError(Exception):
    def __init__(self, message):
        self.message = message
    def __str__(self):
        return repr(self.message)

class Assistant:
    def __init__(self, test_mode=False):
        self.test_mode = test_mode

    def handle_frame(self, header, raw_data):
        if self.test_mode:
            return self.handle_test(header, raw_data)

        # PERFORM Cognitive Assistant Processing
        sys.stdout.write("TEST processing: \n%s\n" % header)
        step = header["step"]

        img_array = np.asarray(bytearray(raw_data), dtype=np.int8)
        cv_image = cv2.imdecode(img_array, -1)

        result = {}
        step = int(float(step))
        print "ste"
        print step
        if step == 0:
            # STEP 0: INITIAL CHECK
            result = self.initial_check(cv_image)
        elif step == 1:
             # STEP 1: CHECK MONITOR UPRIGHT
            print "initi setup"
            result = self.initial_setup(cv_image)
        elif step == 2:
            # STEP 2: MONITOR ON RIGHT HAND, PALM UP
        elif step == 3:
            # STEP 3: DISTANCE BTWN CUFF AND WRIST
        elif step == 4:
            # STEP 4: STRAP TIGHTENED
        elif step == 5:
            # STEP 5: ELBOW ON SURFACE
        elif step == 6:
            # STEP 6: LEFT HAND ON HEART
        elif step == 7:
            # STEP 7: CUFF ON SAME LEVEL AS LEFT HAND / START MEASUREMENT
        elif step == 8:
            # STEP 8: ALL INDICATORS ON
        elif step == 9:
            # STEP 9: STABLE READING ON SCREEN
        else:
            # Invalid step number
            result = {
                "warning": "Invalid step",
                "next_step": -1
            }
        return result

    def handle_test(self, header, raw_data):
        sys.stdout.write("processing: \n%s\n" % header)
        step = int(float(header["step"]))

        if step == 0:
            result = {  "warning" : "Room looks good.",
                      "next_step" : 1 }

        elif step == 1:
            print "initial setup"
            result = {  "warning": "Device looks okay. Please put it on your arm.",
                          "next_step": 1 }
        elif step == 2:
            print "OK"

        return result

    def initial_check(self, img):

        hsv_image = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        avr = np.average(hsv_image)

        ## TODO: check for noise

        if avr < 55:
            return {  "warning" : "Light not good enough.",
                      "next_step" : 0,
              "debug": avr }
        else:
            return {  "warning" : "Room looks good.",
                      "next_step" : 1 }

        return { "warning" : "There is a problem with the application.",
                 "next_step" : -1 }

    def filter_matches(self, kp1, kp2, matches, ratio = 0.75):
        mkp1, mkp2 = [], []
        for m in matches:
            if len(m) == 2 and m[0].distance < m[1].distance * ratio:
                m = m[0]
                mkp1.append( kp1[m.queryIdx] )
                mkp2.append( kp2[m.trainIdx] )
        kp_pairs = zip(mkp1, mkp2)
        return kp_pairs

    def extract_cached_descriptors(self, name):

        fname = name + ".xml"

        try:
            f = open(fname, 'r')
            temp = pickle.load(f)
            kp, desc = CVPickle().unpickle_keypoints(temp)
        except:
            raise CVError("Error getting the cached descriptors")

        f.close()

        return kp, desc

    def extract_image_descriptors(self, img):
        detector = cv2.SURF(400, 5, 5)
        kp, desc = detector.detectAndCompute(img, None)
        return kp, desc

    def initial_setup(self, img):

        # default warning
        warning = "Please make sure that you are holding the device upright \
                    and to the center of your eyesight."

        try:

            img_device = cv2.imread("device.jpg", 0)
            img_scene = img
            # extract device - cached for now
            kp1, desc1 = self.extract_cached_descriptors("device.jpg")
            # extract scene
            kp2, desc2 = self.extract_image_descriptors(img_scene)

            print 'Scene - %d features' % (len(kp2))

            matcher = cv2.BFMatcher(cv2.NORM_L2)
            raw_matches = matcher.knnMatch(desc1, trainDescriptors = desc2, k = 2) #2
            kp_pairs = self.filter_matches(kp1, kp2, raw_matches)

            mkp1, mkp2 = zip(*kp_pairs)

            p1 = np.float32([kp.pt for kp in mkp1])
            p2 = np.float32([kp.pt for kp in mkp2])

            if len(kp_pairs) >= 4:
                H, status = cv2.findHomography(p1, p2, cv2.RANSAC, 5.0)
            else:
                raise CVError("Could not find Homography")

            h1, w1 = img_device.shape[:2]
            h2, w2 = img_scene.shape[:2]
            vis = np.zeros((max(h1, h2), w1+w2), np.uint8)
            vis[:h1, :w1] = img_device
            vis[:h2, w1:w1+w2] = img_scene
            vis = cv2.cvtColor(vis, cv2.COLOR_GRAY2BGR)

            corners = np.float32([[0, 0], [w1, 0], [w1, h1], [0, h1]])
            corners = np.int32( cv2.perspectiveTransform(corners.reshape(1, -1, 2), H).reshape(-1, 2) + (w1, 0) )
            """
            sample corners
            [[2284 1671]
             [2596 1165]
             [3231 1573]
             [2878 2083]]
            """
            ## check correct orientation
            x1,y1 = corners[0]
            x2,y2 = corners[1]
            x3,y3 = corners[2]
            x4,y4 = corners[3]

            if y1 > y4 or y2 > y3:
                warning = "Incorrect device orientation"
                raise CVError("Could not find Homography")
            else:
                return {  "warning": "Device looks okay. Please put it on your arm.",
                          "next_step": 1 }

        except CVError as e:
            print e
            return {  "warning": warning,
                      "next_step": 1  }

<|endoftext|>