<commit_before>#!/usr/bin/python

import argparse
import os
import sys
import time

import runtime

def main(args):
    rec_dir = args.replay_directory
    flags = ""
    if args.flags:
        flags = args.flags
        print(flags)

    if args.args_pass:
        argstr = args.args_pass

        # Split the line at the ,'s
        opts = argstr.split(',')

        # Add the ,'s to the flags
        for itm in opts:
            flags += " -"
            itm = itm.replace('=', ' ')
            flags += itm

    omniplay_path = os.environ['OMNIPLAY_DIR']
    if not 'OMNIPLAY_DIR' in os.environ:
        print("Your OMNIPLAY_DIR environment variable is not setup")
        sys.exit(0)
    runtime_info = runtime.RunTimeInfo(omniplay_location=omniplay_path)

    #if not os.path.exists(args.pin_tool):
    #    print("Pin tool %s does not exist, make sure this is the absolute path" %
    #            args.pin_tool)
    # assert os.path.exists(args.pin_tool)
    # assert os.path.exists(runtime_info.tools_location + "/" + args.pin_tool)

    print("Running replay: %s with tool %s" % (args.replay_directory, args.pin_tool))

    stderr_log = "/tmp/stderr_log"
    if args.stderr_log:
        stderr_log = args.stderr_log
    log_f = open(stderr_log, "w")
    tool_f = open("/tmp/tool_log", "w")

    replay_process = runtime_info.replay(rec_dir, log_f, pin=True)

    time.sleep(1)
    start_time = time.time()

    attach_process = runtime_info.attach_tool_extended(replay_process.pid, args.pin_tool,
                                                        tool_f, flags=flags)

    attach_process.wait()
    replay_process.wait()
    end_time = time.time()

    print("done, took %f secs" % (end_time - start_time))

if __name__ == "__main__":
    parser = argparse.ArgumentParser("Run replay with tool script")
    parser.add_argument("replay_directory", help="Directory of the replay to start the query on")
    parser.add_argument("pin_tool", help="Pin tool to run")
    parser.add_argument("-f", "--flags", dest="flags")
    parser.add_argument("-x", "--extended", dest="extended", action="store_true")
    parser.add_argument("-l", "--log", dest="stderr_log")
    parser.add_argument("-o", dest="stderr_log")
    parser.add_argument("-a", "--arg-pass", dest="args_pass")
    main_args = parser.parse_args()
    main(main_args)

<commit_msg>Added option to follow on replay<commit_after>#!/usr/bin/python

import argparse
import os
import sys
import time

import runtime

def main(args):
    rec_dir = args.replay_directory
    flags = ""
    if args.flags:
        flags = args.flags
        print(flags)
    replay_follow = False
    if args.replay_follow:
        replay_follow = True

    if args.args_pass:
        argstr = args.args_pass

        # Split the line at the ,'s
        opts = argstr.split(',')

        # Add the ,'s to the flags
        for itm in opts:
            flags += " -"
            itm = itm.replace('=', ' ')
            flags += itm

    omniplay_path = os.environ['OMNIPLAY_DIR']
    if not 'OMNIPLAY_DIR' in os.environ:
        print("Your OMNIPLAY_DIR environment variable is not setup")
        sys.exit(0)
    runtime_info = runtime.RunTimeInfo(omniplay_location=omniplay_path)

    if args.pin_tool[0] == "/" and not os.path.exists(args.pin_tool):
        if not os.path.exists(runtime_info.tools_location + "/" + args.pin_tool):
            print("Pin tool %s does not exist")
            sys.exit(0)

    # assert os.path.exists(args.pin_tool)
    # assert os.path.exists(runtime_info.tools_location + "/" + args.pin_tool)

    print("Running replay: %s with tool %s" % (args.replay_directory, args.pin_tool))

    stderr_log = "/tmp/stderr_log"
    if args.stderr_log:
        stderr_log = args.stderr_log
    log_f = open(stderr_log, "w")
    tool_f = open("/tmp/tool_log", "w")

    replay_process = runtime_info.replay(rec_dir, log_f, pin=True,
                                                follow=replay_follow)
    replay_pid = replay_process.pid

    time.sleep(1)
    start_time = time.time()

    attach_process = runtime_info.attach_tool_extended(replay_process.pid, args.pin_tool,
                                                        tool_f, flags=flags)

    attach_process.wait()
    replay_process.wait()
    end_time = time.time()

    print("done, took %f secs" % (end_time - start_time))
    print("replay pid was %d" % replay_pid)

if __name__ == "__main__":
    parser = argparse.ArgumentParser("Run replay with tool script")
    parser.add_argument("replay_directory", help="Directory of the replay to start the query on")
    parser.add_argument("pin_tool", help="Pin tool to run")
    parser.add_argument("-f", "--flags", dest="flags")
    parser.add_argument("-x", "--extended", dest="extended", action="store_true")
    parser.add_argument("-l", "--log", dest="stderr_log")
    parser.add_argument("-o", dest="stderr_log")
    parser.add_argument("-a", "--arg-pass", dest="args_pass")
    parser.add_argument("-w", "--follow", dest="replay_follow", action="store_true")
    main_args = parser.parse_args()
    main(main_args)

<|endoftext|>


<commit_before># Copyright (C) 2011-2013 Red Hat, Inc.
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
#
# Author: tasleson
import hashlib

import os
import unittest
import urlparse
import re

import sys
import syslog
import tty
import termios

import functools

## Get a character from stdin without needing a return key pressed.
# Returns the character pressed
def getch():
    fd = sys.stdin.fileno()
    prev = termios.tcgetattr(fd)
    try:
        tty.setraw(sys.stdin.fileno())
        ch = sys.stdin.read(1)
    finally:
        termios.tcsetattr(fd, termios.TCSADRAIN, prev)
    return ch

## Documentation for Proxy class.
#
# Class to encapsulate the actual class we want to call.  When an attempt is
# made to access an attribute that doesn't exist we will raise an LsmError
# instead of the default keyError.
class Proxy(object):
    """
    Used to provide an unambiguous error when a feature is not implemented.
    """

    ## The constructor.
    # @param    self    The object self
    # @param    obj     The object instance to wrap
    def __init__(self, obj=None):
        """
        Constructor which takes an object to wrap.
        """
        self.proxied_obj = obj

    ## Called each time an attribute is requested of the object
    # @param    self    The object self
    # @param    name    Name of the attribute being accessed
    # @return   The result of the method
    def __getattr__(self, name):
        """
        Called each time an attribute is requested of the object
        """
        if hasattr(self.proxied_obj, name):
            return functools.partial(self.present, name)
        else:
            raise LsmError(ErrorNumber.NO_SUPPORT,
                           "Unsupported operation")

    ## Method which is called to invoke the actual method of interest.
    # @param    self                The object self
    # @param    _proxy_method_name  Method to invoke
    # @param    args                Arguments
    # @param    kwargs              Keyword arguments
    # @return   The result of the method invocation
    def present(self, _proxy_method_name, *args, **kwargs):
        """
        Method which is called to invoke the actual method of interest.
        """
        return getattr(self.proxied_obj, _proxy_method_name)(*args, **kwargs)

# variable in client and specified on the command line for the daemon
UDS_PATH = '/var/run/lsm/ipc'

#Set to True for verbose logging
LOG_VERBOSE = True

##Constant for byte size
SIZE_CONS = {
    'B'  : 1,
    'KiB': 2**10,
    'KB' : 10**3,
    'K'  : 2**10,
    'k'  : 2**10,
    'MiB': 2**20,
    'MB' : 10**6,
    'M'  : 2**20,
    'm'  : 2**20,
    'GiB': 2**30,
    'GB' : 10**9,
    'G'  : 2**30,
    'g'  : 2**30,
    'TiB': 2**40,
    'TB' : 10**12,
    'T'  : 2**40,
    't'  : 2**40,
    'PiB': 2**50,
    'PB' : 10**15,
    'P'  : 2**50,
    'p'  : 2**50,
    'EiB': 2**60,
    'EB' : 10**17,
    'E'  : 2**60,
    'e'  : 2**60,
}
SIZE_CONS_CHK_LST = ['EiB', 'PiB', 'TiB', 'GiB', 'MiB', 'KiB']

##Converts the size into human format.
# @param    size    Size in bytes
# @param    human   True|False
# @return Human representation of size
def sh(size, human=False):
    """
    Convert size in bytes to human readable size
    The return string will follow IEC binary prefixes, e.g. '1.9 KiB'
    For size less than 1024, we do nothing but return the int we get.
    TODO: Need a expect to handle when size is not a int. int() might do.
    """
    units = None
    for key_name in SIZE_CONS_CHK_LST:
        if size >= SIZE_CONS[key_name]:
            size /= float(SIZE_CONS[key_name])
            units = key_name
            break
    if not units:
        units = "B"
    if units:
        return "%.2f %s" % (size, units)
    else:
        return size

##Converts the size into human format.
# @param  size    Size in bytes
# @return Human representation of size in IEC binary size prefixes.
def size_bytes_2_size_human(size):
    """
    Convert integer size in bytes to human readable size.
    We are following rules of IEC binary prefixes on size:
        http://en.wikipedia.org/wiki/Gibibyte
    The biggest of unit this function supported is PiB.
    The precision is 2 which means you will get '1.99 KiB'
    """
    return sh(size, 1)

##Converts the size into human format.
# @param size_human Human readable size string, e.g. '1.9 KiB'
# @return Size in bytes
def size_human_2_size_bytes(size_human):
    """
    Convert human readable size string into integer size in bytes.
    Following rules of IEC binary prefixes on size:
        http://en.wikipedia.org/wiki/Gibibyte
    Supported input size_human in these formats:
        '1.9KiB'        # int(1024*1.9)
        '1 KiB'         # 2**10
        '1B'            # 1
        '2K'            # 2*(2**10), treated as '2KiB'
        '2k'            # 2*(2**10), treated as '2KiB'
        '2KB'           # 2*(10**3)
    """
    regex_size_human = re.compile(r"""
        ^
        ([0-9\.]+)          # 1: number
        [ \t]*              # might have space between number and unit
        ([a-zA-Z]*)         # 2: units
        $
    """, re.X)
    regex_match = regex_size_human.match(size_human)
    units = ''
    number = 0
    size_bytes = 0
    if regex_match:
        number = regex_match.group(1)
        units = regex_match.group(2)
        units = units.replace('IB', 'iB')
        if not units:
            return int(number)
        if units in SIZE_CONS:
            size_bytes = SIZE_CONS[units] * float(number)
    return int(size_bytes)

## Common method used to parse a URI.
# @param    uri         The uri to parse
# @param    requires    Optional list of keys that must be present in output
# @param    required_params Optional list of required parameters that
#           must be present.
# @return   A hash of the parsed values.
def uri_parse(uri, requires=None, required_params=None):
    """
    Common uri parse method that optionally can check for what is needed
    before returning successfully.
    """

    rc = {}
    u = urlparse.urlparse(uri)

    if u.scheme:
        rc['scheme'] = u.scheme

    if u.netloc:
        rc['netloc'] = u.netloc

    if u.port:
        rc['port'] = u.port

    if u.hostname:
        rc['host'] = u.hostname

    if u.username:
        rc['username'] = u.username
    else:
        rc['username'] = None

    rc['parameters'] = uri_parameters(u)

    if requires:
        for r in requires:
            if r not in rc:
                raise LsmError(ErrorNumber.PLUGIN_ERROR,
                               'uri missing \"%s\" or is in invalid form' % r)

    if required_params:
        for r in required_params:
            if r not in rc['parameters']:
                raise LsmError(ErrorNumber.PLUGIN_ERROR,
                               'uri missing query parameter %s' % r)
    return rc

## Parses the parameters (Query string) of the URI
# @param    uri     Full uri
# @returns  hash of the query string parameters.
def uri_parameters(uri):
    url = urlparse.urlparse('http:' + uri[2])
    if len(url) >= 5 and len(url[4]):
        return dict([part.split('=') for part in url[4].split('&')])
    else:
        return {}

## Generates the md5 hex digest of passed in parameter.
# @param    t   Item to generate signature on.
# @returns  md5 hex digest.
def md5(t):
    h = hashlib.md5()
    h.update(t)
    return h.hexdigest()

## Converts a list of arguments to string.
# @param    args    Args to join
# @return string of arguments joined together.
def params_to_string(*args):
    return ''.join([str(e) for e in args])

# Unfortunately the process name remains as 'python' so we are using argv[0] in
# the output to allow us to determine which python exe is indeed logging to
# syslog.
# TODO:  On newer versions of python this is no longer true, need to fix.

## Posts a message to the syslogger.
# @param    level   Logging level
# @param    prg     Program name
# @param    msg     Message to log.
def post_msg(level, prg, msg):
    """
    If a message includes new lines we will create multiple syslog
    entries so that the message is readable.  Otherwise it isn't very readable.
    Hopefully we won't be logging much :-)
    """
    for l in msg.split('\n'):
        if len(l):
            syslog.syslog(level, prg + ": " + l)

def Error(*msg):
    post_msg(syslog.LOG_ERR, os.path.basename(sys.argv[0]),
             params_to_string(*msg))

def Info(*msg):
    if LOG_VERBOSE:
        post_msg(syslog.LOG_INFO, os.path.basename(sys.argv[0]),
                 params_to_string(*msg))

class SocketEOF(Exception):
    """
    Exception class to indicate when we read zero bytes from a socket.
    """
    pass

class LsmError(Exception):
    def __init__(self, code, message, data=None, *args, **kwargs):
        """
        Class represents an error.
        """
        Exception.__init__(self, *args, **kwargs)
        self.code = code
        self.msg = message
        self.data = data

    def __str__(self):
        if self.data is not None:
            return "error: %s msg: %s data: %s" % \
                   (self.code, self.msg, self.data)
        else:
            return "error: %s msg: %s " % (self.code, self.msg)

def addl_error_data(domain, level, exception, debug=None, debug_data=None):
    """
    Used for gathering additional information about an error.
    """
    return {'domain': domain, 'level': level, 'exception': exception,
            'debug': debug, 'debug_data': debug_data}

def get_class(class_name):
    """
    Given a class name it returns the class, caller will then
    need to run the constructor to create.
    """
    parts = class_name.split('.')
    module = ".".join(parts[:-1])
    if len(module):
        m = __import__(module)
        for comp in parts[1:]:
            m = getattr(m, comp)
    else:
        m = __import__('__main__')
        m = getattr(m, class_name)
    return m

class ErrorLevel(object):
    NONE = 0
    WARNING = 1
    ERROR = 2

#Note: Some of these don't make sense for python, but they do for other
#Languages so we will be keeping them consistent even though we won't be
#using them.
class ErrorNumber(object):
    OK = 0
    INTERNAL_ERROR = 1
    JOB_STARTED = 7
    INDEX_BOUNDS = 10
    TIMEOUT = 11
    DAEMON_NOT_RUNNING = 12

    EXISTS_ACCESS_GROUP = 50
    EXISTS_FS = 51
    EXISTS_INITIATOR = 52
    EXISTS_NAME = 53
    FS_NOT_EXPORTED = 54
    INITIATOR_NOT_IN_ACCESS_GROUP = 55

    INVALID_ACCESS_GROUP = 100
    INVALID_ARGUMENT = 101
    INVALID_CONN = 102
    INVALID_ERR = 103
    INVALID_FS = 104
    INVALID_INIT = 105
    INVALID_JOB = 106
    INVALID_NAME = 107
    INVALID_NFS = 108
    INVALID_PLUGIN = 109
    INVALID_POOL = 110
    INVALID_SL = 111
    INVALID_SS = 112
    INVALID_URI = 113
    INVALID_VALUE = 114
    INVALID_VOLUME = 115
    INVALID_CAPABILITY = 116
    INVALID_SYSTEM = 117
    INVALID_IQN = 118

    IS_MAPPED = 125

    NO_CONNECT = 150
    NO_MAPPING = 151
    NO_MEMORY = 152
    NO_SUPPORT = 153

    NOT_FOUND_ACCESS_GROUP = 200
    NOT_FOUND_FS = 201
    NOT_FOUND_JOB = 202
    NOT_FOUND_POOL = 203
    NOT_FOUND_SS = 204
    NOT_FOUND_VOLUME = 205
    NOT_FOUND_NFS_EXPORT = 206
    NOT_FOUND_INITIATOR = 207

    NOT_IMPLEMENTED = 225
    NOT_LICENSED = 226

    OFF_LINE = 250
    ON_LINE = 251

    PLUGIN_AUTH_FAILED = 300
    PLUGIN_DLOPEN = 301
    PLUGIN_DLSYM = 302
    PLUGIN_ERROR = 303
    PLUGIN_MISSING_HOST = 304
    PLUGIN_MISSING_NS = 305
    PLUGIN_MISSING_PORT = 306
    PLUGIN_PERMISSION = 307
    PLUGIN_REGISTRATION = 308
    PLUGIN_UNKNOWN_HOST = 309
    PLUGIN_TIMEOUT = 310
    PLUGIN_NOT_EXIST = 311

    SIZE_INSUFFICIENT_SPACE = 350
    SIZE_SAME = 351
    SIZE_TOO_LARGE = 352
    SIZE_TOO_SMALL = 353
    SIZE_LIMIT_REACHED = 354

    TRANSPORT_COMMUNICATION = 400
    TRANSPORT_SERIALIZATION = 401
    TRANSPORT_INVALID_ARG = 402

    UNSUPPORTED_INITIATOR_TYPE = 450
    UNSUPPORTED_PROVISIONING = 451
    UNSUPPORTED_REPLICATION_TYPE = 452

class JobStatus(object):
    INPROGRESS = 1
    COMPLETE = 2
    STOPPED = 3
    ERROR = 4

class TestCommon(unittest.TestCase):
    def setUp(self):
        pass

    def test_simple(self):

        try:
            raise SocketEOF()
        except SocketEOF as e:
            self.assertTrue(isinstance(e, SocketEOF))

        try:
            raise LsmError(10, 'Message', 'Data')
        except LsmError as e:
            self.assertTrue(e.code == 10 and e.msg == 'Message' and
                            e.data == 'Data')

        ed = addl_error_data('domain', 'level', 'exception', 'debug',
                             'debug_data')
        self.assertTrue(ed['domain'] == 'domain' and ed['level'] == 'level'
                        and ed['debug'] == 'debug'
                        and ed['exception'] == 'exception'
                        and ed['debug_data'] == 'debug_data')

    def tearDown(self):
        pass

if __name__ == '__main__':
    unittest.main()
<commit_msg>common.py: Restore default ability for sizes to be in bytes.<commit_after># Copyright (C) 2011-2013 Red Hat, Inc.
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
#
# Author: tasleson
import hashlib

import os
import unittest
import urlparse
import re

import sys
import syslog
import tty
import termios

import functools

## Get a character from stdin without needing a return key pressed.
# Returns the character pressed
def getch():
    fd = sys.stdin.fileno()
    prev = termios.tcgetattr(fd)
    try:
        tty.setraw(sys.stdin.fileno())
        ch = sys.stdin.read(1)
    finally:
        termios.tcsetattr(fd, termios.TCSADRAIN, prev)
    return ch

## Documentation for Proxy class.
#
# Class to encapsulate the actual class we want to call.  When an attempt is
# made to access an attribute that doesn't exist we will raise an LsmError
# instead of the default keyError.
class Proxy(object):
    """
    Used to provide an unambiguous error when a feature is not implemented.
    """

    ## The constructor.
    # @param    self    The object self
    # @param    obj     The object instance to wrap
    def __init__(self, obj=None):
        """
        Constructor which takes an object to wrap.
        """
        self.proxied_obj = obj

    ## Called each time an attribute is requested of the object
    # @param    self    The object self
    # @param    name    Name of the attribute being accessed
    # @return   The result of the method
    def __getattr__(self, name):
        """
        Called each time an attribute is requested of the object
        """
        if hasattr(self.proxied_obj, name):
            return functools.partial(self.present, name)
        else:
            raise LsmError(ErrorNumber.NO_SUPPORT,
                           "Unsupported operation")

    ## Method which is called to invoke the actual method of interest.
    # @param    self                The object self
    # @param    _proxy_method_name  Method to invoke
    # @param    args                Arguments
    # @param    kwargs              Keyword arguments
    # @return   The result of the method invocation
    def present(self, _proxy_method_name, *args, **kwargs):
        """
        Method which is called to invoke the actual method of interest.
        """
        return getattr(self.proxied_obj, _proxy_method_name)(*args, **kwargs)

# variable in client and specified on the command line for the daemon
UDS_PATH = '/var/run/lsm/ipc'

#Set to True for verbose logging
LOG_VERBOSE = True

##Constant for byte size
SIZE_CONS = {
    'B'  : 1,
    'KiB': 2**10,
    'KB' : 10**3,
    'K'  : 2**10,
    'k'  : 2**10,
    'MiB': 2**20,
    'MB' : 10**6,
    'M'  : 2**20,
    'm'  : 2**20,
    'GiB': 2**30,
    'GB' : 10**9,
    'G'  : 2**30,
    'g'  : 2**30,
    'TiB': 2**40,
    'TB' : 10**12,
    'T'  : 2**40,
    't'  : 2**40,
    'PiB': 2**50,
    'PB' : 10**15,
    'P'  : 2**50,
    'p'  : 2**50,
    'EiB': 2**60,
    'EB' : 10**17,
    'E'  : 2**60,
    'e'  : 2**60,
}
SIZE_CONS_CHK_LST = ['EiB', 'PiB', 'TiB', 'GiB', 'MiB', 'KiB']

##Converts the size into human format.
# @param    size    Size in bytes
# @param    human   True|False
# @return Human representation of size
def sh(size, human=False):
    """
    Convert size in bytes to human readable size
    The return string will follow IEC binary prefixes, e.g. '1.9 KiB'
    For size less than 1024, we do nothing but return the int we get.
    TODO: Need a expect to handle when size is not a int. int() might do.
    """
    units = None

    if human:
        for key_name in SIZE_CONS_CHK_LST:
            if size >= SIZE_CONS[key_name]:
                size /= float(SIZE_CONS[key_name])
                units = key_name
                break
        if not units:
            units = "B"
    if units:
        return "%.2f %s" % (size, units)
    else:
        return size

##Converts the size into human format.
# @param  size    Size in bytes
# @return Human representation of size in IEC binary size prefixes.
def size_bytes_2_size_human(size):
    """
    Convert integer size in bytes to human readable size.
    We are following rules of IEC binary prefixes on size:
        http://en.wikipedia.org/wiki/Gibibyte
    The biggest of unit this function supported is PiB.
    The precision is 2 which means you will get '1.99 KiB'
    """
    return sh(size, True)

##Converts the size into human format.
# @param size_human Human readable size string, e.g. '1.9 KiB'
# @return Size in bytes
def size_human_2_size_bytes(size_human):
    """
    Convert human readable size string into integer size in bytes.
    Following rules of IEC binary prefixes on size:
        http://en.wikipedia.org/wiki/Gibibyte
    Supported input size_human in these formats:
        '1.9KiB'        # int(1024*1.9)
        '1 KiB'         # 2**10
        '1B'            # 1
        '2K'            # 2*(2**10), treated as '2KiB'
        '2k'            # 2*(2**10), treated as '2KiB'
        '2KB'           # 2*(10**3)
    """
    regex_size_human = re.compile(r"""
        ^
        ([0-9\.]+)          # 1: number
        [ \t]*              # might have space between number and unit
        ([a-zA-Z]*)         # 2: units
        $
    """, re.X)
    regex_match = regex_size_human.match(size_human)
    units = ''
    number = 0
    size_bytes = 0
    if regex_match:
        number = regex_match.group(1)
        units = regex_match.group(2)
        units = units.replace('IB', 'iB')
        if not units:
            return int(number)
        if units in SIZE_CONS:
            size_bytes = SIZE_CONS[units] * float(number)
    return int(size_bytes)

## Common method used to parse a URI.
# @param    uri         The uri to parse
# @param    requires    Optional list of keys that must be present in output
# @param    required_params Optional list of required parameters that
#           must be present.
# @return   A hash of the parsed values.
def uri_parse(uri, requires=None, required_params=None):
    """
    Common uri parse method that optionally can check for what is needed
    before returning successfully.
    """

    rc = {}
    u = urlparse.urlparse(uri)

    if u.scheme:
        rc['scheme'] = u.scheme

    if u.netloc:
        rc['netloc'] = u.netloc

    if u.port:
        rc['port'] = u.port

    if u.hostname:
        rc['host'] = u.hostname

    if u.username:
        rc['username'] = u.username
    else:
        rc['username'] = None

    rc['parameters'] = uri_parameters(u)

    if requires:
        for r in requires:
            if r not in rc:
                raise LsmError(ErrorNumber.PLUGIN_ERROR,
                               'uri missing \"%s\" or is in invalid form' % r)

    if required_params:
        for r in required_params:
            if r not in rc['parameters']:
                raise LsmError(ErrorNumber.PLUGIN_ERROR,
                               'uri missing query parameter %s' % r)
    return rc

## Parses the parameters (Query string) of the URI
# @param    uri     Full uri
# @returns  hash of the query string parameters.
def uri_parameters(uri):
    url = urlparse.urlparse('http:' + uri[2])
    if len(url) >= 5 and len(url[4]):
        return dict([part.split('=') for part in url[4].split('&')])
    else:
        return {}

## Generates the md5 hex digest of passed in parameter.
# @param    t   Item to generate signature on.
# @returns  md5 hex digest.
def md5(t):
    h = hashlib.md5()
    h.update(t)
    return h.hexdigest()

## Converts a list of arguments to string.
# @param    args    Args to join
# @return string of arguments joined together.
def params_to_string(*args):
    return ''.join([str(e) for e in args])

# Unfortunately the process name remains as 'python' so we are using argv[0] in
# the output to allow us to determine which python exe is indeed logging to
# syslog.
# TODO:  On newer versions of python this is no longer true, need to fix.

## Posts a message to the syslogger.
# @param    level   Logging level
# @param    prg     Program name
# @param    msg     Message to log.
def post_msg(level, prg, msg):
    """
    If a message includes new lines we will create multiple syslog
    entries so that the message is readable.  Otherwise it isn't very readable.
    Hopefully we won't be logging much :-)
    """
    for l in msg.split('\n'):
        if len(l):
            syslog.syslog(level, prg + ": " + l)

def Error(*msg):
    post_msg(syslog.LOG_ERR, os.path.basename(sys.argv[0]),
             params_to_string(*msg))

def Info(*msg):
    if LOG_VERBOSE:
        post_msg(syslog.LOG_INFO, os.path.basename(sys.argv[0]),
                 params_to_string(*msg))

class SocketEOF(Exception):
    """
    Exception class to indicate when we read zero bytes from a socket.
    """
    pass

class LsmError(Exception):
    def __init__(self, code, message, data=None, *args, **kwargs):
        """
        Class represents an error.
        """
        Exception.__init__(self, *args, **kwargs)
        self.code = code
        self.msg = message
        self.data = data

    def __str__(self):
        if self.data is not None:
            return "error: %s msg: %s data: %s" % \
                   (self.code, self.msg, self.data)
        else:
            return "error: %s msg: %s " % (self.code, self.msg)

def addl_error_data(domain, level, exception, debug=None, debug_data=None):
    """
    Used for gathering additional information about an error.
    """
    return {'domain': domain, 'level': level, 'exception': exception,
            'debug': debug, 'debug_data': debug_data}

def get_class(class_name):
    """
    Given a class name it returns the class, caller will then
    need to run the constructor to create.
    """
    parts = class_name.split('.')
    module = ".".join(parts[:-1])
    if len(module):
        m = __import__(module)
        for comp in parts[1:]:
            m = getattr(m, comp)
    else:
        m = __import__('__main__')
        m = getattr(m, class_name)
    return m

class ErrorLevel(object):
    NONE = 0
    WARNING = 1
    ERROR = 2

#Note: Some of these don't make sense for python, but they do for other
#Languages so we will be keeping them consistent even though we won't be
#using them.
class ErrorNumber(object):
    OK = 0
    INTERNAL_ERROR = 1
    JOB_STARTED = 7
    INDEX_BOUNDS = 10
    TIMEOUT = 11
    DAEMON_NOT_RUNNING = 12

    EXISTS_ACCESS_GROUP = 50
    EXISTS_FS = 51
    EXISTS_INITIATOR = 52
    EXISTS_NAME = 53
    FS_NOT_EXPORTED = 54
    INITIATOR_NOT_IN_ACCESS_GROUP = 55

    INVALID_ACCESS_GROUP = 100
    INVALID_ARGUMENT = 101
    INVALID_CONN = 102
    INVALID_ERR = 103
    INVALID_FS = 104
    INVALID_INIT = 105
    INVALID_JOB = 106
    INVALID_NAME = 107
    INVALID_NFS = 108
    INVALID_PLUGIN = 109
    INVALID_POOL = 110
    INVALID_SL = 111
    INVALID_SS = 112
    INVALID_URI = 113
    INVALID_VALUE = 114
    INVALID_VOLUME = 115
    INVALID_CAPABILITY = 116
    INVALID_SYSTEM = 117
    INVALID_IQN = 118

    IS_MAPPED = 125

    NO_CONNECT = 150
    NO_MAPPING = 151
    NO_MEMORY = 152
    NO_SUPPORT = 153

    NOT_FOUND_ACCESS_GROUP = 200
    NOT_FOUND_FS = 201
    NOT_FOUND_JOB = 202
    NOT_FOUND_POOL = 203
    NOT_FOUND_SS = 204
    NOT_FOUND_VOLUME = 205
    NOT_FOUND_NFS_EXPORT = 206
    NOT_FOUND_INITIATOR = 207

    NOT_IMPLEMENTED = 225
    NOT_LICENSED = 226

    OFF_LINE = 250
    ON_LINE = 251

    PLUGIN_AUTH_FAILED = 300
    PLUGIN_DLOPEN = 301
    PLUGIN_DLSYM = 302
    PLUGIN_ERROR = 303
    PLUGIN_MISSING_HOST = 304
    PLUGIN_MISSING_NS = 305
    PLUGIN_MISSING_PORT = 306
    PLUGIN_PERMISSION = 307
    PLUGIN_REGISTRATION = 308
    PLUGIN_UNKNOWN_HOST = 309
    PLUGIN_TIMEOUT = 310
    PLUGIN_NOT_EXIST = 311

    SIZE_INSUFFICIENT_SPACE = 350
    SIZE_SAME = 351
    SIZE_TOO_LARGE = 352
    SIZE_TOO_SMALL = 353
    SIZE_LIMIT_REACHED = 354

    TRANSPORT_COMMUNICATION = 400
    TRANSPORT_SERIALIZATION = 401
    TRANSPORT_INVALID_ARG = 402

    UNSUPPORTED_INITIATOR_TYPE = 450
    UNSUPPORTED_PROVISIONING = 451
    UNSUPPORTED_REPLICATION_TYPE = 452

class JobStatus(object):
    INPROGRESS = 1
    COMPLETE = 2
    STOPPED = 3
    ERROR = 4

class TestCommon(unittest.TestCase):
    def setUp(self):
        pass

    def test_simple(self):

        try:
            raise SocketEOF()
        except SocketEOF as e:
            self.assertTrue(isinstance(e, SocketEOF))

        try:
            raise LsmError(10, 'Message', 'Data')
        except LsmError as e:
            self.assertTrue(e.code == 10 and e.msg == 'Message' and
                            e.data == 'Data')

        ed = addl_error_data('domain', 'level', 'exception', 'debug',
                             'debug_data')
        self.assertTrue(ed['domain'] == 'domain' and ed['level'] == 'level'
                        and ed['debug'] == 'debug'
                        and ed['exception'] == 'exception'
                        and ed['debug_data'] == 'debug_data')

    def tearDown(self):
        pass

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># Copyright (c) 2012 Spotify AB
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

import task
import file  # wtf @ naming
import rpc
import parameter
import configuration
import interface
import target

Task = task.Task
ExternalTask = task.ExternalTask
WrapperTask = task.WrapperTask
Target = target.Target

File = file.File  # TODO: remove, should be LocalTarget
LocalTarget = File
Parameter = parameter.Parameter
RemoteScheduler = rpc.RemoteScheduler
RPCError = rpc.RPCError

expose = interface.expose
expose_main = interface.expose_main
run = interface.run
build = interface.build

# TODO: how can we get rid of these?
DateHourParameter = parameter.DateHourParameter
DateParameter = parameter.DateParameter
IntParameter = parameter.IntParameter
BooleanParameter = parameter.BooleanParameter
DateIntervalParameter = parameter.DateIntervalParameter

namespace = task.namespace
<commit_msg>make FloatParameter accessible just as the others<commit_after># Copyright (c) 2012 Spotify AB
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

import task
import file  # wtf @ naming
import rpc
import parameter
import configuration
import interface
import target

Task = task.Task
ExternalTask = task.ExternalTask
WrapperTask = task.WrapperTask
Target = target.Target

File = file.File  # TODO: remove, should be LocalTarget
LocalTarget = File
Parameter = parameter.Parameter
RemoteScheduler = rpc.RemoteScheduler
RPCError = rpc.RPCError

expose = interface.expose
expose_main = interface.expose_main
run = interface.run
build = interface.build

# TODO: how can we get rid of these?
DateHourParameter = parameter.DateHourParameter
DateParameter = parameter.DateParameter
IntParameter = parameter.IntParameter
FloatParameter = parameter.FloatParameter
BooleanParameter = parameter.BooleanParameter
DateIntervalParameter = parameter.DateIntervalParameter

namespace = task.namespace
<|endoftext|>


<commit_before>import os
import sonnet as snt
import tensorflow as tf
import click

from .models.fasterrcnn import FasterRCNN
from .models.pretrained import VGG, ResNetV2
from .dataset import TFRecordDataset
from .utils.config import (
    load_config, merge_into, kwargs_to_config, parse_override
)
from .utils.vars import get_saver

MODELS = {
    'fasterrcnn': FasterRCNN,
}

PRETRAINED_MODULES = {
    'vgg': VGG,
    'vgg_16': VGG,
    'resnet': ResNetV2,
    'resnetv2': ResNetV2,
}

OPTIMIZERS = {
    'adam': tf.train.AdamOptimizer,
    'momentum': tf.train.MomentumOptimizer,
}

LEARNING_RATE_DECAY_METHODS = set([
    'piecewise_constant', 'exponential_decay', 'none'
])

@click.command(help='Train models')
@click.argument('model_type', type=click.Choice(MODELS.keys()))
@click.option('config_file', '--config', '-c', type=click.File('r'), help='Config to use.')
@click.option('override_params', '--override', '-o', multiple=True, help='Override model config params.')
@click.option('--continue-training', is_flag=True, help='Continue training using model dir and run name.')
@click.option('--model-dir', default='models/', help='Directory to save the partial trained models.')
@click.option('--checkpoint-file', help='File for the weights of RPN and RCNN for resuming training.')
@click.option('--ignore-scope', help='Used to ignore variables when loading from checkpoint (set to "frcnn" when loading RPN and wanting to train complete network)')
@click.option('--log-dir', default='/tmp/frcnn/', help='Directory for Tensorboard logs.')
@click.option('--save-every', default=1000, help='Save checkpoint after that many batches.')
@click.option('--tf-debug', is_flag=True, help='Create debugging Tensorflow session with tfdb.')
@click.option('--debug', is_flag=True, help='Debug mode (DEBUG log level and intermediate variables are returned)')
@click.option('--run-name', default='train', help='Run name used to log in Tensorboard and isolate checkpoints.')
@click.option('--no-log/--log', default=False, help='Save or don\'t summary logs.')
@click.option('--display-every', default=500, type=int, help='Show image debug information every N batches (debug mode must be activated)')
@click.option('--random-shuffle/--fifo', default=True, help='Ingest data from dataset in random order.')
@click.option('--save-timeline', is_flag=True, help='Save timeline of execution (debug mode must be activated).')
@click.option('--summary-every', default=1, type=int, help='Save summary logs every N batches.')
@click.option('--full-trace', is_flag=True, help='Run graph session with FULL_TRACE config (for memory and running time debugging)')
@click.option('--initial-learning-rate', default=0.0001, type=float, help='Initial learning date.')
@click.option('--learning-rate-decay', default=10000, type=int, help='Decay learning date after N batches.')
@click.option('--learning-rate-decay-method', default='piecewise_constant', type=click.Choice(LEARNING_RATE_DECAY_METHODS), help='Tipo of learning rate decay to use.')
@click.option('optimizer_type', '--optimizer', default='momentum', type=click.Choice(OPTIMIZERS.keys()), help='Optimizer to use.')
@click.option('--momentum', default=0.9, type=float, help='Momentum to use when using the MomentumOptimizer.')
def train(model_type, config_file, override_params, continue_training, **kwargs):

    model_class = MODELS[model_type.lower()]
    config = model_class.base_config

    if config_file:
        # If we have a custom config file overwritting default settings
        # then we merge those values to the base_config.
        custom_config = load_config(config_file)
        config = merge_into(custom_config, config)

    # Load train extra options
    config.train = merge_into(kwargs_to_config(kwargs), config.train)

    if override_params:
        override_config = parse_override(override_params)
        config = merge_into(override_config, config)

    if config.train.debug or config.train.tf_debug:
        tf.logging.set_verbosity(tf.logging.DEBUG)
    else:
        tf.logging.set_verbosity(tf.logging.INFO)

    model = model_class(config)
    pretrained = PRETRAINED_MODULES[config.pretrained.net](
        config.pretrained
    )
    dataset = TFRecordDataset(config)
    train_dataset = dataset()

    train_image = train_dataset['image']
    train_filename = train_dataset['filename']
    train_scale_factor = train_dataset['scale_factor']

    train_bboxes = train_dataset['bboxes']

    # TODO: This is not the best place to configure rank? Why is rank not
    # transmitted through the queue
    train_image.set_shape((None, None, 3))
    # We add fake batch dimension to train data. TODO: DEFINITELY NOT THE BEST
    # PLACE
    train_image = tf.expand_dims(train_image, 0)

    pretrained_dict = pretrained(train_image)
    prediction_dict = model(train_image, pretrained_dict['net'], train_bboxes)

    total_loss = model.loss(prediction_dict)

    initial_global_step = 0
    checkpoint_path = os.path.join(
        config.train.model_dir, config.train.run_name,
        model.scope_name
    )
    if continue_training:
        last_checkpoint = tf.train.latest_checkpoint(
            os.path.dirname(checkpoint_path)
        )
        if not last_checkpoint:
            raise ValueError(
                'Could not find checkpoint in {}. Check run name'.format(
                    checkpoint_path))
        initial_global_step = int(last_checkpoint.split('-')[-1])
        tf.logging.info('Starting training from global_step {}'.format(
            initial_global_step))
        tf.logging.info('Using checkpoint "{}"'.format(last_checkpoint))
        config.train.checkpoint_file = last_checkpoint

    global_step_init = tf.constant_initializer(initial_global_step)

    global_step = tf.get_variable(
        name="global_step", shape=[], dtype=tf.int64,
        initializer=global_step_init, trainable=False,
        collections=[tf.GraphKeys.GLOBAL_VARIABLES, tf.GraphKeys.GLOBAL_STEP]
    )

    # load_weights returns no_op when empty checkpoint_file.
    load_op = pretrained.load_weights(
        checkpoint_file=config.pretrained.weights
    )

    saver = get_saver((model, pretrained, ))
    if config.train.ignore_scope:
        partial_loader = get_saver(
            (model, pretrained, ), ignore_scope=config.train.ignore_scope
        )

    learning_rate_decay_method = config.train.learning_rate_decay_method
    if not learning_rate_decay_method or learning_rate_decay_method == 'none':
        learning_rate = config.train.initial_learning_rate
    elif learning_rate_decay_method == 'piecewise_constant':
        learning_rate = tf.train.piecewise_constant(
            global_step, boundaries=[
                tf.cast(config.train.learning_rate_decay, tf.int64), ],
            values=[
                config.train.initial_learning_rate,
                config.train.initial_learning_rate * 0.1
            ], name='learning_rate_piecewise_constant'
        )
    elif learning_rate_decay_method == 'exponential_decay':
        learning_rate = tf.train.exponential_decay(
            learning_rate=config.initial_learning_rate,
            global_step=global_step,
            decay_steps=config.train.learning_rate_decay, decay_rate=0.96,
            staircase=True, name='learning_rate_with_decay'
        )
    else:
        raise ValueError(
            'Invalid learning_rate method "{}"'.format(
                learning_rate_decay_method))

    tf.summary.scalar('losses/learning_rate', learning_rate)

    optimizer_cls = OPTIMIZERS[config.train.optimizer_type]
    if config.train.optimizer_type == 'momentum':
        optimizer = optimizer_cls(learning_rate, config.train.momentum)
    else:
        optimizer = optimizer_cls(learning_rate)

    trainable_vars = snt.get_variables_in_module(model)
    if config.pretrained.trainable:
        pretrained_trainable_vars = pretrained.get_trainable_vars()
        tf.logging.info('Training {} vars from pretrained module.'.format(
            len(pretrained_trainable_vars)))
        trainable_vars += pretrained_trainable_vars
    else:
        tf.logging.info('Not training variables from pretrained module')

    grads_and_vars = optimizer.compute_gradients(total_loss, trainable_vars)

    # Clip by norm. Grad can be null when not training some modules.
    with tf.name_scope('clip_gradients_by_norm'):
        grads_and_vars = [
            (tf.check_numerics(tf.clip_by_norm(gv[0], 10.), 'Invalid gradient'), gv[1])
            if gv[0] is not None else gv
            for gv in grads_and_vars
        ]

    train_op = optimizer.apply_gradients(
        grads_and_vars, global_step=global_step
    )

    # Create initializer for variables. Queue-related variables need a special
    # initializer.
    init_op = tf.group(
        tf.global_variables_initializer(),
        tf.local_variables_initializer()
    )

    metric_ops = tf.get_collection('metric_ops')
    metrics = tf.get_collection('metrics')

    tf.logging.info('Starting training for {}'.format(model))

    summary_dir = os.path.join(config.train.log_dir, config.train.run_name)
    summarizer = tf.summary.merge([
        tf.summary.merge_all(),
        model.summary,
    ])

    with tf.Session() as sess:

        sess.run(init_op)  # initialized variables
        sess.run(load_op)  # load pretrained weights

        # Restore all variables from checkpoint file
        if config.train.checkpoint_file:
            # If ignore_scope is set, we don't load those variables from checkpoint.
            if config.train.ignore_scope:
                partial_loader.restore(sess, config.train.checkpoint_file)
            else:
                saver.restore(sess, config.train.checkpoint_file)

        if not config.train.no_log:
            writer = tf.summary.FileWriter(summary_dir, sess.graph)

        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess, coord=coord)
        count_images = 0
        step = initial_global_step

        if config.train.tf_debug:
            from tensorflow.python import debug as tensorflow_debug
            sess = tensorflow_debug.LocalCLIDebugWrapperSession(sess)
            sess.add_tensor_filter(
                'has_inf_or_nan', tensorflow_debug.has_inf_or_nan
            )

        try:
            while not coord.should_stop():
                run_metadata = None
                if (step + 1) % config.train.summary_every == 0:
                    run_metadata = tf.RunMetadata()

                run_options = None
                if config.train.full_trace:
                    run_options = tf.RunOptions(
                        trace_level=tf.RunOptions.FULL_TRACE
                    )

                (
                    _, summary, train_loss, step, pred_dict, filename,
                    scale_factor, *_
                ) = sess.run([
                    train_op, summarizer, total_loss, global_step,
                    prediction_dict, train_filename, train_scale_factor,
                    metric_ops
                ], run_metadata=run_metadata, options=run_options)

                write_summary = (
                    not config.train.no_log and
                    step % config.train.summary_every == 0
                )
                if write_summary:
                    writer.add_summary(summary, step)
                    writer.add_run_metadata(
                        run_metadata, str(step)
                    )

                if config.train.debug and step % config.train.display_every == 0:
                    from luminoth.utils.image_vis import (
                        add_images_to_tensoboard
                    )
                    print('Scaled image with {}'.format(scale_factor))
                    print('Image size: {}'.format(pred_dict['image_shape']))
                    add_images_to_tensoboard(
                        pred_dict, step, summary_dir, config.network.with_rcnn
                    )

                    if config.train.save_timeline:
                        from tensorflow.python.client import timeline
                        run_tmln = timeline.Timeline(
                            run_metadata.step_stats)
                        chrome_trace = run_tmln.generate_chrome_trace_format()
                        with open('timeline_{}.json'.format(step), 'w') as f:
                            f.write(chrome_trace)

                count_images += 1

                tf.logging.info('train_loss: {}'.format(train_loss))
                tf.logging.info('step: {}, file: {}'.format(step, filename))

                if not config.train.no_log:
                    if step % config.train.save_every == 0:
                        # We don't support partial saver.
                        saver.save(sess, checkpoint_path, global_step=step)

        except tf.errors.OutOfRangeError:
            tf.logging.info('iter = {}, train_loss = {:.2f}'.format(
                step, train_loss))
            tf.logging.info('finished training -- epoch limit reached')
            tf.logging.info('count_images = {}'.format(count_images))
        finally:
            coord.request_stop()

        # Wait for all threads to stop.
        coord.join(threads)

if __name__ == '__main__':
    train()
<commit_msg>Use get_checkpoint_state instead of latest_checkpoint<commit_after>import os
import sonnet as snt
import tensorflow as tf
import click

from .models.fasterrcnn import FasterRCNN
from .models.pretrained import VGG, ResNetV2
from .dataset import TFRecordDataset
from .utils.config import (
    load_config, merge_into, kwargs_to_config, parse_override
)
from .utils.vars import get_saver

MODELS = {
    'fasterrcnn': FasterRCNN,
}

PRETRAINED_MODULES = {
    'vgg': VGG,
    'vgg_16': VGG,
    'resnet': ResNetV2,
    'resnetv2': ResNetV2,
}

OPTIMIZERS = {
    'adam': tf.train.AdamOptimizer,
    'momentum': tf.train.MomentumOptimizer,
}

LEARNING_RATE_DECAY_METHODS = set([
    'piecewise_constant', 'exponential_decay', 'none'
])

@click.command(help='Train models')
@click.argument('model_type', type=click.Choice(MODELS.keys()))
@click.option('config_file', '--config', '-c', type=click.File('r'), help='Config to use.')
@click.option('override_params', '--override', '-o', multiple=True, help='Override model config params.')
@click.option('--continue-training', is_flag=True, help='Continue training using model dir and run name.')
@click.option('--model-dir', default='models/', help='Directory to save the partial trained models.')
@click.option('--checkpoint-file', help='File for the weights of RPN and RCNN for resuming training.')
@click.option('--ignore-scope', help='Used to ignore variables when loading from checkpoint (set to "frcnn" when loading RPN and wanting to train complete network)')
@click.option('--log-dir', default='/tmp/frcnn/', help='Directory for Tensorboard logs.')
@click.option('--save-every', default=1000, help='Save checkpoint after that many batches.')
@click.option('--tf-debug', is_flag=True, help='Create debugging Tensorflow session with tfdb.')
@click.option('--debug', is_flag=True, help='Debug mode (DEBUG log level and intermediate variables are returned)')
@click.option('--run-name', default='train', help='Run name used to log in Tensorboard and isolate checkpoints.')
@click.option('--no-log/--log', default=False, help='Save or don\'t summary logs.')
@click.option('--display-every', default=500, type=int, help='Show image debug information every N batches (debug mode must be activated)')
@click.option('--random-shuffle/--fifo', default=True, help='Ingest data from dataset in random order.')
@click.option('--save-timeline', is_flag=True, help='Save timeline of execution (debug mode must be activated).')
@click.option('--summary-every', default=1, type=int, help='Save summary logs every N batches.')
@click.option('--full-trace', is_flag=True, help='Run graph session with FULL_TRACE config (for memory and running time debugging)')
@click.option('--initial-learning-rate', default=0.0001, type=float, help='Initial learning date.')
@click.option('--learning-rate-decay', default=10000, type=int, help='Decay learning date after N batches.')
@click.option('--learning-rate-decay-method', default='piecewise_constant', type=click.Choice(LEARNING_RATE_DECAY_METHODS), help='Tipo of learning rate decay to use.')
@click.option('optimizer_type', '--optimizer', default='momentum', type=click.Choice(OPTIMIZERS.keys()), help='Optimizer to use.')
@click.option('--momentum', default=0.9, type=float, help='Momentum to use when using the MomentumOptimizer.')
def train(model_type, config_file, override_params, continue_training, **kwargs):

    model_class = MODELS[model_type.lower()]
    config = model_class.base_config

    if config_file:
        # If we have a custom config file overwritting default settings
        # then we merge those values to the base_config.
        custom_config = load_config(config_file)
        config = merge_into(custom_config, config)

    # Load train extra options
    config.train = merge_into(kwargs_to_config(kwargs), config.train)

    if override_params:
        override_config = parse_override(override_params)
        config = merge_into(override_config, config)

    if config.train.debug or config.train.tf_debug:
        tf.logging.set_verbosity(tf.logging.DEBUG)
    else:
        tf.logging.set_verbosity(tf.logging.INFO)

    model = model_class(config)
    pretrained = PRETRAINED_MODULES[config.pretrained.net](
        config.pretrained
    )
    dataset = TFRecordDataset(config)
    train_dataset = dataset()

    train_image = train_dataset['image']
    train_filename = train_dataset['filename']
    train_scale_factor = train_dataset['scale_factor']

    train_bboxes = train_dataset['bboxes']

    # TODO: This is not the best place to configure rank? Why is rank not
    # transmitted through the queue
    train_image.set_shape((None, None, 3))
    # We add fake batch dimension to train data. TODO: DEFINITELY NOT THE BEST
    # PLACE
    train_image = tf.expand_dims(train_image, 0)

    pretrained_dict = pretrained(train_image)
    prediction_dict = model(train_image, pretrained_dict['net'], train_bboxes)

    total_loss = model.loss(prediction_dict)

    initial_global_step = 0
    checkpoint_path = os.path.join(
        config.train.model_dir, config.train.run_name,
        model.scope_name
    )
    if continue_training:
        last_checkpoint = tf.train.get_checkpoint_state(
            os.path.dirname(checkpoint_path)
        )
        if not last_checkpoint or not last_checkpoint.model_checkpoint_path:
            raise ValueError(
                'Could not find checkpoint in {}. Check run name'.format(
                    checkpoint_path))
        initial_global_step = int(
            last_checkpoint.model_checkpoint_path.split('-')[-1]
        )
        tf.logging.info('Starting training from global_step {}'.format(
            initial_global_step))
        last_checkpoint_path = last_checkpoint.model_checkpoint_path + '.index'
        tf.logging.info('Using checkpoint "{}"'.format(last_checkpoint_path))
        config.train.checkpoint_file = last_checkpoint_path

    global_step_init = tf.constant_initializer(initial_global_step)

    global_step = tf.get_variable(
        name="global_step", shape=[], dtype=tf.int64,
        initializer=global_step_init, trainable=False,
        collections=[tf.GraphKeys.GLOBAL_VARIABLES, tf.GraphKeys.GLOBAL_STEP]
    )

    # load_weights returns no_op when empty checkpoint_file.
    load_op = pretrained.load_weights(
        checkpoint_file=config.pretrained.weights
    )

    saver = get_saver((model, pretrained, ))
    if config.train.ignore_scope:
        partial_loader = get_saver(
            (model, pretrained, ), ignore_scope=config.train.ignore_scope
        )

    learning_rate_decay_method = config.train.learning_rate_decay_method
    if not learning_rate_decay_method or learning_rate_decay_method == 'none':
        learning_rate = config.train.initial_learning_rate
    elif learning_rate_decay_method == 'piecewise_constant':
        learning_rate = tf.train.piecewise_constant(
            global_step, boundaries=[
                tf.cast(config.train.learning_rate_decay, tf.int64), ],
            values=[
                config.train.initial_learning_rate,
                config.train.initial_learning_rate * 0.1
            ], name='learning_rate_piecewise_constant'
        )
    elif learning_rate_decay_method == 'exponential_decay':
        learning_rate = tf.train.exponential_decay(
            learning_rate=config.initial_learning_rate,
            global_step=global_step,
            decay_steps=config.train.learning_rate_decay, decay_rate=0.96,
            staircase=True, name='learning_rate_with_decay'
        )
    else:
        raise ValueError(
            'Invalid learning_rate method "{}"'.format(
                learning_rate_decay_method))

    tf.summary.scalar('losses/learning_rate', learning_rate)

    optimizer_cls = OPTIMIZERS[config.train.optimizer_type]
    if config.train.optimizer_type == 'momentum':
        optimizer = optimizer_cls(learning_rate, config.train.momentum)
    else:
        optimizer = optimizer_cls(learning_rate)

    trainable_vars = snt.get_variables_in_module(model)
    if config.pretrained.trainable:
        pretrained_trainable_vars = pretrained.get_trainable_vars()
        tf.logging.info('Training {} vars from pretrained module.'.format(
            len(pretrained_trainable_vars)))
        trainable_vars += pretrained_trainable_vars
    else:
        tf.logging.info('Not training variables from pretrained module')

    grads_and_vars = optimizer.compute_gradients(total_loss, trainable_vars)

    # Clip by norm. Grad can be null when not training some modules.
    with tf.name_scope('clip_gradients_by_norm'):
        grads_and_vars = [
            (tf.check_numerics(tf.clip_by_norm(gv[0], 10.), 'Invalid gradient'), gv[1])
            if gv[0] is not None else gv
            for gv in grads_and_vars
        ]

    train_op = optimizer.apply_gradients(
        grads_and_vars, global_step=global_step
    )

    # Create initializer for variables. Queue-related variables need a special
    # initializer.
    init_op = tf.group(
        tf.global_variables_initializer(),
        tf.local_variables_initializer()
    )

    metric_ops = tf.get_collection('metric_ops')
    metrics = tf.get_collection('metrics')

    tf.logging.info('Starting training for {}'.format(model))

    summary_dir = os.path.join(config.train.log_dir, config.train.run_name)
    summarizer = tf.summary.merge([
        tf.summary.merge_all(),
        model.summary,
    ])

    with tf.Session() as sess:

        sess.run(init_op)  # initialized variables
        sess.run(load_op)  # load pretrained weights

        # Restore all variables from checkpoint file
        if config.train.checkpoint_file:
            # If ignore_scope is set, we don't load those variables from checkpoint.
            if config.train.ignore_scope:
                partial_loader.restore(sess, config.train.checkpoint_file)
            else:
                saver.restore(sess, config.train.checkpoint_file)

        if not config.train.no_log:
            writer = tf.summary.FileWriter(summary_dir, sess.graph)

        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess, coord=coord)
        count_images = 0
        step = initial_global_step

        if config.train.tf_debug:
            from tensorflow.python import debug as tensorflow_debug
            sess = tensorflow_debug.LocalCLIDebugWrapperSession(sess)
            sess.add_tensor_filter(
                'has_inf_or_nan', tensorflow_debug.has_inf_or_nan
            )

        try:
            while not coord.should_stop():
                run_metadata = None
                if (step + 1) % config.train.summary_every == 0:
                    run_metadata = tf.RunMetadata()

                run_options = None
                if config.train.full_trace:
                    run_options = tf.RunOptions(
                        trace_level=tf.RunOptions.FULL_TRACE
                    )

                (
                    _, summary, train_loss, step, pred_dict, filename,
                    scale_factor, *_
                ) = sess.run([
                    train_op, summarizer, total_loss, global_step,
                    prediction_dict, train_filename, train_scale_factor,
                    metric_ops
                ], run_metadata=run_metadata, options=run_options)

                write_summary = (
                    not config.train.no_log and
                    step % config.train.summary_every == 0
                )
                if write_summary:
                    writer.add_summary(summary, step)
                    writer.add_run_metadata(
                        run_metadata, str(step)
                    )

                if config.train.debug and step % config.train.display_every == 0:
                    from luminoth.utils.image_vis import (
                        add_images_to_tensoboard
                    )
                    print('Scaled image with {}'.format(scale_factor))
                    print('Image size: {}'.format(pred_dict['image_shape']))
                    add_images_to_tensoboard(
                        pred_dict, step, summary_dir, config.network.with_rcnn
                    )

                    if config.train.save_timeline:
                        from tensorflow.python.client import timeline
                        run_tmln = timeline.Timeline(
                            run_metadata.step_stats)
                        chrome_trace = run_tmln.generate_chrome_trace_format()
                        with open('timeline_{}.json'.format(step), 'w') as f:
                            f.write(chrome_trace)

                count_images += 1

                tf.logging.info('train_loss: {}'.format(train_loss))
                tf.logging.info('step: {}, file: {}'.format(step, filename))

                if not config.train.no_log:
                    if step % config.train.save_every == 0:
                        # We don't support partial saver.
                        saver.save(sess, checkpoint_path, global_step=step)

        except tf.errors.OutOfRangeError:
            tf.logging.info('iter = {}, train_loss = {:.2f}'.format(
                step, train_loss))
            tf.logging.info('finished training -- epoch limit reached')
            tf.logging.info('count_images = {}'.format(count_images))
        finally:
            coord.request_stop()

        # Wait for all threads to stop.
        coord.join(threads)

if __name__ == '__main__':
    train()
<|endoftext|>


<commit_before>import eyeD3
import mad

__author__ = 'sabrsorensen'

def __init__(self,name):
    self.tagger = eyeD3.Tag()
    self.m3u_out = open(name+'.m3u', 'w')
    self.m3u_out.write("#Playlist " + name + " generated by gmusic_playlist_export\n#EXTM3U\n")

def M3UEntry(self,path):
    self.tagger.link(path)
    self.m3u_out.write("#EXTINF:," + self.tagger.getArtist() + " - " + self.tagger.getTitle + "\n")
    self.m3u_out.write(path + "\n")

def M3UClose(self):
    self.m3u_out.close()<commit_msg>Removing unnecessary import.<commit_after>import eyeD3

__author__ = 'sabrsorensen'

def __init__(self,name):
    self.tagger = eyeD3.Tag()
    self.m3u_out = open(name+'.m3u', 'w')
    self.m3u_out.write("#Playlist " + name + " generated by gmusic_playlist_export\n#EXTM3U\n")

def M3UEntry(self,path):
    self.tagger.link(path)
    self.m3u_out.write("#EXTINF:," + self.tagger.getArtist() + " - " + self.tagger.getTitle + "\n")
    self.m3u_out.write(path + "\n")

def M3UClose(self):
    self.m3u_out.close()<|endoftext|>


<commit_before>import os.path
import re
import tempfile
from textwrap import dedent

import pytest

from sql.magic import SqlMagic

def runsql(ip_session, statements):
    if isinstance(statements, str):
        statements = [
            statements,
        ]
    for statement in statements:
        result = ip_session.run_line_magic('sql', 'sqlite:// %s' % statement)
    return result  # returns only last result

@pytest.fixture
def ip():
    """Provides an IPython session in which tables have been created"""

    ip_session = get_ipython()
    runsql(ip_session, [
        "CREATE TABLE test (n INT, name TEXT)",
        "INSERT INTO test VALUES (1, 'foo')",
        "INSERT INTO test VALUES (2, 'bar')",
        "CREATE TABLE author (first_name, last_name, year_of_death)",
        "INSERT INTO author VALUES ('William', 'Shakespeare', 1616)",
        "INSERT INTO author VALUES ('Bertold', 'Brecht', 1956)"
    ])
    yield ip_session
    runsql(ip_session, 'DROP TABLE test')
    runsql(ip_session, 'DROP TABLE author')

def test_memory_db(ip):
    assert runsql(ip, "SELECT * FROM test;")[0][0] == 1
    assert runsql(ip, "SELECT * FROM test;")[1]['name'] == 'bar'

def test_html(ip):
    result = runsql(ip, "SELECT * FROM test;")
    assert 'foo	' in result._repr_html_().lower()

def test_print(ip):
    result = runsql(ip, "SELECT * FROM test;")
    assert re.search(r'1\s+\|\s+foo', str(result))

def test_plain_style(ip):
    ip.run_line_magic('config', "SqlMagic.style = 'PLAIN_COLUMNS'")
    result = runsql(ip, "SELECT * FROM test;")
    assert re.search(r'1\s+\|\s+foo', str(result))

def test_multi_sql(ip):
    result = ip.run_cell_magic('sql', '', """
        sqlite://
        SELECT last_name FROM author;
        """)
    assert 'Shakespeare' in str(result) and 'Brecht' in str(result)

def test_result_var(ip):
    ip.run_cell_magic('sql', '', """
        sqlite://
        x <<
        SELECT last_name FROM author;
        """)
    result = ip.user_global_ns['x']
    assert 'Shakespeare' in str(result) and 'Brecht' in str(result)

def test_access_results_by_keys(ip):
    assert runsql(ip,
                  "SELECT * FROM author;")['William'] == (u'William',
                                                          u'Shakespeare', 1616)

def test_duplicate_column_names_accepted(ip):
    result = ip.run_cell_magic('sql', '', """
        sqlite://
        SELECT last_name, last_name FROM author;
        """)
    assert (u'Brecht', u'Brecht') in result

def test_autolimit(ip):
    ip.run_line_magic('config', "SqlMagic.autolimit = 0")
    result = runsql(ip, "SELECT * FROM test;")
    assert len(result) == 2
    ip.run_line_magic('config', "SqlMagic.autolimit = 1")
    result = runsql(ip, "SELECT * FROM test;")
    assert len(result) == 1

def test_persist(ip):
    runsql(ip, "")
    ip.run_cell("results = %sql SELECT * FROM test;")
    ip.runcode("results_dframe = results.DataFrame()")
    runsql(ip, 'PERSIST results_dframe')
    persisted = runsql(ip, 'SELECT * FROM results_dframe')
    assert 'foo' in str(persisted)

def test_persist_nonexistent_raises(ip):
    runsql(ip, "")
    with pytest.raises(NameError):
        runsql(ip, 'PERSIST no_such_dataframe')

def test_persist_non_frame_raises(ip):
    ip.run_cell("not_a_dataframe = 22")
    runsql(ip, "")
    with pytest.raises(TypeError):
        runsql(ip, 'PERSIST not_a_dataframe')

def test_persist_bare(ip):
    ip.run_line_magic('sql', "sqlite://")
    with pytest.raises(SyntaxError):
        runsql(ip, 'PERSIST')

def test_persist_frame_at_its_creation(ip):
    ip.run_cell("results = %sql SELECT * FROM author;")
    runsql(ip, 'PERSIST results.DataFrame()')
    persisted = runsql(ip, 'SELECT * FROM results')
    assert 'Shakespeare' in str(persisted)

# TODO: support
# @with_setup(_setup_author, _teardown_author)
# def test_persist_with_connection_info():
#     ip.run_cell("results = %sql SELECT * FROM author;")
#     ip.run_line_magic('sql', 'sqlite:// PERSIST results.DataFrame()')
#     persisted = ip.run_line_magic('sql', 'SELECT * FROM results')
#     assert 'Shakespeare' in str(persisted)

def test_displaylimit(ip):
    ip.run_line_magic('config', "SqlMagic.autolimit = None")
    ip.run_line_magic('config', "SqlMagic.displaylimit = None")
    result = runsql(
        ip,
        "SELECT * FROM (VALUES ('apple'), ('banana'), ('cherry')) AS Result ORDER BY 1;"
    )
    assert 'apple' in result._repr_html_()
    assert 'banana' in result._repr_html_()
    assert 'cherry' in result._repr_html_()
    ip.run_line_magic('config', "SqlMagic.displaylimit = 1")
    result = runsql(
        ip,
        "SELECT * FROM (VALUES ('apple'), ('banana'), ('cherry')) AS Result ORDER BY 1;"
    )
    assert 'apple' in result._repr_html_()
    assert 'cherry' not in result._repr_html_()

def test_column_local_vars(ip):
    ip.run_line_magic('config', "SqlMagic.column_local_vars = True")
    result = runsql(ip, "SELECT * FROM author;")
    assert result is None
    assert 'William' in ip.user_global_ns['first_name']
    assert 'Shakespeare' in ip.user_global_ns['last_name']
    assert len(ip.user_global_ns['first_name']) == 2
    ip.run_line_magic('config', "SqlMagic.column_local_vars = False")

def test_userns_not_changed(ip):
    ip.run_cell(
        dedent("""
    def function():
        local_var = 'local_val'
        %sql sqlite:// INSERT INTO test VALUES (2, 'bar');
    function()"""))
    assert 'local_var' not in ip.user_ns

def test_bind_vars(ip):
    ip.user_global_ns['x'] = 22
    result = runsql(ip, "SELECT :x")
    assert result[0][0] == 22

def test_autopandas(ip):
    ip.run_line_magic('config', "SqlMagic.autopandas = True")
    dframe = runsql(ip, "SELECT * FROM test;")
    assert not dframe.empty
    assert dframe.ndim == 2
    assert dframe.name[0] == 'foo'

def test_csv(ip):
    ip.run_line_magic('config', "SqlMagic.autopandas = False")  # uh-oh
    result = runsql(ip, "SELECT * FROM test;")
    result = result.csv()
    for row in result.splitlines():
        assert row.count(',') == 1
    assert len(result.splitlines()) == 3

def test_csv_to_file(ip):
    ip.run_line_magic('config', "SqlMagic.autopandas = False")  # uh-oh
    result = runsql(ip, "SELECT * FROM test;")
    with tempfile.TemporaryDirectory() as tempdir:
        fname = os.path.join(tempdir, 'test.csv')
        output = result.csv(fname)
        assert os.path.exists(output.file_path)
        with open(output.file_path) as csvfile:
            content = csvfile.read()
            for row in content.splitlines():
                assert row.count(',') == 1
            assert len(content.splitlines()) == 3

def test_dict(ip):
    result = runsql(ip, "SELECT * FROM author;")
    result = result.dict()
    assert isinstance(result, dict)
    assert 'first_name' in result
    assert 'last_name' in result
    assert 'year_of_death' in result
    assert len(result['last_name']) == 2

def test_dicts(ip):
    result = runsql(ip, "SELECT * FROM author;")
    for row in result.dicts():
        assert isinstance(row, dict)
        assert 'first_name' in row
        assert 'last_name' in row
        assert 'year_of_death' in row
<commit_msg>Fixed failing test code (run_cell not runcode)<commit_after>import os.path
import re
import tempfile
from textwrap import dedent

import pytest

from sql.magic import SqlMagic

def runsql(ip_session, statements):
    if isinstance(statements, str):
        statements = [
            statements,
        ]
    for statement in statements:
        result = ip_session.run_line_magic('sql', 'sqlite:// %s' % statement)
    return result  # returns only last result

@pytest.fixture
def ip():
    """Provides an IPython session in which tables have been created"""

    ip_session = get_ipython()
    runsql(ip_session, [
        "CREATE TABLE test (n INT, name TEXT)",
        "INSERT INTO test VALUES (1, 'foo')",
        "INSERT INTO test VALUES (2, 'bar')",
        "CREATE TABLE author (first_name, last_name, year_of_death)",
        "INSERT INTO author VALUES ('William', 'Shakespeare', 1616)",
        "INSERT INTO author VALUES ('Bertold', 'Brecht', 1956)"
    ])
    yield ip_session
    runsql(ip_session, 'DROP TABLE test')
    runsql(ip_session, 'DROP TABLE author')

def test_memory_db(ip):
    assert runsql(ip, "SELECT * FROM test;")[0][0] == 1
    assert runsql(ip, "SELECT * FROM test;")[1]['name'] == 'bar'

def test_html(ip):
    result = runsql(ip, "SELECT * FROM test;")
    assert 'foo	' in result._repr_html_().lower()

def test_print(ip):
    result = runsql(ip, "SELECT * FROM test;")
    assert re.search(r'1\s+\|\s+foo', str(result))

def test_plain_style(ip):
    ip.run_line_magic('config', "SqlMagic.style = 'PLAIN_COLUMNS'")
    result = runsql(ip, "SELECT * FROM test;")
    assert re.search(r'1\s+\|\s+foo', str(result))

def test_multi_sql(ip):
    result = ip.run_cell_magic('sql', '', """
        sqlite://
        SELECT last_name FROM author;
        """)
    assert 'Shakespeare' in str(result) and 'Brecht' in str(result)

def test_result_var(ip):
    ip.run_cell_magic('sql', '', """
        sqlite://
        x <<
        SELECT last_name FROM author;
        """)
    result = ip.user_global_ns['x']
    assert 'Shakespeare' in str(result) and 'Brecht' in str(result)

def test_access_results_by_keys(ip):
    assert runsql(ip,
                  "SELECT * FROM author;")['William'] == (u'William',
                                                          u'Shakespeare', 1616)

def test_duplicate_column_names_accepted(ip):
    result = ip.run_cell_magic('sql', '', """
        sqlite://
        SELECT last_name, last_name FROM author;
        """)
    assert (u'Brecht', u'Brecht') in result

def test_autolimit(ip):
    ip.run_line_magic('config', "SqlMagic.autolimit = 0")
    result = runsql(ip, "SELECT * FROM test;")
    assert len(result) == 2
    ip.run_line_magic('config', "SqlMagic.autolimit = 1")
    result = runsql(ip, "SELECT * FROM test;")
    assert len(result) == 1

def test_persist(ip):
    runsql(ip, "")
    ip.run_cell("results = %sql SELECT * FROM test;")
    ip.run_cell("results_dframe = results.DataFrame()")
    runsql(ip, 'PERSIST results_dframe')
    persisted = runsql(ip, 'SELECT * FROM results_dframe')
    assert 'foo' in str(persisted)

def test_persist_nonexistent_raises(ip):
    runsql(ip, "")
    with pytest.raises(NameError):
        runsql(ip, 'PERSIST no_such_dataframe')

def test_persist_non_frame_raises(ip):
    ip.run_cell("not_a_dataframe = 22")
    runsql(ip, "")
    with pytest.raises(TypeError):
        runsql(ip, 'PERSIST not_a_dataframe')

def test_persist_bare(ip):
    ip.run_line_magic('sql', "sqlite://")
    with pytest.raises(SyntaxError):
        runsql(ip, 'PERSIST')

def test_persist_frame_at_its_creation(ip):
    ip.run_cell("results = %sql SELECT * FROM author;")
    runsql(ip, 'PERSIST results.DataFrame()')
    persisted = runsql(ip, 'SELECT * FROM results')
    assert 'Shakespeare' in str(persisted)

# TODO: support
# @with_setup(_setup_author, _teardown_author)
# def test_persist_with_connection_info():
#     ip.run_cell("results = %sql SELECT * FROM author;")
#     ip.run_line_magic('sql', 'sqlite:// PERSIST results.DataFrame()')
#     persisted = ip.run_line_magic('sql', 'SELECT * FROM results')
#     assert 'Shakespeare' in str(persisted)

def test_displaylimit(ip):
    ip.run_line_magic('config', "SqlMagic.autolimit = None")
    ip.run_line_magic('config', "SqlMagic.displaylimit = None")
    result = runsql(
        ip,
        "SELECT * FROM (VALUES ('apple'), ('banana'), ('cherry')) AS Result ORDER BY 1;"
    )
    assert 'apple' in result._repr_html_()
    assert 'banana' in result._repr_html_()
    assert 'cherry' in result._repr_html_()
    ip.run_line_magic('config', "SqlMagic.displaylimit = 1")
    result = runsql(
        ip,
        "SELECT * FROM (VALUES ('apple'), ('banana'), ('cherry')) AS Result ORDER BY 1;"
    )
    assert 'apple' in result._repr_html_()
    assert 'cherry' not in result._repr_html_()

def test_column_local_vars(ip):
    ip.run_line_magic('config', "SqlMagic.column_local_vars = True")
    result = runsql(ip, "SELECT * FROM author;")
    assert result is None
    assert 'William' in ip.user_global_ns['first_name']
    assert 'Shakespeare' in ip.user_global_ns['last_name']
    assert len(ip.user_global_ns['first_name']) == 2
    ip.run_line_magic('config', "SqlMagic.column_local_vars = False")

def test_userns_not_changed(ip):
    ip.run_cell(
        dedent("""
    def function():
        local_var = 'local_val'
        %sql sqlite:// INSERT INTO test VALUES (2, 'bar');
    function()"""))
    assert 'local_var' not in ip.user_ns

def test_bind_vars(ip):
    ip.user_global_ns['x'] = 22
    result = runsql(ip, "SELECT :x")
    assert result[0][0] == 22

def test_autopandas(ip):
    ip.run_line_magic('config', "SqlMagic.autopandas = True")
    dframe = runsql(ip, "SELECT * FROM test;")
    assert not dframe.empty
    assert dframe.ndim == 2
    assert dframe.name[0] == 'foo'

def test_csv(ip):
    ip.run_line_magic('config', "SqlMagic.autopandas = False")  # uh-oh
    result = runsql(ip, "SELECT * FROM test;")
    result = result.csv()
    for row in result.splitlines():
        assert row.count(',') == 1
    assert len(result.splitlines()) == 3

def test_csv_to_file(ip):
    ip.run_line_magic('config', "SqlMagic.autopandas = False")  # uh-oh
    result = runsql(ip, "SELECT * FROM test;")
    with tempfile.TemporaryDirectory() as tempdir:
        fname = os.path.join(tempdir, 'test.csv')
        output = result.csv(fname)
        assert os.path.exists(output.file_path)
        with open(output.file_path) as csvfile:
            content = csvfile.read()
            for row in content.splitlines():
                assert row.count(',') == 1
            assert len(content.splitlines()) == 3

def test_dict(ip):
    result = runsql(ip, "SELECT * FROM author;")
    result = result.dict()
    assert isinstance(result, dict)
    assert 'first_name' in result
    assert 'last_name' in result
    assert 'year_of_death' in result
    assert len(result['last_name']) == 2

def test_dicts(ip):
    result = runsql(ip, "SELECT * FROM author;")
    for row in result.dicts():
        assert isinstance(row, dict)
        assert 'first_name' in row
        assert 'last_name' in row
        assert 'year_of_death' in row
<|endoftext|>


<commit_before>'''Parse data in JSON format.'''

import sys
import json

for line in open(sys.argv[1]):
    jsn = json.loads(line.strip())
    print '%d\t%d\t%d\t%s\t%s\t%s\t%s\t%s' % \
                                        (jsn['votes']['cool'],
                                        jsn['votes']['funny'],
                                        jsn['votes']['useful'],
                                        jsn['user_id'],
                                        jsn['name'],
                                        jsn['url'],
                                        jsn['average_stars'],
                                        jsn['review_count'],
                                    )
<commit_msg>output is now written to output.csv in csv format.<commit_after>'''Parse data in JSON format.'''

import sys
import csv
import json

with open('output.csv', 'w') as csvfile:
    writer = csv.writer(csvfile, delimiter=',')
    for line in open(sys.argv[1]):
        jsn = json.loads(line.strip())
        writer.writerow([jsn['votes']['cool'],
                            jsn['votes']['funny'],
                            jsn['votes']['useful'],
                            jsn['user_id'],
                            jsn['name'],
                            jsn['url'],
                            jsn['average_stars'],
                            jsn['review_count'],
                        ])
<|endoftext|>


<commit_before>import tkinter, random

size = 4

def shift(event, d):
    if d == 1:
        for row in range(4):
            s = []
            for col in range(4):
                p = cells[col + 4 * row]['text']
                if p != '':
                    s.append(int(p))
            s = s[::-1]
            i = 0
            while i < len(s) - 1:
                if s[i] == s[i+1]:
                    s[i] *= 2
                    del s[i+1]
                i += 1
            s = s[::-1]
            for col in range(4):
                if col < (4 - len(s)):
                    cells[col + 4 * row]['text'] = ''
                else:
                    cells[col + 4 * row]['text'] = s[col - (4 - len(s))]
        generate()

def generate():
    for i in range(2):
        k = random.randint(0, size**2 - 1)
        while cells[k]['text'] != '':
            k = random.randint(0, size**2 - 1)
        cells[k]['text'] = '2'

root = tkinter.Tk()
frame = tkinter.Frame(root)
frame.focus_set()
frame.pack()
frame.bind('<Up>'   , lambda e: shift(e, 0))
frame.bind('<Right>', lambda e: shift(e, 1))
frame.bind('<Down>' , lambda e: shift(e, 2))
frame.bind('<Left>' , lambda e: shift(e, 3))

cells = []
for i in range(size**2):
    btn = tkinter.Button(frame)
    btn.grid(row = i // size, column = i % size)
    cells.append(btn)

generate()
frame.mainloop()
<commit_msg>правильно обрабатываются сдвиги во все стороны<commit_after>import tkinter, random

size = 4

def shift(event, d):
    if d == 1:
        for row in range(size):
            s = []
            for col in range(size):
                p = cells[col + size * row]['text']
                if p != '':
                    s.append(int(p))
            s = s[::-1]
            i = 0
            while i < len(s) - 1:
                if s[i] == s[i+1]:
                    s[i] *= 2
                    del s[i+1]
                i += 1
            s = s[::-1]
            for col in range(size):
                if col < (size - len(s)):
                    cells[col + size * row]['text'] = ''
                else:
                    cells[col + size * row]['text'] = s[col - (size - len(s))]
    if d == 3:
        for row in range(size):
            s = []
            for col in range(size):
                p = cells[col + size * row]['text']
                if p != '':
                    s.append(int(p))
            i = 0
            while i < len(s) - 1:
                if s[i] == s[i+1]:
                    s[i] *= 2
                    del s[i+1]
                i += 1
            for col in range(size):
                if col < len(s):
                    cells[col + size * row]['text'] = s[col]
                else:
                    cells[col + size * row]['text'] = ''
    if d == 0:
        for col in range(size):
            s = []
            for row in range(size):
                p = cells[col + size * row]['text']
                if p != '':
                    s.append(int(p))
            i = 0
            while i < len(s) - 1:
                if s[i] == s[i+1]:
                    s[i] *= 2
                    del s[i+1]
                i += 1
            for row in range(size):
                if row < len(s):
                    cells[col + size * row]['text'] = s[row]
                else:
                    cells[col + size * row]['text'] = ''
    if d == 2:
        for col in range(size):
            s = []
            for row in range(size):
                p = cells[col + size * row]['text']
                if p != '':
                    s.append(int(p))
            s = s[::-1]
            i = 0
            while i < len(s) - 1:
                if s[i] == s[i+1]:
                    s[i] *= 2
                    del s[i+1]
                i += 1
            s = s[::-1]
            for row in range(size):
                if row < (size - len(s)):
                    cells[col + size * row]['text'] = ''
                else:
                    cells[col + size * row]['text'] = s[row - (size - len(s))]
    generate()

def generate():
    free_cells = []
    for i in range(len(cells)):
        if cells[i]['text'] == '':
            free_cells.append(i)
    if len(free_cells) == 0:
        print("Game over")
    else:
        k = random.choice(free_cells)
        cells[k]['text'] = '2'

root = tkinter.Tk()
frame = tkinter.Frame(root)
frame.focus_set()
frame.pack()
frame.bind('<Up>'   , lambda e: shift(e, 0))
frame.bind('<Right>', lambda e: shift(e, 1))
frame.bind('<Down>' , lambda e: shift(e, 2))
frame.bind('<Left>' , lambda e: shift(e, 3))

cells = []
for i in range(size**2):
    btn = tkinter.Button(frame)
    btn.grid(row = i // size, column = i % size)
    cells.append(btn)

generate()
generate()
frame.mainloop()
<|endoftext|>