<commit_before>import locust
import core
from core import Locust, WebLocust, MasterLocustRunner, SlaveLocustRunner, LocalLocustRunner
from stats import print_stats
import web
import inspectlocust

import gevent
import sys
import os
import inspect
import time
from optparse import OptionParser
from locust.stats import stats_printer, RequestStats, print_percentile_stats, print_error_report

_internals = [Locust, WebLocust]
version = locust.version

def parse_options():
    """
    Handle command-line options with optparse.OptionParser.

    Return list of arguments, largely for use in `parse_arguments`.
    """

    # Initialize
    parser = OptionParser(usage="locust [options] [LocustClass [LocustClassN]] ...")

    parser.add_option(
        '-H', '--host',
        dest="host",
        default=None,
        help="Host to load test"
    )

    parser.add_option(
        '-f', '--locustfile',
        dest='locustfile',
        default='locustfile',
        help="Python module file to import, e.g. '../other.py'. Default: locustfile"
    )

    # Version number (optparse gives you --version but we have to do it
    # ourselves to get -V too. sigh)
    parser.add_option(
        '-V', '--version',
        action='store_true',
        dest='show_version',
        default=False,
        help="show program's version number and exit"
    )

    # List locust commands found in loaded locust files/source files
    parser.add_option(
        '-l', '--list',
        action='store_true',
        dest='list_commands',
        default=False,
        help="print list of possible commands and exit"
    )

    # Display ratio table of all tasks
    parser.add_option(
        '--show-task-ratio',
        action='store_true',
        dest='show_task_ratio',
        default=False,
        help="print table of the locust classes' task execution ratio"
    )

    # if we shgould print stats in the console
    parser.add_option(
        '--print-stats',
        action='store_true',
        dest='print_stats',
        default=False,
        help="Print stats in the console"
    )

    # if we should print stats in the console
    parser.add_option(
        '--web',
        action='store_true',
        dest='web',
        default=False,
        help="Enable the web monitor (port 8089)"
    )

    # if the HTTP client should set gzip headers and try to use gzip decoding
    parser.add_option(
        '--gzip',
        action='store_true',
        dest='gzip',
        default=False,
        help="If present, the HTTP client will set request header Accept-Encoding: gzip, and try to gzip decode the response data."
    )

    # if locust should be run in distributed mode as master
    parser.add_option(
        '--master',
        action='store_true',
        dest='master',
        default=False,
        help="Set locust to run in distributed mode with this process as master"
    )

    # if locust should be run in distributed mode as slave
    parser.add_option(
        '--slave',
        action='store_true',
        dest='slave',
        default=False,
        help="Set locust to run in distributed mode with this process as slave"
    )

    # Number of requests
    parser.add_option(
        '-n', '--num-request',
        action='store',
        type='int',
        dest='num_requests',
        default=None,
        help="Number of requests to perform"
    )

    # Number of clients
    parser.add_option(
        '-c', '--clients',
        action='store',
        type='int',
        dest='num_clients',
        default=1,
        help="Number of concurrent clients"
    )

    # Client hatch rate
    parser.add_option(
        '-r', '--hatch-rate',
        action='store',
        type='int',
        dest='hatch_rate',
        default=1,
        help="The rate per second in which clients are spawned"
    )

    # redis options
    parser.add_option(
        '--redis-host',
        action='store',
        type='str',
        dest='redis_host',
        default="127.0.0.1",
        help="Redis host to use for distributed load testing. Only used when running with --master or --slave. Defaults to 127.0.0.1."
    )
    parser.add_option(
        '--redis-port',
        action='store',
        type='int',
        dest='redis_port',
        default=6379,
        help="Redis port to use for distributed load testing. Only used when running with --master or --slave. Defaults to 6379, which is the default Redis port."
    )

    # Finalize
    # Return three-tuple of parser + the output from parse_args (opt obj, args)
    opts, args = parser.parse_args()
    return parser, opts, args

def _is_package(path):
    """
    Is the given path a Python package?
    """
    return (
        os.path.isdir(path)
        and os.path.exists(os.path.join(path, '__init__.py'))
    )

def find_locustfile(locustfile):
    """
    Attempt to locate a locustfile, either explicitly or by searching parent dirs.
    """
    # Obtain env value
    names = [locustfile]
    # Create .py version if necessary
    if not names[0].endswith('.py'):
        names += [names[0] + '.py']
    # Does the name contain path elements?
    if os.path.dirname(names[0]):
        # If so, expand home-directory markers and test for existence
        for name in names:
            expanded = os.path.expanduser(name)
            if os.path.exists(expanded):
                if name.endswith('.py') or _is_package(expanded):
                    return os.path.abspath(expanded)
    else:
        # Otherwise, start in cwd and work downwards towards filesystem root
        path = '.'
        # Stop before falling off root of filesystem (should be platform
        # agnostic)
        while os.path.split(os.path.abspath(path))[1]:
            for name in names:
                joined = os.path.join(path, name)
                if os.path.exists(joined):
                    if name.endswith('.py') or _is_package(joined):
                        return os.path.abspath(joined)
            path = os.path.join('..', path)
    # Implicit 'return None' if nothing was found

def is_locust(tup):
    """
    Takes (name, object) tuple, returns True if it's a public Locust subclass.
    """
    name, item = tup
    return (
        inspect.isclass(item)
        and issubclass(item, Locust)
        and (item not in _internals)
        and not name.startswith('_')
    )

def load_locustfile(path):
    """
    Import given locustfile path and return (docstring, callables).

    Specifically, the locustfile's ``__doc__`` attribute (a string) and a
    dictionary of ``{'name': callable}`` containing all callables which pass
    the "is a Locust" test.
    """
    # Get directory and locustfile name
    directory, locustfile = os.path.split(path)
    # If the directory isn't in the PYTHONPATH, add it so our import will work
    added_to_path = False
    index = None
    if directory not in sys.path:
        sys.path.insert(0, directory)
        added_to_path = True
    # If the directory IS in the PYTHONPATH, move it to the front temporarily,
    # otherwise other locustfiles -- like Locusts's own -- may scoop the intended
    # one.
    else:
        i = sys.path.index(directory)
        if i != 0:
            # Store index for later restoration
            index = i
            # Add to front, then remove from original position
            sys.path.insert(0, directory)
            del sys.path[i + 1]
    # Perform the import (trimming off the .py)
    imported = __import__(os.path.splitext(locustfile)[0])
    # Remove directory from path if we added it ourselves (just to be neat)
    if added_to_path:
        del sys.path[0]
    # Put back in original index if we moved it
    if index is not None:
        sys.path.insert(index + 1, directory)
        del sys.path[0]
    # Return our two-tuple
    locusts = dict(filter(is_locust, vars(imported).items()))
    return imported.__doc__, locusts

def main():
    parser, options, arguments = parse_options()
    #print "Options:", options, dir(options)
    #print "Arguments:", arguments
    #print "largs:", parser.largs
    #print "rargs:", parser.rargs

    if options.show_version:
        print("Locust %s" % (version))
        sys.exit(0)

    locustfile = find_locustfile(options.locustfile)
    if not locustfile:
        print "Could not find any locustfile!"
        sys.exit(1)

    docstring, locusts = load_locustfile(locustfile)

    if options.list_commands:
        print "Available Locusts:"
        for name in locusts:
            print "    " + name
        sys.exit(0)

    if not locusts:
        sys.stderr.write("No Locust class found!\n")
        sys.exit(1)

    # make sure specified Locust exists
    if arguments:
        missing = set(arguments) - set(locusts.keys())
        if missing:
            sys.stderr.write("Unknown Locust(s): %s\n" % (", ".join(missing)))
            sys.exit(1)
        else:
            names = set(arguments) & set(locusts.keys())
            locust_classes = [locusts[n] for n in names]
    else:
        locust_classes = locusts.values()

    if options.show_task_ratio:
        print "\n Task ratio per locust class"
        print "-" * 80
        inspectlocust.print_task_ratio(locust_classes)
        print "\n Total task ratio"
        print "-" * 80
        inspectlocust.print_task_ratio(locust_classes, total=True)
        sys.exit(0)

    if options.web and not options.slave:
        # spawn web greenlet
        print "Starting web monitor on port 8089"
        main_greenlet = gevent.spawn(web.start, locust_classes, options.hatch_rate, options.num_clients, options.num_requests)

    # enable/disable gzip in WebLocust's HTTP client
    WebLocust.gzip = options.gzip

    if not options.master and not options.slave:
        core.locust_runner = LocalLocustRunner(locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.host)
        # spawn client spawning/hatching greenlet
        if not options.web:
            core.locust_runner.start_hatching()
            main_greenlet = core.locust_runner.greenlet
    elif options.master:
        core.locust_runner = MasterLocustRunner(locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, redis_host=options.redis_host, redis_port=options.redis_port)
    elif options.slave:
        core.locust_runner = SlaveLocustRunner(locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, redis_host=options.redis_host, redis_port=options.redis_port)
        main_greenlet = core.locust_runner.greenlet

    if not options.web or options.print_stats:
        # spawn stats printing greenlet
        gevent.spawn(stats_printer)

    try:
        print
        print "Starting Locust %s" % version
        print
        main_greenlet.join()
    except KeyboardInterrupt, e:
        time.sleep(0.2)
        print "\n"
        print_stats(core.locust_runner.request_stats)
        print_percentile_stats(core.locust_runner.request_stats)
        print_error_report()
        print
        print "Exiting, bye.."
        print

    sys.exit(0)

if __name__ == '__main__':
    main()
<commit_msg>Turned of console stats printing by default when running in slave mode.<commit_after>import locust
import core
from core import Locust, WebLocust, MasterLocustRunner, SlaveLocustRunner, LocalLocustRunner
from stats import print_stats
import web
import inspectlocust

import gevent
import sys
import os
import inspect
import time
from optparse import OptionParser
from locust.stats import stats_printer, RequestStats, print_percentile_stats, print_error_report

_internals = [Locust, WebLocust]
version = locust.version

def parse_options():
    """
    Handle command-line options with optparse.OptionParser.

    Return list of arguments, largely for use in `parse_arguments`.
    """

    # Initialize
    parser = OptionParser(usage="locust [options] [LocustClass [LocustClassN]] ...")

    parser.add_option(
        '-H', '--host',
        dest="host",
        default=None,
        help="Host to load test"
    )

    parser.add_option(
        '-f', '--locustfile',
        dest='locustfile',
        default='locustfile',
        help="Python module file to import, e.g. '../other.py'. Default: locustfile"
    )

    # Version number (optparse gives you --version but we have to do it
    # ourselves to get -V too. sigh)
    parser.add_option(
        '-V', '--version',
        action='store_true',
        dest='show_version',
        default=False,
        help="show program's version number and exit"
    )

    # List locust commands found in loaded locust files/source files
    parser.add_option(
        '-l', '--list',
        action='store_true',
        dest='list_commands',
        default=False,
        help="print list of possible commands and exit"
    )

    # Display ratio table of all tasks
    parser.add_option(
        '--show-task-ratio',
        action='store_true',
        dest='show_task_ratio',
        default=False,
        help="print table of the locust classes' task execution ratio"
    )

    # if we shgould print stats in the console
    parser.add_option(
        '--print-stats',
        action='store_true',
        dest='print_stats',
        default=False,
        help="Print stats in the console"
    )

    # if we should print stats in the console
    parser.add_option(
        '--web',
        action='store_true',
        dest='web',
        default=False,
        help="Enable the web monitor (port 8089)"
    )

    # if the HTTP client should set gzip headers and try to use gzip decoding
    parser.add_option(
        '--gzip',
        action='store_true',
        dest='gzip',
        default=False,
        help="If present, the HTTP client will set request header Accept-Encoding: gzip, and try to gzip decode the response data."
    )

    # if locust should be run in distributed mode as master
    parser.add_option(
        '--master',
        action='store_true',
        dest='master',
        default=False,
        help="Set locust to run in distributed mode with this process as master"
    )

    # if locust should be run in distributed mode as slave
    parser.add_option(
        '--slave',
        action='store_true',
        dest='slave',
        default=False,
        help="Set locust to run in distributed mode with this process as slave"
    )

    # Number of requests
    parser.add_option(
        '-n', '--num-request',
        action='store',
        type='int',
        dest='num_requests',
        default=None,
        help="Number of requests to perform"
    )

    # Number of clients
    parser.add_option(
        '-c', '--clients',
        action='store',
        type='int',
        dest='num_clients',
        default=1,
        help="Number of concurrent clients"
    )

    # Client hatch rate
    parser.add_option(
        '-r', '--hatch-rate',
        action='store',
        type='int',
        dest='hatch_rate',
        default=1,
        help="The rate per second in which clients are spawned"
    )

    # redis options
    parser.add_option(
        '--redis-host',
        action='store',
        type='str',
        dest='redis_host',
        default="127.0.0.1",
        help="Redis host to use for distributed load testing. Only used when running with --master or --slave. Defaults to 127.0.0.1."
    )
    parser.add_option(
        '--redis-port',
        action='store',
        type='int',
        dest='redis_port',
        default=6379,
        help="Redis port to use for distributed load testing. Only used when running with --master or --slave. Defaults to 6379, which is the default Redis port."
    )

    # Finalize
    # Return three-tuple of parser + the output from parse_args (opt obj, args)
    opts, args = parser.parse_args()
    return parser, opts, args

def _is_package(path):
    """
    Is the given path a Python package?
    """
    return (
        os.path.isdir(path)
        and os.path.exists(os.path.join(path, '__init__.py'))
    )

def find_locustfile(locustfile):
    """
    Attempt to locate a locustfile, either explicitly or by searching parent dirs.
    """
    # Obtain env value
    names = [locustfile]
    # Create .py version if necessary
    if not names[0].endswith('.py'):
        names += [names[0] + '.py']
    # Does the name contain path elements?
    if os.path.dirname(names[0]):
        # If so, expand home-directory markers and test for existence
        for name in names:
            expanded = os.path.expanduser(name)
            if os.path.exists(expanded):
                if name.endswith('.py') or _is_package(expanded):
                    return os.path.abspath(expanded)
    else:
        # Otherwise, start in cwd and work downwards towards filesystem root
        path = '.'
        # Stop before falling off root of filesystem (should be platform
        # agnostic)
        while os.path.split(os.path.abspath(path))[1]:
            for name in names:
                joined = os.path.join(path, name)
                if os.path.exists(joined):
                    if name.endswith('.py') or _is_package(joined):
                        return os.path.abspath(joined)
            path = os.path.join('..', path)
    # Implicit 'return None' if nothing was found

def is_locust(tup):
    """
    Takes (name, object) tuple, returns True if it's a public Locust subclass.
    """
    name, item = tup
    return (
        inspect.isclass(item)
        and issubclass(item, Locust)
        and (item not in _internals)
        and not name.startswith('_')
    )

def load_locustfile(path):
    """
    Import given locustfile path and return (docstring, callables).

    Specifically, the locustfile's ``__doc__`` attribute (a string) and a
    dictionary of ``{'name': callable}`` containing all callables which pass
    the "is a Locust" test.
    """
    # Get directory and locustfile name
    directory, locustfile = os.path.split(path)
    # If the directory isn't in the PYTHONPATH, add it so our import will work
    added_to_path = False
    index = None
    if directory not in sys.path:
        sys.path.insert(0, directory)
        added_to_path = True
    # If the directory IS in the PYTHONPATH, move it to the front temporarily,
    # otherwise other locustfiles -- like Locusts's own -- may scoop the intended
    # one.
    else:
        i = sys.path.index(directory)
        if i != 0:
            # Store index for later restoration
            index = i
            # Add to front, then remove from original position
            sys.path.insert(0, directory)
            del sys.path[i + 1]
    # Perform the import (trimming off the .py)
    imported = __import__(os.path.splitext(locustfile)[0])
    # Remove directory from path if we added it ourselves (just to be neat)
    if added_to_path:
        del sys.path[0]
    # Put back in original index if we moved it
    if index is not None:
        sys.path.insert(index + 1, directory)
        del sys.path[0]
    # Return our two-tuple
    locusts = dict(filter(is_locust, vars(imported).items()))
    return imported.__doc__, locusts

def main():
    parser, options, arguments = parse_options()
    #print "Options:", options, dir(options)
    #print "Arguments:", arguments
    #print "largs:", parser.largs
    #print "rargs:", parser.rargs

    if options.show_version:
        print("Locust %s" % (version))
        sys.exit(0)

    locustfile = find_locustfile(options.locustfile)
    if not locustfile:
        print "Could not find any locustfile!"
        sys.exit(1)

    docstring, locusts = load_locustfile(locustfile)

    if options.list_commands:
        print "Available Locusts:"
        for name in locusts:
            print "    " + name
        sys.exit(0)

    if not locusts:
        sys.stderr.write("No Locust class found!\n")
        sys.exit(1)

    # make sure specified Locust exists
    if arguments:
        missing = set(arguments) - set(locusts.keys())
        if missing:
            sys.stderr.write("Unknown Locust(s): %s\n" % (", ".join(missing)))
            sys.exit(1)
        else:
            names = set(arguments) & set(locusts.keys())
            locust_classes = [locusts[n] for n in names]
    else:
        locust_classes = locusts.values()

    if options.show_task_ratio:
        print "\n Task ratio per locust class"
        print "-" * 80
        inspectlocust.print_task_ratio(locust_classes)
        print "\n Total task ratio"
        print "-" * 80
        inspectlocust.print_task_ratio(locust_classes, total=True)
        sys.exit(0)

    if options.web and not options.slave:
        # spawn web greenlet
        print "Starting web monitor on port 8089"
        main_greenlet = gevent.spawn(web.start, locust_classes, options.hatch_rate, options.num_clients, options.num_requests)

    # enable/disable gzip in WebLocust's HTTP client
    WebLocust.gzip = options.gzip

    if not options.master and not options.slave:
        core.locust_runner = LocalLocustRunner(locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.host)
        # spawn client spawning/hatching greenlet
        if not options.web:
            core.locust_runner.start_hatching()
            main_greenlet = core.locust_runner.greenlet
    elif options.master:
        core.locust_runner = MasterLocustRunner(locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, redis_host=options.redis_host, redis_port=options.redis_port)
    elif options.slave:
        core.locust_runner = SlaveLocustRunner(locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, redis_host=options.redis_host, redis_port=options.redis_port)
        main_greenlet = core.locust_runner.greenlet

    if options.print_stats or (not options.web and not options.slave):
        # spawn stats printing greenlet
        gevent.spawn(stats_printer)

    try:
        print
        print "Starting Locust %s" % version
        print
        main_greenlet.join()
    except KeyboardInterrupt, e:
        time.sleep(0.2)
        print "\n"
        print_stats(core.locust_runner.request_stats)
        print_percentile_stats(core.locust_runner.request_stats)
        print_error_report()
        print
        print "Exiting, bye.."
        print

    sys.exit(0)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python
import os
from time import gmtime, strftime
import logging

__author__ = 'Kadir Sert'
__location__ = os.path.dirname(os.path.abspath(__file__))
script_run_time = strftime("%Y-%m-%d_%H-%M-%S", gmtime())
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
handler = logging.FileHandler(os.path.join(__location__, 'log_cleaner.out'))
handler.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.info('')
logger.info('************************ Script Calismaya Basladi... ******************************')

try:
    qf = open(os.path.join(__location__, 'log_cleaner_properties'), 'r')
    for line in qf:
        try:
            line = ''.join(line.split())
            if line == '' or line[0] == '#':
                continue
            lc_props = line.split('|||')
            if lc_props[3] == 'delete':
                lc_message = lc_props[0] + ' - ' + lc_props[1] + ' patternli ' + \
                    lc_props[2] + ' gundur degismemis dosyalar siliniyor!'
                logger.info(lc_message)
                lc_command = 'find ' + lc_props[0] + ' -name ' + lc_props[1] + ' -mtime +' + lc_props[2]
                lc_message = os.popen(lc_command).read()
                logger.info(lc_message)
                lc_command = 'rm -f ' + '$(find ' + lc_props[0] + ' -name ' + lc_props[1] + \
                    ' -mtime +' + lc_props[2] + ')'
                os.system(lc_command)
            if lc_props[3] == 'zip':
                lc_message = lc_props[0] + ' - ' + lc_props[1] + ' patternli ' + \
                    lc_props[2] + ' gundur degismemis dosyalar zipleniyor!'
                logger.info(lc_message)
                lc_command = 'find ' + lc_props[0] + ' -name ' + lc_props[1] + ' -mtime +' + lc_props[2]
                log_files = os.popen(lc_command).read()
                logger.info(log_files)
                if log_files == '':
                    lc_message = lc_props[0] + ' - ' + lc_props[1] + ' patternli dosya bulunamadi!'
                    logger.info(lc_message)
                    continue
                if os.path.isfile(os.path.join(__location__, 'temp_file')):
                    os.remove(os.path.join(__location__, 'temp_file'))
                temp_file = open(os.path.join(__location__, 'temp_file'), 'w')
                temp_file.write(log_files)
                temp_file.close()
                tar_file = os.path.join(os.path.dirname(lc_props[4]),
                                        script_run_time + '_' + os.path.basename(lc_props[4]))
                lc_command = 'tar -czvf ' + tar_file + ' -T ' + os.path.join(__location__, 'temp_file')
                os.system(lc_command)
                if os.path.isfile(os.path.join(__location__, 'temp_file')):
                    os.remove(os.path.join(__location__, 'temp_file'))
                lc_message = lc_props[0] + ' - ' + lc_props[1] + ' patternli ' + \
                    lc_props[2] + ' gundur degismemis dosyalar siliniyor!'
                logger.info(lc_message)
                lc_command = 'find ' + lc_props[0] + ' -name ' + lc_props[1] + ' -mtime +' + lc_props[2]
                lc_message = os.popen(lc_command).read()
                logger.info(lc_message)
                lc_command = 'rm -f ' + '$(find ' + lc_props[0] + ' -name ' + lc_props[1] + \
                    ' -mtime +' + lc_props[2] + ')'
                os.system(lc_command)
        except Exception, e:
            logger.error(e)
finally:
    qf.close()<commit_msg>type -f option added to find<commit_after>#!/usr/bin/python
import os
from time import gmtime, strftime
import logging

__author__ = 'Kadir Sert'
__location__ = os.path.dirname(os.path.abspath(__file__))
script_run_time = strftime("%Y-%m-%d_%H-%M-%S", gmtime())
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
handler = logging.FileHandler(os.path.join(__location__, 'log_cleaner.out'))
handler.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.info('')
logger.info('************************ Script Calismaya Basladi... ******************************')

try:
    qf = open(os.path.join(__location__, 'log_cleaner_properties'), 'r')
    for line in qf:
        try:
            line = ''.join(line.split())
            if line == '' or line[0] == '#':
                continue
            lc_props = line.split('|||')
            if lc_props[3] == 'delete':
                lc_message = lc_props[0] + ' - ' + lc_props[1] + ' patternli ' + \
                    lc_props[2] + ' gundur degismemis dosyalar siliniyor!'
                logger.info(lc_message)
                lc_command = 'find ' + lc_props[0] + ' -name ' + lc_props[1] + ' -mtime +' + lc_props[2] + ' -type f'
                lc_message = os.popen(lc_command).read()
                logger.info(lc_message)
                lc_command = 'rm -f ' + '$(find ' + lc_props[0] + ' -name ' + lc_props[1] + \
                    ' -mtime +' + lc_props[2] + ' -type f' + ')'
                os.system(lc_command)
            if lc_props[3] == 'zip':
                lc_message = lc_props[0] + ' - ' + lc_props[1] + ' patternli ' + \
                    lc_props[2] + ' gundur degismemis dosyalar zipleniyor!'
                logger.info(lc_message)
                lc_command = 'find ' + lc_props[0] + ' -name ' + lc_props[1] + ' -mtime +' + lc_props[2] + ' -type f'
                log_files = os.popen(lc_command).read()
                logger.info(log_files)
                if log_files == '':
                    lc_message = lc_props[0] + ' - ' + lc_props[1] + ' patternli dosya bulunamadi!'
                    logger.info(lc_message)
                    continue
                if os.path.isfile(os.path.join(__location__, 'temp_file')):
                    os.remove(os.path.join(__location__, 'temp_file'))
                temp_file = open(os.path.join(__location__, 'temp_file'), 'w')
                temp_file.write(log_files)
                temp_file.close()
                tar_file = os.path.join(os.path.dirname(lc_props[4]),
                                        script_run_time + '_' + os.path.basename(lc_props[4]))
                lc_command = 'tar -czvf ' + tar_file + ' -T ' + os.path.join(__location__, 'temp_file')
                os.system(lc_command)
                if os.path.isfile(os.path.join(__location__, 'temp_file')):
                    os.remove(os.path.join(__location__, 'temp_file'))
                lc_message = lc_props[0] + ' - ' + lc_props[1] + ' patternli ' + \
                    lc_props[2] + ' gundur degismemis dosyalar siliniyor!'
                logger.info(lc_message)
                lc_command = 'find ' + lc_props[0] + ' -name ' + lc_props[1] + ' -mtime +' + lc_props[2] + ' -type f'
                lc_message = os.popen(lc_command).read()
                logger.info(lc_message)
                lc_command = 'rm -f ' + '$(find ' + lc_props[0] + ' -name ' + lc_props[1] + \
                    ' -mtime +' + lc_props[2] + ' -type f' + ')'
                os.system(lc_command)
        except Exception, e:
            logger.error(e)
finally:
    qf.close()<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Parses CSV output from the loading_measurement and outputs interesting stats.

Example usage:
$ tools/perf/run_measurement --browser=release \
    --output-format=csv --output=/path/to/loading_measurement_output.csv \
    loading_measurement tools/perf/page_sets/top_1m.json
$ tools/perf/perf_tools/loading_measurement_analyzer.py \
    --num-slowest-urls=100 --rank-csv-file=/path/to/top-1m.csv \
    /path/to/loading_measurement_output.csv
"""

import collections
import csv
import heapq
import optparse
import os
import sys

class LoadingMeasurementAnalyzer(object):

  def __init__(self, input_file, options):
    self.ranks = {}
    self.totals = collections.defaultdict(list)
    self.maxes = collections.defaultdict(list)
    self.avgs = collections.defaultdict(list)
    self.num_rows_parsed = 0
    self.num_slowest_urls = options.num_slowest_urls
    self._ParseInputFile(input_file, options)
    if options.rank_csv_file:
      self._ParseRankCsvFile(os.path.expanduser(options.rank_csv_file))

  def _ParseInputFile(self, input_file, options):
    with open(input_file, 'r') as csvfile:
      row_dict = csv.DictReader(csvfile)
      for row in row_dict:
        for key, value in row.iteritems():
          if key in ('url', 'dom_content_loaded_time (ms)', 'load_time (ms)'):
            continue
          if not value:
            continue
          if '_avg' in key:
            self.avgs[key].append((float(value), row['url']))
          elif '_max' in key:
            self.maxes[key].append((float(value), row['url']))
          else:
            self.totals[key].append((float(value), row['url']))
        self.num_rows_parsed += 1
        if options.max_rows and self.num_rows_parsed == int(options.max_rows):
          break

  def _ParseRankCsvFile(self, input_file):
    with open(input_file, 'r') as csvfile:
      for row in csv.reader(csvfile):
        assert len(row) == 2
        self.ranks[row[1]] = int(row[0])

  def _GetRank(self, url):
    url = url.replace('http://', '')
    if url in self.ranks:
      return self.ranks[url]
    return len(self.ranks)

  def PrintSummary(self):
    sum_totals = {}
    for key, values in self.totals.iteritems():
      sum_totals[key] = sum([v[0] for v in values])
    total_time = sum(sum_totals.values())

    print
    print 'Total URLs: ', self.num_rows_parsed
    print 'Total time: %ds' % int(round(total_time / 1000))
    print
    for key, value in sorted(sum_totals.iteritems(), reverse=True,
                             key=lambda i: i[1]):
      output_key = '%30s: ' % key.replace(' (ms)', '')
      output_value = '%10ds ' % (value / 1000)
      output_percent = '%.1f%%' % (100 * value / total_time)
      print output_key, output_value, output_percent

    if not self.num_slowest_urls:
      return

    for key, values in self.totals.iteritems():
      print
      print 'Top %d slowest %s:' % (self.num_slowest_urls,
                                    key.replace(' (ms)', ''))
      slowest = heapq.nlargest(self.num_slowest_urls, values)
      for value, url in slowest:
        print '\t', '%dms\t' % value, url, '(#%s)' % self._GetRank(url)

def main(argv):
  prog_desc = 'Parses CSV output from the loading_measurement'
  parser = optparse.OptionParser(usage=('%prog [options]' + '\n\n' + prog_desc))

  parser.add_option('--max-rows', type='int',
                    help='Only process this many rows')
  parser.add_option('--num-slowest-urls', type='int',
                    help='Output this many slowest URLs for each category')
  parser.add_option('--rank-csv-file', help='A CSV file of <rank,url>')

  options, args = parser.parse_args(argv[1:])

  assert len(args) == 1, 'Must pass exactly one CSV file to analyze'

  LoadingMeasurementAnalyzer(args[0], options).PrintSummary()

  return 0

if __name__ == '__main__':
  sys.exit(main(sys.argv))
<commit_msg>[Telemetry] Add ability to analyze top N sites by rank.<commit_after>#!/usr/bin/env python
# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Parses CSV output from the loading_measurement and outputs interesting stats.

Example usage:
$ tools/perf/run_measurement --browser=release \
    --output-format=csv --output=/path/to/loading_measurement_output.csv \
    loading_measurement tools/perf/page_sets/top_1m.json
$ tools/perf/perf_tools/loading_measurement_analyzer.py \
    --num-slowest-urls=100 --rank-csv-file=/path/to/top-1m.csv \
    /path/to/loading_measurement_output.csv
"""

import collections
import csv
import heapq
import optparse
import os
import sys

class LoadingMeasurementAnalyzer(object):

  def __init__(self, input_file, options):
    self.ranks = {}
    self.totals = collections.defaultdict(list)
    self.maxes = collections.defaultdict(list)
    self.avgs = collections.defaultdict(list)
    self.num_rows_parsed = 0
    self.num_slowest_urls = options.num_slowest_urls
    if options.rank_csv_file:
      self._ParseRankCsvFile(os.path.expanduser(options.rank_csv_file))
    self._ParseInputFile(input_file, options)

  def _ParseInputFile(self, input_file, options):
    with open(input_file, 'r') as csvfile:
      row_dict = csv.DictReader(csvfile)
      for row in row_dict:
        if (options.rank_limit and
            self._GetRank(row['url']) > options.rank_limit):
          continue
        for key, value in row.iteritems():
          if key in ('url', 'dom_content_loaded_time (ms)', 'load_time (ms)'):
            continue
          if not value:
            continue
          if '_avg' in key:
            self.avgs[key].append((float(value), row['url']))
          elif '_max' in key:
            self.maxes[key].append((float(value), row['url']))
          else:
            self.totals[key].append((float(value), row['url']))
        self.num_rows_parsed += 1
        if options.max_rows and self.num_rows_parsed == int(options.max_rows):
          break

  def _ParseRankCsvFile(self, input_file):
    with open(input_file, 'r') as csvfile:
      for row in csv.reader(csvfile):
        assert len(row) == 2
        self.ranks[row[1]] = int(row[0])

  def _GetRank(self, url):
    url = url.replace('http://', '')
    if url in self.ranks:
      return self.ranks[url]
    return len(self.ranks)

  def PrintSummary(self):
    sum_totals = {}
    for key, values in self.totals.iteritems():
      sum_totals[key] = sum([v[0] for v in values])
    total_time = sum(sum_totals.values())

    print
    print 'Total URLs: ', self.num_rows_parsed
    print 'Total time: %ds' % int(round(total_time / 1000))
    print
    for key, value in sorted(sum_totals.iteritems(), reverse=True,
                             key=lambda i: i[1]):
      output_key = '%30s: ' % key.replace(' (ms)', '')
      output_value = '%10ds ' % (value / 1000)
      output_percent = '%.1f%%' % (100 * value / total_time)
      print output_key, output_value, output_percent

    if not self.num_slowest_urls:
      return

    for key, values in self.totals.iteritems():
      print
      print 'Top %d slowest %s:' % (self.num_slowest_urls,
                                    key.replace(' (ms)', ''))
      slowest = heapq.nlargest(self.num_slowest_urls, values)
      for value, url in slowest:
        print '\t', '%dms\t' % value, url, '(#%s)' % self._GetRank(url)

def main(argv):
  prog_desc = 'Parses CSV output from the loading_measurement'
  parser = optparse.OptionParser(usage=('%prog [options]' + '\n\n' + prog_desc))

  parser.add_option('--max-rows', type='int',
                    help='Only process this many rows')
  parser.add_option('--num-slowest-urls', type='int',
                    help='Output this many slowest URLs for each category')
  parser.add_option('--rank-csv-file', help='A CSV file of <rank,url>')
  parser.add_option('--rank-limit', type='int',
                    help='Only process pages higher than this rank')

  options, args = parser.parse_args(argv[1:])

  assert len(args) == 1, 'Must pass exactly one CSV file to analyze'
  if options.rank_limit and not options.rank_csv_file:
    print 'Must pass --rank-csv-file with --rank-limit'
    return 1

  LoadingMeasurementAnalyzer(args[0], options).PrintSummary()

  return 0

if __name__ == '__main__':
  sys.exit(main(sys.argv))
<|endoftext|>


<commit_before>from django.contrib import admin
from django.contrib.auth.admin import UserAdmin as BaseUserAdmin
from django.contrib.auth.models import User

# Register your models here.

from .models import Profile, AmbulancePermission, HospitalPermission

# Define an inline admin descriptor for Profile model
# which acts a bit like a singleton
class ProfileInline(admin.StackedInline):
    model = Profile
    can_delete = False
    verbose_name_plural = 'profile'

# Define a new User admin
class UserAdmin(BaseUserAdmin):
    inlines = (ProfileInline,)

# Re-register UserAdmin
admin.site.unregister(User)
admin.site.register(User, UserAdmin)
<commit_msg>Adding group profiles.<commit_after>from django.contrib import admin
from django.contrib.auth.admin import UserAdmin as BaseUserAdmin
from django.contrib.auth.models import User, Group

# Register your models here.

from .models import Profile, AmbulancePermission, HospitalPermission

# Define an inline admin descriptor for Profile model
# which acts a bit like a singleton
class ProfileInline(admin.StackedInline):
    model = Profile
    can_delete = False
    verbose_name_plural = 'profile'

# Define a new User admin
class UserAdmin(BaseUserAdmin):
    inlines = (ProfileInline,)

# Re-register UserAdmin
admin.site.unregister(User)
admin.site.register(User, UserAdmin)
admin.site.register(Group)
<|endoftext|>


<commit_before>import spacy

from initialize import spacy_nlp
from interfaces.SentenceOperation import SentenceOperation
from interfaces.SentencePairOperation import SentencePairOperation
from tasks.TaskTypes import TaskType

"""
Subject Object Switch.
    Switches with each other the subject and object of English sentences.
"""

def subject_object_switch(sentence, nlp):

    # Tokenize Sentence
    doc = nlp(sentence)

    # Initialize Variables
    changed = False
    chunks = {}
    new_sentence = []
    root_token = list(doc.sents)[0].root

    # Evaluate Tokens
    for token in doc:

        # Evaluate subjects and objects
        if "subj" in token.dep_ or "obj" in token.dep_:

            # Initialize Variables
            current_chunk = []
            check = [token]

            # Evaluate pending tokens
            while check:
                current_token = check.pop(0)
                current_chunk.append(current_token)

                # Get token children
                for child in current_token.children:
                    check.append(child)

            # Sort by token index
            current_chunk.sort(key=lambda x: x.i)

            # Get chunk position
            index = current_chunk[0].i

            # Add chunk to list
            chunks[index] = current_chunk

    # Naively select candidate objects and subjects
    candidate_subjects = [k for k in chunks.keys() if k < root_token.i]
    candidate_objects = [k for k in chunks.keys() if k > root_token.i]

    # If there are both, naively select a subject and an object
    if candidate_subjects and candidate_objects:
        subject_chunk = chunks[max(candidate_subjects)]
        object_chunk = chunks[min(candidate_objects)]
        changed = True

        # Add tokens in the right order
        new_sentence.extend(doc[i] for i in range(0, subject_chunk[0].i))
        new_sentence.extend(
            doc[i] for i in range(object_chunk[0].i, object_chunk[-1].i)
        )
        new_sentence.extend(
            nlp(object_chunk[-1].text + subject_chunk[-1].whitespace_)
        )
        new_sentence.extend(
            doc[i] for i in range(subject_chunk[-1].i + 1, object_chunk[0].i)
        )
        new_sentence.extend(
            doc[i] for i in range(subject_chunk[0].i, subject_chunk[-1].i)
        )
        new_sentence.extend(
            nlp(subject_chunk[-1].text + object_chunk[-1].whitespace_)
        )
        new_sentence.extend(
            doc[i] for i in range(object_chunk[-1].i + 1, len(doc))
        )

    # Rebuild Sentence
    new_sentence = [t.text + t.whitespace_ for t in new_sentence]
    new_sentence = "".join(new_sentence)

    # Naively handle basic capitalization
    if (
        sentence[0] == sentence[0].upper()
        and new_sentence[0] != new_sentence[0].upper()
    ):
        new_sentence = new_sentence[0].upper() + new_sentence[1:]

    return new_sentence, changed

class SentenceSubjectObjectSwitch(SentenceOperation):
    tasks = [TaskType.TEXT_CLASSIFICATION, TaskType.TEXT_TO_TEXT_GENERATION]
    languages = ["en"]
    keywords = [
        "lexical",
        "syntactic",
        "word-order",
        "rule-based",
        "tokenizer-required",
        "chunker-required",
        "meaning-alteration",
        "high-precision",
        "low-coverage",
        "low-generations",
    ]

    def __init__(self, seed=0, max_outputs=1):
        super().__init__(seed, max_outputs=max_outputs)
        # Initialize the spacy model
        self.nlp = spacy_nlp if spacy_nlp else spacy.load("en_core_web_sm")

    def generate(self, sentence: str):

        # Initialize Variables
        output_sentence = sentence

        new_sentence, changed = subject_object_switch(sentence, self.nlp)

        if changed:
            output_sentence = new_sentence

        return [output_sentence]

class PairSubjectObjectSwitch(SentencePairOperation):
    tasks = [TaskType.PARAPHRASE_DETECTION]
    languages = ["en"]
    keywords = [
        "lexical",
        "syntactic",
        "word-order",
        "rule-based",
        "tokenizer-required",
        "chunker-required",
        "meaning-alteration",
        "high-precision",
        "low-coverage",
        "low-generations",
    ]

    def __init__(self, seed=0, max_outputs=3, pos_label="1", neg_label="0"):
        super().__init__(seed, max_outputs=max_outputs)
        # Initialize the spacy model
        self.nlp = spacy_nlp if spacy_nlp else spacy.load("en_core_web_sm")

        self.pos_label = pos_label
        self.neg_label = neg_label

    def generate(self, sentence1: str, sentence2: str, target: str):

        # Initialize Variables
        output_sentences = []
        changed_sentences = {}

        # Only process equivalent pairs
        if target == self.pos_label:

            for n, sentence in enumerate([sentence1, sentence2]):
                new_sentence, changed = subject_object_switch(
                    sentence, self.nlp
                )
                if changed:
                    changed_sentences[n] = new_sentence

        if 0 in changed_sentences.keys():
            output_sentences.append(
                (changed_sentences[0], sentence2, self.neg_label)
            )

        if 1 in changed_sentences.keys():
            output_sentences.append(
                (sentence1, changed_sentences[1], self.neg_label)
            )

        if 0 in changed_sentences.keys() and 1 in changed_sentences.keys():
            output_sentences.append(
                (changed_sentences[0], changed_sentences[1], self.pos_label)
            )

        if not output_sentences:
            output_sentences = [(sentence1, sentence2, target)]

        return output_sentences

"""
# Sample code to demonstrate usage. Can also assist in adding test cases.
# You don't need to keep this code in your transformation.
if __name__ == '__main__':
    import json
    from TestRunner import convert_to_snake_case
    tf = SentenceSubjectObjectSwitch()
    test_cases = []
    for sentence in ["Andrew has not returned the French book to the library.",
                     "John Locke and Adam Smith were born before Karl Marx and Friedrich Engels.",
                     "John loves Mary.",
                     "Ujjal Dev Dosanjh was not the 1st Premier of British Columbia from 1871 to 1872.",
                     "The fighters would not give up the fight."]:
        test_cases.append({
            "class": tf.name(),
            "inputs": {"sentence": sentence},
            "outputs": [{"sentence": o[0]} for o in tf.generate(sentence)]}
        )
    json_file = {"type": convert_to_snake_case(tf.name()), "test_cases": test_cases}
    print(json.dumps(json_file))
    tf = PairSubjectObjectSwitch(max_outputs=3)
    test_cases = []
    for sentence1, sentence2, target in zip(["Andrew has not returned the French book to the library.",
                                     "John Locke and Adam Smith were born before Karl Marx and Friedrich Engels.",
                                     "John loves Mary.",
                                     "Ujjal Dev Dosanjh was not the 1st Premier of British Columbia from 1871 to 1872.",
                                     "The fighters would not give up the fight."],
                                    ["He hasn't brought back the library's books.",
                                     "Karl Marx and Friedrich Engels were born after John Locke and Adam Smith.",
                                     "John really likes Mary.",
                                     "U.D. Dosanjh wasn't the 1st Premier of British Columbia for a year from 1871.",
                                     "The warriors would stay in the battle"],
                                     ["1",
                                      "1",
                                      "1",
                                      "1",
                                      "1"
                                     ]
                                    ):
        test_cases.append({
            "class": tf.name(),
            "inputs": {"sentence1": sentence1, "sentence2": sentence2, "target": target},
            "outputs": [{"sentence1": o[0], "sentence2": o[1], "target": o[2]} for o in tf.generate(sentence1, sentence2, target)]}
        )
    json_file = {"type": convert_to_snake_case(tf.name()), "test_cases": test_cases}
    print(json.dumps(json_file))
"""
<commit_msg>Fix handling of empty sentences, broken by capitalization check<commit_after>import spacy

from initialize import spacy_nlp
from interfaces.SentenceOperation import SentenceOperation
from interfaces.SentencePairOperation import SentencePairOperation
from tasks.TaskTypes import TaskType

"""
Subject Object Switch.
    Switches with each other the subject and object of English sentences.
"""

def subject_object_switch(sentence, nlp):

    # Tokenize Sentence
    doc = nlp(sentence)

    # Initialize Variables
    changed = False
    chunks = {}
    new_sentence = []
    root_token = list(doc.sents)[0].root

    # Evaluate Tokens
    for token in doc:

        # Evaluate subjects and objects
        if "subj" in token.dep_ or "obj" in token.dep_:

            # Initialize Variables
            current_chunk = []
            check = [token]

            # Evaluate pending tokens
            while check:
                current_token = check.pop(0)
                current_chunk.append(current_token)

                # Get token children
                for child in current_token.children:
                    check.append(child)

            # Sort by token index
            current_chunk.sort(key=lambda x: x.i)

            # Get chunk position
            index = current_chunk[0].i

            # Add chunk to list
            chunks[index] = current_chunk

    # Naively select candidate objects and subjects
    candidate_subjects = [k for k in chunks.keys() if k < root_token.i]
    candidate_objects = [k for k in chunks.keys() if k > root_token.i]

    # If there are both, naively select a subject and an object
    if candidate_subjects and candidate_objects:
        subject_chunk = chunks[max(candidate_subjects)]
        object_chunk = chunks[min(candidate_objects)]
        changed = True

        # Add tokens in the right order
        new_sentence.extend(doc[i] for i in range(0, subject_chunk[0].i))
        new_sentence.extend(
            doc[i] for i in range(object_chunk[0].i, object_chunk[-1].i)
        )
        new_sentence.extend(
            nlp(object_chunk[-1].text + subject_chunk[-1].whitespace_)
        )
        new_sentence.extend(
            doc[i] for i in range(subject_chunk[-1].i + 1, object_chunk[0].i)
        )
        new_sentence.extend(
            doc[i] for i in range(subject_chunk[0].i, subject_chunk[-1].i)
        )
        new_sentence.extend(
            nlp(subject_chunk[-1].text + object_chunk[-1].whitespace_)
        )
        new_sentence.extend(
            doc[i] for i in range(object_chunk[-1].i + 1, len(doc))
        )

        # Rebuild Sentence
        new_sentence = [t.text + t.whitespace_ for t in new_sentence]
        new_sentence = "".join(new_sentence)

        # Naively handle basic capitalization
        if (
            sentence[0] == sentence[0].upper()
            and new_sentence[0] != new_sentence[0].upper()
        ):
            new_sentence = new_sentence[0].upper() + new_sentence[1:]

    return new_sentence, changed

class SentenceSubjectObjectSwitch(SentenceOperation):
    tasks = [TaskType.TEXT_CLASSIFICATION, TaskType.TEXT_TO_TEXT_GENERATION]
    languages = ["en"]
    keywords = [
        "lexical",
        "syntactic",
        "word-order",
        "rule-based",
        "tokenizer-required",
        "chunker-required",
        "meaning-alteration",
        "high-precision",
        "low-coverage",
        "low-generations",
    ]

    def __init__(self, seed=0, max_outputs=1):
        super().__init__(seed, max_outputs=max_outputs)
        # Initialize the spacy model
        self.nlp = spacy_nlp if spacy_nlp else spacy.load("en_core_web_sm")

    def generate(self, sentence: str):

        # Initialize Variables
        output_sentence = sentence

        new_sentence, changed = subject_object_switch(sentence, self.nlp)

        if changed:
            output_sentence = new_sentence

        return [output_sentence]

class PairSubjectObjectSwitch(SentencePairOperation):
    tasks = [TaskType.PARAPHRASE_DETECTION]
    languages = ["en"]
    keywords = [
        "lexical",
        "syntactic",
        "word-order",
        "rule-based",
        "tokenizer-required",
        "chunker-required",
        "meaning-alteration",
        "high-precision",
        "low-coverage",
        "low-generations",
    ]

    def __init__(self, seed=0, max_outputs=3, pos_label="1", neg_label="0"):
        super().__init__(seed, max_outputs=max_outputs)
        # Initialize the spacy model
        self.nlp = spacy_nlp if spacy_nlp else spacy.load("en_core_web_sm")

        self.pos_label = pos_label
        self.neg_label = neg_label

    def generate(self, sentence1: str, sentence2: str, target: str):

        # Initialize Variables
        output_sentences = []
        changed_sentences = {}

        # Only process equivalent pairs
        if target == self.pos_label:

            for n, sentence in enumerate([sentence1, sentence2]):
                new_sentence, changed = subject_object_switch(
                    sentence, self.nlp
                )
                if changed:
                    changed_sentences[n] = new_sentence

        if 0 in changed_sentences.keys():
            output_sentences.append(
                (changed_sentences[0], sentence2, self.neg_label)
            )

        if 1 in changed_sentences.keys():
            output_sentences.append(
                (sentence1, changed_sentences[1], self.neg_label)
            )

        if 0 in changed_sentences.keys() and 1 in changed_sentences.keys():
            output_sentences.append(
                (changed_sentences[0], changed_sentences[1], self.pos_label)
            )

        if not output_sentences:
            output_sentences = [(sentence1, sentence2, target)]

        return output_sentences

"""
# Sample code to demonstrate usage. Can also assist in adding test cases.
# You don't need to keep this code in your transformation.
if __name__ == '__main__':
    import json
    from TestRunner import convert_to_snake_case
    tf = SentenceSubjectObjectSwitch()
    test_cases = []
    for sentence in ["Andrew has not returned the French book to the library.",
                     "John Locke and Adam Smith were born before Karl Marx and Friedrich Engels.",
                     "John loves Mary.",
                     "Ujjal Dev Dosanjh was not the 1st Premier of British Columbia from 1871 to 1872.",
                     "The fighters would not give up the fight."]:
        test_cases.append({
            "class": tf.name(),
            "inputs": {"sentence": sentence},
            "outputs": [{"sentence": o[0]} for o in tf.generate(sentence)]}
        )
    json_file = {"type": convert_to_snake_case(tf.name()), "test_cases": test_cases}
    print(json.dumps(json_file))
    tf = PairSubjectObjectSwitch(max_outputs=3)
    test_cases = []
    for sentence1, sentence2, target in zip(["Andrew has not returned the French book to the library.",
                                     "John Locke and Adam Smith were born before Karl Marx and Friedrich Engels.",
                                     "John loves Mary.",
                                     "Ujjal Dev Dosanjh was not the 1st Premier of British Columbia from 1871 to 1872.",
                                     "The fighters would not give up the fight."],
                                    ["He hasn't brought back the library's books.",
                                     "Karl Marx and Friedrich Engels were born after John Locke and Adam Smith.",
                                     "John really likes Mary.",
                                     "U.D. Dosanjh wasn't the 1st Premier of British Columbia for a year from 1871.",
                                     "The warriors would stay in the battle"],
                                     ["1",
                                      "1",
                                      "1",
                                      "1",
                                      "1"
                                     ]
                                    ):
        test_cases.append({
            "class": tf.name(),
            "inputs": {"sentence1": sentence1, "sentence2": sentence2, "target": target},
            "outputs": [{"sentence1": o[0], "sentence2": o[1], "target": o[2]} for o in tf.generate(sentence1, sentence2, target)]}
        )
    json_file = {"type": convert_to_snake_case(tf.name()), "test_cases": test_cases}
    print(json.dumps(json_file))
"""
<|endoftext|>


<commit_before># This is the blocking version of the Slow Poetry Server.

import optparse, os, socket, time

def parse_args():
    usage = """usage: %prog [options] poetry-file

This is the Slow Poetry Server, blocking edition.
Run it like this:

  python slowpoetry.py <path-to-poetry-file>

If you are in the base directory of the twisted-intro package,
you could run it like this:

  python blocking-server/slowpoetry.py poetry/ecstasy.txt

to serve up John Donne's Ecstasy, which I know you want to do.
"""

    parser = optparse.OptionParser(usage)

    help = "The port to listen on. Default to a random available port."
    option = parser.add_option('--port', type='int', help=help)

    help = "The interface to listen on. Default is localhost."
    option = parser.add_option('--iface', help=help, default='')

    help = "The number of seconds between sending bytes."
    option = parser.add_option('--delay', type='float', help=help,
                               default=1)

    help = "The number of bytes to send at a time."
    option = parser.add_option('--num-bytes', type='int', help=help,
                               default=10)

    options, args = parser.parse_args()

    if len(args) != 1:
        parser.error('Provide exactly one poetry file.')

    poetry_file = args[0]

    if not os.path.exists(args[0]):
        parser.error('No such file: %s' % poetry_file)

    return options, poetry_file

def send_poetry(sock, poetry_file, num_bytes, delay):
    """Send some poetry slowly down the socket."""

    inputf = open(poetry_file)

    while True:
        bytes = inputf.read(num_bytes)

        if not bytes: # no more poetry :(
            sock.close()
            inputf.close()
            return

        try:
            sock.sendall(bytes)
        except socket.error:
            sock.close()
            inputf.close()
            return

        time.sleep(delay)

def serve(listen_socket, poetry_file, num_bytes, delay):
    while True:
        sock, addr = listen_socket.accept()

        print 'Somebody at %s wants poetry!' % (addr,)

        send_poetry(sock, poetry_file, num_bytes, delay)

def main():
    options, poetry_file= parse_args()

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    sock.bind((options.iface, options.port or 0))

    sock.listen(5)

    print 'Serving %s on port %s.' % (poetry_file, sock.getsockname()[1])

    serve(sock, poetry_file, options.num_bytes, options.delay)

if __name__ == '__main__':
    main()
<commit_msg>Log bytes sent.<commit_after># This is the blocking version of the Slow Poetry Server.

import optparse, os, socket, time

def parse_args():
    usage = """usage: %prog [options] poetry-file

This is the Slow Poetry Server, blocking edition.
Run it like this:

  python slowpoetry.py <path-to-poetry-file>

If you are in the base directory of the twisted-intro package,
you could run it like this:

  python blocking-server/slowpoetry.py poetry/ecstasy.txt

to serve up John Donne's Ecstasy, which I know you want to do.
"""

    parser = optparse.OptionParser(usage)

    help = "The port to listen on. Default to a random available port."
    option = parser.add_option('--port', type='int', help=help)

    help = "The interface to listen on. Default is localhost."
    option = parser.add_option('--iface', help=help, default='')

    help = "The number of seconds between sending bytes."
    option = parser.add_option('--delay', type='float', help=help,
                               default=1)

    help = "The number of bytes to send at a time."
    option = parser.add_option('--num-bytes', type='int', help=help,
                               default=10)

    options, args = parser.parse_args()

    if len(args) != 1:
        parser.error('Provide exactly one poetry file.')

    poetry_file = args[0]

    if not os.path.exists(args[0]):
        parser.error('No such file: %s' % poetry_file)

    return options, poetry_file

def send_poetry(sock, poetry_file, num_bytes, delay):
    """Send some poetry slowly down the socket."""

    inputf = open(poetry_file)

    while True:
        bytes = inputf.read(num_bytes)

        if not bytes: # no more poetry :(
            sock.close()
            inputf.close()
            return

        print 'Sending %d bytes' % len(bytes)

        try:
            sock.sendall(bytes)
        except socket.error:
            sock.close()
            inputf.close()
            return

        time.sleep(delay)

def serve(listen_socket, poetry_file, num_bytes, delay):
    while True:
        sock, addr = listen_socket.accept()

        print 'Somebody at %s wants poetry!' % (addr,)

        send_poetry(sock, poetry_file, num_bytes, delay)

def main():
    options, poetry_file= parse_args()

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    sock.bind((options.iface, options.port or 0))

    sock.listen(5)

    print 'Serving %s on port %s.' % (poetry_file, sock.getsockname()[1])

    serve(sock, poetry_file, options.num_bytes, options.delay)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Produces load on all available CPU cores.
To quit press CTRL+Z.
"""

from multiprocessing import Pool
from multiprocessing import cpu_count

def f(x):
    while True:
        x*x

if __name__ == '__main__':
    processes = cpu_count()
    print 'utilizing %d cores\n' % processes
    pool = Pool(processes)
    pool.map(f, range(processes))
<commit_msg>Added quit message.<commit_after>#!/usr/bin/env python
"""
Produces load on all available CPU cores.
To quit press CTRL+Z.
"""

from multiprocessing import Pool
from multiprocessing import cpu_count

def f(x):
    while True:
        x*x

if __name__ == '__main__':
    processes = cpu_count()
    print 'utilizing %d cores\n' % processes
    print 'To quit press CTRL + Z'
    pool = Pool(processes)
    pool.map(f, range(processes))
<|endoftext|>


<commit_before>"""
ref: https://cloud.google.com/appengine/articles/update_schema
"""

import webapp2
from google.appengine.ext import ndb
from google.appengine.ext import deferred

from gae.room import ndb_models

program_type_cache = {}
program_cache = {}

class December2017Migration(webapp2.RequestHandler):

  BATCH_SIZE = 100
  PROGRAM_TOKEN_ATTR = 'program'
  PROGRAM_KEY_ATTR = 'program_key'

  def post(self):
    deferred.defer(self.update())
    self.response.write("""
        Schema update started. Check the console for task progress.
        View entities.
        """)

  # exposed for testing
  def update(self):
    self._migrate(ndb_models.NewSite)
    self._migrate(ndb_models.Order)
    self._migrate(ndb_models.CheckRequest)
    self._migrate(ndb_models.Expense)
    self._migrate(ndb_models.StaffTime)
    self._migrate(ndb_models.VendorReceipt)

  def _migrate(self, model_type, cursor=None):
    query = model_type.query()
    results, next_cursor, more = query.fetch_page(December2017Migration.BATCH_SIZE, start_cursor=cursor)
    to_put = []
    for result in results:
      program = self._parse_program_from_program_token_string(getattr(result, December2017Migration.PROGRAM_TOKEN_ATTR))
      if program is not None:
        setattr(result, December2017Migration.PROGRAM_KEY_ATTR, program.key)
        to_put.append(result)

    to_put and ndb.put_multi(to_put)
    more and deferred.defer(obj=self._migrate, model_type=model_type, cursor=next_cursor)

  def _parse_program_from_program_token_string(self, program_token):
    """program_token is like 2017 NRD"""
    try:
      program_string_split = program_token.split()
      assert len(program_string_split) == 2
      year_string, program_type_string = program_string_split
      year = int(year_string)
      program_type_string = program_type_string.upper()
      program_type = program_type_cache.get(program_type_string)
      if program_type is None:
        program_type = ndb_models.ProgramType.get_or_create(program_type_string)
        program_type_cache[program_type_string] = program_type
      program_key = (program_type_string, year)
      program = program_cache.get(program_key)
      if program is None:
        program, _ = ndb_models.Program.get_or_create(program_type.key, year)
        program_cache[program_key] = program
      return program
    except:
      return None
<commit_msg>use string based program key in migration<commit_after>"""
ref: https://cloud.google.com/appengine/articles/update_schema
"""

import webapp2
from google.appengine.ext import ndb
from google.appengine.ext import deferred

from gae.room import ndb_models

program_type_cache = {}
program_cache = {}

class December2017Migration(webapp2.RequestHandler):

  BATCH_SIZE = 100
  PROGRAM_TOKEN_ATTR = 'program'
  PROGRAM_KEY_ATTR = 'program_key'

  def post(self):
    deferred.defer(self.update())
    self.response.write("""
        Schema update started. Check the console for task progress.
        View entities.
        """)

  # exposed for testing
  def update(self):
    self._migrate(ndb_models.NewSite)
    self._migrate(ndb_models.Order)
    self._migrate(ndb_models.CheckRequest)
    self._migrate(ndb_models.Expense)
    self._migrate(ndb_models.StaffTime)
    self._migrate(ndb_models.VendorReceipt)

  def _migrate(self, model_type, cursor=None):
    query = model_type.query()
    results, next_cursor, more = query.fetch_page(December2017Migration.BATCH_SIZE, start_cursor=cursor)
    to_put = []
    for result in results:
      program = self._parse_program_from_program_token_string(getattr(result, December2017Migration.PROGRAM_TOKEN_ATTR))
      if program is not None:
        setattr(result, December2017Migration.PROGRAM_KEY_ATTR, program.key)
        to_put.append(result)

    to_put and ndb.put_multi(to_put)
    more and deferred.defer(obj=self._migrate, model_type=model_type, cursor=next_cursor)

  def _parse_program_from_program_token_string(self, program_token):
    """program_token is like 2017 NRD"""
    try:
      key = ndb.Key(ndb_models.Program, program_token)
      program = key.get()
      if program is None:
        program_string_split = program_token.split()
        assert len(program_string_split) == 2
        year_string, program_type_string = program_string_split
        year = int(year_string)
        program_type = program_type_cache.get(program_type_string)
        if program_type is None:
          program_type = ndb_models.ProgramType.get_or_create(program_type_string)
          program_type_cache[program_type_string] = program_type
        program_key = (program_type_string, year)
        program = program_cache.get(program_key)
        if program is None:
          program, _ = ndb_models.Program.get_or_create(program_type.key, year)
          program_cache[program_key] = program
        return program
    except:
      return None
<|endoftext|>


<commit_before>import discord
from discord.ext import commands
from .utils.dataIO import dataIO, fileIO
import os
import asyncio
from cogs.utils import checks

import sqlite3

class dbHandler:

    def __init__(self, dbPath):
        self.dbPath = dbPath
        self.DB_TABLE_LOG = 'log'
        self.DB_LOG_SERVER_NAME = 'server_name'
        self.DB_LOG_MESSAGE_ID = 'message_id'
        self.DB_LOG_OPERATION = 'operation_type'
        self.DB_LOG_MESSAGE_CONTENT = 'message_content'
        self.DB_LOG_AUTHOR_NAME = 'author_name'
        self.DB_LOG_CHANNEL_NAME = 'channel_name'
        self.DB_LOG_TIMESTAMP = 'timestamp'
        self.DB_TABLE_REACTION = 'reaction'
        self.DB_REACTION_SERVER_NAME = 'server_name'
        self.DB_REACTION_MESSAGE_ID = 'message_id'
        self.DB_REACTION_OPERATION = 'operation_type'
        self.DB_REACTION_MESSAGE = 'reaction_message'
        self.DB_REACTION_EMOJI = 'reaction_emoji'
        self.DB_REACTION_USER = 'reaction_user'
        self.DB_REACTION_CHANNEL_NAME = 'channel_name'
        self.DB_REACTION_TIMESTAMP = 'timestamp'
        self.dbOpen()
        self.createTableIfNotExist()

    def dbOpen(self):
        self.conn = sqlite3.connect(self.dbPath)

    def dbClose(self):
        self.conn.close()

    def createTableIfNotExist(self):
        if self.conn.cursor().execute("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table'").fetchone()[0] == 0:
            c = self.conn.cursor()
            c.execute('CREATE table {0} ({1} TEXT, {2} TEXT, {3} TEXT, {4} TEXT, {5} TEXT, {6} TEXT, {7} TEXT)'.format(self.DB_TABLE_LOG, self.DB_LOG_SERVER_NAME, self.DB_LOG_CHANNEL_NAME, self.DB_LOG_MESSAGE_ID, self.DB_LOG_OPERATION, self.DB_LOG_MESSAGE_CONTENT, self.DB_LOG_AUTHOR_NAME, self.DB_LOG_TIMESTAMP))
            c.execute('CREATE table {0} ({1} TEXT, {2} TEXT, {3} TEXT, {4} TEXT, {5} TEXT, {6} TEXT, {7} TEXT, {8} TEXT)'.format(self.DB_TABLE_REACTION, self.DB_REACTION_SERVER_NAME, self.DB_REACTION_CHANNEL_NAME, self.DB_REACTION_OPERATION, self.DB_REACTION_MESSAGE_ID, self.DB_REACTION_MESSAGE, self.DB_REACTION_EMOJI, self.DB_REACTION_USER, self.DB_REACTION_TIMESTAMP))

            self.conn.commit()
            return True
        return False

    def addLog(self, server_name, message_id, operation, message_content, author_name, channel_name, timestamp):
        c = self.conn.cursor()
        ts = str(timestamp)
        c.execute("INSERT INTO {0} ({1}, {2}, {3}, {4}, {5}, {6}, {7}) VALUES (?, ?, ?, ?, ?, ?, ?)".format(self.DB_TABLE_LOG, self.DB_LOG_SERVER_NAME, self.DB_LOG_MESSAGE_ID, self.DB_LOG_OPERATION, self.DB_LOG_MESSAGE_CONTENT, self.DB_LOG_AUTHOR_NAME, self.DB_LOG_CHANNEL_NAME, self.DB_LOG_TIMESTAMP), (str(server_name), str(message_id), str(operation), str(message_content), str(author_name), str(channel_name), ts[:19]))
        self.conn.commit()
        return True

    def addReaction(self, server_name, message_id, operation, reaction_message, emoji, user, channel_name, timestamp):
        c = self.conn.cursor()
        ts = str(timestamp)
        c.execute("INSERT INTO {0} ({1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}) VALUES (?, ?, ?, ?, ?, ?, ?, ?)".format(self.DB_TABLE_REACTION, self.DB_REACTION_SERVER_NAME, self.DB_REACTION_MESSAGE_ID, self.DB_REACTION_OPERATION, self.DB_REACTION_MESSAGE, self.DB_REACTION_EMOJI, self.DB_REACTION_USER, self.DB_REACTION_CHANNEL_NAME, self.DB_REACTION_TIMESTAMP), (str(server_name), str(message_id), str(operation), str(reaction_message), str(emoji), str(user), str(channel_name), ts[:19]))
        self.conn.commit()
        return True

    def deleteOldLogReaction(self):
        c = self.conn.cursor()
        c.execute("DELETE FROM {0} WHERE ({1} <= datetime('now', '-1 month'))".format(self.DB_TABLE_LOG, self.DB_LOG_TIMESTAMP))
        c.execute("DELETE FROM {0} WHERE ({1} <= datetime('now', '-1 month'))".format(self.DB_TABLE_REACTION, self.DB_REACTION_TIMESTAMP))

        # Test
        #c.execute("DELETE FROM {0} WHERE ({1} <= datetime('now', '-5 minutes'))".format(self.DB_TABLE_LOG, self.DB_LOG_TIMESTAMP))
        #c.execute("DELETE FROM {0} WHERE ({1} <= datetime('now', '-5 minutes'))".format(self.DB_TABLE_REACTION, self.DB_REACTION_TIMESTAMP))
        self.conn.commit()
        return True

class logging:
    """Message Logging!"""

    def __init__(self, bot):
        self.bot = bot
        self.db = dbHandler("data/sqlite/MessageLog.sqlite3")
        self.OPERATION_REACT_ADD = 'A'
        self.OPERATION_REACT_DELETE = 'D'
        self.OPERATION_MESSAGE = 'A'
        self.OPERATION_EDIT = 'E'

    async def on_message(self, message):

        if message.author.bot:
            return

        content = str(message.content) if len(message.content)>0 else ""
        content += str(message.attachments[0]['url']) if message.attachments.[0]['size'] > 0 else ""
        self.db.addLog(message.server, message.id, self.OPERATION_MESSAGE, content, message.author, message.channel, message.timestamp)

    async def on_message_edit(self, before, after):

        if before.author.bot:
            return

        if after.author.bot:
            return

        if before != after:
            content = str(after.content) if len(after.content)>0 else ""
            content += str(after.attachments[0]['url']) if after.attachments.[0]['size'] > 0 else ""
            self.db.addLog(after.server, after.id, self.OPERATION_EDIT, content, after.author, after.channel, after.timestamp)

    async def on_reaction_add(self, reaction, user):

        if user.bot:
            return

        content = str(reaction.message.content) if len(reaction.message.content)>0 else ""
        content += str(reaction.message.attachments[0]['url']) if reaction.message.attachments.[0]['size'] > 0 else ""
        self.db.addReaction(reaction.message.server, reaction.message.id, self.OPERATION_REACT_ADD, content, reaction.emoji, user, reaction.message.channel, reaction.message.timestamp)

    async def on_reaction_remove(self, reaction, user):

        if user.bot:
            return

        content = str(reaction.message.content) if len(reaction.message.content)>0 else ""
        content += str(reaction.message.attachments[0]['url']) if reaction.message.attachments.[0]['size'] > 0 else ""
        self.db.addReaction(reaction.message.server, reaction.message.id, self.OPERATION_REACT_DELETE, content, reaction.emoji, user, reaction.message.channel, reaction.message.timestamp)

    @commands.command(no_pm=True, pass_context=True)
    @checks.admin()
    async def removelogs(self):
        self.db.deleteOldLogReaction()
        await self.bot.say("Logs older than 30 days deleted.")

def check_folders():
    if not os.path.exists("data/sqlite"):
        print("Creating data/Logging folder...")
        os.makedirs("data/sqlite")

def setup(bot):
    check_folders()
    bot.add_cog(logging(bot))<commit_msg>switched IF statement<commit_after>import discord
from discord.ext import commands
from .utils.dataIO import dataIO, fileIO
import os
import asyncio
from cogs.utils import checks

import sqlite3

class dbHandler:

    def __init__(self, dbPath):
        self.dbPath = dbPath
        self.DB_TABLE_LOG = 'log'
        self.DB_LOG_SERVER_NAME = 'server_name'
        self.DB_LOG_MESSAGE_ID = 'message_id'
        self.DB_LOG_OPERATION = 'operation_type'
        self.DB_LOG_MESSAGE_CONTENT = 'message_content'
        self.DB_LOG_AUTHOR_NAME = 'author_name'
        self.DB_LOG_CHANNEL_NAME = 'channel_name'
        self.DB_LOG_TIMESTAMP = 'timestamp'
        self.DB_TABLE_REACTION = 'reaction'
        self.DB_REACTION_SERVER_NAME = 'server_name'
        self.DB_REACTION_MESSAGE_ID = 'message_id'
        self.DB_REACTION_OPERATION = 'operation_type'
        self.DB_REACTION_MESSAGE = 'reaction_message'
        self.DB_REACTION_EMOJI = 'reaction_emoji'
        self.DB_REACTION_USER = 'reaction_user'
        self.DB_REACTION_CHANNEL_NAME = 'channel_name'
        self.DB_REACTION_TIMESTAMP = 'timestamp'
        self.dbOpen()
        self.createTableIfNotExist()

    def dbOpen(self):
        self.conn = sqlite3.connect(self.dbPath)

    def dbClose(self):
        self.conn.close()

    def createTableIfNotExist(self):
        if self.conn.cursor().execute("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table'").fetchone()[0] == 0:
            c = self.conn.cursor()
            c.execute('CREATE table {0} ({1} TEXT, {2} TEXT, {3} TEXT, {4} TEXT, {5} TEXT, {6} TEXT, {7} TEXT)'.format(self.DB_TABLE_LOG, self.DB_LOG_SERVER_NAME, self.DB_LOG_CHANNEL_NAME, self.DB_LOG_MESSAGE_ID, self.DB_LOG_OPERATION, self.DB_LOG_MESSAGE_CONTENT, self.DB_LOG_AUTHOR_NAME, self.DB_LOG_TIMESTAMP))
            c.execute('CREATE table {0} ({1} TEXT, {2} TEXT, {3} TEXT, {4} TEXT, {5} TEXT, {6} TEXT, {7} TEXT, {8} TEXT)'.format(self.DB_TABLE_REACTION, self.DB_REACTION_SERVER_NAME, self.DB_REACTION_CHANNEL_NAME, self.DB_REACTION_OPERATION, self.DB_REACTION_MESSAGE_ID, self.DB_REACTION_MESSAGE, self.DB_REACTION_EMOJI, self.DB_REACTION_USER, self.DB_REACTION_TIMESTAMP))

            self.conn.commit()
            return True
        return False

    def addLog(self, server_name, message_id, operation, message_content, author_name, channel_name, timestamp):
        c = self.conn.cursor()
        ts = str(timestamp)
        c.execute("INSERT INTO {0} ({1}, {2}, {3}, {4}, {5}, {6}, {7}) VALUES (?, ?, ?, ?, ?, ?, ?)".format(self.DB_TABLE_LOG, self.DB_LOG_SERVER_NAME, self.DB_LOG_MESSAGE_ID, self.DB_LOG_OPERATION, self.DB_LOG_MESSAGE_CONTENT, self.DB_LOG_AUTHOR_NAME, self.DB_LOG_CHANNEL_NAME, self.DB_LOG_TIMESTAMP), (str(server_name), str(message_id), str(operation), str(message_content), str(author_name), str(channel_name), ts[:19]))
        self.conn.commit()
        return True

    def addReaction(self, server_name, message_id, operation, reaction_message, emoji, user, channel_name, timestamp):
        c = self.conn.cursor()
        ts = str(timestamp)
        c.execute("INSERT INTO {0} ({1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}) VALUES (?, ?, ?, ?, ?, ?, ?, ?)".format(self.DB_TABLE_REACTION, self.DB_REACTION_SERVER_NAME, self.DB_REACTION_MESSAGE_ID, self.DB_REACTION_OPERATION, self.DB_REACTION_MESSAGE, self.DB_REACTION_EMOJI, self.DB_REACTION_USER, self.DB_REACTION_CHANNEL_NAME, self.DB_REACTION_TIMESTAMP), (str(server_name), str(message_id), str(operation), str(reaction_message), str(emoji), str(user), str(channel_name), ts[:19]))
        self.conn.commit()
        return True

    def deleteOldLogReaction(self):
        c = self.conn.cursor()
        c.execute("DELETE FROM {0} WHERE ({1} <= datetime('now', '-1 month'))".format(self.DB_TABLE_LOG, self.DB_LOG_TIMESTAMP))
        c.execute("DELETE FROM {0} WHERE ({1} <= datetime('now', '-1 month'))".format(self.DB_TABLE_REACTION, self.DB_REACTION_TIMESTAMP))

        # Test
        #c.execute("DELETE FROM {0} WHERE ({1} <= datetime('now', '-5 minutes'))".format(self.DB_TABLE_LOG, self.DB_LOG_TIMESTAMP))
        #c.execute("DELETE FROM {0} WHERE ({1} <= datetime('now', '-5 minutes'))".format(self.DB_TABLE_REACTION, self.DB_REACTION_TIMESTAMP))
        self.conn.commit()
        return True

class logging:
    """Message Logging!"""

    def __init__(self, bot):
        self.bot = bot
        self.db = dbHandler("data/sqlite/MessageLog.sqlite3")
        self.OPERATION_REACT_ADD = 'A'
        self.OPERATION_REACT_DELETE = 'D'
        self.OPERATION_MESSAGE = 'A'
        self.OPERATION_EDIT = 'E'

    async def on_message(self, message):

        if message.author.bot:
            return

        content = str(message.content) if len(message.content)>0 else ""
        content += "" if not message.attachments else str(message.attachments[0]['url'])
        self.db.addLog(message.server, message.id, self.OPERATION_MESSAGE, content, message.author, message.channel, message.timestamp)

    async def on_message_edit(self, before, after):

        if before.author.bot:
            return

        if after.author.bot:
            return

        if before != after:
            content = str(after.content) if len(after.content)>0 else ""
            content += "" if not after.attachments else str(after.attachments[0]['url'])
            self.db.addLog(after.server, after.id, self.OPERATION_EDIT, content, after.author, after.channel, after.timestamp)

    async def on_reaction_add(self, reaction, user):

        if user.bot:
            return

        content = str(reaction.message.content) if len(reaction.message.content)>0 else ""
        content += "" if not reaction.message.attachments else str(reaction.message.attachments[0]['url'])
        self.db.addReaction(reaction.message.server, reaction.message.id, self.OPERATION_REACT_ADD, content, reaction.emoji, user, reaction.message.channel, reaction.message.timestamp)

    async def on_reaction_remove(self, reaction, user):

        if user.bot:
            return

        content = str(reaction.message.content) if len(reaction.message.content)>0 else ""
        content += "" if not reaction.message.attachments else str(reaction.message.attachments[0]['url'])
        self.db.addReaction(reaction.message.server, reaction.message.id, self.OPERATION_REACT_DELETE, content, reaction.emoji, user, reaction.message.channel, reaction.message.timestamp)

    @commands.command(no_pm=True, pass_context=True)
    @checks.admin()
    async def removelogs(self):
        self.db.deleteOldLogReaction()
        await self.bot.say("Logs older than 30 days deleted.")

def check_folders():
    if not os.path.exists("data/sqlite"):
        print("Creating data/Logging folder...")
        os.makedirs("data/sqlite")

def setup(bot):
    check_folders()
    bot.add_cog(logging(bot))<|endoftext|>


<commit_before>from tensorflow.contrib.learn.python.learn.utils import saved_model_export_utils
import tensorflow.contrib.learn as tflearn
import tensorflow.contrib.layers as tflayers
import tensorflow.contrib.metrics as tfmetrics

import tensorflow as tf

CSV_COLUMNS  = ('ontime,dep_delay,taxiout,distance,avg_dep_delay,avg_arr_delay' + \
                ',carrier,dep_lat,dep_lon,arr_lat,arr_lon,origin,dest').split(',')
LABEL_COLUMN = 'ontime'
DEFAULTS     = [[0.0],[0.0],[0.0],[0.0],[0.0],[0.0],\
                ['na'],[0.0],[0.0],[0.0],[0.0],['na'],['na']]

def read_dataset(filename, mode=tf.contrib.learn.ModeKeys.EVAL, batch_size=512, num_training_epochs=5):

  # the actual input function passed to TensorFlow
  def _input_fn():
    num_epochs = num_training_epochs if mode == tf.contrib.learn.ModeKeys.TRAIN else 1

    # could be a path to one file or a file pattern.
    input_file_names = tf.train.match_filenames_once(filename)
    filename_queue = tf.train.string_input_producer(
        input_file_names, num_epochs=num_epochs, shuffle=True)

    # read CSV
    reader = tf.TextLineReader()
    _, value = reader.read_up_to(filename_queue, num_records=batch_size)
    value_column = tf.expand_dims(value, -1)
    columns = tf.decode_csv(value_column, record_defaults=DEFAULTS)
    features = dict(zip(CSV_COLUMNS, columns))
    label = features.pop(LABEL_COLUMN)
    return features, label

  return _input_fn

def get_features_all():
    real = {
      colname : tflayers.real_valued_column(colname) \
          for colname in \
            ('dep_delay,taxiout,distance,avg_dep_delay,avg_arr_delay' +
             ',dep_lat,dep_lon,arr_lat,arr_lon').split(',')
    }
    sparse = {
      'carrier': tflayers.sparse_column_with_keys('carrier',
                  keys='AS,VX,F9,UA,US,WN,HA,EV,MQ,DL,OO,B6,NK,AA'.split(',')),
      'origin' : tflayers.sparse_column_with_hash_bucket('origin', hash_bucket_size=1000), # FIXME
      'dest'   : tflayers.sparse_column_with_hash_bucket('dest', hash_bucket_size=1000) #FIXME
    }
    return real, sparse

def get_features_ch7():
    """Using only the three inputs we originally used in Chapter 7"""
    real = {
      colname : tflayers.real_valued_column(colname) \
          for colname in \
            ('dep_delay,taxiout,distance').split(',')
    }
    sparse = {}
    return real, sparse

def get_features():
    """Using only the three inputs we originally used in Chapter 7"""
    real = {
      colname : tflayers.real_valued_column(colname) \
          for colname in \
            ('dep_delay,taxiout,distance,avg_dep_delay,avg_arr_delay').split(',')
    }
    sparse = {}
    return real, sparse

def wide_and_deep_model(output_dir):
    deep, wide = get_features()
    return \
        tflearn.DNNLinearCombinedClassifier(model_dir=output_dir,
                                           linear_feature_columns=wide,
                                           dnn_feature_columns=deep,
                                           dnn_hidden_units=[64, 32])

def linear_model(output_dir):
    real, sparse = get_features()
    all = {}
    all.update(real)
    all.update(sparse)
    estimator = tflearn.LinearClassifier(model_dir=output_dir, feature_columns=all.values())
    estimator.params["head"]._thresholds = [0.7]  # FIXME: hack
    return estimator

def serving_input_fn():
    real, sparse = get_features()

    feature_placeholders = {
      key : tf.placeholder(tf.float32, [None]) \
        for key in real.keys()
    }
    feature_placeholders.update( {
      key : tf.placeholder(tf.string, [None]) \
        for key in sparse.keys()
    } )

    features = {
      key: tf.expand_dims(tensor, -1)
      for key, tensor in feature_placeholders.items()
    }
    return tflearn.utils.input_fn_utils.InputFnOps(
      features,
      None,
      feature_placeholders)

def make_experiment_fn(traindata, evaldata, **args):
  def _experiment_fn(output_dir):
    return tflearn.Experiment(
        linear_model(output_dir),
        train_input_fn=read_dataset(traindata, mode=tf.contrib.learn.ModeKeys.TRAIN),
        eval_input_fn=read_dataset(evaldata),
        export_strategies=[saved_model_export_utils.make_export_strategy(
            serving_input_fn,
            default_output_alternative_key=None,
            exports_to_keep=1
        )],
        **args
    )
  return _experiment_fn
<commit_msg>Reporting RMSE error<commit_after>from tensorflow.contrib.learn.python.learn.utils import saved_model_export_utils
import tensorflow.contrib.learn as tflearn
import tensorflow.contrib.layers as tflayers
import tensorflow.contrib.metrics as tfmetrics

import tensorflow as tf

CSV_COLUMNS  = ('ontime,dep_delay,taxiout,distance,avg_dep_delay,avg_arr_delay' + \
                ',carrier,dep_lat,dep_lon,arr_lat,arr_lon,origin,dest').split(',')
LABEL_COLUMN = 'ontime'
DEFAULTS     = [[0.0],[0.0],[0.0],[0.0],[0.0],[0.0],\
                ['na'],[0.0],[0.0],[0.0],[0.0],['na'],['na']]

def read_dataset(filename, mode=tf.contrib.learn.ModeKeys.EVAL, batch_size=512, num_training_epochs=5):

  # the actual input function passed to TensorFlow
  def _input_fn():
    num_epochs = num_training_epochs if mode == tf.contrib.learn.ModeKeys.TRAIN else 1

    # could be a path to one file or a file pattern.
    input_file_names = tf.train.match_filenames_once(filename)
    filename_queue = tf.train.string_input_producer(
        input_file_names, num_epochs=num_epochs, shuffle=True)

    # read CSV
    reader = tf.TextLineReader()
    _, value = reader.read_up_to(filename_queue, num_records=batch_size)
    value_column = tf.expand_dims(value, -1)
    columns = tf.decode_csv(value_column, record_defaults=DEFAULTS)
    features = dict(zip(CSV_COLUMNS, columns))
    label = features.pop(LABEL_COLUMN)
    return features, label

  return _input_fn

def get_features_all():
    real = {
      colname : tflayers.real_valued_column(colname) \
          for colname in \
            ('dep_delay,taxiout,distance,avg_dep_delay,avg_arr_delay' +
             ',dep_lat,dep_lon,arr_lat,arr_lon').split(',')
    }
    sparse = {
      'carrier': tflayers.sparse_column_with_keys('carrier',
                  keys='AS,VX,F9,UA,US,WN,HA,EV,MQ,DL,OO,B6,NK,AA'.split(',')),
      'origin' : tflayers.sparse_column_with_hash_bucket('origin', hash_bucket_size=1000), # FIXME
      'dest'   : tflayers.sparse_column_with_hash_bucket('dest', hash_bucket_size=1000) #FIXME
    }
    return real, sparse

def get_features_ch7():
    """Using only the three inputs we originally used in Chapter 7"""
    real = {
      colname : tflayers.real_valued_column(colname) \
          for colname in \
            ('dep_delay,taxiout,distance').split(',')
    }
    sparse = {}
    return real, sparse

def get_features_ch7():
    """Using only the three inputs we originally used in Chapter 7"""
    real = {
      colname : tflayers.real_valued_column(colname) \
          for colname in \
            ('dep_delay,taxiout,distance,avg_dep_delay,avg_arr_delay').split(',')
    }
    sparse = {}
    return real, sparse

def get_features():
    return get_features_all()

def wide_and_deep_model(output_dir):
    deep, wide = get_features()
    return \
        tflearn.DNNLinearCombinedClassifier(model_dir=output_dir,
                                           linear_feature_columns=wide,
                                           dnn_feature_columns=deep,
                                           dnn_hidden_units=[64, 32])

def linear_model(output_dir):
    real, sparse = get_features()
    all = {}
    all.update(real)
    all.update(sparse)
    estimator = tflearn.LinearClassifier(model_dir=output_dir, feature_columns=all.values())
    estimator.params["head"]._thresholds = [0.7]  # FIXME: hack
    return estimator

def serving_input_fn():
    real, sparse = get_features()

    feature_placeholders = {
      key : tf.placeholder(tf.float32, [None]) \
        for key in real.keys()
    }
    feature_placeholders.update( {
      key : tf.placeholder(tf.string, [None]) \
        for key in sparse.keys()
    } )

    features = {
      key: tf.expand_dims(tensor, -1)
      for key, tensor in feature_placeholders.items()
    }
    return tflearn.utils.input_fn_utils.InputFnOps(
      features,
      None,
      feature_placeholders)

def my_rmse(predictions, labels, **args):
  prob_ontime = predictions[:,1]
  return tfmetrics.streaming_root_mean_squared_error(prob_ontime, labels, **args)

def make_experiment_fn(traindata, evaldata, **args):
  def _experiment_fn(output_dir):
    return tflearn.Experiment(
        linear_model(output_dir),
        train_input_fn=read_dataset(traindata, mode=tf.contrib.learn.ModeKeys.TRAIN),
        eval_input_fn=read_dataset(evaldata),
        export_strategies=[saved_model_export_utils.make_export_strategy(
            serving_input_fn,
            default_output_alternative_key=None,
            exports_to_keep=1
        )],
        eval_metrics = {
            'rmse' : tflearn.MetricSpec(metric_fn=my_rmse, prediction_key='probabilities')
        },
        **args
    )
  return _experiment_fn
<|endoftext|>


<commit_before>###

import multiprocessing
import utilities
import forest_age_category
import pandas as pd
import subprocess

### Need to update rasterio package on spot machine before running
### sudo pip install rasterio --upgrade

# Loss, gain, and tree cover density, intact forest landscape, biomass tiles, and continent-ecozone
# All of these are needed for the forest age decision tree
loss = 's3://gfw2-data/forest_change/hansen_2015/Loss_tiles/'
gain = 's3://gfw2-data/forest_change/tree_cover_gain/gaindata_2012/'
tcd = 's3://gfw2-data/forest_cover/2000_treecover/'
ifl = 's3://gfw2-data/climate/carbon_model/other_emissions_inputs/ifl_2000/'
biomass = 's3://gfw2-data/climate/WHRC_biomass/WHRC_V4/Processed/'
cont_eco = 's3://gfw2-data/climate/carbon_model/fao_ecozones/ecozone/20180912/'

# biomass_tile_list = utilities.tile_list(biomass)
# biomass_tile_list = ["00N_000E", "00N_050W", "00N_060W", "00N_010E", "00N_020E", "00N_030E", "00N_040E", "10N_000E", "10N_010E", "10N_010W", "10N_020E", "10N_020W"] # test tiles
biomass_tile_list = ['00N_050W'] # test tile
print biomass_tile_list

download_list = [loss, gain, tcd, ifl, biomass, cont_eco]

# # For downloading all tiles in the folders
# for input in download_list:
#     utilities.s3_folder_download('{}'.format(input), '.')

# For copying individual tiles to spot machine for testing
for tile in biomass_tile_list:

    utilities.s3_file_download('{0}{1}.tif'.format(loss, tile), '.')                                # loss tiles
    utilities.s3_file_download('{0}Hansen_GFC2015_gain_{1}.tif'.format(gain, tile), '.')            # gain tiles
    utilities.s3_file_download('{0}Hansen_GFC2014_treecover2000_{1}.tif'.format(tcd, tile), '.')    # tcd 2000
    utilities.s3_file_download('{0}{1}_res_ifl_2000.tif'.format(ifl, tile), '.')                    # ifl 2000
    utilities.s3_file_download('{0}{1}_biomass.tif'.format(biomass, tile), '.')                     # biomass 2000
    utilities.s3_file_download('{0}fao_ecozones_{1}.tif'.format(cont_eco, tile), '.')               # continents and FAO ecozones 2000

cmd = ['aws', 's3', 'cp', 'http://gfw2-data.s3.amazonaws.com/climate/carbon_model/gain_rate_continent_ecozone_age_20180918.xlsx', '.']
subprocess.check_call(cmd)

# Imports the table with the ecozone-continent codes and the carbon gain rates
gain_table = pd.read_excel("gain_rate_continent_ecozone_age_20180918.xlsx",
                           sheet_name="con-ezn-age gain, for model")

# Removes rows with duplicate codes (N. and S. America for the same ecozone)
gain_table_simplified = gain_table.drop_duplicates(subset='gainEcoCon', keep='first')

# Converts the continent-ecozone codes and young forest gain rates to a dictionary
gain_table_dict = pd.Series(gain_table_simplified.growth_secondary_less_20.values,index=gain_table_simplified.gainEcoCon).to_dict()

# Adds a dictionary entry for where the ecozone-continent code is 0 (not in a continent)
gain_table_dict[0] = 0

# count = multiprocessing.cpu_count()
# pool = multiprocessing.Pool(processes=count/4)
# pool.map(forest_age_category.forest_age_category, biomass_tile_list)

# For single processor use
for tile in biomass_tile_list:

    forest_age_category.forest_age_category(tile, gain_table_dict)

<commit_msg>Testing out the forest age category tile generation on the spot machine using a single processor.<commit_after>###

import multiprocessing
import utilities
import forest_age_category
import pandas as pd
import subprocess

### Need to update rasterio package on spot machine before running
### sudo pip install rasterio --upgrade

# Loss, gain, and tree cover density, intact forest landscape, biomass tiles, and continent-ecozone
# All of these are needed for the forest age decision tree
loss = 's3://gfw2-data/forest_change/hansen_2015/Loss_tiles/'
gain = 's3://gfw2-data/forest_change/tree_cover_gain/gaindata_2012/'
tcd = 's3://gfw2-data/forest_cover/2000_treecover/'
ifl = 's3://gfw2-data/climate/carbon_model/other_emissions_inputs/ifl_2000/'
biomass = 's3://gfw2-data/climate/WHRC_biomass/WHRC_V4/Processed/'
cont_eco = 's3://gfw2-data/climate/carbon_model/fao_ecozones/ecozone/20180912/'

# biomass_tile_list = utilities.tile_list(biomass)
# biomass_tile_list = ["00N_000E", "00N_050W", "00N_060W", "00N_010E", "00N_020E", "00N_030E", "00N_040E", "10N_000E", "10N_010E", "10N_010W", "10N_020E", "10N_020W"] # test tiles
biomass_tile_list = ['00N_050W'] # test tile
print biomass_tile_list

download_list = [loss, gain, tcd, ifl, biomass, cont_eco]

# # For downloading all tiles in the folders
# for input in download_list:
#     utilities.s3_folder_download('{}'.format(input), '.')

# For copying individual tiles to spot machine for testing
for tile in biomass_tile_list:

    utilities.s3_file_download('{0}{1}.tif'.format(loss, tile), '.')                                # loss tiles
    utilities.s3_file_download('{0}Hansen_GFC2015_gain_{1}.tif'.format(gain, tile), '.')            # gain tiles
    utilities.s3_file_download('{0}Hansen_GFC2014_treecover2000_{1}.tif'.format(tcd, tile), '.')    # tcd 2000
    utilities.s3_file_download('{0}{1}_res_ifl_2000.tif'.format(ifl, tile), '.')                    # ifl 2000
    utilities.s3_file_download('{0}{1}_biomass.tif'.format(biomass, tile), '.')                     # biomass 2000
    utilities.s3_file_download('{0}fao_ecozones_{1}.tif'.format(cont_eco, tile), '.')               # continents and FAO ecozones 2000

cmd = ['aws', 's3', 'cp', 's3://gfw2-data/climate/carbon_model/gain_rate_continent_ecozone_age_20180918.xlsx', '.']
subprocess.check_call(cmd)

# Imports the table with the ecozone-continent codes and the carbon gain rates
gain_table = pd.read_excel("gain_rate_continent_ecozone_age_20180918.xlsx",
                           sheet_name="con-ezn-age gain, for model")

# Removes rows with duplicate codes (N. and S. America for the same ecozone)
gain_table_simplified = gain_table.drop_duplicates(subset='gainEcoCon', keep='first')

# Converts the continent-ecozone codes and young forest gain rates to a dictionary
gain_table_dict = pd.Series(gain_table_simplified.growth_secondary_less_20.values,index=gain_table_simplified.gainEcoCon).to_dict()

# Adds a dictionary entry for where the ecozone-continent code is 0 (not in a continent)
gain_table_dict[0] = 0

# count = multiprocessing.cpu_count()
# pool = multiprocessing.Pool(processes=count/4)
# pool.map(forest_age_category.forest_age_category, biomass_tile_list)

# For single processor use
for tile in biomass_tile_list:

    forest_age_category.forest_age_category(tile, gain_table_dict)

<|endoftext|>