<commit_before># Copyright (c) 2005-2006 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Ron Dreslinski

# Splash2 Run Script
#

import m5
from m5.objects import *
import os, optparse, sys
m5.AddToPath('../common')

# --------------------
# Define Command Line Options
# ====================

parser = optparse.OptionParser()

parser.add_option("-d", "--detailed", action="store_true")
parser.add_option("-t", "--timing", action="store_true")
parser.add_option("-m", "--maxtick", type="int")
parser.add_option("-n", "--numcpus",
                  help="Number of cpus in total", type="int")
parser.add_option("-f", "--frequency",
                  default = "1GHz",
                  help="Frequency of each CPU")
parser.add_option("-p", "--protocol",
                  default="moesi",
                  help="The coherence protocol to use for the L1'a (i.e. MOESI, MOSI)")
parser.add_option("--l1size",
                  default = "32kB")
parser.add_option("--l1latency",
                  default = 1)
parser.add_option("--l2size",
                  default = "256kB")
parser.add_option("--l2latency",
                  default = 10)
parser.add_option("--rootdir",
                  help="ROot directory of Splash2",
                  default="/dist/splash2/codes")
parser.add_option("-b", "--benchmark",
                  help="Splash 2 benchmark to run")

(options, args) = parser.parse_args()

if args:
    print "Error: script doesn't take any positional arguments"
    sys.exit(1)

if not options.numcpus:
    print "Specify the number of cpus with -n"
    sys.exit(1)

# --------------------
# Define Splash2 Benchmarks
# ====================
class Cholesky(LiveProcess):
        executable = options.rootdir + '/kernels/cholesky/CHOLESKY'
        cmd = 'CHOLESKY -p' + str(options.numcpus) + ' '\
             + options.rootdir + '/kernels/cholesky/inputs/tk23.O'

class FFT(LiveProcess):
        executable = options.rootdir + '/kernels/fft/FFT'
        cmd = 'FFT -p' + str(options.numcpus) + ' -m18'

class LU_contig(LiveProcess):
        executable = options.rootdir + '/kernels/lu/contiguous_blocks/LU'
        cmd = 'LU -p' + str(options.numcpus)

class LU_noncontig(LiveProcess):
        executable = options.rootdir + '/kernels/lu/non_contiguous_blocks/LU'
        cmd = 'LU -p' + str(options.numcpus)

class Radix(LiveProcess):
        executable = options.rootdir + '/kernels/radix/RADIX'
        cmd = 'RADIX -n524288 -p' + str(options.numcpus)

class Barnes(LiveProcess):
        executable = options.rootdir + '/apps/barnes/BARNES'
        cmd = 'BARNES'
        input = options.rootdir + '/apps/barnes/input.p' + str(options.numcpus)

class FMM(LiveProcess):
        executable = options.rootdir + '/apps/fmm/FMM'
        cmd = 'FMM'
        input = options.rootdir + '/apps/fmm/inputs/input.2048.p' + str(options.numcpus)

class Ocean_contig(LiveProcess):
        executable = options.rootdir + '/apps/ocean/contiguous_partitions/OCEAN'
        cmd = 'OCEAN -p' + str(options.numcpus)

class Ocean_noncontig(LiveProcess):
        executable = options.rootdir + '/apps/ocean/non_contiguous_partitions/OCEAN'
        cmd = 'OCEAN -p' + str(options.numcpus)

class Raytrace(LiveProcess):
        executable = options.rootdir + '/apps/raytrace/RAYTRACE'
        cmd = 'RAYTRACE -p' + str(options.numcpus) + ' ' \
             + options.rootdir + 'apps/raytrace/inputs/teapot.env'

class Water_nsquared(LiveProcess):
        executable = options.rootdir + '/apps/water-nsquared/WATER-NSQUARED'
        cmd = 'WATER-NSQUARED'
        input = options.rootdir + '/apps/water-nsquared/input.p' + str(options.numcpus)

class Water_spatial(LiveProcess):
        executable = options.rootdir + '/apps/water-spatial/WATER-SPATIAL'
        cmd = 'WATER-SPATIAL'
        input = options.rootdir + '/apps/water-spatial/input.p' + str(options.numcpus)

# --------------------
# Base L1 Cache Definition
# ====================

class L1(BaseCache):
    latency = options.l1latency
    block_size = 64
    mshrs = 12
    tgts_per_mshr = 8
    protocol = CoherenceProtocol(protocol=options.protocol)

# ----------------------
# Base L2 Cache Definition
# ----------------------

class L2(BaseCache):
    block_size = 64
    latency = options.l2latency
    mshrs = 92
    tgts_per_mshr = 16
    write_buffers = 8

# ----------------------
# Define the cpus
# ----------------------

busFrequency = Frequency(options.frequency)

if options.timing:
    cpus = [TimingSimpleCPU(cpu_id = i,
                            clock=options.frequency)
            for i in xrange(options.numcpus)]
elif options.detailed:
    cpus = [DerivO3CPU(cpu_id = i,
                       clock=options.frequency)
            for i in xrange(options.numcpus)]
else:
    cpus = [AtomicSimpleCPU(cpu_id = i,
                            clock=options.frequency)
            for i in xrange(options.numcpus)]

# ----------------------
# Create a system, and add system wide objects
# ----------------------
system = System(cpu = cpus, physmem = PhysicalMemory(),
                membus = Bus(clock = busFrequency))

system.toL2bus = Bus(clock = busFrequency)
system.l2 = L2(size = options.l2size, assoc = 8)

# ----------------------
# Connect the L2 cache and memory together
# ----------------------

system.physmem.port = system.membus.port
system.l2.cpu_side = system.toL2bus.port
system.l2.mem_side = system.membus.port

# ----------------------
# Connect the L2 cache and clusters together
# ----------------------
for cpu in cpus:
    cpu.addPrivateSplitL1Caches(L1(size = options.l1size, assoc = 1),
                                L1(size = options.l1size, assoc = 4))
    cpu.mem = cpu.dcache
    # connect cpu level-1 caches to shared level-2 cache
    cpu.connectMemPorts(system.toL2bus)

# ----------------------
# Define the root
# ----------------------

root = Root(system = system)

# --------------------
# Pick the correct Splash2 Benchmarks
# ====================
if options.benchmark == 'Cholesky':
    root.workload = Cholesky()
elif options.benchmark == 'FFT':
    root.workload = FFT()
elif options.benchmark == 'LUContig':
    root.workload = LU_contig()
elif options.benchmark == 'LUNoncontig':
    root.workload = LU_noncontig()
elif options.benchmark == 'Radix':
    root.workload = Radix()
elif options.benchmark == 'Barnes':
    root.workload = Barnes()
elif options.benchmark == 'FMM':
    root.workload = FMM()
elif options.benchmark == 'OceanContig':
    root.workload = Ocean_contig()
elif options.benchmark == 'OceanNoncontig':
    root.workload = Ocean_noncontig()
elif options.benchmark == 'Raytrace':
    root.workload = Raytrace()
elif options.benchmark == 'WaterNSquared':
    root.workload = Water_nsquared()
elif options.benchmark == 'WaterSpatial':
    root.workload = Water_spatial()
else:
    panic("The --benchmark environment variable was set to something" \
          +" improper.\nUse Cholesky, FFT, LUContig, LUNoncontig, Radix" \
          +", Barnes, FMM, OceanContig,\nOceanNoncontig, Raytrace," \
          +" WaterNSquared, or WaterSpatial\n")

# --------------------
# Assign the workload to the cpus
# ====================

for cpu in cpus:
    cpu.workload = root.workload

# ----------------------
# Run the simulation
# ----------------------

if options.timing or options.detailed:
    root.system.mem_mode = 'timing'

# instantiate configuration
m5.instantiate(root)

# simulate until program terminates
if options.maxtick:
    exit_event = m5.simulate(options.maxtick)
else:
    exit_event = m5.simulate(m5.MaxTick)

print 'Exiting @ tick', m5.curTick(), 'because', exit_event.getCause()

<commit_msg>Fix the splash2 run script<commit_after># Copyright (c) 2005-2007 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Ron Dreslinski

# Splash2 Run Script
#

import m5
from m5.objects import *
import os, optparse, sys
m5.AddToPath('../common')

# --------------------
# Define Command Line Options
# ====================

parser = optparse.OptionParser()

parser.add_option("-d", "--detailed", action="store_true")
parser.add_option("-t", "--timing", action="store_true")
parser.add_option("-m", "--maxtick", type="int")
parser.add_option("-n", "--numcpus",
                  help="Number of cpus in total", type="int")
parser.add_option("-f", "--frequency",
                  default = "1GHz",
                  help="Frequency of each CPU")
parser.add_option("-p", "--protocol",
                  default="moesi",
                  help="The coherence protocol to use for the L1'a (i.e. MOESI, MOSI)")
parser.add_option("--l1size",
                  default = "32kB")
parser.add_option("--l1latency",
                  default = 1)
parser.add_option("--l2size",
                  default = "256kB")
parser.add_option("--l2latency",
                  default = 10)
parser.add_option("--rootdir",
                  help="Root directory of Splash2",
                  default="/dist/splash2/codes")
parser.add_option("-b", "--benchmark",
                  help="Splash 2 benchmark to run")

(options, args) = parser.parse_args()

if args:
    print "Error: script doesn't take any positional arguments"
    sys.exit(1)

if not options.numcpus:
    print "Specify the number of cpus with -n"
    sys.exit(1)

# --------------------
# Define Splash2 Benchmarks
# ====================
class Cholesky(LiveProcess):
    cwd = options.rootdir + '/kernels/cholesky'
    executable = options.rootdir + '/kernels/cholesky/CHOLESKY'
    cmd = 'CHOLESKY -p' + str(options.numcpus) + ' '\
          + options.rootdir + '/kernels/cholesky/inputs/tk23.O'

class FFT(LiveProcess):
    cwd = options.rootdir + '/kernels/fft'
    executable = options.rootdir + '/kernels/fft/FFT'
    cmd = 'FFT -p' + str(options.numcpus) + ' -m18'

class LU_contig(LiveProcess):
    executable = options.rootdir + '/kernels/lu/contiguous_blocks/LU'
    cmd = 'LU -p' + str(options.numcpus)
    cwd = options.rootdir + '/kernels/lu/contiguous_blocks'

class LU_noncontig(LiveProcess):
    executable = options.rootdir + '/kernels/lu/non_contiguous_blocks/LU'
    cmd = 'LU -p' + str(options.numcpus)
    cwd = options.rootdir + '/kernels/lu/non_contiguous_blocks'

class Radix(LiveProcess):
    executable = options.rootdir + '/kernels/radix/RADIX'
    cmd = 'RADIX -n524288 -p' + str(options.numcpus)
    cwd = options.rootdir + '/kernels/radix'

class Barnes(LiveProcess):
    executable = options.rootdir + '/apps/barnes/BARNES'
    cmd = 'BARNES'
    input = options.rootdir + '/apps/barnes/input.p' + str(options.numcpus)
    cwd = options.rootdir + '/apps/barnes'

class FMM(LiveProcess):
    executable = options.rootdir + '/apps/fmm/FMM'
    cmd = 'FMM'
    if str(options.numcpus) == '1':
        input = options.rootdir + '/apps/fmm/inputs/input.2048'
    else:
        input = options.rootdir + '/apps/fmm/inputs/input.2048.p' + str(options.numcpus)
    cwd = options.rootdir + '/apps/fmm'

class Ocean_contig(LiveProcess):
    executable = options.rootdir + '/apps/ocean/contiguous_partitions/OCEAN'
    cmd = 'OCEAN -p' + str(options.numcpus)
    cwd = options.rootdir + '/apps/ocean/contiguous_partitions'

class Ocean_noncontig(LiveProcess):
    executable = options.rootdir + '/apps/ocean/non_contiguous_partitions/OCEAN'
    cmd = 'OCEAN -p' + str(options.numcpus)
    cwd = options.rootdir + '/apps/ocean/non_contiguous_partitions'

class Raytrace(LiveProcess):
    executable = options.rootdir + '/apps/raytrace/RAYTRACE'
    cmd = 'RAYTRACE -p' + str(options.numcpus) + ' ' \
          + options.rootdir + '/apps/raytrace/inputs/teapot.env'
    cwd = options.rootdir + '/apps/raytrace'

class Water_nsquared(LiveProcess):
    executable = options.rootdir + '/apps/water-nsquared/WATER-NSQUARED'
    cmd = 'WATER-NSQUARED'
    if options.numcpus==1:
        input = options.rootdir + '/apps/water-nsquared/input'
    else:
        input = options.rootdir + '/apps/water-nsquared/input.p' + str(options.numcpus)
    cwd = options.rootdir + '/apps/water-nsquared'

class Water_spatial(LiveProcess):
    executable = options.rootdir + '/apps/water-spatial/WATER-SPATIAL'
    cmd = 'WATER-SPATIAL'
    if options.numcpus==1:
        input = options.rootdir + '/apps/water-spatial/input'
    else:
        input = options.rootdir + '/apps/water-spatial/input.p' + str(options.numcpus)
    cwd = options.rootdir + '/apps/water-spatial'

# --------------------
# Base L1 Cache Definition
# ====================

class L1(BaseCache):
    latency = options.l1latency
    block_size = 64
    mshrs = 12
    tgts_per_mshr = 8
    protocol = CoherenceProtocol(protocol=options.protocol)

# ----------------------
# Base L2 Cache Definition
# ----------------------

class L2(BaseCache):
    block_size = 64
    latency = options.l2latency
    mshrs = 92
    tgts_per_mshr = 16
    write_buffers = 8

# ----------------------
# Define the cpus
# ----------------------

busFrequency = Frequency(options.frequency)

if options.timing:
    cpus = [TimingSimpleCPU(cpu_id = i,
                            clock=options.frequency)
            for i in xrange(options.numcpus)]
elif options.detailed:
    cpus = [DerivO3CPU(cpu_id = i,
                       clock=options.frequency)
            for i in xrange(options.numcpus)]
else:
    cpus = [AtomicSimpleCPU(cpu_id = i,
                            clock=options.frequency)
            for i in xrange(options.numcpus)]

# ----------------------
# Create a system, and add system wide objects
# ----------------------
system = System(cpu = cpus, physmem = PhysicalMemory(),
                membus = Bus(clock = busFrequency))

system.toL2bus = Bus(clock = busFrequency)
system.l2 = L2(size = options.l2size, assoc = 8)

# ----------------------
# Connect the L2 cache and memory together
# ----------------------

system.physmem.port = system.membus.port
system.l2.cpu_side = system.toL2bus.port
system.l2.mem_side = system.membus.port

# ----------------------
# Connect the L2 cache and clusters together
# ----------------------
for cpu in cpus:
    cpu.addPrivateSplitL1Caches(L1(size = options.l1size, assoc = 1),
                                L1(size = options.l1size, assoc = 4))
    cpu.mem = cpu.dcache
    # connect cpu level-1 caches to shared level-2 cache
    cpu.connectMemPorts(system.toL2bus)

# ----------------------
# Define the root
# ----------------------

root = Root(system = system)

# --------------------
# Pick the correct Splash2 Benchmarks
# ====================
if options.benchmark == 'Cholesky':
    root.workload = Cholesky()
elif options.benchmark == 'FFT':
    root.workload = FFT()
elif options.benchmark == 'LUContig':
    root.workload = LU_contig()
elif options.benchmark == 'LUNoncontig':
    root.workload = LU_noncontig()
elif options.benchmark == 'Radix':
    root.workload = Radix()
elif options.benchmark == 'Barnes':
    root.workload = Barnes()
elif options.benchmark == 'FMM':
    root.workload = FMM()
elif options.benchmark == 'OceanContig':
    root.workload = Ocean_contig()
elif options.benchmark == 'OceanNoncontig':
    root.workload = Ocean_noncontig()
elif options.benchmark == 'Raytrace':
    root.workload = Raytrace()
elif options.benchmark == 'WaterNSquared':
    root.workload = Water_nsquared()
elif options.benchmark == 'WaterSpatial':
    root.workload = Water_spatial()
else:
    panic("The --benchmark environment variable was set to something" \
          +" improper.\nUse Cholesky, FFT, LUContig, LUNoncontig, Radix" \
          +", Barnes, FMM, OceanContig,\nOceanNoncontig, Raytrace," \
          +" WaterNSquared, or WaterSpatial\n")

# --------------------
# Assign the workload to the cpus
# ====================

for cpu in cpus:
    cpu.workload = root.workload

# ----------------------
# Run the simulation
# ----------------------

if options.timing or options.detailed:
    root.system.mem_mode = 'timing'

# instantiate configuration
m5.instantiate(root)

# simulate until program terminates
if options.maxtick:
    exit_event = m5.simulate(options.maxtick)
else:
    exit_event = m5.simulate(m5.MaxTick)

print 'Exiting @ tick', m5.curTick(), 'because', exit_event.getCause()

<|endoftext|>


<commit_before>'''
A hypermedia REST API for Salt using the CherryPy framework
'''
# Import Python libs
import itertools
import signal
import os
import json

# Import third-party libs
import cherrypy
import cherrypy.wsgiserver as wsgiserver
import cherrypy.wsgiserver.ssl_builtin

import jinja2

# Import Salt libs
import salt.auth
import salt.log
import salt.output
from salt.utils import yaml

# Import salt-api libs
import saltapi

logger = salt.log.logging.getLogger(__name__)

jenv = jinja2.Environment(loader=jinja2.FileSystemLoader([
    os.path.join(os.path.dirname(__file__), 'tmpl'),
]))

def __virtual__():
    if 'port' in __opts__.get(__name__.rsplit('.')[-1], {}):
        return 'rest'
    return False

def fmt_lowdata(data):
    '''
    Take CherryPy body data from a POST (et al) request and format it into
    lowdata. It will accept repeated parameters and pair and format those
    into multiple lowdata chunks.
    '''
    pairs = []
    for k,v in data.items():
        # Ensure parameter is a list
        argl = v if isinstance(v, list) else [v]
        # Make pairs of (key, value) from {key: [*value]}
        pairs.append(zip([k] * len(argl), argl))

    lowdata = []
    for i in itertools.izip_longest(*pairs):
        if not all(i):
            msg = "Error pairing parameters: %s"
            raise Exception(msg % str(i))
        lowdata.append(dict(i))

    return lowdata

def salt_auth_tool():
    ignore_urls = ('/login',)

    # Grab the session via a cookie (for browsers) or via a custom header
    sid = (cherrypy.session.get('token', None) or
            cherrypy.request.headers.get('X-Auth-Token', None))

    if not cherrypy.request.path_info in ignore_urls and not sid :
        raise cherrypy.InternalRedirect('/login')

    cherrypy.response.headers['Cache-Control'] = 'private'

cherrypy.tools.salt_auth = cherrypy.Tool('before_request_body', salt_auth_tool)

# Be conservative in what you send
ct_out_map = {
    'application/json': 'json',
    'application/x-yaml': 'yaml',
    'text/html': 'jinja',
}

def hypermedia_handler(*args, **kwargs):
    best = cherrypy.lib.cptools.accept(ct_out_map.keys()) # raises 406
    out = ct_out_map[best]

    cherrypy.response.headers['Content-Type'] = best

    try:
        ret = cherrypy.serving.request._hypermedia_inner_handler(*args, **kwargs)
    except Exception as exc:
        logger.debug("Error while processing request for: %s",
                cherrypy.request.path_info,
                exc_info=True)

        cherrypy.response.status = 500
        cherrypy.response._tmpl = '500.html'

        ret = {
            'success': False,
            'status': cherrypy.response.status,
            'message': '{0}'.format(cherrypy._cperror.format_exc())}

    # Skip the outputters and use jinja instead
    if out == 'jinja':
        tmpl = jenv.get_template(cherrypy.response._tmpl)
        return tmpl.render(**ret)

    return salt.output.out_format(ret, out, __opts__)

def hypermedia_out():
    request = cherrypy.serving.request
    request._hypermedia_inner_handler = request.handler
    request.handler = hypermedia_handler

    # cherrypy.response.headers['Alternates'] = self.ct_out_map.keys()
    # TODO: add 'negotiate' to Vary header and 'list' to TCN header
    # Alternates: {"paper.1" 0.9 {type text/html} {language en}},
    #          {"paper.2" 0.7 {type text/html} {language fr}},
    #          {"paper.3" 1.0 {type application/postscript} {language en}}

cherrypy.tools.hypermedia_out = cherrypy.Tool('before_handler', hypermedia_out)

# Be liberal in what you accept
ct_in_map = {
    'application/x-www-form-urlencoded': fmt_lowdata,
    'application/json': json.loads,
    'application/x-yaml': yaml.load,
    'text/yaml': yaml.load,
}

def hypermedia_in():
    '''
    Unserialize POST/PUT data of a specified content type, if possible
    '''
    if cherrypy.request.method in ('POST', 'PUT'):
        ct_type = cherrypy.request.headers['content-type'].lower()
        ct_in = ct_in_map.get(ct_type, None)

        if not ct_in:
            raise cherrypy.HTTPError(406, 'Content type not supported')

        body = cherrypy.request.body.read()
        cherrypy.request.params['body'] = ct_in(body)

cherrypy.tools.hypermedia_in = cherrypy.Tool('before_handler', hypermedia_in)

class LowDataAdapter(object):
    '''
    '''
    exposed = True

    def __init__(self, opts):
        self.opts = opts
        self.api = saltapi.APIClient(opts)

    def exec_lowdata(self, lowdata):
        '''
        Pass lowdata to Salt to be executed
        '''
        # FIXME: change this to yield results from one of Salt's iterative returns
        logger.debug("SaltAPI is passing low-data: %s", lowdata)
        return [self.api.run(chunk) for chunk in lowdata]

    def GET(self):
        lowdata = [{'client': 'local', 'tgt': '*',
                'fun': ['grains.items', 'sys.list_functions'],
                'arg': [[], []],
        }]
        return self.exec_lowdata(lowdata)

    def POST(self, **kwargs):
        '''
        Run a given function in a given client with the given args
        '''
        return self.exec_lowdata(self.fmt_lowdata(kwargs))

class Login(LowDataAdapter):
    '''
    '''
    exposed = True

    def GET(self):
        cherrypy.response._tmpl = 'login.html'
        cherrypy.response.status = '401 Unauthorized'
        cherrypy.response.headers['WWW-Authenticate'] = 'Session'

        return {
            'success': False,
            'status': cherrypy.response.status,
            'message': "Please log in",
        }

    def POST(self, **kwargs):
        auth = salt.auth.LoadAuth(self.opts)
        token = auth.mk_token(kwargs).get('token', False)
        cherrypy.response.headers['X-Auth-Token'] = cherrypy.session.id
        cherrypy.session['token'] = token
        raise cherrypy.HTTPRedirect('/', 302)

class API(object):
    url_map = {
        'index': LowDataAdapter,
        'login': Login,
    }

    def __init__(self, opts):
        self.opts = opts
        for url, cls in self.url_map.items():
            setattr(self, url, cls(self.opts))

    def verify_certs(self, *args):
        msg = ("Could not find a certificate: {0}\n"
                "If you want to quickly generate a self-signed certificate, "
                "use the tls.create_self_signed_cert function in Salt")

        for arg in args:
            if not os.path.exists(arg):
                raise Exception(msg.format(arg))

    def get_conf(self):
        # Grab config opts
        apiopts = self.opts.get(__name__.rsplit('.', 1)[-1], {})

        conf = {
            'global': {
                'server.socket_host': '0.0.0.0',
                'server.socket_port': apiopts.pop('port', 8000),
                'debug': apiopts.pop('debug', False),
            },
            '/': {
                'request.dispatch': cherrypy.dispatch.MethodDispatcher(),

                'tools.trailing_slash.on': True,
                'tools.gzip.on': True,

                'tools.sessions.on': True,
                'tools.sessions.timeout': 60 * 10, # 10 hours
                'tools.salt_auth.on': True,

                # 'tools.autovary.on': True,
                'tools.hypermedia_out.on': True,
                'tools.hypermedia_in.on': True,
            },
        }

        conf['global'].update(apiopts)
        return conf

def start():
    '''
    Server loop here. Started in a multiprocess.
    '''
    root = API(__opts__)
    conf = root.get_conf()
    gconf = conf.get('global', {})

    if gconf.get('debug', False):
        cherrypy.quickstart(root, '/', conf)
    else:
        root.verify_certs(gconf['ssl_crt'], gconf['ssl_key'])

        ssl_a = wsgiserver.ssl_builtin.BuiltinSSLAdapter(
                gconf['ssl_crt'], gconf['ssl_key'])
        wsgi_d = wsgiserver.WSGIPathInfoDispatcher({'/': root})
        server = wsgiserver.CherryPyWSGIServer(
                ('0.0.0.0', gconf['server.socket_port']),
                wsgi_app=wsgi_d,
                ssl_adapter=ssl_a)

        signal.signal(signal.SIGINT, lambda *args: server.stop())
        server.start()
<commit_msg>Don't catch CherryPy exceptions<commit_after>'''
A hypermedia REST API for Salt using the CherryPy framework
'''
# Import Python libs
import itertools
import signal
import os
import json

# Import third-party libs
import cherrypy
import cherrypy.wsgiserver as wsgiserver
import cherrypy.wsgiserver.ssl_builtin

import jinja2

# Import Salt libs
import salt.auth
import salt.log
import salt.output
from salt.utils import yaml

# Import salt-api libs
import saltapi

logger = salt.log.logging.getLogger(__name__)

jenv = jinja2.Environment(loader=jinja2.FileSystemLoader([
    os.path.join(os.path.dirname(__file__), 'tmpl'),
]))

def __virtual__():
    if 'port' in __opts__.get(__name__.rsplit('.')[-1], {}):
        return 'rest'
    return False

def fmt_lowdata(data):
    '''
    Take CherryPy body data from a POST (et al) request and format it into
    lowdata. It will accept repeated parameters and pair and format those
    into multiple lowdata chunks.
    '''
    pairs = []
    for k,v in data.items():
        # Ensure parameter is a list
        argl = v if isinstance(v, list) else [v]
        # Make pairs of (key, value) from {key: [*value]}
        pairs.append(zip([k] * len(argl), argl))

    lowdata = []
    for i in itertools.izip_longest(*pairs):
        if not all(i):
            msg = "Error pairing parameters: %s"
            raise Exception(msg % str(i))
        lowdata.append(dict(i))

    return lowdata

def salt_auth_tool():
    ignore_urls = ('/login',)

    # Grab the session via a cookie (for browsers) or via a custom header
    sid = (cherrypy.session.get('token', None) or
            cherrypy.request.headers.get('X-Auth-Token', None))

    if not cherrypy.request.path_info in ignore_urls and not sid :
        raise cherrypy.InternalRedirect('/login')

    cherrypy.response.headers['Cache-Control'] = 'private'

cherrypy.tools.salt_auth = cherrypy.Tool('before_request_body', salt_auth_tool)

# Be conservative in what you send
ct_out_map = {
    'application/json': 'json',
    'application/x-yaml': 'yaml',
    'text/html': 'jinja',
}

def hypermedia_handler(*args, **kwargs):
    best = cherrypy.lib.cptools.accept(ct_out_map.keys()) # raises 406
    out = ct_out_map[best]

    cherrypy.response.headers['Content-Type'] = best

    try:
        ret = cherrypy.serving.request._hypermedia_inner_handler(*args, **kwargs)
    except cherrypy.CherryPyException:
        raise
    except Exception as exc:
        logger.debug("Error while processing request for: %s",
                cherrypy.request.path_info,
                exc_info=True)

        cherrypy.response.status = 500
        cherrypy.response._tmpl = '500.html'

        ret = {
            'success': False,
            'status': cherrypy.response.status,
            'message': '{0}'.format(cherrypy._cperror.format_exc())}

    # Skip the outputters and use jinja instead
    if out == 'jinja':
        tmpl = jenv.get_template(cherrypy.response._tmpl)
        return tmpl.render(**ret)

    return salt.output.out_format(ret, out, __opts__)

def hypermedia_out():
    request = cherrypy.serving.request
    request._hypermedia_inner_handler = request.handler
    request.handler = hypermedia_handler

    # cherrypy.response.headers['Alternates'] = self.ct_out_map.keys()
    # TODO: add 'negotiate' to Vary header and 'list' to TCN header
    # Alternates: {"paper.1" 0.9 {type text/html} {language en}},
    #          {"paper.2" 0.7 {type text/html} {language fr}},
    #          {"paper.3" 1.0 {type application/postscript} {language en}}

cherrypy.tools.hypermedia_out = cherrypy.Tool('before_handler', hypermedia_out)

# Be liberal in what you accept
ct_in_map = {
    'application/x-www-form-urlencoded': fmt_lowdata,
    'application/json': json.loads,
    'application/x-yaml': yaml.load,
    'text/yaml': yaml.load,
}

def hypermedia_in():
    '''
    Unserialize POST/PUT data of a specified content type, if possible
    '''
    if cherrypy.request.method in ('POST', 'PUT'):
        ct_type = cherrypy.request.headers['content-type'].lower()
        ct_in = ct_in_map.get(ct_type, None)

        if not ct_in:
            raise cherrypy.HTTPError(406, 'Content type not supported')

        body = cherrypy.request.body.read()
        cherrypy.request.params['body'] = ct_in(body)

cherrypy.tools.hypermedia_in = cherrypy.Tool('before_handler', hypermedia_in)

class LowDataAdapter(object):
    '''
    '''
    exposed = True

    def __init__(self, opts):
        self.opts = opts
        self.api = saltapi.APIClient(opts)

    def exec_lowdata(self, lowdata):
        '''
        Pass lowdata to Salt to be executed
        '''
        # FIXME: change this to yield results from one of Salt's iterative returns
        logger.debug("SaltAPI is passing low-data: %s", lowdata)
        return [self.api.run(chunk) for chunk in lowdata]

    def GET(self):
        lowdata = [{'client': 'local', 'tgt': '*',
                'fun': ['grains.items', 'sys.list_functions'],
                'arg': [[], []],
        }]
        return self.exec_lowdata(lowdata)

    def POST(self, **kwargs):
        '''
        Run a given function in a given client with the given args
        '''
        return self.exec_lowdata(self.fmt_lowdata(kwargs))

class Login(LowDataAdapter):
    '''
    '''
    exposed = True

    def GET(self):
        cherrypy.response._tmpl = 'login.html'
        cherrypy.response.status = '401 Unauthorized'
        cherrypy.response.headers['WWW-Authenticate'] = 'Session'

        return {
            'success': False,
            'status': cherrypy.response.status,
            'message': "Please log in",
        }

    def POST(self, **kwargs):
        auth = salt.auth.LoadAuth(self.opts)
        token = auth.mk_token(kwargs).get('token', False)
        cherrypy.response.headers['X-Auth-Token'] = cherrypy.session.id
        cherrypy.session['token'] = token
        raise cherrypy.HTTPRedirect('/', 302)

class API(object):
    url_map = {
        'index': LowDataAdapter,
        'login': Login,
    }

    def __init__(self, opts):
        self.opts = opts
        for url, cls in self.url_map.items():
            setattr(self, url, cls(self.opts))

    def verify_certs(self, *args):
        msg = ("Could not find a certificate: {0}\n"
                "If you want to quickly generate a self-signed certificate, "
                "use the tls.create_self_signed_cert function in Salt")

        for arg in args:
            if not os.path.exists(arg):
                raise Exception(msg.format(arg))

    def get_conf(self):
        # Grab config opts
        apiopts = self.opts.get(__name__.rsplit('.', 1)[-1], {})

        conf = {
            'global': {
                'server.socket_host': '0.0.0.0',
                'server.socket_port': apiopts.pop('port', 8000),
                'debug': apiopts.pop('debug', False),
            },
            '/': {
                'request.dispatch': cherrypy.dispatch.MethodDispatcher(),

                'tools.trailing_slash.on': True,
                'tools.gzip.on': True,

                'tools.sessions.on': True,
                'tools.sessions.timeout': 60 * 10, # 10 hours
                'tools.salt_auth.on': True,

                # 'tools.autovary.on': True,
                'tools.hypermedia_out.on': True,
                'tools.hypermedia_in.on': True,
            },
        }

        conf['global'].update(apiopts)
        return conf

def start():
    '''
    Server loop here. Started in a multiprocess.
    '''
    root = API(__opts__)
    conf = root.get_conf()
    gconf = conf.get('global', {})

    if gconf.get('debug', False):
        cherrypy.quickstart(root, '/', conf)
    else:
        root.verify_certs(gconf['ssl_crt'], gconf['ssl_key'])

        ssl_a = wsgiserver.ssl_builtin.BuiltinSSLAdapter(
                gconf['ssl_crt'], gconf['ssl_key'])
        wsgi_d = wsgiserver.WSGIPathInfoDispatcher({'/': root})
        server = wsgiserver.CherryPyWSGIServer(
                ('0.0.0.0', gconf['server.socket_port']),
                wsgi_app=wsgi_d,
                ssl_adapter=ssl_a)

        signal.signal(signal.SIGINT, lambda *args: server.stop())
        server.start()
<|endoftext|>


<commit_before># Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import time
import uuid

import google.api_core.exceptions
import google.auth
from google.cloud import compute_v1
import pytest

from sample_firewall import (
    create_firewall_rule,
    delete_firewall_rule,
    get_firewall_rule,
    list_firewall_rules,
    patch_firewall_priority,
)

PROJECT = google.auth.default()[1]

@pytest.fixture
def firewall_rule():
    firewall_rule = compute_v1.Firewall()
    firewall_rule.name = "firewall-sample-test" + uuid.uuid4().hex[:10]
    firewall_rule.direction = "INGRESS"
    allowed_ports = compute_v1.Allowed()
    allowed_ports.I_p_protocol = "tcp"
    allowed_ports.ports = ["80"]
    firewall_rule.allowed = [allowed_ports]
    firewall_rule.source_ranges = ["0.0.0.0/0"]
    firewall_rule.network = "global/networks/default"
    firewall_rule.description = "Rule generated by Python sample test fixture."
    firewall_rule.target_tags = ["web"]

    firewall_client = compute_v1.FirewallsClient()
    op = firewall_client.insert_unary(project=PROJECT, firewall_resource=firewall_rule)

    op_client = compute_v1.GlobalOperationsClient()
    op_client.wait(project=PROJECT, operation=op.name)

    yield firewall_client.get(project=PROJECT, firewall=firewall_rule.name)

    try:
        op = firewall_client.delete_unary(project=PROJECT, firewall=firewall_rule.name)
        op_client.wait(project=PROJECT, operation=op.name)
    except google.api_core.exceptions.BadRequest as err:
        if err.code == 400 and "is not ready" in err.message:
            # This means GCE enforcer has already deleted that rule.
            pass
        else:
            raise err

def test_create_delete():
    rule_name = "firewall-sample-test-" + uuid.uuid4().hex[:10]
    create_firewall_rule(PROJECT, rule_name)
    rule = get_firewall_rule(PROJECT, rule_name)
    assert rule.name == rule_name
    assert "web" in rule.target_tags
    delete_firewall_rule(PROJECT, rule_name)
    assert all(rule.name != rule_name for rule in list_firewall_rules(PROJECT))

def test_patch_rule(firewall_rule):
    fw_client = compute_v1.FirewallsClient()
    assert firewall_rule.priority == 1000
    patch_firewall_priority(PROJECT, firewall_rule.name, 500)
    time.sleep(2)
    updated_firewall_rule = fw_client.get(project=PROJECT, firewall=firewall_rule.name)
    assert updated_firewall_rule.priority == 500
<commit_msg>chore(samples): Removing some leaked instances. (#179)<commit_after># Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import time
import uuid

import google.api_core.exceptions
import google.auth
from google.cloud import compute_v1
import pytest

from sample_firewall import (
    create_firewall_rule,
    delete_firewall_rule,
    get_firewall_rule,
    patch_firewall_priority,
)

PROJECT = google.auth.default()[1]

@pytest.fixture
def firewall_rule():
    firewall_rule = compute_v1.Firewall()
    firewall_rule.name = "firewall-sample-test" + uuid.uuid4().hex[:10]
    firewall_rule.direction = "INGRESS"
    allowed_ports = compute_v1.Allowed()
    allowed_ports.I_p_protocol = "tcp"
    allowed_ports.ports = ["80"]
    firewall_rule.allowed = [allowed_ports]
    firewall_rule.source_ranges = ["0.0.0.0/0"]
    firewall_rule.network = "global/networks/default"
    firewall_rule.description = "Rule generated by Python sample test fixture."
    firewall_rule.target_tags = ["web"]

    firewall_client = compute_v1.FirewallsClient()
    op = firewall_client.insert_unary(project=PROJECT, firewall_resource=firewall_rule)

    op_client = compute_v1.GlobalOperationsClient()
    op_client.wait(project=PROJECT, operation=op.name)

    yield firewall_client.get(project=PROJECT, firewall=firewall_rule.name)

    try:
        op = firewall_client.delete_unary(project=PROJECT, firewall=firewall_rule.name)
        op_client.wait(project=PROJECT, operation=op.name)
    except google.api_core.exceptions.BadRequest as err:
        if err.code == 400 and "is not ready" in err.message:
            # This means GCE enforcer has already deleted that rule.
            pass
        else:
            raise err

@pytest.fixture
def autodelete_firewall_name():
    """
    Provide a name for a firewall rule and then delete the rule.
    """
    rule_name = "firewall-sample-test-" + uuid.uuid4().hex[:10]
    yield rule_name
    try:
        delete_firewall_rule(PROJECT, rule_name)
    except google.api_core.exceptions.BadRequest as err:
        if err.code == 400 and "is not ready" in err.message:
            # We can ignore this, this is most likely GCE Enforcer removing the rule before us.
            pass
        else:
            # Something else went wrong, let's escalate it.
            raise err

def test_create(autodelete_firewall_name):
    create_firewall_rule(PROJECT, autodelete_firewall_name)
    rule = get_firewall_rule(PROJECT, autodelete_firewall_name)
    assert rule.name == autodelete_firewall_name
    assert "web" in rule.target_tags

def test_patch_rule(firewall_rule):
    fw_client = compute_v1.FirewallsClient()
    assert firewall_rule.priority == 1000
    patch_firewall_priority(PROJECT, firewall_rule.name, 500)
    time.sleep(2)
    updated_firewall_rule = fw_client.get(project=PROJECT, firewall=firewall_rule.name)
    assert updated_firewall_rule.priority == 500
<|endoftext|>


<commit_before>#!/usr/bin/python3

#  Copyright (C) 2012 Hai Bison
#
#  See the file LICENSE at the root directory of this project for copying
#  permission.

'''
This tool parse messages.properties and generate all strings to their
respective IDs, which can be put into class R.string.
'''

import re

SOURCE_FILE = '../code/src/group/pals/desktop/app/apksigner/i18n/messages.properties'

# PARSE SOURCE FILE

# Should start from 1...
count = 1

with open(SOURCE_FILE, 'r') as f:
    for line in f:
        line = line.strip()
        if not line.startswith('#') and len(line) > 0:
            print('public static final int {} = 0x{:08x};'\
                   .format(re.sub(r'(?s)=.*', '', line).strip(), count))
            count += 1
<commit_msg>Comments.<commit_after>#!/usr/bin/python3

#  Copyright (C) 2012 Hai Bison
#
#  See the file LICENSE at the root directory of this project for copying
#  permission.

'''
This tool parses `messages.properties` and generates all strings to their
respective IDs, which can be put into class R.string.
'''

import re

SOURCE_FILE = '../code/src/group/pals/desktop/app/apksigner/i18n/messages.properties'

# PARSE SOURCE FILE

# Should start from 1...
count = 1

with open(SOURCE_FILE, 'r') as f:
    for line in f:
        line = line.strip()
        if not line.startswith('#') and len(line) > 0:
            print('public static final int {} = 0x{:08x};'\
                   .format(re.sub(r'(?s)=.*', '', line).strip(), count))
            count += 1
<|endoftext|>


<commit_before># Copyright (c) 2013 Henry S. Harrison

import numpy as np
import pandas as pd
import itertools
import functools
import collections
import logging

class SortError(Exception):
    pass

class DesignError(Exception):
    pass

class VariableError(Exception):
    pass

def make_sort_function(array, repeats, method):
    """
    array: iterable to be sorted
    repeats: number of times for each element in array to appear
    method: either a string {'random', more options to be implemented} or a list of indices
    """
    if method == 'random':
        return functools.partial(np.random.permutation, repeats * array)
    #TODO: More sorts (e.g. counterbalance)
    elif isinstance(method, str):
        raise SortError('Unrecognized sort method {}.'.format(method))
    else:
            return lambda: repeats * np.array(array)[method]

class Variable():
    def __init__(self, name):
        logging.debug('Creating variable %s of type %s.', name, type(self))
        self.name = name

    def __str__(self):
        return self.name

    def value(self, *args, **kwargs):
        return None

class ConstantVariable(Variable):
    def __init__(self, name, value):
        super(ConstantVariable, self).__init__(name)
        self._value = value

    def value(self):
        return self._value

class IndependentVariable(Variable):
    """
    Additional args: levels (list of values for the IV)
    Additional kwargs: design (between or within), change_by (trial, block, session, participant)
        design='between' is equivalent to change_by='participant'
    """
    def __init__(self, name, levels, design='within', change_by='trial'):
        super(IndependentVariable, self).__init__(name)
        self.levels = levels
        self.design = design
        if self.design == 'between':
            self.change_by = 'participant'
        elif self.design == 'within' and change_by == 'participant':
            raise DesignError('Cannot have a within-subjects IV change by participant.')
        else:
            self.change_by = change_by

    def value(self, idx, *args, **kwargs):
        return self.levels[idx]

class CustomVariable(Variable):
    def __init__(self, name, func):
        super(CustomVariable, self).__init__(name)
        self.func = func

    def value(self):
        return self.func()

class RandomVariable(CustomVariable):
    def __init__(self, name, lower, upper):
        super(RandomVariable, self).__init__(name, lambda: (upper - lower)*np.random.random() + lower)

def new_variable(name, levels):
    logging.debug('Creating new variable from kwarg %s=%s...', name, levels)
    if callable(levels):
        return CustomVariable(name, levels)
    elif np.isscalar(levels):
        return ConstantVariable(name, levels)
    elif isinstance(levels, collections.abc.Iterable):
        return IndependentVariable(name, levels)
    else:
        raise VariableError('Cannot create variable {} = {}'.format(name, levels))

class Experiment():
    """
    Experiments should subclass this and override, at minimum, the method run_trial(trial_idx, **trial_settings).
    Other methods to override:
       session_start()
       session_end()
       block_start(block_idx, block) where block is a DataFrame with rows = trials
       block_end(block_idx, block)
       inter_block(block_idx, block) where block_idx and block refer to the next block
       inter_trial(trial_idx, **trial_settings) where trial_idx and trial_settings refer to the next trial

    Inputs to Experiment.__init__:
       *args: Variable objects
       **kwargs: output_names (column names in saved data, length = number of outputs returned by run_trial)
                 trial list settings:
                     trials_per_type_per_block {default = 1}
                     blocks_per_type {default = 1}
                     trial_sort {'random' (default), array of indices}
                     block_sort {'random' (default), array of indices}
                 Any number of name = value pairs, creating Variables.
                    ConstantVariable if value = constant
                    CustomVariable if value is callable
                    IndependentVariable (within-subjects) if value is iterable
    """
    # TODO: between-subjects design
    # TODO: multi-session experiments
    def __init__(self, *args, output_names=None, **kwargs):
        logging.info('Constructing new experiment...')
        self.output_names = output_names

        trial_list_settings_defaults = {'trials_per_type_per_block': 1,
                                        'blocks_per_type': 1,
                                        'trial_sort': 'random',
                                        'block_sort': 'random'}
        self.trial_list_settings = {key: kwargs.pop(key, default)
                                    for key, default in trial_list_settings_defaults.items()}

        logging.info('Constructing variables...')
        self.variables = list(args)
        for k, v in kwargs.items():
            self.variables.append(new_variable(k, v))

        logging.debug('Sorting variables by type...')
        self.trial_ivs = [v for v in self.variables if isinstance(v, IndependentVariable) and v.change_by == 'trial']
        self.block_ivs = [v for v in self.variables if isinstance(v, IndependentVariable) and v.change_by == 'block']
        self.constants = [v for v in self.variables if isinstance(v, ConstantVariable)]
        self.custom_vars = [v for v in self.variables if isinstance(v, CustomVariable)]

        self.n_blocks = 0
        self.n_trials = 0

        logging.info('Constructing blocks and trials...')
        self.blocks = list(self.block_list(**self.trial_list_settings))
        self.raw_results = []

    def block_list(self, trials_per_type_per_block=1, blocks_per_type=1, trial_sort='random', block_sort='random'):
        # In this and the next block, we cross the indices of each IV's levels rather than the actual values.
        # This allows for subclasses to override the value method and do stuff besides indexing to determine the value.
        if self.trial_ivs:
            logging.info('Crossing IVs that vary by trial...')
            iv_idxs = itertools.product(*[range(len(v)) for v in self.trial_ivs])
            trial_types = [{iv.name: iv.value(condition[idx]) for idx, iv in enumerate(self.trial_ivs)}
                           for condition in iv_idxs]
        else:
            logging.info('No IVs that vary by trial.')
            trial_types = [{}]

        if self.block_ivs:
            logging.info('Crossing IVs that vary by block...')
            iv_idxs = itertools.product(*[range(len(v)) for v in self.block_ivs])
            block_types = [{iv.name: iv.value(condition[idx]) for idx, iv in enumerate(self.block_ivs)}
                           for condition in iv_idxs]
        else:
            logging.info('No IVs that vary by block.')
            block_types = [{}]

        # TODO: Pass any args/kwargs to custom_vars.value?
        more_vars = lambda idx: {v.name: v.value() for v in np.concatenate((self.custom_vars, self.constants))}

        # Constructing sort functions, rather than directly sorting, allows for a different sort for each call
        logging.debug('Creating sort method for trials within a block...')
        sort_block = make_sort_function(trial_types, trials_per_type_per_block, trial_sort)
        logging.debug('Creating sort method for blocks within a session...')
        sort_session = make_sort_function(block_types, blocks_per_type, block_sort)

        logging.debug('Constructing blocks within session...')
        blocks = list(sort_session())
        self.n_trials = len(blocks) * len(sort_block())
        self.n_blocks = len(blocks)
        for block_idx, block in enumerate(blocks):
            logging.debug('Constructing trials within block %s...', block_idx)
            trials = list(sort_block())
            for trial_idx, trial in enumerate(trials):
                # Add block-specific IVs, custom vars, and constants
                logging.debug('Constructing trial %s...', trial_idx)
                trial.update({k: v for k, v in block.items()})
                trial.update(more_vars(block_idx*len(trials) + trial_idx))
            yield pd.DataFrame(trials, index=block_idx*len(trials) + np.arange(len(trials)))

    def save_data(self, output_file):
        """
        Trial settings and results are combined into one DataFrame, which is pickled.
        """
        # Concatenate trial settings
        logging.debug('Combining trial inputs and outputs...')
        trial_inputs = pd.concat(self.blocks)

        results = pd.DataFrame(self.raw_results, columns=self.output_names)

        logging.debug('Writing data to file %s...', output_file)
        pd.concat([trial_inputs, results], axis=1).to_pickle(output_file)

    def run_session(self, output_file):
        logging.info('Running session...')
        logging.debug('Running start_session()...')
        self.session_start()

        for block_idx, block in enumerate(self.blocks):
            logging.debug('Block %s:', block_idx)
            if block_idx > 1:
                logging.debug('Running inter_block()...')
                self.inter_block(block_idx, block)
            logging.debug('Running block_start()')
            self.block_start(block_idx, block)

            for trial_idx, trial in block.iterrows():
                if trial_idx > 0:
                    logging.debug('Running inter_trial()...')
                    self.inter_trial(trial_idx, **dict(trial))
                logging.info('Running trial %s...', trial_idx)
                self.raw_results.append(self.run_trial(trial_idx, **dict(trial)))

            logging.debug('Running block_end()')
            self.block_end(block_idx, block)
        logging.debug('Running session_end()')
        self.session_end()

        logging.info('Saving data...')
        self.save_data(output_file)

    def run_trial(self, trial_idx, **kwargs):
        return None

    def block_start(self, block_idx, block):
        pass

    def block_end(self, block_idx, block):
        pass

    def inter_block(self, block_idx, block):
        pass

    def session_start(self):
        pass

    def session_end(self):
        pass

    def inter_trial(self, current_trial, **kwargs):
        pass<commit_msg>Implemented metaclasses.<commit_after># Copyright (c) 2013 Henry S. Harrison

import numpy as np
import pandas as pd
import itertools
import functools
import collections
import logging

class SortError(Exception):
    pass

class DesignError(Exception):
    pass

class VariableError(Exception):
    pass

def make_sort_function(array, repeats, method):
    """
    array: iterable to be sorted
    repeats: number of times for each element in array to appear
    method: either a string {'random', more options to be implemented} or a list of indices
    """
    if method == 'random':
        return functools.partial(np.random.permutation, repeats * array)
    #TODO: More sorts (e.g. counterbalance)
    elif isinstance(method, str):
        raise SortError('Unrecognized sort method {}.'.format(method))
    else:
            return lambda: repeats * np.array(array)[method]

class Variable(metaclass=collections.ABCMeta):
    def __init__(self, name):
        logging.debug('Creating variable %s of type %s.', name, type(self))
        self.name = name

    def __str__(self):
        return self.name

    @collections.abstractmethod
    def value(self, *args, **kwargs):
        return None

class ConstantVariable(Variable):
    def __init__(self, name, value):
        super(ConstantVariable, self).__init__(name)
        self._value = value

    def value(self):
        return self._value

class IndependentVariable(Variable):
    """
    Additional args: levels (list of values for the IV)
    Additional kwargs: design (between or within), change_by (trial, block, session, participant)
        design='between' is equivalent to change_by='participant'
    """
    def __init__(self, name, levels, design='within', change_by='trial'):
        super(IndependentVariable, self).__init__(name)
        self.levels = levels
        self.design = design
        if self.design == 'between':
            self.change_by = 'participant'
        elif self.design == 'within' and change_by == 'participant':
            raise DesignError('Cannot have a within-subjects IV change by participant.')
        else:
            self.change_by = change_by

    def value(self, idx, *args, **kwargs):
        return self.levels[idx]

class CustomVariable(Variable):
    def __init__(self, name, func):
        super(CustomVariable, self).__init__(name)
        self.func = func

    def value(self):
        return self.func()

class RandomVariable(CustomVariable):
    def __init__(self, name, lower, upper):
        super(RandomVariable, self).__init__(name, lambda: (upper - lower)*np.random.random() + lower)

def new_variable(name, levels):
    logging.debug('Creating new variable from kwarg %s=%s...', name, levels)
    if callable(levels):
        return CustomVariable(name, levels)
    elif np.isscalar(levels):
        return ConstantVariable(name, levels)
    elif isinstance(levels, collections.abc.Iterable):
        return IndependentVariable(name, levels)
    else:
        raise VariableError('Cannot create variable {} = {}'.format(name, levels))

class Experiment(metaclass=collections.ABCMeta):
    """
    Experiments should subclass this and override, at minimum, the method run_trial(trial_idx, **trial_settings).
    Other methods to override:
       session_start()
       session_end()
       block_start(block_idx, block) where block is a DataFrame with rows = trials
       block_end(block_idx, block)
       inter_block(block_idx, block) where block_idx and block refer to the next block
       inter_trial(trial_idx, **trial_settings) where trial_idx and trial_settings refer to the next trial

    Inputs to Experiment.__init__:
       *args: Variable objects
       **kwargs: output_names (column names in saved data, length = number of outputs returned by run_trial)
                 trial list settings:
                     trials_per_type_per_block {default = 1}
                     blocks_per_type {default = 1}
                     trial_sort {'random' (default), array of indices}
                     block_sort {'random' (default), array of indices}
                 Any number of name = value pairs, creating Variables.
                    ConstantVariable if value = constant
                    CustomVariable if value is callable
                    IndependentVariable (within-subjects) if value is iterable
    """
    # TODO: between-subjects design
    # TODO: multi-session experiments
    def __init__(self, *args, output_names=None, **kwargs):
        logging.info('Constructing new experiment...')
        self.output_names = output_names

        trial_list_settings_defaults = {'trials_per_type_per_block': 1,
                                        'blocks_per_type': 1,
                                        'trial_sort': 'random',
                                        'block_sort': 'random'}
        self.trial_list_settings = {key: kwargs.pop(key, default)
                                    for key, default in trial_list_settings_defaults.items()}

        logging.info('Constructing variables...')
        self.variables = list(args)
        for k, v in kwargs.items():
            self.variables.append(new_variable(k, v))

        logging.debug('Sorting variables by type...')
        self.trial_ivs = [v for v in self.variables if isinstance(v, IndependentVariable) and v.change_by == 'trial']
        self.block_ivs = [v for v in self.variables if isinstance(v, IndependentVariable) and v.change_by == 'block']
        self.constants = [v for v in self.variables if isinstance(v, ConstantVariable)]
        self.custom_vars = [v for v in self.variables if isinstance(v, CustomVariable)]

        self.n_blocks = 0
        self.n_trials = 0

        logging.info('Constructing blocks and trials...')
        self.blocks = list(self.block_list(**self.trial_list_settings))
        self.raw_results = []

    def block_list(self, trials_per_type_per_block=1, blocks_per_type=1, trial_sort='random', block_sort='random'):
        # In this and the next block, we cross the indices of each IV's levels rather than the actual values.
        # This allows for subclasses to override the value method and do stuff besides indexing to determine the value.
        if self.trial_ivs:
            logging.info('Crossing IVs that vary by trial...')
            iv_idxs = itertools.product(*[range(len(v)) for v in self.trial_ivs])
            trial_types = [{iv.name: iv.value(condition[idx]) for idx, iv in enumerate(self.trial_ivs)}
                           for condition in iv_idxs]
        else:
            logging.info('No IVs that vary by trial.')
            trial_types = [{}]

        if self.block_ivs:
            logging.info('Crossing IVs that vary by block...')
            iv_idxs = itertools.product(*[range(len(v)) for v in self.block_ivs])
            block_types = [{iv.name: iv.value(condition[idx]) for idx, iv in enumerate(self.block_ivs)}
                           for condition in iv_idxs]
        else:
            logging.info('No IVs that vary by block.')
            block_types = [{}]

        # TODO: Pass any args/kwargs to custom_vars.value?
        more_vars = lambda idx: {v.name: v.value() for v in np.concatenate((self.custom_vars, self.constants))}

        # Constructing sort functions, rather than directly sorting, allows for a different sort for each call
        logging.debug('Creating sort method for trials within a block...')
        sort_block = make_sort_function(trial_types, trials_per_type_per_block, trial_sort)
        logging.debug('Creating sort method for blocks within a session...')
        sort_session = make_sort_function(block_types, blocks_per_type, block_sort)

        logging.debug('Constructing blocks within session...')
        blocks = list(sort_session())
        self.n_trials = len(blocks) * len(sort_block())
        self.n_blocks = len(blocks)
        for block_idx, block in enumerate(blocks):
            logging.debug('Constructing trials within block %s...', block_idx)
            trials = list(sort_block())
            for trial_idx, trial in enumerate(trials):
                # Add block-specific IVs, custom vars, and constants
                logging.debug('Constructing trial %s...', trial_idx)
                trial.update({k: v for k, v in block.items()})
                trial.update(more_vars(block_idx*len(trials) + trial_idx))
            yield pd.DataFrame(trials, index=block_idx*len(trials) + np.arange(len(trials)))

    def save_data(self, output_file):
        """
        Trial settings and results are combined into one DataFrame, which is pickled.
        """
        # Concatenate trial settings
        logging.debug('Combining trial inputs and outputs...')
        trial_inputs = pd.concat(self.blocks)

        results = pd.DataFrame(self.raw_results, columns=self.output_names)

        logging.debug('Writing data to file %s...', output_file)
        pd.concat([trial_inputs, results], axis=1).to_pickle(output_file)

    def run_session(self, output_file):
        logging.info('Running session...')
        logging.debug('Running start_session()...')
        self.session_start()

        for block_idx, block in enumerate(self.blocks):
            logging.debug('Block %s:', block_idx)
            if block_idx > 1:
                logging.debug('Running inter_block()...')
                self.inter_block(block_idx, block)
            logging.debug('Running block_start()')
            self.block_start(block_idx, block)

            for trial_idx, trial in block.iterrows():
                if trial_idx > 0:
                    logging.debug('Running inter_trial()...')
                    self.inter_trial(trial_idx, **dict(trial))
                logging.info('Running trial %s...', trial_idx)
                self.raw_results.append(self.run_trial(trial_idx, **dict(trial)))

            logging.debug('Running block_end()')
            self.block_end(block_idx, block)
        logging.debug('Running session_end()')
        self.session_end()

        logging.info('Saving data...')
        self.save_data(output_file)

    @collections.abstractmethod
    def run_trial(self, trial_idx, **kwargs):
        return None

    def block_start(self, block_idx, block):
        pass

    def block_end(self, block_idx, block):
        pass

    def inter_block(self, block_idx, block):
        pass

    def session_start(self):
        pass

    def session_end(self):
        pass

    def inter_trial(self, current_trial, **kwargs):
        pass<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2015 Open Connectome Project (http://openconnecto.me)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# extract_b0.py
# Created by Greg Kiar on 2015-02-23.
# Email: gkiar@jhu.edu
# Copyright (c) 2015. All rights reserved.

from argparse import ArgumentParser
from nibabel import load, save, Nifti1Image
from numpy import where, loadtxt
from os import system

def register_vol(dti_img, bvals, aligned_img, b0_img):
  print "Loading dti data..."
  dti_vol = load(dti_img)
  dti_data = dti_vol.get_data()

  print "Loading bvals file..."
  b = loadtxt(bvals)

  print "Extracting B0 volume..."
  b0_vol=dti_data[:,:,:,int(where(b==0)[0])]
  b0_head = dti_vol.get_header()
  b0_head.set_data_shape(b0_head.get_data_shape()[0:3])
  b0_out = Nifti1Image( data=b0_vol, affine=dti_vol.get_affine(), header=b0_head )
  save(b0_out, b0_img)

  directions = dti_vol.get_shape()[3]
  print "Number of 3D volumes:", directions

  aligned_vol = dti_vol
  aligned_data = aligned_vol.get_data()
  print "Beginning registration..."
  temp_in_img = 'temp_in.nii'
  temp_out_img = 'temp_out.nii'
  temp_head = dti_vol.get_header()
  temp_head.set_data_shape(temp_head.get_data_shape()[0:3])

  for ii in range(directions):
    print "Volume ", ii, " of ", directions
    currentvol = aligned_data[:,:,:,ii]
    #print "Writing current volume to disk..."
    out = Nifti1Image( data=currentvol, affine=dti_vol.get_affine(), header=temp_head )
    save(out, temp_in_img)
    system('antsRegistration -d 3 -o [affine.mat,'+temp_out_img+'] -r ['+b0_img+', '+temp_in_img+',1] -m Mattes[ '+b0_img+', '+temp_in_img+',1,12] -t Affine[0.75] -c [25, 1e-4, 5] --smoothing-sigmas 1 -f 1')
    temp_vol = load(temp_out_img)
    currentvol = temp_vol.get_data()
    aligned_data[:,:,:,ii] = currentvol
    system('rm -f '+temp_out_img+';rm -f'+temp_in_img)

  system('rm -f affine0GenerticAffine.mat')
  out = Nifti1Image( data=aligned_vol, affine=dti_vol.get_affine(), header=dt_vol.get_header() )
  save(out, aligned_img)

def main():
  parser = ArgumentParser(description="")
  parser.add_argument("DTI", action="store", help="The DTI image we want to extract B0 from (.nii, .nii.gz)")
  parser.add_argument("bvals", action="store", help="The b-value file corresponding to the DTI image (.b)")
  parser.add_argument("alignedDTI", action="store", help="The output file location of the B0 scan (.nii, .nii.gz)")

  result = parser.parse_args()

  register_vol(result.DTI, result.bvals, result.alignedDTI)

if __name__ == "__main__":
    main()

<commit_msg>Code for extracting B0 volume from DTI stack, as well as aligning DTI stack to the B0 volume. Stores both B0 and aligned DTI as data derivatives<commit_after>#!/usr/bin/env python

# Copyright 2015 Open Connectome Project (http://openconnecto.me)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# extract_b0.py
# Created by Greg Kiar on 2015-02-23.
# Email: gkiar@jhu.edu
# Copyright (c) 2015. All rights reserved.

from argparse import ArgumentParser
from nibabel import load, save, Nifti1Image
from numpy import where, loadtxt
from os import system

def register_vol(dti_img, bvals, aligned_img, b0_img):
  print "Loading dti data..."
  dti_vol = load(dti_img)
  dti_data = dti_vol.get_data()

  print "Loading bvals file..."
  b = loadtxt(bvals)

  print "Extracting B0 volume..."
  b0_vol=dti_data[:,:,:,int(where(b==0)[0])]
  b0_head = dti_vol.get_header()
  b0_head.set_data_shape(b0_head.get_data_shape()[0:3])
  b0_out = Nifti1Image( data=b0_vol, affine=dti_vol.get_affine(), header=b0_head )
  save(b0_out, b0_img)

  directions = dti_vol.get_shape()[3]
  print "Number of 3D volumes:", directions

  aligned_vol = dti_vol
  aligned_data = aligned_vol.get_data()
  print "Beginning registration..."
  temp_in_img = 'temp_in.nii'
  temp_out_img = 'temp_out.nii'
  temp_head = dti_vol.get_header()
  temp_head.set_data_shape(temp_head.get_data_shape()[0:3])

  for ii in range(directions):
    print "Volume ", ii+1, " of ", directions
    currentvol = aligned_data[:,:,:,ii]
    #print "Writing current volume to disk..."
    out = Nifti1Image( data=currentvol, affine=dti_vol.get_affine(), header=temp_head )
    save(out, temp_in_img)
    system('antsRegistration -d 3 -o [affine,'+temp_out_img+'] -r ['+b0_img+', '+temp_in_img+',1] -m Mattes[ '+b0_img+', '+temp_in_img+',1,12] -t Affine[0.75] -c [25, 1e-4, 5] --smoothing-sigmas 1 -f 1 > /dev/null')
    temp_vol = load(temp_out_img)
    currentvol = temp_vol.get_data()
    aligned_data[:,:,:,ii] = currentvol
    system('rm '+temp_out_img+';rm '+temp_in_img)

  system('rm  affine0GenericAffine.mat')

  dims = dti_vol.get_header().get_data_shape()
  dti_vol.get_header().set_data_shape((dims[0], dims[1], dims[2], directions))
  out = Nifti1Image( data=aligned_data, affine=dti_vol.get_affine(), header=dti_vol.get_header() )

  save(out, aligned_img)

def main():
  parser = ArgumentParser(description="")
  parser.add_argument("DTI", action="store", help="The DTI image we want to extract B0 from (.nii, .nii.gz)")
  parser.add_argument("bvals", action="store", help="The b-value file corresponding to the DTI image (.b)")
  parser.add_argument("alignedDTI", action="store", help="The output file location of the aligned DTI volume  scan (.nii, .nii.gz)")
  parser.add_argument("B0", action="store", help="The output file location of the B0 scan (.nii, .nii.gz)")
  result = parser.parse_args()

  register_vol(result.DTI, result.bvals, result.alignedDTI, result.B0)

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before>#!/usr/bin/python

import sys, getopt, scipy, os, h5py
import numpy as np, pylab as pl
from sklearn.utils.arpack import eigsh
from sklearn.cluster import KMeans

def prepMat(data):
    K = (data + 1) / 2.
    v = np.sqrt(np.sum(K, axis=1))
    A = K/(v[:, None] * v[None, :])
    del K
    A = np.squeeze(A * [A > 0])
    return A

def runEmbed(data, n_components):
    lambdas, vectors = eigsh(data, k=n_components)
    lambdas = lambdas[::-1]
    vectors = vectors[:, ::-1]
    psi = vectors/vectors[:, 0][:, None]
    lambdas = lambdas[1:] / (1 - lambdas[1:])
    embedding = psi[:, 1:(n_components + 1)] * lambdas[:n_components][None, :]
    #embedding_sorted = np.argsort(embedding[:], axis=1)
    return embedding

def kmeans(embedding, n_components):
    est = KMeans(n_clusters=n_components, n_jobs=-1, init='k-means++', n_init=300)
    est.fit_transform(embedding)
    labels = est.labels_
    data = labels.astype(np.float)
    return data

def recort(data, cortex):
    d = ([0] * np.shape(data))
    count = 0
    for i in cortex:
        d[i] = data[count] + 1
        count = count + 1
    return d

<commit_msg>cleaning up individual_distance analysis<commit_after><|endoftext|>


<commit_before>B1;3400;0c"""
This script is a collection of objects and methods used for MLPs: inspired from Theano MLP tutorial http://deeplearning.net/tutorial
"""
__docformat__ = 'restructedtext en'

import cPickle
import os
import sys
import time

import numpy as np

import theano
import theano.tensor as T

class NetworkLayer(object):
    def __init__(self, rng, input, n_in, n_out, W=None, b=None,
                 activation=T.tanh):
        """
        Typical layer of a MLP: units are fully-connected and have
        tanh activation function. Weight matrix W is of shape (n_in,n_out)
        and the bias vector b is of shape (n_out,).

        :type rng: numpy.random.RandomState
        :param rng: a random number generator used to initialize weights

        :type input: theano.tensor.dmatrix
        :param input: a symbolic tensor of shape (n_examples, n_in)

        :type n_in: int
        :param n_in: dimensionality of input

        :type n_out: int
        :param n_out: number of hidden units

        :type activation: theano.Op or function
        :param activation: Non linearity to be applied in the hidden
                           layer
        """
        self.input = input

        if W is None:
            W_values = np.asarray(rng.uniform(
                    low=-np.sqrt(6. / (n_in + n_out)),
                    high=np.sqrt(6. / (n_in + n_out)),
                    size=(n_in, n_out)), dtype=theano.config.floatX)
            if activation == theano.tensor.nnet.sigmoid:
                W_values *= 4

            W = theano.shared(value=W_values, name='W', borrow=True)

        if b is None:
            b_values = np.zeros((n_out,), dtype=theano.config.floatX)
            b = theano.shared(value=b_values, name='b', borrow=True)

        self.W = W
        self.b = b

        lin_output = T.dot(input, self.W) + self.b

        self.output = (lin_output if activation is None
                       else activation(lin_output))

        # parameters of the model
        self.params = [self.W, self.b]

    def set_symbolic_input(self, input):
        """We use this function to bind a symbolic variable with the input
        of the network layer. Added to specify that in training time."""
        self.input = input

class MLP(object):
    """Multi-Layer Perceptron class.
    Inspired from Theano MLP tutorial http://deeplearning.net/tutorial/mlp.html
    """

    def __init__(self, input, rng, n_in, n_hiddens, hid_activations,
                 n_out, out_activation):
        """Initialize the parameters for the multilayer perceptron

        :type rng: np.random.RandomState
        :param rng: a random number generator used to initialize weights

        :type n_in: int
        :param n_in: number of input units, the dimension of the space in
        which the datapoints lie

        :type n_hidden_list: list of int
        :param n_hidden_list: a list of number of units in each hidden layer

        :type activations_list: list of lambdas
        :param n_hidden_list: a list of activations used in each hidden layer

        :type n_out: int
        :param n_out: number of output units, the dimension of the space in
        which the labels lie

        """

        self.input = input

        # We are dealing with multiple hidden layers MLP
        layer0 = NetworkLayer(rng=rng, input=input,
                              n_in=n_in, n_out=n_hiddens[0],
                              activation=hid_activations[0])

        h_layers = [('hiddenLayer0',layer0)]

        for i in range(1,len(n_hiddens)):
            h_layers.append(('hiddenLayer%d'%i,
                        NetworkLayer(rng=rng, input=h_layers[i-1][1].output,
                                     n_in=n_hiddens[i-1], n_out=n_hiddens[i],
                                     activation=hid_activations[i])))

        self.__dict__.update(dict(h_layers))

        # The logistic regression layer gets as input the hidden units
        # of the hidden layer
        self.outputLayer =  NetworkLayer(rng=rng,
                                         input=h_layers[-1][1].output,
                                         n_in=n_hiddens[-1],
                                         n_out=n_out,
                                         activation=out_activation)

        # square of L2 norm ; one regularization option is to enforce
        # square of L2 norm to be small
        self.L2_sqr = (self.outputLayer.W ** 2).sum()
        for i in range(len(n_hiddens)):
            self.L2_sqr += (self.__dict__['hiddenLayer%d'%i].W ** 2).sum()

        # the prediction is simply the output of the output layer
        self.y_pred = self.outputLayer.output

        # the parameters of the model are the parameters of the all layers it
        # is made out of
        params = self.outputLayer.params
        for layer in h_layers:
            params.extend(layer[1].params)
        self.params = params

    def set_symbolic_input(self, input):
        """We use this function to bind a symbolic variable with the input
        of the network layer. Added to specify that in training time."""
        self.input = input

#    def train(self, x, y, training_loss, learning_rate,
    def train(self, y, training_loss, learning_rate,
              n_epochs, train_x, train_y, valid_x, valid_y, batch_size):
        """Train the MLP using SGD"""

        index = T.iscalar()  # index to a [mini]batch
        lr = T.scalar() # learning rate symbolic

        #index.tag.test_value = 1
        gparams = []
        for param in self.params:
            gparam = T.grad(training_loss, param)
            gparams.append(gparam)

        updates = []

        for param, gparam in zip(self.params, gparams):
            updates.append((param, param - gparam * \
                            T.cast(lr,dtype=theano.config.floatX)))

        train_model = theano.function(inputs=[index, lr],
            outputs=[training_loss,MSE(y_pred=self.y_pred, y=y)],
            updates=updates,
            givens={
                self.input: train_x[index * batch_size:(index+1) * batch_size],
                y: train_y[index * batch_size:(index + 1) * batch_size]})
        validate_model = theano.function(inputs=[index],
            outputs=MSE(y_pred=self.y_pred, y=y),
            givens={
                self.input: valid_x[index * batch_size:(index+1) * batch_size],
                y: valid_y[index * batch_size:(index + 1) * batch_size]})

        compute_var = theano.function(inputs=[index],
            outputs=MSE(y_pred=self.y_pred, y=y),
            givens={
                self.input: train_x[index * batch_size:(index+1) * batch_size],
                y: train_y[index * batch_size:(index + 1) * batch_size]})

        # compute number of minibatches for training and validation
        n_train_batches = train_x.get_value(borrow=True).shape[0] / batch_size
        n_valid_batches = valid_x.get_value(borrow=True).shape[0] / batch_size

        print 'training...'

        start_time = time.clock()
        epoch = 0

        total_training_costs = []
        total_validation_costs = []
        total_validation_NLL = []

        lr_time = 0
        lr_step = learning_rate / ((train_x.get_value().shape[0]*1.0/batch_size)*(n_epochs-30))
        lr_val = learning_rate

        while (epoch < n_epochs):
            epoch = epoch + 1
            epoch_training_costs = []
            #import pdb; pdb.set_trace()
            for minibatch_index in xrange(n_train_batches):

                # linear annealing after 30 epochs...
                if epoch > 30:
                    # lr_val = learning_rate / (1.0+lr_time)
                    # lr_time = lr_time + 1
                    lr_val = lr_val - lr_step
                else:
                    lr_val = learning_rate

                loss_value,minibatch_avg_cost = \
                                train_model(minibatch_index, lr_val)
                epoch_training_costs.append(minibatch_avg_cost)

            this_training_cost = np.mean(epoch_training_costs)
            this_validation_cost = np.mean([validate_model(i) for i
                                            in xrange(n_valid_batches)])
            error = 0
            for minibatch_index in xrange(n_train_batches):
                error += compute_var(minibatch_index)
            error /= n_train_batches
            model_var = error
            NLL = .5*np.log(2*np.pi*model_var) + .5

            #pdb.set_trace()
            total_training_costs.append(this_training_cost)
            total_validation_costs.append(this_validation_cost)
            total_validation_NLL.append(NLL)

            print 'epoch %i, training MSE %f, validation MSE %f, NLL %f' %\
            (epoch, this_training_cost, this_validation_cost, NLL)

        end_time = time.clock()

        print "Training took %.2f minutes..."%((end_time-start_time)/60.)

        #return losses and parameters..
        return total_training_costs, total_validation_costs,total_validation_NLL

    def save_model(self,filename='MLP.save',
                   output_folder='output_folder'):
        """
        This function pickles the paramaters in a file for later usage
        """
        storage_file = open(os.path.join(output_folder,filename), 'wb')
        cPickle.dump(self, storage_file , protocol=cPickle.HIGHEST_PROTOCOL)
        storage_file.close()

    @staticmethod
    def load_model(filename='MLP.save',
                   output_folder='output_folder'):
        """
        This function loads pickled paramaters from a file
        """
        storage_file = open(os.path.join(output_folder,filename), 'rb')
        model = cPickle.load(storage_file)
        storage_file.close()
        return model

def Rectifier(x):
    """Implementation of the rectifier activation function"""
    return T.switch(x>0, x, 0)

def MSE(y_pred, y):
    """Return the mean of squared error, i.e. the difference between
    the predicted output and target

    :type y_pred: theano.tensor.TensorType
    :param y: corresponds to a vector that gives the prediction fo each example

    :type y: theano.tensor.TensorType
    :param y: corresponds to a vector that gives the target for each example

    """
    return T.mean((y_pred - y) ** 2)

<commit_msg>adding small modif<commit_after>"""
This script is a collection of objects and methods used for MLPs: inspired from Theano MLP tutorial http://deeplearning.net/tutorial
"""
__docformat__ = 'restructedtext en'

import cPickle
import os
import sys
import time

import numpy as np

import theano
import theano.tensor as T

class NetworkLayer(object):
    def __init__(self, rng, input, n_in, n_out, W=None, b=None,
                 activation=T.tanh):
        """
        Typical layer of a MLP: units are fully-connected and have
        tanh activation function. Weight matrix W is of shape (n_in,n_out)
        and the bias vector b is of shape (n_out,).

        :type rng: numpy.random.RandomState
        :param rng: a random number generator used to initialize weights

        :type input: theano.tensor.dmatrix
        :param input: a symbolic tensor of shape (n_examples, n_in)

        :type n_in: int
        :param n_in: dimensionality of input

        :type n_out: int
        :param n_out: number of hidden units

        :type activation: theano.Op or function
        :param activation: Non linearity to be applied in the hidden
                           layer
        """
        self.input = input

        if W is None:
            W_values = np.asarray(rng.uniform(
                    low=-np.sqrt(6. / (n_in + n_out)),
                    high=np.sqrt(6. / (n_in + n_out)),
                    size=(n_in, n_out)), dtype=theano.config.floatX)
            if activation == theano.tensor.nnet.sigmoid:
                W_values *= 4

            W = theano.shared(value=W_values, name='W', borrow=True)

        if b is None:
            b_values = np.zeros((n_out,), dtype=theano.config.floatX)
            b = theano.shared(value=b_values, name='b', borrow=True)

        self.W = W
        self.b = b

        lin_output = T.dot(input, self.W) + self.b

        self.output = (lin_output if activation is None
                       else activation(lin_output))

        # parameters of the model
        self.params = [self.W, self.b]

    def set_symbolic_input(self, input):
        """We use this function to bind a symbolic variable with the input
        of the network layer. Added to specify that in training time."""
        self.input = input

class MLP(object):
    """Multi-Layer Perceptron class.
    Inspired from Theano MLP tutorial http://deeplearning.net/tutorial/mlp.html
    """

    def __init__(self, input, rng, n_in, n_hiddens, hid_activations,
                 n_out, out_activation):
        """Initialize the parameters for the multilayer perceptron

        :type rng: np.random.RandomState
        :param rng: a random number generator used to initialize weights

        :type n_in: int
        :param n_in: number of input units, the dimension of the space in
        which the datapoints lie

        :type n_hidden_list: list of int
        :param n_hidden_list: a list of number of units in each hidden layer

        :type activations_list: list of lambdas
        :param n_hidden_list: a list of activations used in each hidden layer

        :type n_out: int
        :param n_out: number of output units, the dimension of the space in
        which the labels lie

        """

        self.input = input

        # We are dealing with multiple hidden layers MLP
        layer0 = NetworkLayer(rng=rng, input=input,
                              n_in=n_in, n_out=n_hiddens[0],
                              activation=hid_activations[0])

        h_layers = [('hiddenLayer0',layer0)]

        for i in range(1,len(n_hiddens)):
            h_layers.append(('hiddenLayer%d'%i,
                        NetworkLayer(rng=rng, input=h_layers[i-1][1].output,
                                     n_in=n_hiddens[i-1], n_out=n_hiddens[i],
                                     activation=hid_activations[i])))

        self.__dict__.update(dict(h_layers))

        # The logistic regression layer gets as input the hidden units
        # of the hidden layer
        self.outputLayer =  NetworkLayer(rng=rng,
                                         input=h_layers[-1][1].output,
                                         n_in=n_hiddens[-1],
                                         n_out=n_out,
                                         activation=out_activation)

        # square of L2 norm ; one regularization option is to enforce
        # square of L2 norm to be small
        self.L2_sqr = (self.outputLayer.W ** 2).sum()
        for i in range(len(n_hiddens)):
            self.L2_sqr += (self.__dict__['hiddenLayer%d'%i].W ** 2).sum()

        # the prediction is simply the output of the output layer
        self.y_pred = self.outputLayer.output

        # the parameters of the model are the parameters of the all layers it
        # is made out of
        params = self.outputLayer.params
        for layer in h_layers:
            params.extend(layer[1].params)
        self.params = params

    def set_symbolic_input(self, input):
        """We use this function to bind a symbolic variable with the input
        of the network layer. Added to specify that in training time."""
        self.input = input

#    def train(self, x, y, training_loss, learning_rate,
    def train(self, y, training_loss, learning_rate,
              n_epochs, train_x, train_y, valid_x, valid_y, batch_size):
        """Train the MLP using SGD"""

        index = T.iscalar()  # index to a [mini]batch
        lr = T.scalar() # learning rate symbolic

        #index.tag.test_value = 1
        gparams = []
        for param in self.params:
            gparam = T.grad(training_loss, param)
            gparams.append(gparam)

        updates = []

        for param, gparam in zip(self.params, gparams):
            updates.append((param, param - gparam * \
                            T.cast(lr,dtype=theano.config.floatX)))

        train_model = theano.function(inputs=[index, lr],
            outputs=[training_loss,MSE(y_pred=self.y_pred, y=y)],
            updates=updates,
            givens={
                self.input: train_x[index * batch_size:(index+1) * batch_size],
                y: train_y[index * batch_size:(index + 1) * batch_size]})
        validate_model = theano.function(inputs=[index],
            outputs=MSE(y_pred=self.y_pred, y=y),
            givens={
                self.input: valid_x[index * batch_size:(index+1) * batch_size],
                y: valid_y[index * batch_size:(index + 1) * batch_size]})

        compute_var = theano.function(inputs=[index],
            outputs=MSE(y_pred=self.y_pred, y=y),
            givens={
                self.input: train_x[index * batch_size:(index+1) * batch_size],
                y: train_y[index * batch_size:(index + 1) * batch_size]})

        # compute number of minibatches for training and validation
        n_train_batches = train_x.get_value(borrow=True).shape[0] / batch_size
        n_valid_batches = valid_x.get_value(borrow=True).shape[0] / batch_size

        print 'training...'

        start_time = time.clock()
        epoch = 0

        total_training_costs = []
        total_validation_costs = []
        total_validation_NLL = []

        lr_time = 0
        lr_step = learning_rate / ((train_x.get_value().shape[0]*1.0/batch_size)*(n_epochs-30))
        lr_val = learning_rate

        while (epoch < n_epochs):
            epoch = epoch + 1
            epoch_training_costs = []
            #import pdb; pdb.set_trace()
            for minibatch_index in xrange(n_train_batches):

                # linear annealing after 30 epochs...
                if epoch > 30:
                    # lr_val = learning_rate / (1.0+lr_time)
                    # lr_time = lr_time + 1
                    lr_val = lr_val - lr_step
                else:
                    lr_val = learning_rate

                loss_value,minibatch_avg_cost = \
                                train_model(minibatch_index, lr_val)
                epoch_training_costs.append(minibatch_avg_cost)

            this_training_cost = np.mean(epoch_training_costs)
            this_validation_cost = np.mean([validate_model(i) for i
                                            in xrange(n_valid_batches)])
            error = 0
            for minibatch_index in xrange(n_train_batches):
                error += compute_var(minibatch_index)
            error /= n_train_batches
            model_var = error
            NLL = .5*np.log(2*np.pi*model_var) + .5

            #pdb.set_trace()
            total_training_costs.append(this_training_cost)
            total_validation_costs.append(this_validation_cost)
            total_validation_NLL.append(NLL)

            print 'epoch %i, training MSE %f, validation MSE %f, NLL %f' %\
            (epoch, this_training_cost, this_validation_cost, NLL)

        end_time = time.clock()

        print "Training took %.2f minutes..."%((end_time-start_time)/60.)

        #return losses and parameters..
        return total_training_costs, total_validation_costs,total_validation_NLL

    def save_model(self,filename='MLP.save',
                   output_folder='output_folder'):
        """
        This function pickles the paramaters in a file for later usage
        """
        storage_file = open(os.path.join(output_folder,filename), 'wb')
        cPickle.dump(self, storage_file , protocol=cPickle.HIGHEST_PROTOCOL)
        storage_file.close()

    @staticmethod
    def load_model(filename='MLP.save',
                   output_folder='output_folder'):
        """
        This function loads pickled paramaters from a file
        """
        storage_file = open(os.path.join(output_folder,filename), 'rb')
        model = cPickle.load(storage_file)
        storage_file.close()
        return model

def Rectifier(x):
    """Implementation of the rectifier activation function"""
    return T.switch(x>0, x, 0)

def MSE(y_pred, y):
    """Return the mean of squared error, i.e. the difference between
    the predicted output and target

    :type y_pred: theano.tensor.TensorType
    :param y: corresponds to a vector that gives the prediction fo each example

    :type y: theano.tensor.TensorType
    :param y: corresponds to a vector that gives the target for each example

    """
    return T.mean((y_pred - y) ** 2)

<|endoftext|>