<commit_before># Copyright (c) 2005-2007 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Ron Dreslinski

# Splash2 Run Script
#

import m5
from m5.objects import *
import os, optparse, sys
m5.AddToPath('../common')

# --------------------
# Define Command Line Options
# ====================

parser = optparse.OptionParser()

parser.add_option("-d", "--detailed", action="store_true")
parser.add_option("-t", "--timing", action="store_true")
parser.add_option("-m", "--maxtick", type="int")
parser.add_option("-n", "--numcpus",
                  help="Number of cpus in total", type="int")
parser.add_option("-f", "--frequency",
                  default = "1GHz",
                  help="Frequency of each CPU")
parser.add_option("--l1size",
                  default = "32kB")
parser.add_option("--l1latency",
                  default = "1ns")
parser.add_option("--l2size",
                  default = "256kB")
parser.add_option("--l2latency",
                  default = "10ns")
parser.add_option("--rootdir",
                  help="Root directory of Splash2",
                  default="/dist/splash2/codes")
parser.add_option("-b", "--benchmark",
                  help="Splash 2 benchmark to run")

(options, args) = parser.parse_args()

if args:
    print "Error: script doesn't take any positional arguments"
    sys.exit(1)

if not options.numcpus:
    print "Specify the number of cpus with -n"
    sys.exit(1)

# --------------------
# Define Splash2 Benchmarks
# ====================
class Cholesky(LiveProcess):
    cwd = options.rootdir + '/kernels/cholesky'
    executable = options.rootdir + '/kernels/cholesky/CHOLESKY'
    cmd = ['CHOLESKY', '-p', str(options.numcpus),
            options.rootdir + '/kernels/cholesky/inputs/tk23.O']

class FFT(LiveProcess):
    cwd = options.rootdir + '/kernels/fft'
    executable = options.rootdir + '/kernels/fft/FFT'
    cmd = ['FFT', '-p', str(options.numcpus), '-m18']

class LU_contig(LiveProcess):
    executable = options.rootdir + '/kernels/lu/contiguous_blocks/LU'
    cmd = ['LU', '-p', str(options.numcpus)]
    cwd = options.rootdir + '/kernels/lu/contiguous_blocks'

class LU_noncontig(LiveProcess):
    executable = options.rootdir + '/kernels/lu/non_contiguous_blocks/LU'
    cmd = ['LU', '-p', str(options.numcpus)]
    cwd = options.rootdir + '/kernels/lu/non_contiguous_blocks'

class Radix(LiveProcess):
    executable = options.rootdir + '/kernels/radix/RADIX'
    cmd = ['RADIX', '-n524288', '-p', str(options.numcpus)]
    cwd = options.rootdir + '/kernels/radix'

class Barnes(LiveProcess):
    executable = options.rootdir + '/apps/barnes/BARNES'
    cmd = ['BARNES']
    input = options.rootdir + '/apps/barnes/input.p' + str(options.numcpus)
    cwd = options.rootdir + '/apps/barnes'

class FMM(LiveProcess):
    executable = options.rootdir + '/apps/fmm/FMM'
    cmd = ['FMM']
    if str(options.numcpus) == '1':
        input = options.rootdir + '/apps/fmm/inputs/input.2048'
    else:
        input = options.rootdir + '/apps/fmm/inputs/input.2048.p' + str(options.numcpus)
    cwd = options.rootdir + '/apps/fmm'

class Ocean_contig(LiveProcess):
    executable = options.rootdir + '/apps/ocean/contiguous_partitions/OCEAN'
    cmd = ['OCEAN', '-p', str(options.numcpus)]
    cwd = options.rootdir + '/apps/ocean/contiguous_partitions'

class Ocean_noncontig(LiveProcess):
    executable = options.rootdir + '/apps/ocean/non_contiguous_partitions/OCEAN'
    cmd = ['OCEAN', '-p', str(options.numcpus)]
    cwd = options.rootdir + '/apps/ocean/non_contiguous_partitions'

class Raytrace(LiveProcess):
    executable = options.rootdir + '/apps/raytrace/RAYTRACE'
    cmd = ['RAYTRACE', '-p', str(options.numcpus),
           options.rootdir + '/apps/raytrace/inputs/teapot.env']
    cwd = options.rootdir + '/apps/raytrace'

class Water_nsquared(LiveProcess):
    executable = options.rootdir + '/apps/water-nsquared/WATER-NSQUARED'
    cmd = ['WATER-NSQUARED']
    if options.numcpus==1:
        input = options.rootdir + '/apps/water-nsquared/input'
    else:
        input = options.rootdir + '/apps/water-nsquared/input.p' + str(options.numcpus)
    cwd = options.rootdir + '/apps/water-nsquared'

class Water_spatial(LiveProcess):
    executable = options.rootdir + '/apps/water-spatial/WATER-SPATIAL'
    cmd = ['WATER-SPATIAL']
    if options.numcpus==1:
        input = options.rootdir + '/apps/water-spatial/input'
    else:
        input = options.rootdir + '/apps/water-spatial/input.p' + str(options.numcpus)
    cwd = options.rootdir + '/apps/water-spatial'

# --------------------
# Base L1 Cache Definition
# ====================

class L1(BaseCache):
    latency = options.l1latency
    block_size = 64
    mshrs = 12
    tgts_per_mshr = 8

# ----------------------
# Base L2 Cache Definition
# ----------------------

class L2(BaseCache):
    block_size = 64
    latency = options.l2latency
    mshrs = 92
    tgts_per_mshr = 16
    write_buffers = 8

# ----------------------
# Define the cpus
# ----------------------

busFrequency = Frequency(options.frequency)

if options.timing:
    cpus = [TimingSimpleCPU(cpu_id = i,
                            clock=options.frequency)
            for i in xrange(options.numcpus)]
elif options.detailed:
    cpus = [DerivO3CPU(cpu_id = i,
                       clock=options.frequency)
            for i in xrange(options.numcpus)]
else:
    cpus = [AtomicSimpleCPU(cpu_id = i,
                            clock=options.frequency)
            for i in xrange(options.numcpus)]

# ----------------------
# Create a system, and add system wide objects
# ----------------------
system = System(cpu = cpus, physmem = PhysicalMemory(),
                membus = Bus(clock = busFrequency))

system.toL2bus = Bus(clock = busFrequency)
system.l2 = L2(size = options.l2size, assoc = 8)

# ----------------------
# Connect the L2 cache and memory together
# ----------------------

system.physmem.port = system.membus.port
system.l2.cpu_side = system.toL2bus.port
system.l2.mem_side = system.membus.port

# ----------------------
# Connect the L2 cache and clusters together
# ----------------------
for cpu in cpus:
    cpu.addPrivateSplitL1Caches(L1(size = options.l1size, assoc = 1),
                                L1(size = options.l1size, assoc = 4))
    cpu.mem = cpu.dcache
    # connect cpu level-1 caches to shared level-2 cache
    cpu.connectMemPorts(system.toL2bus)

# ----------------------
# Define the root
# ----------------------

root = Root(system = system)

# --------------------
# Pick the correct Splash2 Benchmarks
# ====================
if options.benchmark == 'Cholesky':
    root.workload = Cholesky()
elif options.benchmark == 'FFT':
    root.workload = FFT()
elif options.benchmark == 'LUContig':
    root.workload = LU_contig()
elif options.benchmark == 'LUNoncontig':
    root.workload = LU_noncontig()
elif options.benchmark == 'Radix':
    root.workload = Radix()
elif options.benchmark == 'Barnes':
    root.workload = Barnes()
elif options.benchmark == 'FMM':
    root.workload = FMM()
elif options.benchmark == 'OceanContig':
    root.workload = Ocean_contig()
elif options.benchmark == 'OceanNoncontig':
    root.workload = Ocean_noncontig()
elif options.benchmark == 'Raytrace':
    root.workload = Raytrace()
elif options.benchmark == 'WaterNSquared':
    root.workload = Water_nsquared()
elif options.benchmark == 'WaterSpatial':
    root.workload = Water_spatial()
else:
    panic("The --benchmark environment variable was set to something" \
          +" improper.\nUse Cholesky, FFT, LUContig, LUNoncontig, Radix" \
          +", Barnes, FMM, OceanContig,\nOceanNoncontig, Raytrace," \
          +" WaterNSquared, or WaterSpatial\n")

# --------------------
# Assign the workload to the cpus
# ====================

for cpu in cpus:
    cpu.workload = root.workload

# ----------------------
# Run the simulation
# ----------------------

if options.timing or options.detailed:
    root.system.mem_mode = 'timing'

# instantiate configuration
m5.instantiate(root)

# simulate until program terminates
if options.maxtick:
    exit_event = m5.simulate(options.maxtick)
else:
    exit_event = m5.simulate(m5.MaxTick)

print 'Exiting @ tick', m5.curTick(), 'because', exit_event.getCause()

<commit_msg>Configs: Fix for benchmarks that don't use getopt.<commit_after># Copyright (c) 2005-2007 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Ron Dreslinski

# Splash2 Run Script
#

import m5
from m5.objects import *
import os, optparse, sys
m5.AddToPath('../common')

# --------------------
# Define Command Line Options
# ====================

parser = optparse.OptionParser()

parser.add_option("-d", "--detailed", action="store_true")
parser.add_option("-t", "--timing", action="store_true")
parser.add_option("-m", "--maxtick", type="int")
parser.add_option("-n", "--numcpus",
                  help="Number of cpus in total", type="int")
parser.add_option("-f", "--frequency",
                  default = "1GHz",
                  help="Frequency of each CPU")
parser.add_option("--l1size",
                  default = "32kB")
parser.add_option("--l1latency",
                  default = "1ns")
parser.add_option("--l2size",
                  default = "256kB")
parser.add_option("--l2latency",
                  default = "10ns")
parser.add_option("--rootdir",
                  help="Root directory of Splash2",
                  default="/dist/splash2/codes")
parser.add_option("-b", "--benchmark",
                  help="Splash 2 benchmark to run")

(options, args) = parser.parse_args()

if args:
    print "Error: script doesn't take any positional arguments"
    sys.exit(1)

if not options.numcpus:
    print "Specify the number of cpus with -n"
    sys.exit(1)

# --------------------
# Define Splash2 Benchmarks
# ====================
class Cholesky(LiveProcess):
    cwd = options.rootdir + '/kernels/cholesky'
    executable = options.rootdir + '/kernels/cholesky/CHOLESKY'
    cmd = ['CHOLESKY', '-p' +  str(options.numcpus),
            options.rootdir + '/kernels/cholesky/inputs/tk23.O']

class FFT(LiveProcess):
    cwd = options.rootdir + '/kernels/fft'
    executable = options.rootdir + '/kernels/fft/FFT'
    cmd = ['FFT', '-p', str(options.numcpus), '-m18']

class LU_contig(LiveProcess):
    executable = options.rootdir + '/kernels/lu/contiguous_blocks/LU'
    cmd = ['LU', '-p', str(options.numcpus)]
    cwd = options.rootdir + '/kernels/lu/contiguous_blocks'

class LU_noncontig(LiveProcess):
    executable = options.rootdir + '/kernels/lu/non_contiguous_blocks/LU'
    cmd = ['LU', '-p', str(options.numcpus)]
    cwd = options.rootdir + '/kernels/lu/non_contiguous_blocks'

class Radix(LiveProcess):
    executable = options.rootdir + '/kernels/radix/RADIX'
    cmd = ['RADIX', '-n524288', '-p', str(options.numcpus)]
    cwd = options.rootdir + '/kernels/radix'

class Barnes(LiveProcess):
    executable = options.rootdir + '/apps/barnes/BARNES'
    cmd = ['BARNES']
    input = options.rootdir + '/apps/barnes/input.p' + str(options.numcpus)
    cwd = options.rootdir + '/apps/barnes'

class FMM(LiveProcess):
    executable = options.rootdir + '/apps/fmm/FMM'
    cmd = ['FMM']
    if str(options.numcpus) == '1':
        input = options.rootdir + '/apps/fmm/inputs/input.2048'
    else:
        input = options.rootdir + '/apps/fmm/inputs/input.2048.p' + str(options.numcpus)
    cwd = options.rootdir + '/apps/fmm'

class Ocean_contig(LiveProcess):
    executable = options.rootdir + '/apps/ocean/contiguous_partitions/OCEAN'
    cmd = ['OCEAN', '-p', str(options.numcpus)]
    cwd = options.rootdir + '/apps/ocean/contiguous_partitions'

class Ocean_noncontig(LiveProcess):
    executable = options.rootdir + '/apps/ocean/non_contiguous_partitions/OCEAN'
    cmd = ['OCEAN', '-p', str(options.numcpus)]
    cwd = options.rootdir + '/apps/ocean/non_contiguous_partitions'

class Raytrace(LiveProcess):
    executable = options.rootdir + '/apps/raytrace/RAYTRACE'
    cmd = ['RAYTRACE', '-p' + str(options.numcpus),
           options.rootdir + '/apps/raytrace/inputs/teapot.env']
    cwd = options.rootdir + '/apps/raytrace'

class Water_nsquared(LiveProcess):
    executable = options.rootdir + '/apps/water-nsquared/WATER-NSQUARED'
    cmd = ['WATER-NSQUARED']
    if options.numcpus==1:
        input = options.rootdir + '/apps/water-nsquared/input'
    else:
        input = options.rootdir + '/apps/water-nsquared/input.p' + str(options.numcpus)
    cwd = options.rootdir + '/apps/water-nsquared'

class Water_spatial(LiveProcess):
    executable = options.rootdir + '/apps/water-spatial/WATER-SPATIAL'
    cmd = ['WATER-SPATIAL']
    if options.numcpus==1:
        input = options.rootdir + '/apps/water-spatial/input'
    else:
        input = options.rootdir + '/apps/water-spatial/input.p' + str(options.numcpus)
    cwd = options.rootdir + '/apps/water-spatial'

# --------------------
# Base L1 Cache Definition
# ====================

class L1(BaseCache):
    latency = options.l1latency
    block_size = 64
    mshrs = 12
    tgts_per_mshr = 8

# ----------------------
# Base L2 Cache Definition
# ----------------------

class L2(BaseCache):
    block_size = 64
    latency = options.l2latency
    mshrs = 92
    tgts_per_mshr = 16
    write_buffers = 8

# ----------------------
# Define the cpus
# ----------------------

busFrequency = Frequency(options.frequency)

if options.timing:
    cpus = [TimingSimpleCPU(cpu_id = i,
                            clock=options.frequency)
            for i in xrange(options.numcpus)]
elif options.detailed:
    cpus = [DerivO3CPU(cpu_id = i,
                       clock=options.frequency)
            for i in xrange(options.numcpus)]
else:
    cpus = [AtomicSimpleCPU(cpu_id = i,
                            clock=options.frequency)
            for i in xrange(options.numcpus)]

# ----------------------
# Create a system, and add system wide objects
# ----------------------
system = System(cpu = cpus, physmem = PhysicalMemory(),
                membus = Bus(clock = busFrequency))

system.toL2bus = Bus(clock = busFrequency)
system.l2 = L2(size = options.l2size, assoc = 8)

# ----------------------
# Connect the L2 cache and memory together
# ----------------------

system.physmem.port = system.membus.port
system.l2.cpu_side = system.toL2bus.port
system.l2.mem_side = system.membus.port

# ----------------------
# Connect the L2 cache and clusters together
# ----------------------
for cpu in cpus:
    cpu.addPrivateSplitL1Caches(L1(size = options.l1size, assoc = 1),
                                L1(size = options.l1size, assoc = 4))
    cpu.mem = cpu.dcache
    # connect cpu level-1 caches to shared level-2 cache
    cpu.connectMemPorts(system.toL2bus)

# ----------------------
# Define the root
# ----------------------

root = Root(system = system)

# --------------------
# Pick the correct Splash2 Benchmarks
# ====================
if options.benchmark == 'Cholesky':
    root.workload = Cholesky()
elif options.benchmark == 'FFT':
    root.workload = FFT()
elif options.benchmark == 'LUContig':
    root.workload = LU_contig()
elif options.benchmark == 'LUNoncontig':
    root.workload = LU_noncontig()
elif options.benchmark == 'Radix':
    root.workload = Radix()
elif options.benchmark == 'Barnes':
    root.workload = Barnes()
elif options.benchmark == 'FMM':
    root.workload = FMM()
elif options.benchmark == 'OceanContig':
    root.workload = Ocean_contig()
elif options.benchmark == 'OceanNoncontig':
    root.workload = Ocean_noncontig()
elif options.benchmark == 'Raytrace':
    root.workload = Raytrace()
elif options.benchmark == 'WaterNSquared':
    root.workload = Water_nsquared()
elif options.benchmark == 'WaterSpatial':
    root.workload = Water_spatial()
else:
    print >> sys.stderr, """The --benchmark environment variable was set to something improper.
Use Cholesky, FFT, LUContig, LUNoncontig, Radix, Barnes, FMM, OceanContig,
OceanNoncontig, Raytrace, WaterNSquared, or WaterSpatial"""
    sys.exit(1)

# --------------------
# Assign the workload to the cpus
# ====================

for cpu in cpus:
    cpu.workload = root.workload

# ----------------------
# Run the simulation
# ----------------------

if options.timing or options.detailed:
    root.system.mem_mode = 'timing'

# instantiate configuration
m5.instantiate(root)

# simulate until program terminates
if options.maxtick:
    exit_event = m5.simulate(options.maxtick)
else:
    exit_event = m5.simulate(m5.MaxTick)

print 'Exiting @ tick', m5.curTick(), 'because', exit_event.getCause()

<|endoftext|>


<commit_before>"""
file        :   fabfile.py
date        :   2016-04-01
module      :   deploy
description :   fabric fabfile for deploying to staging/production
"""

import os
import random

from fabric.contrib.files import exists, sudo, append
from fabric.api import env, local, run, get, put

env.use_ssh_config = True

REPO_URL = 'https://github.com/nicorellius/fiblist.git'
SETTINGS_FOLDER = 'fiblist/conf/settings'
PROJECT = 'fiblist'
VIRTENV_FOLDER = '/home/{0}/dev/virtenvs/{1}'.format(env.user, PROJECT)

def deploy():

    site_folder = '/home/{0}/sites/{1}'.format(env.user, PROJECT)
    source_folder = '{0}/source'.format(site_folder)
    secret_key_file = '/etc/prv/{0}/secret_key.txt'.format(PROJECT)
    http_server = 'nginx'
    uwsgi_server = 'uwsgi'

    _create_directory_structure_if_necessary(site_folder)
    _get_latest_source(site_folder, source_folder)
    _generate_secret_key(source_folder, secret_key_file)
    _update_virtenv(site_folder)
    _update_static_files(source_folder)
    _update_database(source_folder)
    _restart_servers(http_server, uwsgi_server)

def _create_directory_structure_if_necessary(site_folder):

    for subfolder in ('database', 'static', 'source'):
        run('mkdir -p {0}/{1}'.format(site_folder, subfolder))

def _get_latest_source(site_folder, source_folder):

    if exists('{0}/{1}'.format(site_folder, '.git')):
        run('cd {0} && git fetch'.format(source_folder))

    else:
        run('git clone {0} {1}'.format(REPO_URL, source_folder))

    current_commit = local('git log -n 1 --format=%H', capture=True)
    run('cd {0} && git reset --hard {1}'.format(source_folder, current_commit))

def _generate_secret_key(source_folder, secret_key_file):

    settings_file = '{0}/{1}/staging.py'.format(source_folder, SETTINGS_FOLDER)

    if exists(secret_key_file):
        get(local_path='/tmp/secret_key.txt', remote_path=secret_key_file)

        tmp_key_path = '/tmp/secret_key.txt'
        with open(tmp_key_path, 'r') as key_file:
            data = key_file.read().replace('\n', '')

        # tmp_key_file = data

        if data is not '':
            append(settings_file, '\nSECRET_KEY = "{0}"'.format(data))

        sudo('rm -rf /tmp/secret_key.txt')

    else:
        print("[localhost] print: Remote key file does not exist. Making one now.")
        chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
        generated_key = ''.join([random.SystemRandom().choice(chars) for _ in range(50)])

        append(settings_file, '\nSECRET_KEY = "{0}"'.format(generated_key))

        with open('secret_key.txt', 'w') as text_file:
            print('{0}'.format(generated_key), file=text_file)
            tmp_key_file = os.path.join('/tmp/', tmp_key_file)

        put(local_path=tmp_key_file, remote_path='/etc/prv/fiblist/secret_key.txt')

        sudo('rm -rf /tmp/secret_key.txt')

def _update_virtenv(site_folder):

    virtenv_folder = VIRTENV_FOLDER

    if not exists('{0}/bin/pip'.format(virtenv_folder)):
        run('mkvirtualenv --python=/usr/bin/python3 {0}'.format(virtenv_folder))

    run('{0}/bin/pip install -r {1}/requirements.txt'.format(
        virtenv_folder,
        site_folder
    ))

def _update_static_files(source_folder):
    run('cd {0} && {1}/bin/python3 manage.py collectstatic --noinput --settings={2}.conf.settings.staging'.format(
        source_folder,
        VIRTENV_FOLDER,
        PROJECT
    ))

def _update_database(source_folder):
    run('cd {0} && {1}/bin/python3 manage.py migrate --noinput --settings={2}.conf.settings.staging'.format(
        source_folder,
        VIRTENV_FOLDER,
        PROJECT
    ))

def _restart_servers(http_server, uwsgi_server):
    sudo('service {0} restart'.format(http_server))
    sudo('restart {0}'.format(uwsgi_server))
<commit_msg>fixed bug in print to file, used incorrect variable.<commit_after>"""
file        :   fabfile.py
date        :   2016-04-01
module      :   deploy
description :   fabric fabfile for deploying to staging/production
"""

import os
import random

from fabric.contrib.files import exists, sudo, append
from fabric.api import env, local, run, get, put

env.use_ssh_config = True

REPO_URL = 'https://github.com/nicorellius/fiblist.git'
SETTINGS_FOLDER = 'fiblist/conf/settings'
PROJECT = 'fiblist'
VIRTENV_FOLDER = '/home/{0}/dev/virtenvs/{1}'.format(env.user, PROJECT)

def deploy():

    site_folder = '/home/{0}/sites/{1}'.format(env.user, PROJECT)
    source_folder = '{0}/source'.format(site_folder)
    secret_key_file = '/etc/prv/{0}/secret_key.txt'.format(PROJECT)
    http_server = 'nginx'
    uwsgi_server = 'uwsgi'

    _create_directory_structure_if_necessary(site_folder)
    _get_latest_source(site_folder, source_folder)
    _generate_secret_key(source_folder, secret_key_file)
    _update_virtenv(site_folder)
    _update_static_files(source_folder)
    _update_database(source_folder)
    _restart_servers(http_server, uwsgi_server)

def _create_directory_structure_if_necessary(site_folder):

    for subfolder in ('database', 'static', 'source'):
        run('mkdir -p {0}/{1}'.format(site_folder, subfolder))

def _get_latest_source(site_folder, source_folder):

    if exists('{0}/{1}'.format(site_folder, '.git')):
        run('cd {0} && git fetch'.format(source_folder))

    else:
        run('git clone {0} {1}'.format(REPO_URL, source_folder))

    current_commit = local('git log -n 1 --format=%H', capture=True)
    run('cd {0} && git reset --hard {1}'.format(source_folder, current_commit))

def _generate_secret_key(source_folder, secret_key_file):

    settings_file = '{0}/{1}/staging.py'.format(source_folder, SETTINGS_FOLDER)

    if exists(secret_key_file):
        get(local_path='/tmp/secret_key.txt', remote_path=secret_key_file)

        tmp_key_path = '/tmp/secret_key.txt'
        with open(tmp_key_path, 'r') as key_file:
            data = key_file.read().replace('\n', '')

        # tmp_key_file = data

        if data is not '':
            append(settings_file, '\nSECRET_KEY = "{0}"'.format(data))

        sudo('rm -rf /tmp/secret_key.txt')

    else:
        print("[localhost] print: Remote key file does not exist. Making one now.")
        chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)'
        generated_key = ''.join([random.SystemRandom().choice(chars) for _ in range(50)])

        append(settings_file, '\nSECRET_KEY = "{0}"'.format(generated_key))

        with open('secret_key.txt', 'w') as text_file:
            print('{0}'.format(generated_key), file=text_file)
            tmp_key_file = os.path.join('/tmp/', text_file)

        put(local_path=tmp_key_file, remote_path='/etc/prv/fiblist/secret_key.txt')

        sudo('rm -rf /tmp/secret_key.txt')

def _update_virtenv(site_folder):

    virtenv_folder = VIRTENV_FOLDER

    if not exists('{0}/bin/pip'.format(virtenv_folder)):
        run('mkvirtualenv --python=/usr/bin/python3 {0}'.format(virtenv_folder))

    run('{0}/bin/pip install -r {1}/requirements.txt'.format(
        virtenv_folder,
        site_folder
    ))

def _update_static_files(source_folder):
    run('cd {0} && {1}/bin/python3 manage.py collectstatic --noinput --settings={2}.conf.settings.staging'.format(
        source_folder,
        VIRTENV_FOLDER,
        PROJECT
    ))

def _update_database(source_folder):
    run('cd {0} && {1}/bin/python3 manage.py migrate --noinput --settings={2}.conf.settings.staging'.format(
        source_folder,
        VIRTENV_FOLDER,
        PROJECT
    ))

def _restart_servers(http_server, uwsgi_server):
    sudo('service {0} restart'.format(http_server))
    sudo('restart {0}'.format(uwsgi_server))
<|endoftext|>


<commit_before>"""
Expected error reduction framework for active learning.
"""

from typing import Tuple

import numpy as np

from scipy.stats import entropy

from sklearn.base import clone
from sklearn.exceptions import NotFittedError

from modAL.models import ActiveLearner
from modAL.utils.data import modALinput, data_vstack
from modAL.utils.selection import multi_argmax

def expected_error_reduction(learner: ActiveLearner, X: modALinput,
                             p_subsample: np.float = 1.0, n_instances: int = 1) -> Tuple[np.ndarray, modALinput]:
    """
    Expected error reduction query strategy.

    References:
        Roy and McCallum, 2001 (http://groups.csail.mit.edu/rrg/papers/icml01.pdf)

    Args:
        learner: The ActiveLearner object for which the expected error is to be estimated.
        X: The samples.
        p_subsample: Probability of keeping a sample from the pool when calculating expected error.
            Significantly improves runtime for large sample pools.
        n_instances: The number of instances to be sampled.

    Returns:
        The indices of the instances from X chosen to be labelled; the instances from X chosen to be labelled.
    """

    assert 0.0 <= p_subsample <= 1.0, 'p_subsample subsampling keep ratio must be between 0.0 and 1.0'

    expected_error = np.zeros(shape=(len(X), ))
    possible_labels = np.unique(learner.y_training)

    try:
        X_proba = learner.predict_proba(X)
    except NotFittedError:
        # TODO: implement a proper cold-start
        return 0, X[0]

    for x_idx, x in enumerate(X):
        # subsample the data if needed
        if np.random.rand() <= p_subsample:
            # estimate the expected error
            for y_idx, y in enumerate(possible_labels):
                X_new = data_vstack((learner.X_training, x.reshape(1, -1)))
                y_new = data_vstack((learner.y_training, np.array(y).reshape(1, )))

                refitted_estimator = clone(learner.estimator).fit(X_new, y_new)
                uncertainty = 1 - np.max(refitted_estimator.predict_proba(X), axis=1)

                expected_error[x_idx] += np.sum(uncertainty)*X_proba[x_idx, y_idx]

        else:
            expected_error[x_idx] -np.nan

    query_idx = multi_argmax(expected_error, n_instances)

    return query_idx, X[query_idx]

def expected_log_loss_reduction(learner: ActiveLearner, X: modALinput,
                                p_subsample: np.float = 1.0, n_instances: int = 1) -> Tuple[np.ndarray, modALinput]:
    """
    Expected log loss reduction query strategy.

    References:
        Roy and McCallum, 2001 (http://groups.csail.mit.edu/rrg/papers/icml01.pdf)

    Args:
        learner: The ActiveLearner object for which the expected log loss is to be estimated.
        X: The samples.
        p_subsample: Probability of keeping a sample from the pool when calculating expected log loss.
            Significantly improves runtime for large sample pools.
        n_instances: The number of instances to be sampled.

    Returns:
        The indices of the instances from X chosen to be labelled; the instances from X chosen to be labelled.
    """

    assert 0.0 <= p_subsample <= 1.0, 'p_subsample subsampling keep ratio must be between 0.0 and 1.0'

    expected_log_loss = np.zeros(shape=(len(X), ))
    possible_labels = np.unique(learner.y_training)

    try:
        X_proba = learner.predict_proba(X)
    except NotFittedError:
        # TODO: implement a proper cold-start
        return 0, X[0]

    for x_idx, x in enumerate(X):
        # subsample the data if needed
        if np.random.rand() <= p_subsample:
            # estimate the expected error
            for y_idx, y in enumerate(possible_labels):
                X_new = data_vstack((learner.X_training, x.reshape(1, -1)))
                y_new = data_vstack((learner.y_training, np.array(y).reshape(1, )))

                refitted_estimator = clone(learner.estimator).fit(X_new, y_new)
                refitted_proba = refitted_estimator.predict_proba(X)
                entr = np.transpose(entropy(np.transpose(refitted_proba)))

                expected_log_loss[x_idx] += np.sum(entr)*X_proba[x_idx, y_idx]

        else:
            expected_log_loss[x_idx] -np.nan

    query_idx = multi_argmax(expected_log_loss, n_instances)

    return query_idx, X[query_idx]
<commit_msg>fix: bugfix for subsampled eer<commit_after>"""
Expected error reduction framework for active learning.
"""

from typing import Tuple

import numpy as np

from scipy.stats import entropy

from sklearn.base import clone
from sklearn.exceptions import NotFittedError

from modAL.models import ActiveLearner
from modAL.utils.data import modALinput, data_vstack
from modAL.utils.selection import multi_argmax

def expected_error_reduction(learner: ActiveLearner, X: modALinput,
                             p_subsample: np.float = 1.0, n_instances: int = 1) -> Tuple[np.ndarray, modALinput]:
    """
    Expected error reduction query strategy.

    References:
        Roy and McCallum, 2001 (http://groups.csail.mit.edu/rrg/papers/icml01.pdf)

    Args:
        learner: The ActiveLearner object for which the expected error is to be estimated.
        X: The samples.
        p_subsample: Probability of keeping a sample from the pool when calculating expected error.
            Significantly improves runtime for large sample pools.
        n_instances: The number of instances to be sampled.

    Returns:
        The indices of the instances from X chosen to be labelled; the instances from X chosen to be labelled.
    """

    assert 0.0 <= p_subsample <= 1.0, 'p_subsample subsampling keep ratio must be between 0.0 and 1.0'

    expected_error = np.zeros(shape=(len(X), ))
    possible_labels = np.unique(learner.y_training)

    try:
        X_proba = learner.predict_proba(X)
    except NotFittedError:
        # TODO: implement a proper cold-start
        return 0, X[0]

    for x_idx, x in enumerate(X):
        # subsample the data if needed
        if np.random.rand() <= p_subsample:
            # estimate the expected error
            for y_idx, y in enumerate(possible_labels):
                X_new = data_vstack((learner.X_training, x.reshape(1, -1)))
                y_new = data_vstack((learner.y_training, np.array(y).reshape(1, )))

                refitted_estimator = clone(learner.estimator).fit(X_new, y_new)
                uncertainty = 1 - np.max(refitted_estimator.predict_proba(X), axis=1)

                expected_error[x_idx] += np.sum(uncertainty)*X_proba[x_idx, y_idx]

        else:
            expected_error[x_idx] = np.inf

    query_idx = multi_argmax(expected_error, n_instances)

    return query_idx, X[query_idx]

def expected_log_loss_reduction(learner: ActiveLearner, X: modALinput,
                                p_subsample: np.float = 1.0, n_instances: int = 1) -> Tuple[np.ndarray, modALinput]:
    """
    Expected log loss reduction query strategy.

    References:
        Roy and McCallum, 2001 (http://groups.csail.mit.edu/rrg/papers/icml01.pdf)

    Args:
        learner: The ActiveLearner object for which the expected log loss is to be estimated.
        X: The samples.
        p_subsample: Probability of keeping a sample from the pool when calculating expected log loss.
            Significantly improves runtime for large sample pools.
        n_instances: The number of instances to be sampled.

    Returns:
        The indices of the instances from X chosen to be labelled; the instances from X chosen to be labelled.
    """

    assert 0.0 <= p_subsample <= 1.0, 'p_subsample subsampling keep ratio must be between 0.0 and 1.0'

    expected_log_loss = np.zeros(shape=(len(X), ))
    possible_labels = np.unique(learner.y_training)

    try:
        X_proba = learner.predict_proba(X)
    except NotFittedError:
        # TODO: implement a proper cold-start
        return 0, X[0]

    for x_idx, x in enumerate(X):
        # subsample the data if needed
        if np.random.rand() <= p_subsample:
            # estimate the expected error
            for y_idx, y in enumerate(possible_labels):
                X_new = data_vstack((learner.X_training, x.reshape(1, -1)))
                y_new = data_vstack((learner.y_training, np.array(y).reshape(1, )))

                refitted_estimator = clone(learner.estimator).fit(X_new, y_new)
                refitted_proba = refitted_estimator.predict_proba(X)
                entr = np.transpose(entropy(np.transpose(refitted_proba)))

                expected_log_loss[x_idx] += np.sum(entr)*X_proba[x_idx, y_idx]

        else:
            expected_log_loss[x_idx] = np.inf

    query_idx = multi_argmax(expected_log_loss, n_instances)

    return query_idx, X[query_idx]
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""Frames, arguments and predicates."""

import unittest
import verbnetprepclasses

class Frame:
    """A frame extracted from the corpus

    :var sentence: Sentence in which the frame appears
    :var predicate: Predicate object representing the frame's predicate
    :var args: Arg list containing the predicate's arguments

    """

    def __init__(self, sentence, predicate, args, words, frame_name):
        self.frame_name = frame_name
        self.sentence = sentence
        self.predicate = predicate
        self.args = sorted(args)
        self.words = words

    def get_word(self, word):
        return self.sentence[word.begin:word.end + 1]

    def __eq__(self, other):
        return (isinstance(other, self.__class__) and
            self.sentence == other.sentence and
            self.predicate == other.predicate and
            self.args == other.args and
            self.words == other.words)

class VerbnetFrame:
    """A representation of a frame syntaxic structure

    :var structure: String containing a VerbNet-style representation of the structure
    :var roles: List of the possible VerbNet roles for each structure's slot
    :var num_slots: Number of argument slots in :structure
    :var verbnet_class: For VerbNet-extracted frames, the vnclass

    """

    phrase_replacements = {
        "Poss":"NP", "QUO":"S",
        "Sinterrog":"S", "Sfin":"S",
        "VPbrst":"S", "VPing":"S_ING", "VPto":"to S"
    }

    def __init__(self, structure, roles, vnclass = None):
        self.structure = structure

        # Transform "a" in {"a"} and keep everything else unchanged
        self.roles = [{x} if isinstance(x, str) else x for x in roles]

        self.num_slots = len(self.roles)

        # Used to retrieve vnclass and map roles to framenet roles
        self.vnclass = vnclass

    def __eq__(self, other):
        return (isinstance(other, self.__class__) and
            self.structure == other.structure and
            self.roles == other.roles and
            self.num_slots == other.num_slots)

    def __repr__(self):
        return "VerbnetFrame({}, {}, {})".format(
                self.structure, self.roles, self.vnclass)

    @staticmethod
    def build_from_frame(frame):
        """Build a VerbNet frame from a Frame object

        :param frame: The original Frame.
        :type frame: Frame.
        :returns: VerbnetFrame -- the built frame, without the roles
        """

        # The main job is to build the VerbNet structure representation
        # from the Frame object data

        num_slots = 0

        # First, delete everything that is before or after the frame
        begin = frame.predicate.begin
        end = frame.predicate.end

        for argument in frame.args:
            if not argument.instanciated: continue
            num_slots += 1
            if argument.begin < begin: begin = argument.begin
            if argument.end > end: end = argument.end

        structure = frame.sentence[begin:end + 1]
        # Then, replace the predicate/arguments by their phrase type
        structure = VerbnetFrame._reduce_args(frame, structure, begin)
        # And delete everything else, except some keywords
        structure = VerbnetFrame._keep_only_keywords(structure)
        # Transform the structure into a list
        structure = structure.split(" ")
        # Delete every keyword before the verb
        structure = VerbnetFrame._strip_leftpart_keywords(structure)

        result = VerbnetFrame(structure, [])
        result.num_slots = num_slots

        # Fill the role list with None value
        result.roles = [None] * num_slots

        return result

    @staticmethod
    def _reduce_args(frame, structure, new_begin):
        """Replace the predicate and the argument of a frame by phrase type marks

        :param frame: The original Frame.
        :type frame: Frame.
        :param structure: The current structure representation.
        :type structure: str.
        :param new_begin: The left offset cause by previous manipulations.
        :type new_begin: int.
        :returns: String -- the reduced string
        """
        predicate_begin = frame.predicate.begin - new_begin
        predicate_end = frame.predicate.end - new_begin

        for argument in reversed(frame.args):
            if not argument.instanciated: continue

            before = structure[0:argument.begin - new_begin]
            after = structure[1 + argument.end - new_begin:]
            # Replace every "PP" by "prep NP"
            if argument.phrase_type == "PP":
                prep = argument.text.split(" ")[0].lower()
                added_length = 6 + len(prep)
                structure = "{}{} < NP>{}".format(before, prep, after)
            # Replace every "Swhether" by "if S" or "whether S"
            elif argument.phrase_type == "Swhether":
                sub = argument.text.split(" ")[0].lower()
                added_length = 5 + len(sub)
                structure = "{}{} < S>{}".format(before, sub, after)
            # Handle simple phrase replacements
            elif argument.phrase_type in VerbnetFrame.phrase_replacements:
                phrase = VerbnetFrame.phrase_replacements[argument.phrase_type]
                added_length = len(phrase)
                structure = "{} < {}>{}".format(before, phrase, after)
            else:
                added_length = 3 + len(argument.phrase_type)
                structure = "{}< {}>{}".format(before, argument.phrase_type, after)

            # Compute the new position of the predicate if we reduced an argument before it
            if argument.begin - new_begin < predicate_begin:
                offset = (argument.end - argument.begin + 1) - added_length
                predicate_begin -= offset
                predicate_end -= offset

        structure = "{}< V>{}".format(
            structure[0:predicate_begin], structure[1+predicate_end:])

        return structure

    @staticmethod
    def _keep_only_keywords(sentence):
        """Keep only keywords and phrase type markers in the structure

        :param sentence: The structure to reduce.
        :type sentence: str.
        :returns: String -- the reduced string
        """
        pos = 0
        last_pos = len(sentence) - 1
        inside_tag = False
        result = ""

        while pos < last_pos:
            if sentence[pos] == ">": inside_tag = False
            if inside_tag:
                result += sentence[pos]
                pos += 1
                continue
            if sentence[pos] == "<": inside_tag = True

            for search in verbnetprepclasses.keywords:
                if (search == sentence[pos:pos + len(search)].lower() and
                    (pos == 0 or sentence[pos - 1] == " ") and
                    (pos + len(search) == len(sentence) or
                        sentence[pos + len(search)] == " ")
                ):
                    pos += len(search) - 1
                    result += " "+search

            pos += 1

        if result[0] == " ": result = result[1:]
        if result[-1] == " ": result = result[:-1]

        return result

    @staticmethod
    def _strip_leftpart_keywords(sentence):
        result = []
        found_verb = False
        for elem in sentence:
            if elem == "V": found_verb = True
            if found_verb or elem[0].isupper():
                result.append(elem)

        return result

class Arg:

    """An argument of a frame

    :var begin: integer, position of the argument's first character in the sentence
    :var end: integer, position of the argument's last character in the sentence
    :var text: string containing the argument's text
    :var role: string containing the argument's role
    :var instanciated: boolean that marks wether the argument is instanciated

    """

    def __init__(self, begin, end, text, role, instanciated, phrase_type):
        self.begin = begin
        self.end = end
        self.text = text
        self.role = role
        self.instanciated = instanciated
        self.phrase_type = phrase_type

    def __eq__(self, other):
        return (isinstance(other, self.__class__)  and
            ((self.begin == other.begin and self.end == other.end) or
                (self.instanciated == False and other.instanciated == False)) and
            self.role == other.role and
            self.phrase_type == other.phrase_type)

    def __cmp__(self, other):
        if not self.instanciated:
            if other.instanciated: return 1
            if self.role < other.role: return -1
            if self.role > other.role: return 1
            return 0
        if not other.instanciated: return -1
        if self.begin < other.begin: return -1
        if self.begin > other.begin: return 1
        return 0

    def __lt__(self, other):
        return self.__cmp__(other) < 0

    def __le__(self, other):
        return self.__cmp__(other) <= 0

    def __ge__(self, other):
        return self.__cmp__(other) >= 0

    def __gt__(self, other):
        return self.__cmp__(other) > 0

class Predicate:

    """A frame's predicate

    :var begin: integer, position of the predicate's first character in the sentence
    :var end: integer, position of the predicate's last character in the sentence
    :var text: string containing the predicate's text
    :var lemma: string containing the predicate's lemma

    """

    def __init__(self, begin, end, text, lemma):
        self.begin = begin
        self.end = end
        self.text = text
        self.lemma = lemma

    def __eq__(self, other):
        return (isinstance(other, self.__class__) and
            self.begin == other.begin and
            self.end == other.end and
            self.lemma == other.lemma)

class Word:
    """A frame's word

    :var begin: integer, position of the predicate's first character in the sentence
    :var end: integer, position of the predicate's last character in the sentence
    :var text: string containing the predicate's text
    :var pos: string containing the predicate's part-of-speech

    """

    def __init__(self, begin, end, pos):
        self.begin = begin
        self.end = end
        if pos == 'sent': pos = '.'
        self.pos = pos.upper()

    def __eq__(self, other):
        return (isinstance(other, self.__class__) and
            self.begin == other.begin and
            self.end == other.end and
            self.pos == other.pos)

    def __repr__(self):
        return "Word({}, {}, \"{}\")".format(self.begin, self.end, self.pos)

class VerbnetFrameTest(unittest.TestCase):
    def test_conversion(self):
        tested_frames = [
            Frame(
                "Rep . Tony Hall , D- Ohio , urges the United Nations to allow"+\
                " a freer flow of food and medicine into Iraq .",
                Predicate(28, 32, "urges", "urge"),
                [
                    Arg(34, 51, "the United Nations", "Addressee", True, "NP"),
                    Arg(53, 104,
                        "to allow a freer flow of food and medicine into Iraq",
                        "Content", True, "VPto"),
                    Arg(0, 26, "Rep . Tony Hall , D- Ohio", "Speaker", True, "NP")
                ],
                [
                    Word(0, 2, "NN"), Word(4, 4, "."), Word(6, 9, "NP"),
                    Word(11, 14, "NP"), Word(16, 16, ","), Word(18, 19, "NN"),
                    Word(21, 24, "NP"), Word(26, 26, ","), Word(28, 32, "VVZ"),
                    Word(34, 36, "DT"), Word(38, 43, "NP"), Word(45, 51, "NPS"),
                    Word(53, 54, "TO"), Word(56, 60, "VV"), Word(62, 62, "DT"),
                    Word(64, 68, "JJR"), Word(70, 73, "NN"), Word(75, 76, "IN"),
                    Word(78, 81, "NN"), Word(83, 85, "CC"), Word(87, 94, "NN"),
                    Word(96, 99, "IN"), Word(101, 104, "NP"), Word(106, 106, ".")
                ] ),
            Frame(
                "Rep . Tony Hall , D- Ohio , urges the United Nations to allow"+\
                " a freer flow of food and medicine into Iraq .",
                 Predicate(56, 60, "allow", "allow"),
                 [
                    Arg(62, 104,
                        "a freer flow of food and medicine into Iraq",
                        "Action", True, "NP"),
                    Arg(34, 51, "the United Nations", "Grantee", True, "NP"),
                    Arg(0, -1, "", "Grantor", False, "")
                 ],
                 [
                    Word(0, 2, "NN"), Word(4, 4, "."), Word(6, 9, "NP"),
                    Word(11, 14, "NP"), Word(16, 16, ","), Word(18, 19, "NN"),
                    Word(21, 24, "NP"), Word(26, 26, ","), Word(28, 32, "VVZ"),
                    Word(34, 36, "DT"), Word(38, 43, "NP"), Word(45, 51, "NPS"),
                    Word(53, 54, "TO"), Word(56, 60, "VV"), Word(62, 62, "DT"),
                    Word(64, 68, "JJR"), Word(70, 73, "NN"), Word(75, 76, "IN"),
                    Word(78, 81, "NN"), Word(83, 85, "CC"), Word(87, 94, "NN"),
                    Word(96, 99, "IN"), Word(101, 104, "NP"), Word(106, 106, ".")
                 ] ) ]

        expected_results = [
            VerbnetFrame(["NP", "V", "NP", "to", "S"], [None, None, None]),
            VerbnetFrame(["NP", "V", "NP"], [None, None])
        ]

        verbnet_frame = VerbnetFrame.build_from_frame(tested_frames[0])
        self.assertEqual(expected_results[0], verbnet_frame)
        verbnet_frame = VerbnetFrame.build_from_frame(tested_frames[1])
        self.assertEqual(expected_results[1], verbnet_frame)

if __name__ == "__main__":
    unittest.main()
<commit_msg>Update unit test of framestructure<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""Frames, arguments and predicates."""

import unittest
import verbnetprepclasses

class Frame:
    """A frame extracted from the corpus

    :var sentence: Sentence in which the frame appears
    :var predicate: Predicate object representing the frame's predicate
    :var args: Arg list containing the predicate's arguments

    """

    def __init__(self, sentence, predicate, args, words, frame_name):
        self.frame_name = frame_name
        self.sentence = sentence
        self.predicate = predicate
        self.args = sorted(args)
        self.words = words

    def get_word(self, word):
        return self.sentence[word.begin:word.end + 1]

    def __eq__(self, other):
        return (isinstance(other, self.__class__) and
            self.sentence == other.sentence and
            self.predicate == other.predicate and
            self.args == other.args and
            self.words == other.words)

class VerbnetFrame:
    """A representation of a frame syntaxic structure

    :var structure: String containing a VerbNet-style representation of the structure
    :var roles: List of the possible VerbNet roles for each structure's slot
    :var num_slots: Number of argument slots in :structure
    :var verbnet_class: For VerbNet-extracted frames, the vnclass

    """

    phrase_replacements = {
        "Poss":"NP", "QUO":"S",
        "Sinterrog":"S", "Sfin":"S",
        "VPbrst":"S", "VPing":"S_ING", "VPto":"to S"
    }

    def __init__(self, structure, roles, vnclass = None):
        self.structure = structure

        # Transform "a" in {"a"} and keep everything else unchanged
        self.roles = [{x} if isinstance(x, str) else x for x in roles]

        self.num_slots = len(self.roles)

        # Used to retrieve vnclass and map roles to framenet roles
        self.vnclass = vnclass

    def __eq__(self, other):
        return (isinstance(other, self.__class__) and
            self.structure == other.structure and
            self.roles == other.roles and
            self.num_slots == other.num_slots)

    def __repr__(self):
        return "VerbnetFrame({}, {}, {})".format(
                self.structure, self.roles, self.vnclass)

    @staticmethod
    def build_from_frame(frame):
        """Build a VerbNet frame from a Frame object

        :param frame: The original Frame.
        :type frame: Frame.
        :returns: VerbnetFrame -- the built frame, without the roles
        """

        # The main job is to build the VerbNet structure representation
        # from the Frame object data

        num_slots = 0

        # First, delete everything that is before or after the frame
        begin = frame.predicate.begin
        end = frame.predicate.end

        for argument in frame.args:
            if not argument.instanciated: continue
            num_slots += 1
            if argument.begin < begin: begin = argument.begin
            if argument.end > end: end = argument.end

        structure = frame.sentence[begin:end + 1]
        # Then, replace the predicate/arguments by their phrase type
        structure = VerbnetFrame._reduce_args(frame, structure, begin)
        # And delete everything else, except some keywords
        structure = VerbnetFrame._keep_only_keywords(structure)
        # Transform the structure into a list
        structure = structure.split(" ")
        # Delete every keyword before the verb
        structure = VerbnetFrame._strip_leftpart_keywords(structure)

        result = VerbnetFrame(structure, [])
        result.num_slots = num_slots

        # Fill the role list with None value
        result.roles = [None] * num_slots

        return result

    @staticmethod
    def _reduce_args(frame, structure, new_begin):
        """Replace the predicate and the argument of a frame by phrase type marks

        :param frame: The original Frame.
        :type frame: Frame.
        :param structure: The current structure representation.
        :type structure: str.
        :param new_begin: The left offset cause by previous manipulations.
        :type new_begin: int.
        :returns: String -- the reduced string
        """
        predicate_begin = frame.predicate.begin - new_begin
        predicate_end = frame.predicate.end - new_begin

        for argument in reversed(frame.args):
            if not argument.instanciated: continue

            before = structure[0:argument.begin - new_begin]
            after = structure[1 + argument.end - new_begin:]
            # Replace every "PP" by "prep NP"
            if argument.phrase_type == "PP":
                prep = argument.text.split(" ")[0].lower()
                added_length = 6 + len(prep)
                structure = "{}{} < NP>{}".format(before, prep, after)
            # Replace every "Swhether" by "if S" or "whether S"
            elif argument.phrase_type == "Swhether":
                sub = argument.text.split(" ")[0].lower()
                added_length = 5 + len(sub)
                structure = "{}{} < S>{}".format(before, sub, after)
            # Handle simple phrase replacements
            elif argument.phrase_type in VerbnetFrame.phrase_replacements:
                phrase = VerbnetFrame.phrase_replacements[argument.phrase_type]
                added_length = len(phrase)
                structure = "{} < {}>{}".format(before, phrase, after)
            else:
                added_length = 3 + len(argument.phrase_type)
                structure = "{}< {}>{}".format(before, argument.phrase_type, after)

            # Compute the new position of the predicate if we reduced an argument before it
            if argument.begin - new_begin < predicate_begin:
                offset = (argument.end - argument.begin + 1) - added_length
                predicate_begin -= offset
                predicate_end -= offset

        structure = "{}< V>{}".format(
            structure[0:predicate_begin], structure[1+predicate_end:])

        return structure

    @staticmethod
    def _keep_only_keywords(sentence):
        """Keep only keywords and phrase type markers in the structure

        :param sentence: The structure to reduce.
        :type sentence: str.
        :returns: String -- the reduced string
        """
        pos = 0
        last_pos = len(sentence) - 1
        inside_tag = False
        result = ""

        while pos < last_pos:
            if sentence[pos] == ">": inside_tag = False
            if inside_tag:
                result += sentence[pos]
                pos += 1
                continue
            if sentence[pos] == "<": inside_tag = True

            for search in verbnetprepclasses.keywords:
                if (search == sentence[pos:pos + len(search)].lower() and
                    (pos == 0 or sentence[pos - 1] == " ") and
                    (pos + len(search) == len(sentence) or
                        sentence[pos + len(search)] == " ")
                ):
                    pos += len(search) - 1
                    result += " "+search

            pos += 1

        if result[0] == " ": result = result[1:]
        if result[-1] == " ": result = result[:-1]

        return result

    @staticmethod
    def _strip_leftpart_keywords(sentence):
        result = []
        found_verb = False
        for elem in sentence:
            if elem == "V": found_verb = True
            if found_verb or elem[0].isupper():
                result.append(elem)

        return result

class Arg:

    """An argument of a frame

    :var begin: integer, position of the argument's first character in the sentence
    :var end: integer, position of the argument's last character in the sentence
    :var text: string containing the argument's text
    :var role: string containing the argument's role
    :var instanciated: boolean that marks wether the argument is instanciated

    """

    def __init__(self, begin, end, text, role, instanciated, phrase_type):
        self.begin = begin
        self.end = end
        self.text = text
        self.role = role
        self.instanciated = instanciated
        self.phrase_type = phrase_type

    def __eq__(self, other):
        return (isinstance(other, self.__class__)  and
            ((self.begin == other.begin and self.end == other.end) or
                (self.instanciated == False and other.instanciated == False)) and
            self.role == other.role and
            self.phrase_type == other.phrase_type)

    def __cmp__(self, other):
        if not self.instanciated:
            if other.instanciated: return 1
            if self.role < other.role: return -1
            if self.role > other.role: return 1
            return 0
        if not other.instanciated: return -1
        if self.begin < other.begin: return -1
        if self.begin > other.begin: return 1
        return 0

    def __lt__(self, other):
        return self.__cmp__(other) < 0

    def __le__(self, other):
        return self.__cmp__(other) <= 0

    def __ge__(self, other):
        return self.__cmp__(other) >= 0

    def __gt__(self, other):
        return self.__cmp__(other) > 0

class Predicate:

    """A frame's predicate

    :var begin: integer, position of the predicate's first character in the sentence
    :var end: integer, position of the predicate's last character in the sentence
    :var text: string containing the predicate's text
    :var lemma: string containing the predicate's lemma

    """

    def __init__(self, begin, end, text, lemma):
        self.begin = begin
        self.end = end
        self.text = text
        self.lemma = lemma

    def __eq__(self, other):
        return (isinstance(other, self.__class__) and
            self.begin == other.begin and
            self.end == other.end and
            self.lemma == other.lemma)

class Word:
    """A frame's word

    :var begin: integer, position of the predicate's first character in the sentence
    :var end: integer, position of the predicate's last character in the sentence
    :var text: string containing the predicate's text
    :var pos: string containing the predicate's part-of-speech

    """

    def __init__(self, begin, end, pos):
        self.begin = begin
        self.end = end
        if pos == 'sent': pos = '.'
        self.pos = pos.upper()

    def __eq__(self, other):
        return (isinstance(other, self.__class__) and
            self.begin == other.begin and
            self.end == other.end and
            self.pos == other.pos)

    def __repr__(self):
        return "Word({}, {}, \"{}\")".format(self.begin, self.end, self.pos)

class VerbnetFrameTest(unittest.TestCase):
    def test_conversion(self):
        tested_frames = [
            Frame(
                "Rep . Tony Hall , D- Ohio , urges the United Nations to allow"+\
                " a freer flow of food and medicine into Iraq .",
                Predicate(28, 32, "urges", "urge"),
                [
                    Arg(34, 51, "the United Nations", "Addressee", True, "NP"),
                    Arg(53, 104,
                        "to allow a freer flow of food and medicine into Iraq",
                        "Content", True, "VPto"),
                    Arg(0, 26, "Rep . Tony Hall , D- Ohio", "Speaker", True, "NP")
                ],
                [
                    Word(0, 2, "NN"), Word(4, 4, "."), Word(6, 9, "NP"),
                    Word(11, 14, "NP"), Word(16, 16, ","), Word(18, 19, "NN"),
                    Word(21, 24, "NP"), Word(26, 26, ","), Word(28, 32, "VVZ"),
                    Word(34, 36, "DT"), Word(38, 43, "NP"), Word(45, 51, "NPS"),
                    Word(53, 54, "TO"), Word(56, 60, "VV"), Word(62, 62, "DT"),
                    Word(64, 68, "JJR"), Word(70, 73, "NN"), Word(75, 76, "IN"),
                    Word(78, 81, "NN"), Word(83, 85, "CC"), Word(87, 94, "NN"),
                    Word(96, 99, "IN"), Word(101, 104, "NP"), Word(106, 106, ".")
                ],
                "Attempt_suasion" ),
            Frame(
                "Rep . Tony Hall , D- Ohio , urges the United Nations to allow"+\
                " a freer flow of food and medicine into Iraq .",
                 Predicate(56, 60, "allow", "allow"),
                 [
                    Arg(62, 104,
                        "a freer flow of food and medicine into Iraq",
                        "Action", True, "NP"),
                    Arg(34, 51, "the United Nations", "Grantee", True, "NP"),
                    Arg(0, -1, "", "Grantor", False, "")
                 ],
                 [
                    Word(0, 2, "NN"), Word(4, 4, "."), Word(6, 9, "NP"),
                    Word(11, 14, "NP"), Word(16, 16, ","), Word(18, 19, "NN"),
                    Word(21, 24, "NP"), Word(26, 26, ","), Word(28, 32, "VVZ"),
                    Word(34, 36, "DT"), Word(38, 43, "NP"), Word(45, 51, "NPS"),
                    Word(53, 54, "TO"), Word(56, 60, "VV"), Word(62, 62, "DT"),
                    Word(64, 68, "JJR"), Word(70, 73, "NN"), Word(75, 76, "IN"),
                    Word(78, 81, "NN"), Word(83, 85, "CC"), Word(87, 94, "NN"),
                    Word(96, 99, "IN"), Word(101, 104, "NP"), Word(106, 106, ".")
                 ],
                 "Grant_permission" ) ]

        expected_results = [
            VerbnetFrame(["NP", "V", "NP", "to", "S"], [None, None, None]),
            VerbnetFrame(["NP", "V", "NP"], [None, None])
        ]

        verbnet_frame = VerbnetFrame.build_from_frame(tested_frames[0])
        self.assertEqual(expected_results[0], verbnet_frame)
        verbnet_frame = VerbnetFrame.build_from_frame(tested_frames[1])
        self.assertEqual(expected_results[1], verbnet_frame)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright: (c) 2008, Jarek Zgoda <jarek.zgoda@gmail.com>

__revision__ = '$Id: models.py 28 2009-10-22 15:03:02Z jarek.zgoda $'

import datetime

from django.db import models
from django.core.urlresolvers import reverse
from django.conf import settings
from django.contrib.sites.models import Site
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.fields import GenericForeignKey
from django.http import HttpRequest, HttpResponse
from django.shortcuts import render
from django.utils.timezone import now as timezone_now

from zerver.lib.send_email import send_email
from zerver.lib.utils import generate_random_token
from zerver.models import PreregistrationUser, EmailChangeStatus, MultiuseInvite
from random import SystemRandom
import string
from typing import Any, Dict, Optional, Text, Union

class ConfirmationKeyException(Exception):
    WRONG_LENGTH = 1
    EXPIRED = 2
    DOES_NOT_EXIST = 3

    def __init__(self, error_type):
        # type: (int) -> None
        super().__init__()
        self.error_type = error_type

def render_confirmation_key_error(request, exception):
    # type: (HttpRequest, ConfirmationKeyException) -> HttpResponse
    if exception.error_type == ConfirmationKeyException.WRONG_LENGTH:
        return render(request, 'confirmation/link_malformed.html')
    if exception.error_type == ConfirmationKeyException.EXPIRED:
        return render(request, 'confirmation/link_expired.html')
    return render(request, 'confirmation/link_does_not_exist.html')

def generate_key():
    # type: () -> str
    generator = SystemRandom()
    # 24 characters * 5 bits of entropy/character = 120 bits of entropy
    return ''.join(generator.choice(string.ascii_lowercase + string.digits) for _ in range(24))

def get_object_from_key(confirmation_key):
    # type: (str) -> Union[MultiuseInvite, PreregistrationUser, EmailChangeStatus]
    # Confirmation keys used to be 40 characters
    if len(confirmation_key) not in (24, 40):
        raise ConfirmationKeyException(ConfirmationKeyException.WRONG_LENGTH)
    try:
        confirmation = Confirmation.objects.get(confirmation_key=confirmation_key)
    except Confirmation.DoesNotExist:
        raise ConfirmationKeyException(ConfirmationKeyException.DOES_NOT_EXIST)

    time_elapsed = timezone_now() - confirmation.date_sent
    if time_elapsed.total_seconds() > _properties[confirmation.type].validity_in_days * 24 * 3600:
        raise ConfirmationKeyException(ConfirmationKeyException.EXPIRED)

    obj = confirmation.content_object
    if hasattr(obj, "status"):
        obj.status = getattr(settings, 'STATUS_ACTIVE', 1)
        obj.save(update_fields=['status'])
    return obj

def create_confirmation_link(obj, host, confirmation_type, url_args=None):
    # type: (Union[ContentType, int], str, int, Optional[Dict[str, str]]) -> str
    key = generate_key()
    Confirmation.objects.create(content_object=obj, date_sent=timezone_now(), confirmation_key=key,
                                type=confirmation_type)
    return confirmation_url(key, host, confirmation_type, url_args)

def confirmation_url(confirmation_key, host, confirmation_type, url_args=None):
    # type: (str, str, int, Optional[Dict[str, str]]) -> str
    if url_args is None:
        url_args = {}
    url_args['confirmation_key'] = confirmation_key
    return '%s%s%s' % (settings.EXTERNAL_URI_SCHEME, host,
                       reverse(_properties[confirmation_type].url_name, kwargs=url_args))

class Confirmation(models.Model):
    content_type = models.ForeignKey(ContentType)
    object_id = models.PositiveIntegerField()  # type: int
    content_object = GenericForeignKey('content_type', 'object_id')
    date_sent = models.DateTimeField()  # type: datetime.datetime
    confirmation_key = models.CharField(max_length=40)  # type: str

    # The following list is the set of valid types
    USER_REGISTRATION = 1
    INVITATION = 2
    EMAIL_CHANGE = 3
    UNSUBSCRIBE = 4
    SERVER_REGISTRATION = 5
    MULTIUSE_INVITE = 6
    type = models.PositiveSmallIntegerField()  # type: int

    def __unicode__(self):
        # type: () -> Text
        return '<Confirmation: %s>' % (self.content_object,)

class ConfirmationType(object):
    def __init__(self, url_name, validity_in_days=settings.CONFIRMATION_LINK_DEFAULT_VALIDITY_DAYS):
        # type: (str, int) -> None
        self.url_name = url_name
        self.validity_in_days = validity_in_days

_properties = {
    Confirmation.USER_REGISTRATION: ConfirmationType('confirmation.views.confirm'),
    Confirmation.INVITATION: ConfirmationType('confirmation.views.confirm',
                                              validity_in_days=settings.INVITATION_LINK_VALIDITY_DAYS),
    Confirmation.EMAIL_CHANGE: ConfirmationType('zerver.views.user_settings.confirm_email_change'),
    Confirmation.UNSUBSCRIBE: ConfirmationType('zerver.views.unsubscribe.email_unsubscribe',
                                               validity_in_days=1000000),  # should never expire
    Confirmation.MULTIUSE_INVITE: ConfirmationType('zerver.views.registration.accounts_home_from_multiuse_invite',
                                                   validity_in_days=settings.INVITATION_LINK_VALIDITY_DAYS)
}

# Conirmation pathways for which there is no content_object that we need to
# keep track of.

def check_key_is_valid(creation_key):
    # type: (Text) -> bool
    if not RealmCreationKey.objects.filter(creation_key=creation_key).exists():
        return False
    days_sofar = (timezone_now() - RealmCreationKey.objects.get(creation_key=creation_key).date_created).days
    # Realm creation link expires after settings.REALM_CREATION_LINK_VALIDITY_DAYS
    if days_sofar <= settings.REALM_CREATION_LINK_VALIDITY_DAYS:
        return True
    return False

def generate_realm_creation_url():
    # type: () -> Text
    key = generate_key()
    RealmCreationKey.objects.create(creation_key=key, date_created=timezone_now())
    return u'%s%s%s' % (settings.EXTERNAL_URI_SCHEME,
                        settings.EXTERNAL_HOST,
                        reverse('zerver.views.create_realm',
                                kwargs={'creation_key': key}))

class RealmCreationKey(models.Model):
    creation_key = models.CharField('activation key', max_length=40)
    date_created = models.DateTimeField('created', default=timezone_now)
<commit_msg>models: Fix typo in comment.<commit_after># -*- coding: utf-8 -*-

# Copyright: (c) 2008, Jarek Zgoda <jarek.zgoda@gmail.com>

__revision__ = '$Id: models.py 28 2009-10-22 15:03:02Z jarek.zgoda $'

import datetime

from django.db import models
from django.core.urlresolvers import reverse
from django.conf import settings
from django.contrib.sites.models import Site
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.fields import GenericForeignKey
from django.http import HttpRequest, HttpResponse
from django.shortcuts import render
from django.utils.timezone import now as timezone_now

from zerver.lib.send_email import send_email
from zerver.lib.utils import generate_random_token
from zerver.models import PreregistrationUser, EmailChangeStatus, MultiuseInvite
from random import SystemRandom
import string
from typing import Any, Dict, Optional, Text, Union

class ConfirmationKeyException(Exception):
    WRONG_LENGTH = 1
    EXPIRED = 2
    DOES_NOT_EXIST = 3

    def __init__(self, error_type):
        # type: (int) -> None
        super().__init__()
        self.error_type = error_type

def render_confirmation_key_error(request, exception):
    # type: (HttpRequest, ConfirmationKeyException) -> HttpResponse
    if exception.error_type == ConfirmationKeyException.WRONG_LENGTH:
        return render(request, 'confirmation/link_malformed.html')
    if exception.error_type == ConfirmationKeyException.EXPIRED:
        return render(request, 'confirmation/link_expired.html')
    return render(request, 'confirmation/link_does_not_exist.html')

def generate_key():
    # type: () -> str
    generator = SystemRandom()
    # 24 characters * 5 bits of entropy/character = 120 bits of entropy
    return ''.join(generator.choice(string.ascii_lowercase + string.digits) for _ in range(24))

def get_object_from_key(confirmation_key):
    # type: (str) -> Union[MultiuseInvite, PreregistrationUser, EmailChangeStatus]
    # Confirmation keys used to be 40 characters
    if len(confirmation_key) not in (24, 40):
        raise ConfirmationKeyException(ConfirmationKeyException.WRONG_LENGTH)
    try:
        confirmation = Confirmation.objects.get(confirmation_key=confirmation_key)
    except Confirmation.DoesNotExist:
        raise ConfirmationKeyException(ConfirmationKeyException.DOES_NOT_EXIST)

    time_elapsed = timezone_now() - confirmation.date_sent
    if time_elapsed.total_seconds() > _properties[confirmation.type].validity_in_days * 24 * 3600:
        raise ConfirmationKeyException(ConfirmationKeyException.EXPIRED)

    obj = confirmation.content_object
    if hasattr(obj, "status"):
        obj.status = getattr(settings, 'STATUS_ACTIVE', 1)
        obj.save(update_fields=['status'])
    return obj

def create_confirmation_link(obj, host, confirmation_type, url_args=None):
    # type: (Union[ContentType, int], str, int, Optional[Dict[str, str]]) -> str
    key = generate_key()
    Confirmation.objects.create(content_object=obj, date_sent=timezone_now(), confirmation_key=key,
                                type=confirmation_type)
    return confirmation_url(key, host, confirmation_type, url_args)

def confirmation_url(confirmation_key, host, confirmation_type, url_args=None):
    # type: (str, str, int, Optional[Dict[str, str]]) -> str
    if url_args is None:
        url_args = {}
    url_args['confirmation_key'] = confirmation_key
    return '%s%s%s' % (settings.EXTERNAL_URI_SCHEME, host,
                       reverse(_properties[confirmation_type].url_name, kwargs=url_args))

class Confirmation(models.Model):
    content_type = models.ForeignKey(ContentType)
    object_id = models.PositiveIntegerField()  # type: int
    content_object = GenericForeignKey('content_type', 'object_id')
    date_sent = models.DateTimeField()  # type: datetime.datetime
    confirmation_key = models.CharField(max_length=40)  # type: str

    # The following list is the set of valid types
    USER_REGISTRATION = 1
    INVITATION = 2
    EMAIL_CHANGE = 3
    UNSUBSCRIBE = 4
    SERVER_REGISTRATION = 5
    MULTIUSE_INVITE = 6
    type = models.PositiveSmallIntegerField()  # type: int

    def __unicode__(self):
        # type: () -> Text
        return '<Confirmation: %s>' % (self.content_object,)

class ConfirmationType(object):
    def __init__(self, url_name, validity_in_days=settings.CONFIRMATION_LINK_DEFAULT_VALIDITY_DAYS):
        # type: (str, int) -> None
        self.url_name = url_name
        self.validity_in_days = validity_in_days

_properties = {
    Confirmation.USER_REGISTRATION: ConfirmationType('confirmation.views.confirm'),
    Confirmation.INVITATION: ConfirmationType('confirmation.views.confirm',
                                              validity_in_days=settings.INVITATION_LINK_VALIDITY_DAYS),
    Confirmation.EMAIL_CHANGE: ConfirmationType('zerver.views.user_settings.confirm_email_change'),
    Confirmation.UNSUBSCRIBE: ConfirmationType('zerver.views.unsubscribe.email_unsubscribe',
                                               validity_in_days=1000000),  # should never expire
    Confirmation.MULTIUSE_INVITE: ConfirmationType('zerver.views.registration.accounts_home_from_multiuse_invite',
                                                   validity_in_days=settings.INVITATION_LINK_VALIDITY_DAYS)
}

# Confirmation pathways for which there is no content_object that we need to
# keep track of.

def check_key_is_valid(creation_key):
    # type: (Text) -> bool
    if not RealmCreationKey.objects.filter(creation_key=creation_key).exists():
        return False
    days_sofar = (timezone_now() - RealmCreationKey.objects.get(creation_key=creation_key).date_created).days
    # Realm creation link expires after settings.REALM_CREATION_LINK_VALIDITY_DAYS
    if days_sofar <= settings.REALM_CREATION_LINK_VALIDITY_DAYS:
        return True
    return False

def generate_realm_creation_url():
    # type: () -> Text
    key = generate_key()
    RealmCreationKey.objects.create(creation_key=key, date_created=timezone_now())
    return u'%s%s%s' % (settings.EXTERNAL_URI_SCHEME,
                        settings.EXTERNAL_HOST,
                        reverse('zerver.views.create_realm',
                                kwargs={'creation_key': key}))

class RealmCreationKey(models.Model):
    creation_key = models.CharField('activation key', max_length=40)
    date_created = models.DateTimeField('created', default=timezone_now)
<|endoftext|>


<commit_before>"""
Expected error reduction framework for active learning.
"""

from typing import Tuple

import numpy as np

from sklearn.base import clone
from sklearn.exceptions import NotFittedError

from modAL.models import ActiveLearner
from modAL.utils.data import modALinput, data_vstack
from modAL.utils.selection import multi_argmax, shuffled_argmax
from modAL.uncertainty import _proba_uncertainty, _proba_entropy

def expected_error_reduction(learner: ActiveLearner, X: modALinput, loss: str = 'binary',
                             p_subsample: np.float = 1.0, n_instances: int = 1,
                             random_tie_break: bool = False) -> Tuple[np.ndarray, modALinput]:
    """
    Expected error reduction query strategy.

    References:
        Roy and McCallum, 2001 (http://groups.csail.mit.edu/rrg/papers/icml01.pdf)

    Args:
        learner: The ActiveLearner object for which the expected error
            is to be estimated.
        X: The samples.
        loss: The loss function to be used. Can be 'binary' or 'log'.
        p_subsample: Probability of keeping a sample from the pool when
            calculating expected error. Significantly improves runtime
            for large sample pools.
        n_instances: The number of instances to be sampled.
        random_tie_break: If True, shuffles utility scores to randomize the order. This
            can be used to break the tie when the highest utility score is not unique.

    Returns:
        The indices of the instances from X chosen to be labelled;
        the instances from X chosen to be labelled.
    """

    assert 0.0 <= p_subsample <= 1.0, 'p_subsample subsampling keep ratio must be between 0.0 and 1.0'
    assert loss in ['binary', 'log'], 'loss must be \'binary\' or \'log\''

    expected_error = np.zeros(shape=(len(X), ))
    possible_labels = np.unique(learner.y_training)

    try:
        X_proba = learner.predict_proba(X)
    except NotFittedError:
        # TODO: implement a proper cold-start
        return 0, X[0]

    cloned_estimator = clone(learner.estimator)

    for x_idx, x in enumerate(X):
        # subsample the data if needed
        if np.random.rand() <= p_subsample:
            # estimate the expected error
            for y_idx, y in enumerate(possible_labels):
                X_new = data_vstack((learner.X_training, np.expand_dims(x, axis=0)))
                y_new = data_vstack((learner.y_training, np.array(y).reshape(1,)))

                cloned_estimator.fit(X_new, y_new)
                refitted_proba = cloned_estimator.predict_proba(X)
                if loss is 'binary':
                    nloss = _proba_uncertainty(refitted_proba)
                elif loss is 'log':
                    nloss = _proba_entropy(refitted_proba)

                expected_error[x_idx] += np.sum(nloss)*X_proba[x_idx, y_idx]

        else:
            expected_error[x_idx] = np.inf

    if not random_tie_break:
        query_idx = multi_argmax(-expected_error, n_instances)
    else:
        query_idx = shuffled_argmax(-expected_error, n_instances)

    return query_idx, X[query_idx]
<commit_msg>queried instance removed from the loss calculation<commit_after>"""
Expected error reduction framework for active learning.
"""

from typing import Tuple

import numpy as np

from sklearn.base import clone
from sklearn.exceptions import NotFittedError

from modAL.models import ActiveLearner
from modAL.utils.data import modALinput, data_vstack
from modAL.utils.selection import multi_argmax, shuffled_argmax
from modAL.uncertainty import _proba_uncertainty, _proba_entropy

def expected_error_reduction(learner: ActiveLearner, X: modALinput, loss: str = 'binary',
                             p_subsample: np.float = 1.0, n_instances: int = 1,
                             random_tie_break: bool = False) -> Tuple[np.ndarray, modALinput]:
    """
    Expected error reduction query strategy.

    References:
        Roy and McCallum, 2001 (http://groups.csail.mit.edu/rrg/papers/icml01.pdf)

    Args:
        learner: The ActiveLearner object for which the expected error
            is to be estimated.
        X: The samples.
        loss: The loss function to be used. Can be 'binary' or 'log'.
        p_subsample: Probability of keeping a sample from the pool when
            calculating expected error. Significantly improves runtime
            for large sample pools.
        n_instances: The number of instances to be sampled.
        random_tie_break: If True, shuffles utility scores to randomize the order. This
            can be used to break the tie when the highest utility score is not unique.

    Returns:
        The indices of the instances from X chosen to be labelled;
        the instances from X chosen to be labelled.
    """

    assert 0.0 <= p_subsample <= 1.0, 'p_subsample subsampling keep ratio must be between 0.0 and 1.0'
    assert loss in ['binary', 'log'], 'loss must be \'binary\' or \'log\''

    expected_error = np.zeros(shape=(len(X), ))
    possible_labels = np.unique(learner.y_training)

    try:
        X_proba = learner.predict_proba(X)
    except NotFittedError:
        # TODO: implement a proper cold-start
        return 0, X[0]

    cloned_estimator = clone(learner.estimator)

    for x_idx, x in enumerate(X):
        # subsample the data if needed
        if np.random.rand() <= p_subsample:
            X_reduced = np.delete(X, x_idx, axis=0)
            # estimate the expected error
            for y_idx, y in enumerate(possible_labels):
                X_new = data_vstack((learner.X_training, np.expand_dims(x, axis=0)))
                y_new = data_vstack((learner.y_training, np.array(y).reshape(1,)))

                cloned_estimator.fit(X_new, y_new)
                refitted_proba = cloned_estimator.predict_proba(X_reduced)
                if loss is 'binary':
                    nloss = _proba_uncertainty(refitted_proba)
                elif loss is 'log':
                    nloss = _proba_entropy(refitted_proba)

                expected_error[x_idx] += np.sum(nloss)*X_proba[x_idx, y_idx]

        else:
            expected_error[x_idx] = np.inf

    if not random_tie_break:
        query_idx = multi_argmax(-expected_error, n_instances)
    else:
        query_idx = shuffled_argmax(-expected_error, n_instances)

    return query_idx, X[query_idx]
<|endoftext|>


<commit_before>import time
import serial

import BlinkyTapeUnitTest
import TestRig
import UserInterface
import BlinkyTape
import DetectPlatform

class TestFunctionalTests(BlinkyTapeUnitTest.BlinkyTapeTestCase):
  def __init__(self, methodName):
    super(TestFunctionalTests, self).__init__(methodName)

  @classmethod
  def setUpClass(self):
    # TODO: Why can't access self.testRig from __init__ here?
    self.testRig = TestRig.testRig
    self.testRig.resetState()

    self.i = UserInterface.interface
    self.dut = BlinkyTape.BlinkyTape()

  @classmethod
  def tearDownClass(self):
    self.dut = None
    self.testRig.resetState()

  def setUp(self):
    self.stopMe = True

  def tearDown(self):
    if self.stopMe:
      self.Stop()

  def test_010_dutDisconnectedCurrent(self):
    self.i.DisplayMessage("Checking disconnected current...")
    MIN_OFF_CURRENT = -1
    MAX_OFF_CURRENT = 1

    current = self.testRig.measure('DUT_CURRENT')

    self.i.DisplayMessage("DUT disconnectd current: %0.2f < %0.2f < %0.2f." % (MIN_OFF_CURRENT, current, MAX_OFF_CURRENT))
    self.StoreTestResultData("%0.2f" % current)

    self.assertTrue(current> MIN_OFF_CURRENT
                    and current < MAX_OFF_CURRENT)
    self.stopMe = False

  def test_020_usbEnumeration(self):
    self.i.DisplayMessage("Waiting for device to enumerate on USB...")

    MAX_ENUMERATION_TIME_S = 5
    # Scan for all connected devices; platform dependent
    originalPorts = set(DetectPlatform.ListSerialPorts())

    self.testRig.enableRelay('EN_USB_GND')
    self.testRig.enableRelay('EN_USB_VCC')
    self.testRig.enableRelay('EN_USB_DATA')

    # Wait for the device to enumerate
    startTime = time.time()
    while(time.time() < startTime + 5):
      finalPorts = set(DetectPlatform.ListSerialPorts())
      newPorts = finalPorts - originalPorts
      if len(newPorts) == 1:
        self.dut.port = list(newPorts)[0]
        break

    self.LogDataPoint("DUT Usb port:", self.dut.port)
    self.assertTrue(self.dut.port != '')
    self.stopMe = False

  def test_030_dutConnected(self):
    self.i.DisplayMessage("Connecting to DUT")

    connected = False
    try:
      self.dut.connect(self.dut.port)
    except serial.SerialException:
      pass
    else:
      connected = True

    self.assertTrue(connected)
    self.stopMe = False

  def test_040_dutConnectedCurrent(self):
    self.i.DisplayMessage("Checking connected current...")

    MIN_CONNECTED_CURRENT = 20
    MAX_CONNECTED_CURRENT = 40

    current = self.testRig.measure('DUT_CURRENT')

    self.i.DisplayMessage("DUT connected current: %0.2f < %0.2f < %0.2f." % (MIN_CONNECTED_CURRENT, current, MAX_CONNECTED_CURRENT))
    self.StoreTestResultData("%0.2f" % current)

    self.assertTrue(current > MIN_CONNECTED_CURRENT
                    and current < MAX_CONNECTED_CURRENT)
    self.stopMe = False

  def test_045_LedsConnectedCurrent(self):
    self.i.DisplayMessage("Checking LEDs connected current...")

    MIN_CONNECTED_CURRENT = 30
    MAX_CONNECTED_CURRENT = 50

    self.testRig.enableRelay('EN_LED_OUT')

    current = self.testRig.measure('DUT_CURRENT')

    self.i.DisplayMessage("LEDs connected current: %0.2f < %0.2f < %0.2f." % (MIN_CONNECTED_CURRENT, current, MAX_CONNECTED_CURRENT))
    self.StoreTestResultData("%0.2f" % current)

    self.assertTrue(current > MIN_CONNECTED_CURRENT
                    and current < MAX_CONNECTED_CURRENT)
    self.stopMe = False

  def test_050_redLedsOnCurrent(self):
    self.i.DisplayMessage("Checking red LEDs on...")

    MIN_RED_CURRENT = 130
    MAX_RED_CURRENT = 150
    # TODO: Why send this twice?
    for j in range (0, 2):
      for x in range(0, 60):
        self.dut.sendPixel(255,0,0)
      self.dut.show();

    current = self.testRig.measure('DUT_CURRENT')

    self.i.DisplayMessage("Red LEDs current: %0.2f < %0.2f < %0.2f." % (MIN_RED_CURRENT, current, MAX_RED_CURRENT))
    self.StoreTestResultData("%0.2f" % current)

    self.assertTrue(current > MIN_RED_CURRENT
                    and current < MAX_RED_CURRENT)
    self.stopMe = False

  def skip_test_060_greenLedsOnCurrent(self):
    self.i.DisplayMessage("Checking green LEDs on...")

    MIN_GREEN_CURRENT = 130
    MAX_GREEN_CURRENT = 150
    # TODO: Why send this twice?
    for j in range (0, 2):
      for x in range(0, 60):
        self.dut.sendPixel(0,255,0)
      self.dut.show();

    current = self.testRig.measure('DUT_CURRENT')

    self.i.DisplayMessage("Green LEDs current: %0.2f < %0.2f < %0.2f." % (MIN_GREEN_CURRENT, current, MAX_GREEN_CURRENT))
    self.StoreTestResultData("%0.2f" % current)

    self.assertTrue(current > MIN_GREEN_CURRENT
                    and current < MAX_GREEN_CURRENT)
    self.stopMe = False

  def skip_test_070_blueLedsOnCurrent(self):
    self.i.DisplayMessage("Checking blue LEDs on...")

    MIN_BLUE_CURRENT = 130
    MAX_BLUE_CURRENT = 150
    # TODO: Why send this twice?
    for j in range (0, 2):
      for x in range(0, 60):
        self.dut.sendPixel(0,0,255)
      self.dut.show();

    current = self.testRig.measure('DUT_CURRENT')

    self.i.DisplayMessage("Blue LEDs current: %0.2f < %0.2f < %0.2f." % (MIN_BLUE_CURRENT, current, MAX_BLUE_CURRENT))
    self.StoreTestResultData("%0.2f" % current)

    self.assertTrue(current > MIN_BLUE_CURRENT
                    and current < MAX_BLUE_CURRENT)
    self.stopMe = False

  def test_080_whiteLedsOnCurrent(self):
    self.i.DisplayMessage("Checking white LEDs on...")

    MIN_WHITE_CURRENT = 250
    MAX_WHITE_CURRENT = 350
    # TODO: Why send this twice?
    for j in range (0, 2):
      for x in range(0, 60):
        self.dut.sendPixel(255,255,255)
      self.dut.show();

    current = self.testRig.measure('DUT_CURRENT')

    self.i.DisplayMessage("White LEDs current: %0.2f < %0.2f < %0.2f." % (MIN_WHITE_CURRENT, current, MAX_WHITE_CURRENT))
    self.StoreTestResultData("%0.2f" % current)

    self.assertTrue(current > MIN_WHITE_CURRENT
                    and current < MAX_WHITE_CURRENT)
    self.stopMe = False

  def test_090_D7_connected(self):
    self.i.DisplayMessage("Checking D7 input works...")
    self.testRig.setOutputLow('DUT_D7')

    pinStates = 0
    for j in range (0, 2):
      for x in range(0, 60):
        self.dut.sendPixel(0,0,0)
      pinStates = self.dut.show();

    self.testRig.setInput('DUT_D7')

    self.assertTrue(ord(pinStates[0]) == 11)
    self.stopMe = False

  def test_091_D11_connected(self):
    self.i.DisplayMessage("Checking D11 input works...")
    self.testRig.setOutputLow('DUT_D11')

    pinStates = 0
    for j in range (0, 2):
      for x in range(0, 60):
        self.dut.sendPixel(0,0,0)
      pinStates = self.dut.show();

    self.testRig.setInput('DUT_D11')

    self.assertTrue(ord(pinStates[0]) == 13)
    self.stopMe = False

  def test_092_A9_connected(self):
    self.i.DisplayMessage("Checking A9 input works...")
    self.testRig.setOutputLow('DUT_A9')

    pinStates = 0
    for j in range (0, 2):
      for x in range(0, 60):
        self.dut.sendPixel(0,0,0)
      pinStates = self.dut.show();

    self.testRig.setInput('DUT_A9')

    self.assertTrue(ord(pinStates[0]) == 14)
    self.stopMe = False

  def test_100_button_connected(self):
    self.i.DisplayMessage("Checking button input works...")

    MAX_TIME_SECONDS = 60;

    startTime = time.time()
    found = False
    mode = True

    while((not found) and (time.time() < startTime + MAX_TIME_SECONDS)):
      pinStates = 0
      for j in range (0, 2):
        for x in range(0, 60):
	  if mode:
            self.dut.sendPixel(0,0,100)
          else:
            self.dut.sendPixel(0,0,0)
        pinStates = self.dut.show();

      if ord(pinStates[0]) == 0x07:
        found = True

      mode = not mode
      time.sleep(.5)

    self.assertTrue(found)
    self.stopMe = False

<commit_msg>Bring in most fixes to get the rig working on the factory floor.<commit_after># NOTE: The Producion version of this is slighly different. The guard bands for current
# measurement were changed, and the LED strip output is turned on before USB power, to
# prevent the LEDs from being damaged. This version is an attempt to bring these changes
# in, however they have to be retrieved from the factory test rig.

import time
import serial

import BlinkyTapeUnitTest
import TestRig
import UserInterface
import BlinkyTape
import DetectPlatform

class TestFunctionalTests(BlinkyTapeUnitTest.BlinkyTapeTestCase):
  def __init__(self, methodName):
    super(TestFunctionalTests, self).__init__(methodName)

  @classmethod
  def setUpClass(self):
    # TODO: Why can't access self.testRig from __init__ here?
    self.testRig = TestRig.testRig
    self.testRig.resetState()

    self.i = UserInterface.interface
    self.dut = BlinkyTape.BlinkyTape()

  @classmethod
  def tearDownClass(self):
    self.dut = None
    self.testRig.resetState()

  def setUp(self):
    self.stopMe = True

  def tearDown(self):
    if self.stopMe:
      self.Stop()

  def test_010_dutDisconnectedCurrent(self):
    self.i.DisplayMessage("Checking disconnected current...")
    MIN_OFF_CURRENT = -1
    MAX_OFF_CURRENT = 1

    current = self.testRig.measure('DUT_CURRENT')

    self.i.DisplayMessage("DUT disconnectd current: %0.2f < %0.2f < %0.2f." % (MIN_OFF_CURRENT, current, MAX_OFF_CURRENT))
    self.StoreTestResultData("%0.2f" % current)

    self.assertTrue(current> MIN_OFF_CURRENT
                    and current < MAX_OFF_CURRENT)
    self.stopMe = False

  def test_020_usbEnumeration(self):
    self.i.DisplayMessage("Waiting for device to enumerate on USB...")

    MAX_ENUMERATION_TIME_S = 5
    # Scan for all connected devices; platform dependent
    originalPorts = set(DetectPlatform.ListSerialPorts())

    self.testRig.enableRelay('EN_LED_OUT')

    self.testRig.enableRelay('EN_USB_GND')
    self.testRig.enableRelay('EN_USB_VCC')
    self.testRig.enableRelay('EN_USB_DATA')

    # Wait for the device to enumerate
    startTime = time.time()
    while(time.time() < startTime + 5):
      finalPorts = set(DetectPlatform.ListSerialPorts())
      newPorts = finalPorts - originalPorts
      if len(newPorts) == 1:
        self.dut.port = list(newPorts)[0]
        break

    self.LogDataPoint("DUT Usb port:", self.dut.port)
    self.assertTrue(self.dut.port != '')
    self.stopMe = False

  def test_030_dutConnected(self):
    self.i.DisplayMessage("Connecting to DUT")

    connected = False
    try:
      self.dut.connect(self.dut.port)
    except serial.SerialException:
      pass
    else:
      connected = True

    self.assertTrue(connected)
    self.stopMe = False

  def test_040_dutConnectedCurrent(self):
    self.i.DisplayMessage("Checking connected current...")

    MIN_CONNECTED_CURRENT = 20
    MAX_CONNECTED_CURRENT = 40

    current = self.testRig.measure('DUT_CURRENT')

    self.i.DisplayMessage("DUT connected current: %0.2f < %0.2f < %0.2f." % (MIN_CONNECTED_CURRENT, current, MAX_CONNECTED_CURRENT))
    self.StoreTestResultData("%0.2f" % current)

    self.assertTrue(current > MIN_CONNECTED_CURRENT
                    and current < MAX_CONNECTED_CURRENT)
    self.stopMe = False

  def test_045_LedsConnectedCurrent(self):
    self.i.DisplayMessage("Checking LEDs connected current...")

    MIN_CONNECTED_CURRENT = 30
    MAX_CONNECTED_CURRENT = 50

    current = self.testRig.measure('DUT_CURRENT')

    self.i.DisplayMessage("LEDs connected current: %0.2f < %0.2f < %0.2f." % (MIN_CONNECTED_CURRENT, current, MAX_CONNECTED_CURRENT))
    self.StoreTestResultData("%0.2f" % current)

    self.assertTrue(current > MIN_CONNECTED_CURRENT
                    and current < MAX_CONNECTED_CURRENT)
    self.stopMe = False

  def test_050_redLedsOnCurrent(self):
    self.i.DisplayMessage("Checking red LEDs on...")

    MIN_RED_CURRENT = 50
    MAX_RED_CURRENT = 100
    # TODO: Why send this twice?
    for j in range (0, 2):
      for x in range(0, 60):
        self.dut.sendPixel(255,0,0)
      self.dut.show();

    current = self.testRig.measure('DUT_CURRENT')

    self.i.DisplayMessage("Red LEDs current: %0.2f < %0.2f < %0.2f." % (MIN_RED_CURRENT, current, MAX_RED_CURRENT))
    self.StoreTestResultData("%0.2f" % current)

    self.assertTrue(current > MIN_RED_CURRENT
                    and current < MAX_RED_CURRENT)
    self.stopMe = False

  def skip_test_060_greenLedsOnCurrent(self):
    self.i.DisplayMessage("Checking green LEDs on...")

    MIN_GREEN_CURRENT = 50
    MAX_GREEN_CURRENT = 100
    # TODO: Why send this twice?
    for j in range (0, 2):
      for x in range(0, 60):
        self.dut.sendPixel(0,255,0)
      self.dut.show();

    current = self.testRig.measure('DUT_CURRENT')

    self.i.DisplayMessage("Green LEDs current: %0.2f < %0.2f < %0.2f." % (MIN_GREEN_CURRENT, current, MAX_GREEN_CURRENT))
    self.StoreTestResultData("%0.2f" % current)

    self.assertTrue(current > MIN_GREEN_CURRENT
                    and current < MAX_GREEN_CURRENT)
    self.stopMe = False

  def skip_test_070_blueLedsOnCurrent(self):
    self.i.DisplayMessage("Checking blue LEDs on...")

    MIN_BLUE_CURRENT = 50
    MAX_BLUE_CURRENT = 100
    # TODO: Why send this twice?
    for j in range (0, 2):
      for x in range(0, 60):
        self.dut.sendPixel(0,0,255)
      self.dut.show();

    current = self.testRig.measure('DUT_CURRENT')

    self.i.DisplayMessage("Blue LEDs current: %0.2f < %0.2f < %0.2f." % (MIN_BLUE_CURRENT, current, MAX_BLUE_CURRENT))
    self.StoreTestResultData("%0.2f" % current)

    self.assertTrue(current > MIN_BLUE_CURRENT
                    and current < MAX_BLUE_CURRENT)
    self.stopMe = False

  def test_080_whiteLedsOnCurrent(self):
    self.i.DisplayMessage("Checking white LEDs on...")

    MIN_WHITE_CURRENT = 100
    MAX_WHITE_CURRENT = 300
    # TODO: Why send this twice?
    for j in range (0, 2):
      for x in range(0, 60):
        self.dut.sendPixel(255,255,255)
      self.dut.show();

    current = self.testRig.measure('DUT_CURRENT')

    self.i.DisplayMessage("White LEDs current: %0.2f < %0.2f < %0.2f." % (MIN_WHITE_CURRENT, current, MAX_WHITE_CURRENT))
    self.StoreTestResultData("%0.2f" % current)

    self.assertTrue(current > MIN_WHITE_CURRENT
                    and current < MAX_WHITE_CURRENT)
    self.stopMe = False

  def test_090_D7_connected(self):
    self.i.DisplayMessage("Checking D7 input works...")
    self.testRig.setOutputLow('DUT_D7')

    pinStates = 0
    for j in range (0, 2):
      for x in range(0, 60):
        self.dut.sendPixel(0,0,0)
      pinStates = self.dut.show();

    self.testRig.setInput('DUT_D7')

    self.assertTrue(ord(pinStates[0]) == 11)
    self.stopMe = False

  def test_091_D11_connected(self):
    self.i.DisplayMessage("Checking D11 input works...")
    self.testRig.setOutputLow('DUT_D11')

    pinStates = 0
    for j in range (0, 2):
      for x in range(0, 60):
        self.dut.sendPixel(0,0,0)
      pinStates = self.dut.show();

    self.testRig.setInput('DUT_D11')

    self.assertTrue(ord(pinStates[0]) == 13)
    self.stopMe = False

  def test_092_A9_connected(self):
    self.i.DisplayMessage("Checking A9 input works...")
    self.testRig.setOutputLow('DUT_A9')

    pinStates = 0
    for j in range (0, 2):
      for x in range(0, 60):
        self.dut.sendPixel(0,0,0)
      pinStates = self.dut.show();

    self.testRig.setInput('DUT_A9')

    self.assertTrue(ord(pinStates[0]) == 14)
    self.stopMe = False

  def test_100_button_connected(self):
    self.i.DisplayMessage("Checking button input works...")

    MAX_TIME_SECONDS = 60;

    startTime = time.time()
    found = False
    mode = True

    while((not found) and (time.time() < startTime + MAX_TIME_SECONDS)):
      pinStates = 0
      for j in range (0, 2):
        for x in range(0, 60):
	  if mode:
            self.dut.sendPixel(0,0,100)
          else:
            self.dut.sendPixel(0,0,0)
        pinStates = self.dut.show();

      if ord(pinStates[0]) == 0x07:
        found = True

      mode = not mode
      time.sleep(.5)

    self.assertTrue(found)
    self.stopMe = False

<|endoftext|>