<commit_before># -*- coding: utf-8 -*-
import os
import glob
import pathlib2 as pathlib
from pybtex.database.input import bibtex
from pybtex.exceptions import PybtexError

def handle_files_args(*paths_args):
    """
    Handle file(s) arguments from command line

    This method takes the string(s) which were passed to the cli which indicate the files on which to operate. It expands the path arguments and creates a list of `pathlib.Path` objects which unambiguously point to the files indicated by the cli arguments.

    :param str *paths_args: Paths to files.
    :rtype: list
    """
    paths = []

    for paths_arg in paths_args:
        # Handle paths implicitly rooted at user home dir
        paths_arg = os.path.expanduser(paths_arg)

        # Expand wildcards
        paths_arg = glob.glob(paths_arg)

        # Create list of pathlib.Path objects
        paths.extend([pathlib.Path(path_arg) for path_arg in paths_arg])

    return paths

def import_bib_files(*paths):
    """
    List of dicts for use in refmanage

    For each argument passed to this method, a dictionary of relevant information is built. These dicts are assembled into a list and returned.

    :param pathlib.Path *paths: Path to BibTeX file.
    :rtype list:
    """
    bibs = [construct_bib_dict(path) for path in paths]
    return bibs

def construct_bib_dict(path):
    """
    Create dict containing data for use in refmanage

    This method creates dicts containing information for use in refmanage corresponding to the file indicated by `path`. Each dictionary contains the following data:

    * path: The `pathlib.Path` object passed as the argument to this method.
    * bib: BibTeX data, stored as a `pybtex.database.BibliographyData` object, parsed from the file indicated by the `path` key. If the file was unparseable, the exception raised during parsing is stored in this variable.
    * terse_msg: Message corresponding to the file to be printed to STDOUT when the "--verbose" flag is absent from the command line.
    * verbose_msg: Message corresponding to the file to be printed to STDOUT when the "--verbose" flag is passed on the command line.

    :param pathlib.Path path: Path to BibTeX file.
    """
    bib = parse_bib_file(path)
    terse_msg = generate_terse_output_message(path)
    verbose_msg = generate_verbose_err_output_message(bib)

    bib_dict = {"path": path,
        "bib": bib,
        "terse_msg": terse_msg,
        "verbose_msg": verbose_msg,}

    return bib_dict

def parse_bib_file(path):
    """
    Parse BibTeX file located at `path`

    This method attempts to parse the BibTeX file located at `path`. If the file is parseable, a `pybtex.database.BibliographyData` object is returned, containing the bibliography data contained in the file. If the file is unparseable, the exception raised by the parser is returned.

    :param pathlib.Path path: Path to BibTeX file.
    """
    parser = bibtex.Parser()
    try:
        bib = parser.parse_file(str(path.resolve()))
    except PybtexError, e:
        bib = e

    return bib

def bib_subdict(bibs, val_type):
    """
    Subdict with keys of instance `val_type`

    :param dict bibs:
    :param type val_type:
    :rtype dict:
    """
    subdict = {key: val for (key, val) in bibs.iteritems() if isinstance(val, val_type)}
    return subdict

def generate_terse_output_message(path):
    """
    Generate non-verbose output message
    """
    terse_msg = str(path.resolve())
    return terse_msg

def generate_verbose_err_output_message(err):
    """
    Generate output message for an error
    """
    msg = ""
    try:
        msg += val.error_type + "\n"
        msg += val.message + "\n"
        msg += val.linno + "\n"
        msg += get_context()
    except:
        pass

    return msg
<commit_msg>Reorg code; purely cosmetic change<commit_after># -*- coding: utf-8 -*-
import os
import glob
import pathlib2 as pathlib
from pybtex.database.input import bibtex
from pybtex.exceptions import PybtexError

def handle_files_args(*paths_args):
    """
    Handle file(s) arguments from command line

    This method takes the string(s) which were passed to the cli which indicate the files on which to operate. It expands the path arguments and creates a list of `pathlib.Path` objects which unambiguously point to the files indicated by the cli arguments.

    :param str *paths_args: Paths to files.
    :rtype: list
    """
    paths = []

    for paths_arg in paths_args:
        # Handle paths implicitly rooted at user home dir
        paths_arg = os.path.expanduser(paths_arg)

        # Expand wildcards
        paths_arg = glob.glob(paths_arg)

        # Create list of pathlib.Path objects
        paths.extend([pathlib.Path(path_arg) for path_arg in paths_arg])

    return paths

def import_bib_files(*paths):
    """
    List of dicts for use in refmanage

    For each argument passed to this method, a dictionary of relevant information is built. These dicts are assembled into a list and returned.

    :param pathlib.Path *paths: Path to BibTeX file.
    :rtype list:
    """
    bibs = [construct_bib_dict(path) for path in paths]
    return bibs

def construct_bib_dict(path):
    """
    Create dict containing data for use in refmanage

    This method creates dicts containing information for use in refmanage corresponding to the file indicated by `path`. Each dictionary contains the following data:

    * path: The `pathlib.Path` object passed as the argument to this method.
    * bib: BibTeX data, stored as a `pybtex.database.BibliographyData` object, parsed from the file indicated by the `path` key. If the file was unparseable, the exception raised during parsing is stored in this variable.
    * terse_msg: Message corresponding to the file to be printed to STDOUT when the "--verbose" flag is absent from the command line.
    * verbose_msg: Message corresponding to the file to be printed to STDOUT when the "--verbose" flag is passed on the command line.

    :param pathlib.Path path: Path to BibTeX file.
    """
    bib = parse_bib_file(path)
    terse_msg = generate_terse_output_message(path)
    verbose_msg = generate_verbose_err_output_message(bib)

    bib_dict = {"path": path,
        "bib": bib,
        "terse_msg": terse_msg,
        "verbose_msg": verbose_msg,}

    return bib_dict

def parse_bib_file(path):
    """
    Parse BibTeX file located at `path`

    This method attempts to parse the BibTeX file located at `path`. If the file is parseable, a `pybtex.database.BibliographyData` object is returned, containing the bibliography data contained in the file. If the file is unparseable, the exception raised by the parser is returned.

    :param pathlib.Path path: Path to BibTeX file.
    """
    parser = bibtex.Parser()
    try:
        bib = parser.parse_file(str(path.resolve()))
    except PybtexError, e:
        bib = e

    return bib

def generate_terse_output_message(path):
    """
    Generate non-verbose output message
    """
    terse_msg = str(path.resolve())
    return terse_msg

def generate_verbose_err_output_message(err):
    """
    Generate output message for an error
    """
    msg = ""
    try:
        msg += val.error_type + "\n"
        msg += val.message + "\n"
        msg += val.linno + "\n"
        msg += get_context()
    except:
        pass

    return msg

def bib_subdict(bibs, val_type):
    """
    Subdict with keys of instance `val_type`

    :param dict bibs:
    :param type val_type:
    :rtype dict:
    """
    subdict = {key: val for (key, val) in bibs.iteritems() if isinstance(val, val_type)}
    return subdict
<|endoftext|>


<commit_before># Copyright (c) 2006 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Ron Dreslinski

import m5
from m5.objects import *
import os, optparse, sys
m5.AddToPath('../common')

parser = optparse.OptionParser()

parser.add_option("--caches", action="store_true")
parser.add_option("-t", "--timing", action="store_true")
parser.add_option("-m", "--maxtick", type="int")
parser.add_option("-l", "--maxloads", default = "1000000000000", type="int")
parser.add_option("-n", "--numtesters", default = "8", type="int")
parser.add_option("-p", "--protocol",
                  default="moesi",
                  help="The coherence protocol to use for the L1'a (i.e. MOESI, MOSI)")

(options, args) = parser.parse_args()

if args:
     print "Error: script doesn't take any positional arguments"
     sys.exit(1)

# --------------------
# Base L1 Cache
# ====================

class L1(BaseCache):
    latency = 1
    block_size = 64
    mshrs = 12
    tgts_per_mshr = 8
    protocol = CoherenceProtocol(protocol=options.protocol)

# ----------------------
# Base L2 Cache
# ----------------------

class L2(BaseCache):
    block_size = 64
    latency = 10
    mshrs = 92
    tgts_per_mshr = 16
    write_buffers = 8

#MAX CORES IS 8 with the false sharing method
if options.numtesters > 8:
     print "Error: NUmber of testers limited to 8 because of false sharing"
     sys,exit(1)

if options.timing:
     cpus = [ MemTest(atomic=False, max_loads=options.maxloads, percent_functional=50,
                      percent_uncacheable=10, progress_interval=1000)
              for i in xrange(options.numtesters) ]
else:
     cpus = [ MemTest(atomic=True, max_loads=options.maxloads, percent_functional=50,
                      percent_uncacheable=10, progress_interval=1000)
              for i in xrange(options.numtesters) ]
# system simulated
system = System(cpu = cpus, funcmem = PhysicalMemory(),
                physmem = PhysicalMemory(), membus = Bus(clock="500GHz", width=16))

# l2cache & bus
if options.caches:
    system.toL2Bus = Bus(clock="500GHz", width=16)
    system.l2c = L2(size='64kB', assoc=8)
    system.l2c.cpu_side = system.toL2Bus.port

    # connect l2c to membus
    system.l2c.mem_side = system.membus.port

which_port = 0
# add L1 caches
for cpu in cpus:
    if options.caches:
         cpu.l1c = L1(size = '32kB', assoc = 4)
         cpu.test = cpu.l1c.cpu_side
         cpu.l1c.mem_side = system.toL2Bus.port
    else:
         cpu.test = system.membus.port
    if  which_port == 0:
         system.funcmem.port = cpu.functional
         which_port = 1
    else:
         system.funcmem.functional = cpu.functional

# connect memory to membus
system.physmem.port = system.membus.port

# -----------------------
# run simulation
# -----------------------

root = Root( system = system )
if options.timing:
    root.system.mem_mode = 'timing'
else:
    root.system.mem_mode = 'atomic'

# instantiate configuration
m5.instantiate(root)

# simulate until program terminates
if options.maxtick:
    exit_event = m5.simulate(options.maxtick)
else:
    exit_event = m5.simulate()

print 'Exiting @ tick', m5.curTick(), 'because', exit_event.getCause()
<commit_msg>Give physical memory some latency to stress the system<commit_after># Copyright (c) 2006 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Ron Dreslinski

import m5
from m5.objects import *
import os, optparse, sys
m5.AddToPath('../common')

parser = optparse.OptionParser()

parser.add_option("--caches", action="store_true")
parser.add_option("-t", "--timing", action="store_true")
parser.add_option("-m", "--maxtick", type="int")
parser.add_option("-l", "--maxloads", default = "1000000000000", type="int")
parser.add_option("-n", "--numtesters", default = "8", type="int")
parser.add_option("-p", "--protocol",
                  default="moesi",
                  help="The coherence protocol to use for the L1'a (i.e. MOESI, MOSI)")

(options, args) = parser.parse_args()

if args:
     print "Error: script doesn't take any positional arguments"
     sys.exit(1)

# --------------------
# Base L1 Cache
# ====================

class L1(BaseCache):
    latency = 1
    block_size = 64
    mshrs = 12
    tgts_per_mshr = 8
    protocol = CoherenceProtocol(protocol=options.protocol)

# ----------------------
# Base L2 Cache
# ----------------------

class L2(BaseCache):
    block_size = 64
    latency = 10
    mshrs = 92
    tgts_per_mshr = 16
    write_buffers = 8

#MAX CORES IS 8 with the false sharing method
if options.numtesters > 8:
     print "Error: NUmber of testers limited to 8 because of false sharing"
     sys,exit(1)

if options.timing:
     cpus = [ MemTest(atomic=False, max_loads=options.maxloads, percent_functional=50,
                      percent_uncacheable=10, progress_interval=1000)
              for i in xrange(options.numtesters) ]
else:
     cpus = [ MemTest(atomic=True, max_loads=options.maxloads, percent_functional=50,
                      percent_uncacheable=10, progress_interval=1000)
              for i in xrange(options.numtesters) ]
# system simulated
system = System(cpu = cpus, funcmem = PhysicalMemory(),
                physmem = PhysicalMemory(latency = "50ps"), membus = Bus(clock="500GHz", width=16))

# l2cache & bus
if options.caches:
    system.toL2Bus = Bus(clock="500GHz", width=16)
    system.l2c = L2(size='64kB', assoc=8)
    system.l2c.cpu_side = system.toL2Bus.port

    # connect l2c to membus
    system.l2c.mem_side = system.membus.port

which_port = 0
# add L1 caches
for cpu in cpus:
    if options.caches:
         cpu.l1c = L1(size = '32kB', assoc = 4)
         cpu.test = cpu.l1c.cpu_side
         cpu.l1c.mem_side = system.toL2Bus.port
    else:
         cpu.test = system.membus.port
    if  which_port == 0:
         system.funcmem.port = cpu.functional
         which_port = 1
    else:
         system.funcmem.functional = cpu.functional

# connect memory to membus
system.physmem.port = system.membus.port

# -----------------------
# run simulation
# -----------------------

root = Root( system = system )
if options.timing:
    root.system.mem_mode = 'timing'
else:
    root.system.mem_mode = 'atomic'

# instantiate configuration
m5.instantiate(root)

# simulate until program terminates
if options.maxtick:
    exit_event = m5.simulate(options.maxtick)
else:
    exit_event = m5.simulate()

print 'Exiting @ tick', m5.curTick(), 'because', exit_event.getCause()
<|endoftext|>


<commit_before># Copyright (c) 2009-2011 Advanced Micro Devices, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Brad Beckmann

#
# Full system configuraiton for ruby
#

import os
import optparse
import sys
from os.path import join as joinpath

import m5
from m5.defines import buildEnv
from m5.objects import *
from m5.util import addToPath, panic

if not buildEnv['FULL_SYSTEM']:
    panic("This script requires full-system mode (*_FS).")

addToPath('../common')
addToPath('../ruby')

import Ruby

from FSConfig import *
from SysPaths import *
from Benchmarks import *
import Simulation
from Caches import *

# Get paths we might need.  It's expected this file is in m5/configs/example.
config_path = os.path.dirname(os.path.abspath(__file__))
config_root = os.path.dirname(config_path)
m5_root = os.path.dirname(config_root)

parser = optparse.OptionParser()

# Benchmark options
parser.add_option("-b", "--benchmark", action="store", type="string",
                  dest="benchmark",
                  help="Specify the benchmark to run. Available benchmarks: %s"\
                  % DefinedBenchmarks)
parser.add_option("-o", "--options", default="",
    help='The options to pass to the binary, use " " around the entire string')
parser.add_option("-i", "--input", default="", help="Read stdin from a file.")
parser.add_option("--output", default="", help="Redirect stdout to a file.")
parser.add_option("--errout", default="", help="Redirect stderr to a file.")

#
# Add the ruby specific and protocol specific options
#
Ruby.define_options(parser)

execfile(os.path.join(config_root, "common", "Options.py"))

(options, args) = parser.parse_args()

if args:
    print "Error: script doesn't take any positional arguments"
    sys.exit(1)

if options.benchmark:
    try:
        bm = Benchmarks[options.benchmark]
    except KeyError:
        print "Error benchmark %s has not been defined." % options.benchmark
        print "Valid benchmarks are: %s" % DefinedBenchmarks
        sys.exit(1)
else:
    bm = [SysConfig()]

#
# currently ruby fs only works in simple timing mode because ruby does not
# support atomic accesses by devices.  Also ruby_fs currently assumes
# that is running a checkpoints that were created by ALPHA_FS under atomic
# mode.  Since switch cpus are not defined in these checkpoints, we don't
# fast forward with the atomic cpu and instead set the FutureClass to None.
# Therefore the cpus resolve to the correct names and unserialize correctly.
#
class CPUClass(TimingSimpleCPU): pass
test_mem_mode = 'timing'
FutureClass = None

CPUClass.clock = options.clock

if buildEnv['TARGET_ISA'] == "alpha":
    system = makeLinuxAlphaRubySystem(test_mem_mode, bm[0])
elif buildEnv['TARGET_ISA'] == "x86":
    system = makeLinuxX86System(test_mem_mode, options.num_cpus, bm[0], True)
    setWorkCountOptions(system, options)
else:
    fatal("incapable of building non-alpha or non-x86 full system!")

Ruby.create_system(options, system, system.piobus, system._dma_devices)

system.cpu = [CPUClass(cpu_id=i) for i in xrange(options.num_cpus)]

for (i, cpu) in enumerate(system.cpu):
    #
    # Tie the cpu ports to the correct ruby system ports
    #
    cpu.icache_port = system.ruby._cpu_ruby_ports[i].port
    cpu.dcache_port = system.ruby._cpu_ruby_ports[i].port
    if buildEnv['TARGET_ISA'] == "x86":
        cpu.itb.walker.port = system.ruby._cpu_ruby_ports[i].port
        cpu.dtb.walker.port = system.ruby._cpu_ruby_ports[i].port
        cpu.interrupts.pio = system.piobus.port
        cpu.interrupts.int_port = system.piobus.port

root = Root(system = system)

Simulation.run(options, root, system, FutureClass)
<commit_msg>Ruby FS: Add the options for kernel and simulation script These options were missing from the script ruby_fs.py. This patch adds these options to the script.<commit_after># Copyright (c) 2009-2011 Advanced Micro Devices, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Brad Beckmann

#
# Full system configuraiton for ruby
#

import optparse
import os
import sys
from os.path import join as joinpath

import m5
from m5.defines import buildEnv
from m5.objects import *
from m5.util import addToPath, fatal

if not buildEnv['FULL_SYSTEM']:
    fatal("This script requires full-system mode (*_FS).")

addToPath('../common')
addToPath('../ruby')

import Ruby

from FSConfig import *
from SysPaths import *
from Benchmarks import *
import Simulation
from Caches import *

# Get paths we might need.  It's expected this file is in m5/configs/example.
config_path = os.path.dirname(os.path.abspath(__file__))
config_root = os.path.dirname(config_path)
m5_root = os.path.dirname(config_root)

parser = optparse.OptionParser()
# System options
parser.add_option("--kernel", action="store", type="string")
parser.add_option("--script", action="store", type="string")
# Benchmark options
parser.add_option("-b", "--benchmark", action="store", type="string",
                  dest="benchmark",
                  help="Specify the benchmark to run. Available benchmarks: %s"\
                  % DefinedBenchmarks)
parser.add_option("-o", "--options", default="",
    help='The options to pass to the binary, use " " around the entire string')
parser.add_option("-i", "--input", default="", help="Read stdin from a file.")
parser.add_option("--output", default="", help="Redirect stdout to a file.")
parser.add_option("--errout", default="", help="Redirect stderr to a file.")

#
# Add the ruby specific and protocol specific options
#
Ruby.define_options(parser)

execfile(os.path.join(config_root, "common", "Options.py"))

(options, args) = parser.parse_args()

if args:
    print "Error: script doesn't take any positional arguments"
    sys.exit(1)

if options.benchmark:
    try:
        bm = Benchmarks[options.benchmark]
    except KeyError:
        print "Error benchmark %s has not been defined." % options.benchmark
        print "Valid benchmarks are: %s" % DefinedBenchmarks
        sys.exit(1)
else:
    bm = [SysConfig()]

#
# currently ruby fs only works in simple timing mode because ruby does not
# support atomic accesses by devices.  Also ruby_fs currently assumes
# that is running a checkpoints that were created by ALPHA_FS under atomic
# mode.  Since switch cpus are not defined in these checkpoints, we don't
# fast forward with the atomic cpu and instead set the FutureClass to None.
# Therefore the cpus resolve to the correct names and unserialize correctly.
#
class CPUClass(TimingSimpleCPU): pass
test_mem_mode = 'timing'
FutureClass = None

CPUClass.clock = options.clock

if buildEnv['TARGET_ISA'] == "alpha":
    system = makeLinuxAlphaRubySystem(test_mem_mode, bm[0])
elif buildEnv['TARGET_ISA'] == "x86":
    system = makeLinuxX86System(test_mem_mode, options.num_cpus, bm[0], True)
    setWorkCountOptions(system, options)
else:
    fatal("incapable of building non-alpha or non-x86 full system!")

if options.kernel is not None:
    system.kernel = binary(options.kernel)

if options.script is not None:
    system.readfile = options.script

system.cpu = [CPUClass(cpu_id=i) for i in xrange(options.num_cpus)]
Ruby.create_system(options, system, system.piobus, system._dma_devices)

for (i, cpu) in enumerate(system.cpu):
    #
    # Tie the cpu ports to the correct ruby system ports
    #
    cpu.icache_port = system.ruby._cpu_ruby_ports[i].port
    cpu.dcache_port = system.ruby._cpu_ruby_ports[i].port
    if buildEnv['TARGET_ISA'] == "x86":
        cpu.itb.walker.port = system.ruby._cpu_ruby_ports[i].port
        cpu.dtb.walker.port = system.ruby._cpu_ruby_ports[i].port
        cpu.interrupts.pio = system.piobus.port
        cpu.interrupts.int_port = system.piobus.port

root = Root(system = system)

Simulation.run(options, root, system, FutureClass)
<|endoftext|>


<commit_before>""" generic mechanism for marking and selecting python functions. """
import py

class MarkerError(Exception):
    """Error in use of a pytest marker/attribute"""

def pytest_namespace():
    return {'mark': MarkGenerator()}

def pytest_addoption(parser):
    group = parser.getgroup("general")
    group._addoption(
        '-k',
        action="store", dest="keyword", default='', metavar="EXPRESSION",
        help="only run tests which match the given substring expression. "
             "An expression is a python evaluatable expression "
             "where all names are substring-matched against test names "
             "and their parent classes. Example: -k 'test_method or test "
             "other' matches all test functions and classes whose name "
             "contains 'test_method' or 'test_other'. "
             "Additionally keywords are matched to classes and functions "
             "containing extra names in their 'extra_keyword_matches' set, "
             "as well as functions which have names assigned directly to them."
    )

    group._addoption(
        "-m",
        action="store", dest="markexpr", default="", metavar="MARKEXPR",
        help="only run tests matching given mark expression.  "
             "example: -m 'mark1 and not mark2'."
    )

    group.addoption(
        "--markers", action="store_true",
        help="show markers (builtin, plugin and per-project ones)."
    )

    parser.addini("markers", "markers for test functions", 'linelist')

def pytest_cmdline_main(config):
    if config.option.markers:
        config.do_configure()
        tw = py.io.TerminalWriter()
        for line in config.getini("markers"):
            name, rest = line.split(":", 1)
            tw.write("@pytest.mark.%s:" % name, bold=True)
            tw.line(rest)
            tw.line()
        config.do_unconfigure()
        return 0
pytest_cmdline_main.tryfirst = True

def pytest_collection_modifyitems(items, config):
    keywordexpr = config.option.keyword
    matchexpr = config.option.markexpr
    if not keywordexpr and not matchexpr:
        return
    # pytest used to allow "-" for negating
    # but today we just allow "-" at the beginning, use "not" instead
    # we probably remove "-" alltogether soon
    if keywordexpr.startswith("-"):
        keywordexpr = "not " + keywordexpr[1:]
    selectuntil = False
    if keywordexpr[-1:] == ":":
        selectuntil = True
        keywordexpr = keywordexpr[:-1]

    remaining = []
    deselected = []
    for colitem in items:
        if keywordexpr and not matchkeyword(colitem, keywordexpr):
            deselected.append(colitem)
        else:
            if selectuntil:
                keywordexpr = None
            if matchexpr:
                if not matchmark(colitem, matchexpr):
                    deselected.append(colitem)
                    continue
            remaining.append(colitem)

    if deselected:
        config.hook.pytest_deselected(items=deselected)
        items[:] = remaining

class MarkMapping:
    """Provides a local mapping for markers where item access
    resolves to True if the marker is present. """
    def __init__(self, keywords):
        mymarks = set()
        for key, value in keywords.items():
            if isinstance(value, MarkInfo) or isinstance(value, MarkDecorator):
                mymarks.add(key)
        self._mymarks = mymarks

    def __getitem__(self, name):
        return name in self._mymarks

class KeywordMapping:
    """Provides a local mapping for keywords.
    Given a list of names, map any substring of one of these names to True.
    """
    def __init__(self, names):
        self._names = names

    def __getitem__(self, subname):
        for name in self._names:
            if subname in name:
                return True
        return False

def matchmark(colitem, markexpr):
    """Tries to match on any marker names, attached to the given colitem."""
    return eval(markexpr, {}, MarkMapping(colitem.keywords))

def matchkeyword(colitem, keywordexpr):
    """Tries to match given keyword expression to given collector item.

    Will match on the name of colitem, including the names of its parents.
    Only matches names of items which are either a :class:`Class` or a
    :class:`Function`.
    Additionally, matches on names in the 'extra_keyword_matches' set of
    any item, as well as names directly assigned to test functions.
    """
    mapped_names = set()

    # Add the names of the current item and any parent items
    import pytest
    for item in colitem.listchain():
        if not isinstance(item, pytest.Instance):
            mapped_names.add(item.name)

    # Add the names added as extra keywords to current or parent items
    for name in colitem.listextrakeywords():
        mapped_names.add(name)

    # Add the names attached to the current function through direct assignment
    if hasattr(colitem, 'function'):
        for name in colitem.function.__dict__:
            mapped_names.add(name)

    mapping = KeywordMapping(mapped_names)
    if " " not in keywordexpr:
        # special case to allow for simple "-k pass" and "-k 1.3"
        return mapping[keywordexpr]
    elif keywordexpr.startswith("not ") and " " not in keywordexpr[4:]:
        return not mapping[keywordexpr[4:]]
    return eval(keywordexpr, {}, mapping)

def pytest_configure(config):
    import pytest
    if config.option.strict:
        pytest.mark._config = config

class MarkGenerator:
    """ Factory for :class:`MarkDecorator` objects - exposed as
    a ``pytest.mark`` singleton instance.  Example::

         import py
         @pytest.mark.slowtest
         def test_function():
            pass

    will set a 'slowtest' :class:`MarkInfo` object
    on the ``test_function`` object. """

    def __getattr__(self, name):
        if name[0] == "_":
            raise AttributeError("Marker name must NOT start with underscore")
        if hasattr(self, '_config'):
            self._check(name)
        return MarkDecorator(name)

    def _check(self, name):
        try:
            if name in self._markers:
                return
        except AttributeError:
            pass
        self._markers = l = set()
        for line in self._config.getini("markers"):
            beginning = line.split(":", 1)
            x = beginning[0].split("(", 1)[0]
            l.add(x)
        if name not in self._markers:
            raise AttributeError("%r not a registered marker" % (name,))

def istestfunc(func):
    return hasattr(func, "__call__") and \
        getattr(func, "__name__", "<lambda>") != "<lambda>"

class MarkDecorator:
    """ A decorator for test functions and test classes.  When applied
    it will create :class:`MarkInfo` objects which may be
    :ref:`retrieved by hooks as item keywords <excontrolskip>`.
    MarkDecorator instances are often created like this::

        mark1 = pytest.mark.NAME              # simple MarkDecorator
        mark2 = pytest.mark.NAME(name1=value) # parametrized MarkDecorator

    and can then be applied as decorators to test functions::

        @mark2
        def test_function():
            pass

    When a MarkDecorator instance is called it does the following:
      1. If called with a single class as its only positional argument and no
         additional keyword arguments, it attaches itself to the class so it
         gets applied automatically to all test cases found in that class.
      2. If called with a single function as its only positional argument and
         no additional keyword arguments, it attaches a MarkInfo object to the
         function, containing all the arguments already stored internally in
         the MarkDecorator.
      3. When called in any other case, it performs a 'fake construction' call,
         i.e. it returns a new MarkDecorator instance with the original
         MarkDecorator's content updated with the arguments passed to this
         call.

    Note: The rules above prevent MarkDecorator objects from storing only a
    single function or class reference as their positional argument with no
    additional keyword or positional arguments.

    """
    def __init__(self, name, args=None, kwargs=None):
        self.name = name
        self.args = args or ()
        self.kwargs = kwargs or {}

    @property
    def markname(self):
        return self.name # for backward-compat (2.4.1 had this attr)

    def __repr__(self):
        d = self.__dict__.copy()
        name = d.pop('name')
        return "<MarkDecorator %r %r>" % (name, d)

    def __call__(self, *args, **kwargs):
        """ if passed a single callable argument: decorate it with mark info.
            otherwise add *args/**kwargs in-place to mark information. """
        if args and not kwargs:
            func = args[0]
            if len(args) == 1 and (istestfunc(func) or
                                   hasattr(func, '__bases__')):
                if hasattr(func, '__bases__'):
                    if hasattr(func, 'pytestmark'):
                        l = func.pytestmark
                        if not isinstance(l, list):
                            func.pytestmark = [l, self]
                        else:
                            l.append(self)
                    else:
                        func.pytestmark = [self]
                else:
                    holder = getattr(func, self.name, None)
                    if holder is None:
                        holder = MarkInfo(
                            self.name, self.args, self.kwargs
                        )
                        setattr(func, self.name, holder)
                    else:
                        holder.add(self.args, self.kwargs)
                return func
        kw = self.kwargs.copy()
        kw.update(kwargs)
        args = self.args + args
        return self.__class__(self.name, args=args, kwargs=kw)

class MarkInfo:
    """ Marking object created by :class:`MarkDecorator` instances. """
    def __init__(self, name, args, kwargs):
        #: name of attribute
        self.name = name
        #: positional argument list, empty if none specified
        self.args = args
        #: keyword argument dictionary, empty if nothing specified
        self.kwargs = kwargs
        self._arglist = [(args, kwargs.copy())]

    def __repr__(self):
        return "<MarkInfo %r args=%r kwargs=%r>" % (
            self.name, self.args, self.kwargs
        )

    def add(self, args, kwargs):
        """ add a MarkInfo with the given args and kwargs. """
        self._arglist.append((args, kwargs))
        self.args += args
        self.kwargs.update(kwargs)

    def __iter__(self):
        """ yield MarkInfo objects each relating to a marking-call. """
        for args, kwargs in self._arglist:
            yield MarkInfo(self.name, args, kwargs)
<commit_msg>Change docstring style<commit_after>""" generic mechanism for marking and selecting python functions. """
import py

class MarkerError(Exception):
    """
    Error in use of a pytest marker/attribute.
    """

def pytest_namespace():
    return {'mark': MarkGenerator()}

def pytest_addoption(parser):
    group = parser.getgroup("general")
    group._addoption(
        '-k',
        action="store", dest="keyword", default='', metavar="EXPRESSION",
        help="only run tests which match the given substring expression. "
             "An expression is a python evaluatable expression "
             "where all names are substring-matched against test names "
             "and their parent classes. Example: -k 'test_method or test "
             "other' matches all test functions and classes whose name "
             "contains 'test_method' or 'test_other'. "
             "Additionally keywords are matched to classes and functions "
             "containing extra names in their 'extra_keyword_matches' set, "
             "as well as functions which have names assigned directly to them."
    )

    group._addoption(
        "-m",
        action="store", dest="markexpr", default="", metavar="MARKEXPR",
        help="only run tests matching given mark expression.  "
             "example: -m 'mark1 and not mark2'."
    )

    group.addoption(
        "--markers", action="store_true",
        help="show markers (builtin, plugin and per-project ones)."
    )

    parser.addini("markers", "markers for test functions", 'linelist')

def pytest_cmdline_main(config):
    if config.option.markers:
        config.do_configure()
        tw = py.io.TerminalWriter()
        for line in config.getini("markers"):
            name, rest = line.split(":", 1)
            tw.write("@pytest.mark.%s:" % name, bold=True)
            tw.line(rest)
            tw.line()
        config.do_unconfigure()
        return 0
pytest_cmdline_main.tryfirst = True

def pytest_collection_modifyitems(items, config):
    keywordexpr = config.option.keyword
    matchexpr = config.option.markexpr
    if not keywordexpr and not matchexpr:
        return
    # pytest used to allow "-" for negating
    # but today we just allow "-" at the beginning, use "not" instead
    # we probably remove "-" alltogether soon
    if keywordexpr.startswith("-"):
        keywordexpr = "not " + keywordexpr[1:]
    selectuntil = False
    if keywordexpr[-1:] == ":":
        selectuntil = True
        keywordexpr = keywordexpr[:-1]

    remaining = []
    deselected = []
    for colitem in items:
        if keywordexpr and not matchkeyword(colitem, keywordexpr):
            deselected.append(colitem)
        else:
            if selectuntil:
                keywordexpr = None
            if matchexpr:
                if not matchmark(colitem, matchexpr):
                    deselected.append(colitem)
                    continue
            remaining.append(colitem)

    if deselected:
        config.hook.pytest_deselected(items=deselected)
        items[:] = remaining

class MarkMapping:
    """Provides a local mapping for markers where item access
    resolves to True if the marker is present. """
    def __init__(self, keywords):
        mymarks = set()
        for key, value in keywords.items():
            if isinstance(value, MarkInfo) or isinstance(value, MarkDecorator):
                mymarks.add(key)
        self._mymarks = mymarks

    def __getitem__(self, name):
        return name in self._mymarks

class KeywordMapping:
    """Provides a local mapping for keywords.
    Given a list of names, map any substring of one of these names to True.
    """
    def __init__(self, names):
        self._names = names

    def __getitem__(self, subname):
        for name in self._names:
            if subname in name:
                return True
        return False

def matchmark(colitem, markexpr):
    """Tries to match on any marker names, attached to the given colitem."""
    return eval(markexpr, {}, MarkMapping(colitem.keywords))

def matchkeyword(colitem, keywordexpr):
    """Tries to match given keyword expression to given collector item.

    Will match on the name of colitem, including the names of its parents.
    Only matches names of items which are either a :class:`Class` or a
    :class:`Function`.
    Additionally, matches on names in the 'extra_keyword_matches' set of
    any item, as well as names directly assigned to test functions.
    """
    mapped_names = set()

    # Add the names of the current item and any parent items
    import pytest
    for item in colitem.listchain():
        if not isinstance(item, pytest.Instance):
            mapped_names.add(item.name)

    # Add the names added as extra keywords to current or parent items
    for name in colitem.listextrakeywords():
        mapped_names.add(name)

    # Add the names attached to the current function through direct assignment
    if hasattr(colitem, 'function'):
        for name in colitem.function.__dict__:
            mapped_names.add(name)

    mapping = KeywordMapping(mapped_names)
    if " " not in keywordexpr:
        # special case to allow for simple "-k pass" and "-k 1.3"
        return mapping[keywordexpr]
    elif keywordexpr.startswith("not ") and " " not in keywordexpr[4:]:
        return not mapping[keywordexpr[4:]]
    return eval(keywordexpr, {}, mapping)

def pytest_configure(config):
    import pytest
    if config.option.strict:
        pytest.mark._config = config

class MarkGenerator:
    """ Factory for :class:`MarkDecorator` objects - exposed as
    a ``pytest.mark`` singleton instance.  Example::

         import py
         @pytest.mark.slowtest
         def test_function():
            pass

    will set a 'slowtest' :class:`MarkInfo` object
    on the ``test_function`` object. """

    def __getattr__(self, name):
        if name[0] == "_":
            raise AttributeError("Marker name must NOT start with underscore")
        if hasattr(self, '_config'):
            self._check(name)
        return MarkDecorator(name)

    def _check(self, name):
        try:
            if name in self._markers:
                return
        except AttributeError:
            pass
        self._markers = l = set()
        for line in self._config.getini("markers"):
            beginning = line.split(":", 1)
            x = beginning[0].split("(", 1)[0]
            l.add(x)
        if name not in self._markers:
            raise AttributeError("%r not a registered marker" % (name,))

def istestfunc(func):
    return hasattr(func, "__call__") and \
        getattr(func, "__name__", "<lambda>") != "<lambda>"

class MarkDecorator:
    """ A decorator for test functions and test classes.  When applied
    it will create :class:`MarkInfo` objects which may be
    :ref:`retrieved by hooks as item keywords <excontrolskip>`.
    MarkDecorator instances are often created like this::

        mark1 = pytest.mark.NAME              # simple MarkDecorator
        mark2 = pytest.mark.NAME(name1=value) # parametrized MarkDecorator

    and can then be applied as decorators to test functions::

        @mark2
        def test_function():
            pass

    When a MarkDecorator instance is called it does the following:
      1. If called with a single class as its only positional argument and no
         additional keyword arguments, it attaches itself to the class so it
         gets applied automatically to all test cases found in that class.
      2. If called with a single function as its only positional argument and
         no additional keyword arguments, it attaches a MarkInfo object to the
         function, containing all the arguments already stored internally in
         the MarkDecorator.
      3. When called in any other case, it performs a 'fake construction' call,
         i.e. it returns a new MarkDecorator instance with the original
         MarkDecorator's content updated with the arguments passed to this
         call.

    Note: The rules above prevent MarkDecorator objects from storing only a
    single function or class reference as their positional argument with no
    additional keyword or positional arguments.

    """
    def __init__(self, name, args=None, kwargs=None):
        self.name = name
        self.args = args or ()
        self.kwargs = kwargs or {}

    @property
    def markname(self):
        return self.name # for backward-compat (2.4.1 had this attr)

    def __repr__(self):
        d = self.__dict__.copy()
        name = d.pop('name')
        return "<MarkDecorator %r %r>" % (name, d)

    def __call__(self, *args, **kwargs):
        """ if passed a single callable argument: decorate it with mark info.
            otherwise add *args/**kwargs in-place to mark information. """
        if args and not kwargs:
            func = args[0]
            if len(args) == 1 and (istestfunc(func) or
                                   hasattr(func, '__bases__')):
                if hasattr(func, '__bases__'):
                    if hasattr(func, 'pytestmark'):
                        l = func.pytestmark
                        if not isinstance(l, list):
                            func.pytestmark = [l, self]
                        else:
                            l.append(self)
                    else:
                        func.pytestmark = [self]
                else:
                    holder = getattr(func, self.name, None)
                    if holder is None:
                        holder = MarkInfo(
                            self.name, self.args, self.kwargs
                        )
                        setattr(func, self.name, holder)
                    else:
                        holder.add(self.args, self.kwargs)
                return func
        kw = self.kwargs.copy()
        kw.update(kwargs)
        args = self.args + args
        return self.__class__(self.name, args=args, kwargs=kw)

class MarkInfo:
    """ Marking object created by :class:`MarkDecorator` instances. """
    def __init__(self, name, args, kwargs):
        #: name of attribute
        self.name = name
        #: positional argument list, empty if none specified
        self.args = args
        #: keyword argument dictionary, empty if nothing specified
        self.kwargs = kwargs
        self._arglist = [(args, kwargs.copy())]

    def __repr__(self):
        return "<MarkInfo %r args=%r kwargs=%r>" % (
            self.name, self.args, self.kwargs
        )

    def add(self, args, kwargs):
        """ add a MarkInfo with the given args and kwargs. """
        self._arglist.append((args, kwargs))
        self.args += args
        self.kwargs.update(kwargs)

    def __iter__(self):
        """ yield MarkInfo objects each relating to a marking-call. """
        for args, kwargs in self._arglist:
            yield MarkInfo(self.name, args, kwargs)
<|endoftext|>


<commit_before># coding=utf-8
"""
I18n.

To mark strings for transalation::

    from abilian.i18n import _
    _(u'message to translate')

Use :data:`_` for gettext, :data:`_l` for lazy_gettext, :data:`_n` for
ngettext.

:data:`Babel extension<babel>` support multiple translation paths. This
allows to add more catalogs to search for translations, in LIFO
order. This feature can be used to override some translations in a
custom application, be providing a catalog with messages to override::

    current_app.extensions['babel'].add_translations('abilian.core')

See :meth:`add_translations<Babel.add_translations>`.

To extract messages to build the message catalog template (.pot), use the
following "`-k`" parameters:

.. code-block:: bash

    $ pybabel extract -F babel.cfg -k "_n:1,2" -k "_l" -o "msg.pot" "src"

This can be made easier by placing in `setup.cfg`:

.. code-block:: ini

    [extract_messages]
    mapping_file = babel.cfg
    keywords = _n:1,2 _l
    output-file = msg.pot
    input-dirs = src

And just type:

.. code-block:: bash

    $ python setup.py extract_messages

"""
from __future__ import absolute_import, print_function, division

import os
import re
import importlib
import unicodedata
from pathlib import Path
from contextlib import contextmanager
from datetime import datetime

import pytz
from babel import Locale
from babel.localedata import locale_identifiers
from babel.support import Translations
from babel.dates import LOCALTZ, get_timezone, get_timezone_gmt
from flask import g, request, _request_ctx_stack, current_app, render_template
import flask_babel
from flask_babel import (
    Babel as BabelBase,
    gettext, lazy_gettext, ngettext,
)

__all__ = [
  'babel',
  'gettext', '_',
  'lazy_gettext', '_l', 'localeselector',
  'ngettext', '_n',
  'set_locale',
  'timezoneselector',
  'VALID_LANGUAGES_CODE',
  'render_template_i18n'
]

#: gettext alias
_ = gettext

#: lazy_gettext alias
_l = lazy_gettext
#: ngettext alias
_n = ngettext

#: accepted languages codes
VALID_LANGUAGES_CODE = frozenset(lang for lang in locale_identifiers()
                                 if len(lang) == 2)

def get_default_locale():
  return current_app.extensions['babel'].default_locale

def _get_locale():
  locale = flask_babel.get_locale()
  if locale is None:
    locale = get_default_locale()
  return locale

def __gettext_territory(code):
  locale = _get_locale()
  return (
      locale.territories.get(code)
      or get_default_locale().territories.get(code))

#: get localized territory name
def country_name(code):
  return __gettext_territory(code)

#: lazy version of :func:`country_name`
def lazy_country_name(code):
  from speaklater import make_lazy_string
  return make_lazy_string(__gettext_territory, code)

def default_country():
  return current_app.config.get('DEFAULT_COUNTRY')

def country_choices(first=None, default_country_first=True):
  """
  Return a list of (code, countries), alphabetically sorted on localized
  country name.

  :param first: Country code to be placed at the top
  """
  locale = _get_locale()
  territories = [(code, name) for code, name in locale.territories.iteritems()
                 if len(code) == 2] # skip 3-digit regions

  if first is None and default_country_first:
    first = default_country()

  def sortkey(item):
    if first is not None and item[0] == first:
      return u'0'
    return to_lower_ascii(item[1])

  territories.sort(key=sortkey)
  return territories

def supported_app_locales():
  """
  language codes and labels supported by current application

  :return: an iterable of `(:class:`babel.Locale`, label)`, label being the
  locale language human name in current locale.
  """
  locale = _get_locale()
  codes = current_app.config['BABEL_ACCEPT_LANGUAGES']
  return ((Locale.parse(code), locale.languages.get(code, code))
          for code in codes)

def timezones_choices():
  """
  Timezones values and their labels for current locale.

  :return: an iterable of `(code, label)`, code being a timezone code and label
  the timezone name in current locale.
  """
  utcnow = pytz.utc.localize(datetime.utcnow())
  locale = _get_locale()
  for tz in sorted(pytz.common_timezones):
    tz = get_timezone(tz)
    now = tz.normalize(utcnow.astimezone(tz))
    label = u'({}) {}'.format(get_timezone_gmt(now, locale=locale), tz.zone)
    yield (tz, label) #get_timezone_name(tz, locale=locale))

class Babel(BabelBase):
  """
  Allow to load translations from other modules
  """
  _translations_paths = None

  def __init__(self, *args, **kwargs):
    BabelBase.__init__(self, *args, **kwargs)

  def init_app(self, app):
    super(Babel, self).init_app(app)
    self._translations_paths = [
      (os.path.join(app.root_path, 'translations'), 'messages')
    ]

  def add_translations(self, module_name,
                       translations_dir='translations',
                       domain='messages'):
    """
    Adds translations from external module. For example::

        babel.add_translations('abilian.core')

    Will add translations files from `abilian.core` module.
    """
    module = importlib.import_module(module_name)
    for path in (Path(p, translations_dir) for p in module.__path__):
      if not (path.exists() and path.is_dir()):
        continue

      if not os.access(unicode(path), os.R_OK):
        self.app.logger.warning(
          "Babel translations: read access not allowed {}, skipping."
          "".format(
            repr(unicode((path)).encode('utf-8')))
        )
        continue

      self._translations_paths.append((unicode(path), domain))

def _get_translations_multi_paths():
  """
  Returns the correct gettext translations that should be used for this
  request. This will never fail and return a dummy translation object
  if used outside of the request or if a translation cannot be found.
  """
  ctx = _request_ctx_stack.top
  if ctx is None:
      return None

  translations = getattr(ctx, 'babel_translations', None)
  if translations is None:
    babel_ext = ctx.app.extensions['babel']
    translations = None
    trs = None

    # reverse order: thus the application catalog is loaded last, so that
    # translations from libraries can be overriden
    for (dirname, domain) in reversed(babel_ext._translations_paths):
      trs = Translations.load(dirname,
                              locales=[flask_babel.get_locale()],
                              domain=domain)

      # babel.support.Translations is a subclass of
      # babel.support.NullTranslations, so we test if object has a 'merge'
      # method

      if not trs or not hasattr(trs, 'merge'):
        # got None or NullTranslations instance
        continue
      elif (translations is not None and hasattr(translations, 'merge')):
          translations.merge(trs)
      else:
          translations = trs

    # ensure translations is at least a NullTranslations object
    if translations is None:
      translations = trs

    ctx.babel_translations = translations

  return translations

# monkey patch flask-babel
flask_babel.get_translations = _get_translations_multi_paths

#: importable instance of :class:`Babel`
babel = Babel()

def localeselector():
  """
  Default locale selector used in abilian applications
  """
  # if a user is logged in, use the locale from the user settings
  user = getattr(g, 'user', None)
  if user is not None:
    locale = getattr(user, 'locale', None)
    if locale:
      return locale

  # Otherwise, try to guess the language from the user accept header the browser
  # transmits.  By default we support en/fr. The best match wins.
  return request.accept_languages.best_match(
    current_app.config['BABEL_ACCEPT_LANGUAGES']
  )

def timezoneselector():
  """
  Default timezone selector used in abilian applications
  """
  return LOCALTZ

@contextmanager
def set_locale(locale):
  """
  Change current locale.

  Can be used as a context manager to temporary change locale::

      with set_locale('fr') as fr_locale:
          ...

  :type locale: :class:`babel.core.Locale` or `str`
  :param locale: locale to use. If it's a string if must be a valid locale
                 specification
  :rtype: :class:`babel.core.Locale`
  :return: locale set
  """
  ctx = _request_ctx_stack.top
  if ctx is None:
    yield

  if not isinstance(locale, Locale):
    locale = Locale.parse(locale)

  current_locale = getattr(ctx, 'babel_locale', None)
  ctx.babel_locale = locale
  yield locale
  ctx.babel_locale = current_locale

def get_template_i18n(template_name, locale):
  """
    Build template list with preceding locale if found
  """
  if locale is None:
    return template_name

  template_list = []
  parts = template_name.rsplit('.', 1)
  root = parts[0]
  suffix = parts[1]

  if locale.territory is not None:
    locale_string = '_'.join([locale.language, locale.territory])
    localized_template_path = u'.'.join([root, locale_string, suffix])
    template_list.append(localized_template_path)

  localized_template_path = u'.'.join([root, locale.language, suffix])
  template_list.append(localized_template_path)

  # append the default
  template_list.append(template_name)
  return template_list

def render_template_i18n(template_name_or_list, **context):
  """
    Try to build an ordered list of template to satisfy the current locale
  """
  template_list = []
  # Use locale if present in **context
  if 'locale' in context:
    locale = Locale.parse(context['locale'])
  else:
    # Use get_locale() or default_locale
    locale = flask_babel.get_locale()

  if isinstance(template_name_or_list, basestring):
    template_list = get_template_i18n(template_name_or_list, locale)
  else:
    # Search for locale for each member of the list, do not bypass
    for template in template_name_or_list:
        template_list.append(get_template_i18n(template, locale))

  with set_locale(locale):
    return render_template(template_list, **context)

_NOT_WORD_RE = re.compile(r'[^\w\s]+', flags=re.UNICODE)

def to_lower_ascii(value):
  value = unicode(value)
  value = _NOT_WORD_RE.sub(u' ', value)
  value = unicodedata.normalize('NFKD', value)
  value = value.encode('ascii', 'ignore')
  value = value.decode('ascii')
  value = value.strip().lower()
  value = re.sub(r'[_\s]+', u' ', value)
  return value
<commit_msg>i18n: non-destructive catalog merge<commit_after># coding=utf-8
"""
I18n.

To mark strings for transalation::

    from abilian.i18n import _
    _(u'message to translate')

Use :data:`_` for gettext, :data:`_l` for lazy_gettext, :data:`_n` for
ngettext.

:data:`Babel extension<babel>` support multiple translation paths. This
allows to add more catalogs to search for translations, in LIFO
order. This feature can be used to override some translations in a
custom application, be providing a catalog with messages to override::

    current_app.extensions['babel'].add_translations('abilian.core')

See :meth:`add_translations<Babel.add_translations>`.

To extract messages to build the message catalog template (.pot), use the
following "`-k`" parameters:

.. code-block:: bash

    $ pybabel extract -F babel.cfg -k "_n:1,2" -k "_l" -o "msg.pot" "src"

This can be made easier by placing in `setup.cfg`:

.. code-block:: ini

    [extract_messages]
    mapping_file = babel.cfg
    keywords = _n:1,2 _l
    output-file = msg.pot
    input-dirs = src

And just type:

.. code-block:: bash

    $ python setup.py extract_messages

"""
from __future__ import absolute_import, print_function, division

import os
import re
from gettext import GNUTranslations
import importlib
import unicodedata
from pathlib import Path
from contextlib import contextmanager
from datetime import datetime

import pytz
from babel import Locale
from babel.localedata import locale_identifiers
from babel.support import Translations as BaseTranslations
from babel.dates import LOCALTZ, get_timezone, get_timezone_gmt
from flask import g, request, _request_ctx_stack, current_app, render_template
import flask_babel
from flask_babel import (
    Babel as BabelBase,
    gettext, lazy_gettext, ngettext,
)

__all__ = [
  'babel',
  'gettext', '_',
  'lazy_gettext', '_l', 'localeselector',
  'ngettext', '_n',
  'set_locale',
  'timezoneselector',
  'VALID_LANGUAGES_CODE',
  'render_template_i18n'
]

#: gettext alias
_ = gettext

#: lazy_gettext alias
_l = lazy_gettext
#: ngettext alias
_n = ngettext

#: accepted languages codes
VALID_LANGUAGES_CODE = frozenset(lang for lang in locale_identifiers()
                                 if len(lang) == 2)

def get_default_locale():
  return current_app.extensions['babel'].default_locale

def _get_locale():
  locale = flask_babel.get_locale()
  if locale is None:
    locale = get_default_locale()
  return locale

def __gettext_territory(code):
  locale = _get_locale()
  return (
      locale.territories.get(code)
      or get_default_locale().territories.get(code))

#: get localized territory name
def country_name(code):
  return __gettext_territory(code)

#: lazy version of :func:`country_name`
def lazy_country_name(code):
  from speaklater import make_lazy_string
  return make_lazy_string(__gettext_territory, code)

def default_country():
  return current_app.config.get('DEFAULT_COUNTRY')

def country_choices(first=None, default_country_first=True):
  """
  Return a list of (code, countries), alphabetically sorted on localized
  country name.

  :param first: Country code to be placed at the top
  """
  locale = _get_locale()
  territories = [(code, name) for code, name in locale.territories.iteritems()
                 if len(code) == 2] # skip 3-digit regions

  if first is None and default_country_first:
    first = default_country()

  def sortkey(item):
    if first is not None and item[0] == first:
      return u'0'
    return to_lower_ascii(item[1])

  territories.sort(key=sortkey)
  return territories

def supported_app_locales():
  """
  language codes and labels supported by current application

  :return: an iterable of `(:class:`babel.Locale`, label)`, label being the
  locale language human name in current locale.
  """
  locale = _get_locale()
  codes = current_app.config['BABEL_ACCEPT_LANGUAGES']
  return ((Locale.parse(code), locale.languages.get(code, code))
          for code in codes)

def timezones_choices():
  """
  Timezones values and their labels for current locale.

  :return: an iterable of `(code, label)`, code being a timezone code and label
  the timezone name in current locale.
  """
  utcnow = pytz.utc.localize(datetime.utcnow())
  locale = _get_locale()
  for tz in sorted(pytz.common_timezones):
    tz = get_timezone(tz)
    now = tz.normalize(utcnow.astimezone(tz))
    label = u'({}) {}'.format(get_timezone_gmt(now, locale=locale), tz.zone)
    yield (tz, label) #get_timezone_name(tz, locale=locale))

class Babel(BabelBase):
  """
  Allow to load translations from other modules
  """
  _translations_paths = None

  def __init__(self, *args, **kwargs):
    BabelBase.__init__(self, *args, **kwargs)

  def init_app(self, app):
    super(Babel, self).init_app(app)
    self._translations_paths = [
      (os.path.join(app.root_path, 'translations'), 'messages')
    ]

  def add_translations(self, module_name,
                       translations_dir='translations',
                       domain='messages'):
    """
    Adds translations from external module. For example::

        babel.add_translations('abilian.core')

    Will add translations files from `abilian.core` module.
    """
    module = importlib.import_module(module_name)
    for path in (Path(p, translations_dir) for p in module.__path__):
      if not (path.exists() and path.is_dir()):
        continue

      if not os.access(unicode(path), os.R_OK):
        self.app.logger.warning(
          "Babel translations: read access not allowed {}, skipping."
          "".format(
            repr(unicode((path)).encode('utf-8')))
        )
        continue

      self._translations_paths.append((unicode(path), domain))

class Translations(BaseTranslations):
  """
  Merge only non-empty translations.

  This avoids having uncomplete catalog that "clear" existing translations, when
  used with :func:`_get_translations_multi_paths`.
  """
  def merge(self, translations):
    if isinstance(translations, GNUTranslations):

      for msgkey, msgstr in translations._catalog.items():
        msgid = msgkey

        if isinstance(msgkey, tuple):
          msgid = msgkey[0]

        msgstr = msgstr.strip()
        if msgkey in self._catalog and (msgid == msgstr):
          # when msgstr is empty, compile_catalog sets msgstr = msgid so this is
          # probable an existing translation that would be "erased" by msgid
          # string: skip it.

          # logger.debug('Catalog: %r, skip msgkey: %r, existing: %r',
          #                translations, msgkey, self._catalog[msgkey])
          continue

        self._catalog[msgkey] = msgstr

      if isinstance(translations, BaseTranslations):
        self.files.extend(translations.files)

    return self

def _get_translations_multi_paths():
  """
  Returns the correct gettext translations that should be used for this
  request. This will never fail and return a dummy translation object
  if used outside of the request or if a translation cannot be found.
  """
  ctx = _request_ctx_stack.top
  if ctx is None:
      return None

  translations = getattr(ctx, 'babel_translations', None)
  if translations is None:
    babel_ext = ctx.app.extensions['babel']
    translations = None
    trs = None

    # reverse order: thus the application catalog is loaded last, so that
    # translations from libraries can be overriden
    for (dirname, domain) in reversed(babel_ext._translations_paths):
      trs = Translations.load(dirname,
                              locales=[flask_babel.get_locale()],
                              domain=domain)

      # babel.support.Translations is a subclass of
      # babel.support.NullTranslations, so we test if object has a 'merge'
      # method

      if not trs or not hasattr(trs, 'merge'):
        # got None or NullTranslations instance
        continue
      elif (translations is not None and hasattr(translations, 'merge')):
          translations.merge(trs)
      else:
          translations = trs

    # ensure translations is at least a NullTranslations object
    if translations is None:
      translations = trs

    ctx.babel_translations = translations

  return translations

# monkey patch flask-babel
flask_babel.get_translations = _get_translations_multi_paths

#: importable instance of :class:`Babel`
babel = Babel()

def localeselector():
  """
  Default locale selector used in abilian applications
  """
  # if a user is logged in, use the locale from the user settings
  user = getattr(g, 'user', None)
  if user is not None:
    locale = getattr(user, 'locale', None)
    if locale:
      return locale

  # Otherwise, try to guess the language from the user accept header the browser
  # transmits.  By default we support en/fr. The best match wins.
  return request.accept_languages.best_match(
    current_app.config['BABEL_ACCEPT_LANGUAGES']
  )

def timezoneselector():
  """
  Default timezone selector used in abilian applications
  """
  return LOCALTZ

@contextmanager
def set_locale(locale):
  """
  Change current locale.

  Can be used as a context manager to temporary change locale::

      with set_locale('fr') as fr_locale:
          ...

  :type locale: :class:`babel.core.Locale` or `str`
  :param locale: locale to use. If it's a string if must be a valid locale
                 specification
  :rtype: :class:`babel.core.Locale`
  :return: locale set
  """
  ctx = _request_ctx_stack.top
  if ctx is None:
    yield

  if not isinstance(locale, Locale):
    locale = Locale.parse(locale)

  current_locale = getattr(ctx, 'babel_locale', None)
  ctx.babel_locale = locale
  yield locale
  ctx.babel_locale = current_locale

def get_template_i18n(template_name, locale):
  """
    Build template list with preceding locale if found
  """
  if locale is None:
    return template_name

  template_list = []
  parts = template_name.rsplit('.', 1)
  root = parts[0]
  suffix = parts[1]

  if locale.territory is not None:
    locale_string = '_'.join([locale.language, locale.territory])
    localized_template_path = u'.'.join([root, locale_string, suffix])
    template_list.append(localized_template_path)

  localized_template_path = u'.'.join([root, locale.language, suffix])
  template_list.append(localized_template_path)

  # append the default
  template_list.append(template_name)
  return template_list

def render_template_i18n(template_name_or_list, **context):
  """
    Try to build an ordered list of template to satisfy the current locale
  """
  template_list = []
  # Use locale if present in **context
  if 'locale' in context:
    locale = Locale.parse(context['locale'])
  else:
    # Use get_locale() or default_locale
    locale = flask_babel.get_locale()

  if isinstance(template_name_or_list, basestring):
    template_list = get_template_i18n(template_name_or_list, locale)
  else:
    # Search for locale for each member of the list, do not bypass
    for template in template_name_or_list:
        template_list.append(get_template_i18n(template, locale))

  with set_locale(locale):
    return render_template(template_list, **context)

_NOT_WORD_RE = re.compile(r'[^\w\s]+', flags=re.UNICODE)

def to_lower_ascii(value):
  value = unicode(value)
  value = _NOT_WORD_RE.sub(u' ', value)
  value = unicodedata.normalize('NFKD', value)
  value = value.encode('ascii', 'ignore')
  value = value.decode('ascii')
  value = value.strip().lower()
  value = re.sub(r'[_\s]+', u' ', value)
  return value
<|endoftext|>


<commit_before># Copyright 2017 Max Planck Society
# Distributed under the BSD-3 Software license,
# (See accompanying file ./LICENSE.txt or copy at
# https://opensource.org/licenses/BSD-3-Clause)
"""Training AdaGAN on various datasets.

Refer to the arXiv paper 'AdaGAN: Boosting Generative Models'
Coded by Ilya Tolstikhin, Carl-Johann Simon-Gabriel
"""

import os
import argparse
import logging
import tensorflow as tf
import numpy as np
from datahandler import DataHandler
from adagan import AdaGan
from metrics import Metrics
import utils

flags = tf.app.flags
flags.DEFINE_float("g_learning_rate", 0.001,
                   "Learning rate for Generator optimizers [16e-4]")
flags.DEFINE_float("d_learning_rate", 0.0005,
                   "Learning rate for Discriminator optimizers [4e-4]")
flags.DEFINE_float("learning_rate", 0.003,
                   "Learning rate for other optimizers [8e-4]")
flags.DEFINE_float("adam_beta1", 0.5, "Beta1 parameter for Adam optimizer [0.5]")
flags.DEFINE_integer("zdim", 8, "Dimensionality of the latent space [100]")
flags.DEFINE_float("init_std", 0.0099999, "Initial variance for weights [0.02]")
flags.DEFINE_string("workdir", 'results_mnist_pot', "Working directory ['results']")
flags.DEFINE_bool("unrolled", False, "Use unrolled GAN training [True]")
flags.DEFINE_bool("vae", False, "Use VAE instead of GAN")
flags.DEFINE_bool("pot", True, "Use POT instead of GAN")
flags.DEFINE_float("pot_lambda", 10., "POT regularization")
flags.DEFINE_bool("is_bagging", False, "Do we want to use bagging instead of adagan? [False]")
FLAGS = flags.FLAGS

def main():
    opts = {}
    # Utility
    opts['random_seed'] = 66
    opts['dataset'] = 'mnist' # gmm, circle_gmm,  mnist, mnist3 ...
    opts['data_dir'] = 'mnist'
    opts['trained_model_path'] = None #'models'
    opts['mnist_trained_model_file'] = None #'mnist_trainSteps_19999_yhat' # 'mnist_trainSteps_20000'
    opts['work_dir'] = FLAGS.workdir
    opts['ckpt_dir'] = 'checkpoints'
    opts["verbose"] = 1
    opts['tf_run_batch_size'] = 128
    opts["early_stop"] = -1 # set -1 to run normally
    opts["plot_every"] = 50
    opts["save_every_epoch"] = 10
    opts['gmm_max_val'] = 15.

    # Datasets
    opts['toy_dataset_size'] = 10000
    opts['toy_dataset_dim'] = 2
    opts['mnist3_dataset_size'] = 2 * 64 # 64 * 2500
    opts['mnist3_to_channels'] = False # Hide 3 digits of MNIST to channels
    opts['input_normalize_sym'] = False # Normalize data to [-1, 1]
    opts['gmm_modes_num'] = 5

    # AdaGAN parameters
    opts['adagan_steps_total'] = 1
    opts['samples_per_component'] = 1000
    opts['is_bagging'] = FLAGS.is_bagging
    opts['beta_heur'] = 'uniform' # uniform, constant
    opts['weights_heur'] = 'theory_star' # theory_star, theory_dagger, topk
    opts['beta_constant'] = 0.5
    opts['topk_constant'] = 0.5
    opts["mixture_c_epoch_num"] = 5
    opts["eval_points_num"] = 25600
    opts['digit_classification_threshold'] = 0.999
    opts['inverse_metric'] = False # Use metric from the Unrolled GAN paper?
    opts['inverse_num'] = 100 # Number of real points to inverse.
    opts['objective'] = None

    # Generative model parameters
    opts["init_std"] = FLAGS.init_std
    opts["init_bias"] = 0.0
    opts['latent_space_distr'] = 'normal' # uniform, normal
    opts['latent_space_dim'] = FLAGS.zdim
    opts["gan_epoch_num"] = 100
    opts['convolutions'] = True # If False then encoder is MLP of 3 layers
    opts['d_num_filters'] = 512
    opts['d_num_layers'] = 4
    opts['g_num_filters'] = 1024
    opts['g_num_layers'] = 3
    opts['e_is_random'] = False
    opts['e_pretrain'] = False
    opts['e_pretrain_bsize'] = 1000
    opts['e_num_filters'] = 1024
    opts['e_num_layers'] = 4
    opts['g_arch'] = 'dcgan_mod'
    opts['g_stride1_deconv'] = False
    opts['g_3x3_conv'] = 0
    opts['e_arch'] = 'dcgan'
    opts['e_3x3_conv'] = 0
    opts['conv_filters_dim'] = 4
    # --GAN specific:
    opts['conditional'] = False
    opts['unrolled'] = FLAGS.unrolled # Use Unrolled GAN? (only for images)
    opts['unrolling_steps'] = 5 # Used only if unrolled = True
    # --VAE specific
    opts['vae'] = FLAGS.vae
    opts['vae_sigma'] = 0.01
    # --POT specific
    opts['pot'] = FLAGS.pot
    opts['pot_pz_std'] = 2.
    opts['pot_lambda'] = FLAGS.pot_lambda
    opts['adv_c_loss'] = 'none'
    opts['vgg_layer'] = 'pool2'
    opts['adv_c_patches_size'] = 5
    opts['adv_c_num_units'] = 32
    opts['adv_c_loss_w'] = 1.0
    opts['cross_p_w'] = 0.0
    opts['diag_p_w'] = 0.0
    opts['emb_c_loss_w'] = 1.0
    opts['reconstr_w'] = 1.0
    opts['z_test'] = 'gan'
    opts['gan_p_trick'] = False
    opts['z_test_corr_w'] = 1.0
    opts['z_test_proj_dim'] = 10

    # Optimizer parameters
    opts['optimizer'] = 'adam' # sgd, adam
    opts["batch_size"] = 100
    opts["d_steps"] = 1
    opts['d_new_minibatch'] = False
    opts["g_steps"] = 2
    opts['batch_norm'] = True
    opts['dropout'] = False
    opts['dropout_keep_prob'] = 0.5
    opts['recon_loss'] = 'l2sq'
    # "manual" or number (float or int) giving the number of epochs to divide
    # the learning rate by 10 (converted into an exp decay per epoch).
    opts['decay_schedule'] = 'manual'
    opts['opt_learning_rate'] = FLAGS.learning_rate
    opts['opt_d_learning_rate'] = FLAGS.d_learning_rate
    opts['opt_g_learning_rate'] = FLAGS.g_learning_rate
    opts["opt_beta1"] = FLAGS.adam_beta1
    opts['batch_norm_eps'] = 1e-05
    opts['batch_norm_decay'] = 0.9

    if opts['e_is_random']:
        assert opts['latent_space_distr'] == 'normal',\
            'Random encoders currently work only with Gaussian Pz'
    # Data augmentation
    opts['data_augm'] = False

    if opts['verbose']:
        logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(message)s')
    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')

    utils.create_dir(opts['work_dir'])
    utils.create_dir(os.path.join(opts['work_dir'], opts['ckpt_dir']))

    with utils.o_gfile((opts['work_dir'], 'params.txt'), 'w') as text:
        text.write('Parameters:\n')
        for key in opts:
            text.write('%s : %s\n' % (key, opts[key]))

    data = DataHandler(opts)
    assert data.num_points >= opts['batch_size'], 'Training set too small'
    adagan = AdaGan(opts, data)
    metrics = Metrics()

    train_size = data.num_points
    random_idx = np.random.choice(train_size, 4*320, replace=False)
    metrics.make_plots(opts, 0, data.data,
            data.data[random_idx], adagan._data_weights, prefix='dataset_')

    for step in range(opts["adagan_steps_total"]):
        logging.info('Running step {} of AdaGAN'.format(step + 1))
        adagan.make_step(opts, data)
        num_fake = opts['eval_points_num']
        logging.debug('Sampling fake points')
        fake_points = adagan.sample_mixture(num_fake)
        logging.debug('Sampling more fake points')
        more_fake_points = adagan.sample_mixture(500)
        logging.debug('Plotting results')
        if opts['dataset'] == 'gmm':
            metrics.make_plots(opts, step, data.data[:500],
                    fake_points[0:100], adagan._data_weights[:500])
            logging.debug('Evaluating results')
            (likelihood, C) = metrics.evaluate(
                opts, step, data.data[:500],
                fake_points, more_fake_points, prefix='')
        else:
            metrics.make_plots(opts, step, data.data,
                    fake_points[:320], adagan._data_weights)
            if opts['inverse_metric']:
                logging.debug('Evaluating results')
                l2 = np.min(adagan._invert_losses[:step + 1], axis=0)
                logging.debug('MSE=%.5f, STD=%.5f' % (np.mean(l2), np.std(l2)))
            res = metrics.evaluate(
                opts, step, data.data[:500],
                fake_points, more_fake_points, prefix='')
    logging.debug("AdaGan finished working!")

if __name__ == '__main__':
    main()
<commit_msg>nothing<commit_after># Copyright 2017 Max Planck Society
# Distributed under the BSD-3 Software license,
# (See accompanying file ./LICENSE.txt or copy at
# https://opensource.org/licenses/BSD-3-Clause)
"""Training AdaGAN on various datasets.

Refer to the arXiv paper 'AdaGAN: Boosting Generative Models'
Coded by Ilya Tolstikhin, Carl-Johann Simon-Gabriel
"""

import os
import argparse
import logging
import tensorflow as tf
import numpy as np
from datahandler import DataHandler
from adagan import AdaGan
from metrics import Metrics
import utils

flags = tf.app.flags
flags.DEFINE_float("g_learning_rate", 0.001,
                   "Learning rate for Generator optimizers [16e-4]")
flags.DEFINE_float("d_learning_rate", 0.0005,
                   "Learning rate for Discriminator optimizers [4e-4]")
flags.DEFINE_float("learning_rate", 0.003,
                   "Learning rate for other optimizers [8e-4]")
flags.DEFINE_float("adam_beta1", 0.5, "Beta1 parameter for Adam optimizer [0.5]")
flags.DEFINE_integer("zdim", 8, "Dimensionality of the latent space [100]")
flags.DEFINE_float("init_std", 0.0099999, "Initial variance for weights [0.02]")
flags.DEFINE_string("workdir", 'results_mnist_pot', "Working directory ['results']")
flags.DEFINE_bool("unrolled", False, "Use unrolled GAN training [True]")
flags.DEFINE_bool("vae", False, "Use VAE instead of GAN")
flags.DEFINE_bool("pot", True, "Use POT instead of GAN")
flags.DEFINE_float("pot_lambda", 10., "POT regularization")
flags.DEFINE_bool("is_bagging", False, "Do we want to use bagging instead of adagan? [False]")
FLAGS = flags.FLAGS

def main():
    opts = {}
    # Utility
    opts['random_seed'] = 66
    opts['dataset'] = 'mnist' # gmm, circle_gmm,  mnist, mnist3 ...
    opts['data_dir'] = 'mnist'
    opts['trained_model_path'] = None #'models'
    opts['mnist_trained_model_file'] = None #'mnist_trainSteps_19999_yhat' # 'mnist_trainSteps_20000'
    opts['work_dir'] = FLAGS.workdir
    opts['ckpt_dir'] = 'checkpoints'
    opts["verbose"] = 1
    opts['tf_run_batch_size'] = 128
    opts["early_stop"] = -1 # set -1 to run normally
    opts["plot_every"] = 50
    opts["save_every_epoch"] = 10
    opts['gmm_max_val'] = 15.

    # Datasets
    opts['toy_dataset_size'] = 10000
    opts['toy_dataset_dim'] = 2
    opts['mnist3_dataset_size'] = 2 * 64 # 64 * 2500
    opts['mnist3_to_channels'] = False # Hide 3 digits of MNIST to channels
    opts['input_normalize_sym'] = False # Normalize data to [-1, 1]
    opts['gmm_modes_num'] = 5

    # AdaGAN parameters
    opts['adagan_steps_total'] = 1
    opts['samples_per_component'] = 1000
    opts['is_bagging'] = FLAGS.is_bagging
    opts['beta_heur'] = 'uniform' # uniform, constant
    opts['weights_heur'] = 'theory_star' # theory_star, theory_dagger, topk
    opts['beta_constant'] = 0.5
    opts['topk_constant'] = 0.5
    opts["mixture_c_epoch_num"] = 5
    opts["eval_points_num"] = 25600
    opts['digit_classification_threshold'] = 0.999
    opts['inverse_metric'] = False # Use metric from the Unrolled GAN paper?
    opts['inverse_num'] = 100 # Number of real points to inverse.
    opts['objective'] = None

    # Generative model parameters
    opts["init_std"] = FLAGS.init_std
    opts["init_bias"] = 0.0
    opts['latent_space_distr'] = 'normal' # uniform, normal
    opts['latent_space_dim'] = FLAGS.zdim
    opts["gan_epoch_num"] = 100
    opts['convolutions'] = True # If False then encoder is MLP of 3 layers
    opts['d_num_filters'] = 512
    opts['d_num_layers'] = 4
    opts['g_num_filters'] = 1024
    opts['g_num_layers'] = 3
    opts['e_is_random'] = False
    opts['e_pretrain'] = False
    opts['e_pretrain_bsize'] = 1000
    opts['e_num_filters'] = 1024
    opts['e_num_layers'] = 4
    opts['g_arch'] = 'dcgan_mod'
    opts['g_stride1_deconv'] = False
    opts['g_3x3_conv'] = 0
    opts['e_arch'] = 'dcgan'
    opts['e_3x3_conv'] = 0
    opts['conv_filters_dim'] = 4
    # --GAN specific:
    opts['conditional'] = False
    opts['unrolled'] = FLAGS.unrolled # Use Unrolled GAN? (only for images)
    opts['unrolling_steps'] = 5 # Used only if unrolled = True
    # --VAE specific
    opts['vae'] = FLAGS.vae
    opts['vae_sigma'] = 0.01
    # --POT specific
    opts['pot'] = FLAGS.pot
    opts['pot_pz_std'] = 2.
    opts['pot_lambda'] = FLAGS.pot_lambda
    opts['adv_c_loss'] = 'none'
    opts['vgg_layer'] = 'pool2'
    opts['adv_c_patches_size'] = 5
    opts['adv_c_num_units'] = 32
    opts['adv_c_loss_w'] = 1.0
    opts['cross_p_w'] = 0.0
    opts['diag_p_w'] = 0.0
    opts['emb_c_loss_w'] = 1.0
    opts['reconstr_w'] = 1.0
    opts['z_test'] = 'gan'
    opts['gan_p_trick'] = False
    opts['pz_transform'] = False
    opts['z_test_corr_w'] = 1.0
    opts['z_test_proj_dim'] = 10

    # Optimizer parameters
    opts['optimizer'] = 'adam' # sgd, adam
    opts["batch_size"] = 100
    opts["d_steps"] = 1
    opts['d_new_minibatch'] = False
    opts["g_steps"] = 2
    opts['batch_norm'] = True
    opts['dropout'] = False
    opts['dropout_keep_prob'] = 0.5
    opts['recon_loss'] = 'l2sq'
    # "manual" or number (float or int) giving the number of epochs to divide
    # the learning rate by 10 (converted into an exp decay per epoch).
    opts['decay_schedule'] = 'manual'
    opts['opt_learning_rate'] = FLAGS.learning_rate
    opts['opt_d_learning_rate'] = FLAGS.d_learning_rate
    opts['opt_g_learning_rate'] = FLAGS.g_learning_rate
    opts["opt_beta1"] = FLAGS.adam_beta1
    opts['batch_norm_eps'] = 1e-05
    opts['batch_norm_decay'] = 0.9

    if opts['e_is_random']:
        assert opts['latent_space_distr'] == 'normal',\
            'Random encoders currently work only with Gaussian Pz'
    # Data augmentation
    opts['data_augm'] = False

    if opts['verbose']:
        logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(message)s')
    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')

    utils.create_dir(opts['work_dir'])
    utils.create_dir(os.path.join(opts['work_dir'], opts['ckpt_dir']))

    with utils.o_gfile((opts['work_dir'], 'params.txt'), 'w') as text:
        text.write('Parameters:\n')
        for key in opts:
            text.write('%s : %s\n' % (key, opts[key]))

    data = DataHandler(opts)
    assert data.num_points >= opts['batch_size'], 'Training set too small'
    adagan = AdaGan(opts, data)
    metrics = Metrics()

    train_size = data.num_points
    random_idx = np.random.choice(train_size, 4*320, replace=False)
    metrics.make_plots(opts, 0, data.data,
            data.data[random_idx], adagan._data_weights, prefix='dataset_')

    for step in range(opts["adagan_steps_total"]):
        logging.info('Running step {} of AdaGAN'.format(step + 1))
        adagan.make_step(opts, data)
        num_fake = opts['eval_points_num']
        logging.debug('Sampling fake points')
        fake_points = adagan.sample_mixture(num_fake)
        logging.debug('Sampling more fake points')
        more_fake_points = adagan.sample_mixture(500)
        logging.debug('Plotting results')
        if opts['dataset'] == 'gmm':
            metrics.make_plots(opts, step, data.data[:500],
                    fake_points[0:100], adagan._data_weights[:500])
            logging.debug('Evaluating results')
            (likelihood, C) = metrics.evaluate(
                opts, step, data.data[:500],
                fake_points, more_fake_points, prefix='')
        else:
            metrics.make_plots(opts, step, data.data,
                    fake_points[:320], adagan._data_weights)
            if opts['inverse_metric']:
                logging.debug('Evaluating results')
                l2 = np.min(adagan._invert_losses[:step + 1], axis=0)
                logging.debug('MSE=%.5f, STD=%.5f' % (np.mean(l2), np.std(l2)))
            res = metrics.evaluate(
                opts, step, data.data[:500],
                fake_points, more_fake_points, prefix='')
    logging.debug("AdaGan finished working!")

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># _*_ encoding: utf-8 _*_
"""Binary Search Tree Implementation."""
import random
import time

class Bst(object):
    """Implement Binary Search Tree."""

    def __init__(self, value=None, parent=None, left_child=None,
                 right_child=None):
        """Init Tree."""
        self.value = value
        self.parent = parent
        self.left_child = left_child
        self.right_child = right_child

    @property
    def left_child(self):
        """Left child."""
        return self._left_child

    @left_child.setter
    def left_child(self, other_node):
        self._left_child = other_node
        if other_node is not None:
            other_node.parent = self

    @property
    def right_child(self):
        """Right child."""
        return self._right_child

    def insert(self, value):
        """Insert value into tree if not present."""
        if self.contains(value):
            return
        if self.value is None:
            self.value = value
        try:
            if value > self.value:
                if not self.right_child:
                    self.right_child = Bst(parent=self, value=value)
                else:
                    self.right_child.insert(value)
            elif value < self.value:
                if not self.left_child:
                    self.left_child = Bst(parent=self, value=value)
                else:
                    self.left_child.insert(value)
        except TypeError:
            raise TypeError("Cannot mix types in a binary search tree.")

    def contains(self, value):
    """Return True if value in tree."""
    if self.value == value:
        return True
    left_contains = False
    right_contains = False
    if self.left_child is not None:
        left_contains = self.left_child.contains(value)
    if self.right_child is not None:
        right_contains = self.right_child.contains(value)
    return left_contains or right_contains

    def size(self):
        """Return size of tree."""
        if self.value is None:
            return 0
        if not self.left_child and not self.right_child:
            return 1
        sizes = [child.size()
                 for child in (self.left_child, self.right_child)
                 if child is not None]
        return sum(sizes) + 1

    def depth(self):
        """Return number of levels in the tree."""
        if self.value is None:
            return 0
        if not self.left_child and not self.right_child:
            return 1
        depths = [child.depth()
                  for child in (self.left_child, self.right_child)
                  if child is not None]
<commit_msg>added return max<commit_after># _*_ encoding: utf-8 _*_
"""Binary Search Tree Implementation."""
import random
import time

class Bst(object):
    """Implement Binary Search Tree."""

    def __init__(self, value=None, parent=None, left_child=None,
                 right_child=None):
        """Init Tree."""
        self.value = value
        self.parent = parent
        self.left_child = left_child
        self.right_child = right_child

    @property
    def left_child(self):
        """Left child."""
        return self._left_child

    @left_child.setter
    def left_child(self, other_node):
        self._left_child = other_node
        if other_node is not None:
            other_node.parent = self

    @property
    def right_child(self):
        """Right child."""
        return self._right_child

    def insert(self, value):
        """Insert value into tree if not present."""
        if self.contains(value):
            return
        if self.value is None:
            self.value = value
        try:
            if value > self.value:
                if not self.right_child:
                    self.right_child = Bst(parent=self, value=value)
                else:
                    self.right_child.insert(value)
            elif value < self.value:
                if not self.left_child:
                    self.left_child = Bst(parent=self, value=value)
                else:
                    self.left_child.insert(value)
        except TypeError:
            raise TypeError("Cannot mix types in a binary search tree.")

    def contains(self, value):
    """Return True if value in tree."""
    if self.value == value:
        return True
    left_contains = False
    right_contains = False
    if self.left_child is not None:
        left_contains = self.left_child.contains(value)
    if self.right_child is not None:
        right_contains = self.right_child.contains(value)
    return left_contains or right_contains

    def size(self):
        """Return size of tree."""
        if self.value is None:
            return 0
        if not self.left_child and not self.right_child:
            return 1
        sizes = [child.size()
                 for child in (self.left_child, self.right_child)
                 if child is not None]
        return sum(sizes) + 1

    def depth(self):
        """Return number of levels in the tree."""
        if self.value is None:
            return 0
        if not self.left_child and not self.right_child:
            return 1
        depths = [child.depth()
                  for child in (self.left_child, self.right_child)
                  if child is not None]
        return max(depths) + 1
<|endoftext|>