<commit_before>"""
Script to install Souma on OsX, Windows, and Unix

Usage:
    python package.py py2app
    python package.py py2exe
"""
import ez_setup
import numpy  # important for py2exe to work
ez_setup.use_setuptools()

import sys
import os
from esky.bdist_esky import Executable
from setuptools import setup

if sys.platform == 'win32':
    import py2exe

APP = ['run.py', ]

""" Read current version identifier as recorded in `souma/__init__.py` """
with open("__init__.py", 'rb') as f:
    VERSION = f.readline().split("=")[1].strip().replace('"', '')

""" Compile .less files """
filenames = ["main", ]
for fn in filenames:
    rv = os.system("touch ./static/css/{}.css".format(fn))
    rv += os.system("lesscpy ./static/css/{fn}.less > ./static/css/{fn}.css".format(fn=fn))

""" Compile list of static files """
with open(".gitignore") as f:
    ignorefiles = [l.strip() for l in f.readlines()]

DATA_FILES = [('', ['__init__.py'])]
for datadir in ['templates', 'static']:
    for root, dirs, files in os.walk(datadir):
        DATA_FILES.append((root, [os.path.join(root, fn) for fn in files if fn not in ignorefiles]))

""" Modules imported using import() need to be manually specified here """
INCLUDES = [
    "web_ui",
    "jinja2.ext",
    "sklearn",
    "sklearn.utils",
    "sqlalchemy.dialects.sqlite",
    "sqlalchemy.ext.declarative",
    "wtforms.ext",
    "wtforms.ext.csrf",
    "flask",
    "flask_sqlalchemy",
    "flask.views",
    "flask.signals",
    "flask.helpers",
    "flask.ext",
    "flaskext",
    "flaskext.uploads",
    "flask_wtf",
    "sqlalchemy.orm",
    "sqlalchemy.event",
    "sqlalchemy.ext.declarative",
    "sqlalchemy.engine.url",
    "sqlalchemy.connectors.mxodbc",
    "sqlalchemy.connectors.mysqldb",
    "sqlalchemy.connectors.zxJDBC",
    "sqlalchemy.dialects.sqlite.base",
    "sqlalchemy.dialects.sybase.base",
    "sqlalchemy.dialects.sybase.mxodbc",
    "sqlalchemy.engine.base",
    "sqlalchemy.engine.default",
    "sqlalchemy.engine.interfaces",
    "sqlalchemy.engine.reflection",
    "sqlalchemy.engine.result",
    "sqlalchemy.engine.strategies",
    "sqlalchemy.engine.threadlocal",
    "sqlalchemy.engine.url",
    "sqlalchemy.engine.util",
    "sqlalchemy.event.api",
    "sqlalchemy.event.attr",
    "sqlalchemy.event.base",
    "sqlalchemy.event.legacy",
    "sqlalchemy.event.registry",
    "sqlalchemy.events",
    "sqlalchemy.exc",
    "sqlalchemy.ext.associationproxy",
    "sqlalchemy.ext.automap",
    "sqlalchemy.ext.compiler",
    "sqlalchemy.ext.declarative.api",
    "sqlalchemy.ext.declarative.base",
    "sqlalchemy.ext.declarative.clsregistry",
    "sqlalchemy.ext.horizontal_shard",
    "sqlalchemy.ext.hybrid",
    "sqlalchemy.ext.instrumentation",
    "sqlalchemy.ext.mutable",
    "sqlalchemy.ext.orderinglist",
    "sqlalchemy.ext.serializer",
    "sqlalchemy.inspection",
    "sqlalchemy.interfaces",
    "sqlalchemy.log",
    "sqlalchemy.orm.attributes",
    "sqlalchemy.orm.base",
    "sqlalchemy.orm.collections",
    "sqlalchemy.orm.dependency",
    "sqlalchemy.orm.deprecated_interfaces",
    "sqlalchemy.orm.descriptor_props",
    "sqlalchemy.orm.dynamic",
    "sqlalchemy.orm.evaluator",
    "sqlalchemy.orm.events",
    "sqlalchemy.orm.exc",
    "sqlalchemy.orm.identity",
    "sqlalchemy.orm.instrumentation",
    "sqlalchemy.orm.interfaces",
    "sqlalchemy.orm.loading",
    "sqlalchemy.orm.mapper",
    "sqlalchemy.orm.path_registry",
    "sqlalchemy.orm.persistence",
    "sqlalchemy.orm.properties",
    "sqlalchemy.orm.query",
    "sqlalchemy.orm.relationships",
    "sqlalchemy.orm.scoping",
    "sqlalchemy.orm.session",
    "sqlalchemy.orm.state",
    "sqlalchemy.orm.strategies",
    "sqlalchemy.orm.strategy_options",
    "sqlalchemy.orm.sync",
    "sqlalchemy.orm.unitofwork",
    "sqlalchemy.orm.util",
    "sqlalchemy.pool",
    "sqlalchemy.processors",
    "sqlalchemy.schema",
    "sqlalchemy.sql.annotation",
    "sqlalchemy.sql.base",
    "sqlalchemy.sql.compiler",
    "sqlalchemy.sql.ddl",
    "sqlalchemy.sql.default_comparator",
    "sqlalchemy.sql.dml",
    "sqlalchemy.sql.elements",
    "sqlalchemy.sql.expression",
    "sqlalchemy.sql.functions",
    "sqlalchemy.sql.naming",
    "sqlalchemy.sql.operators",
    "sqlalchemy.sql.schema",
    "sqlalchemy.sql.selectable",
    "sqlalchemy.sql.sqltypes",
    "sqlalchemy.sql.type_api",
    "sqlalchemy.sql.util",
    "sqlalchemy.sql.visitors",
    "sqlalchemy.types",
    "sqlalchemy.util._collections",
    "sqlalchemy.util.compat",
    "sqlalchemy.util.deprecations",
    "sqlalchemy.util.langhelpers",
    "sqlalchemy.util.queue",
    "sqlalchemy.util.topological",
    "flask_sqlalchemy._compat",
    "lxml._elementpath",
    "lxml.etree",
    "scipy.sparse.csgraph._validation",
    "gzip",
    "scipy.special._ufuncs_cxx",
    "sklearn.utils.sparsetools._graph_validation",
    "gevent",
    "gevent.core",
    "logging",
    "Crypto",
    "Crypto.Hash"
]

# might need to explicitly include dll:
# data_files=[('.', 'libmmd.dll')
# also:
# http://stackoverflow.com/questions/10060765/create-python-exe-without-msvcp90-dll

WIN_OPTIONS = {
    "dist_dir": "../dist",
    "includes": INCLUDES,
    "packages": ["nucleus", "web_ui", "synapse", "astrolab"],
    "dll_excludes": [],
    'bundle_files': 1
}

DARWIN_OPTIONS = {
    "argv_emulation": True,
    "bdist_base": "../build",
    "dist_dir": "../dist",
    "iconfile": "static/images/icon_osx.icns",
    "includes": INCLUDES,
    "packages": ["nucleus", "web_ui", "synapse", "astrolab"],
    "site_packages": True,
    "plist": {
        "CFBundleVersion": VERSION,
        "LSBackgroundOnly": True,
        "LSUIElement": True
    },
}

def find_data_files(source, target, patterns):
    """Locates the specified data-files and returns the matches
    in a data_files compatible format.

    Parameters:
        source(String): Root of the source data tree.
            Use '' or '.' for current directory.
        target(String): Root of the target data tree.
            Use '' or '.' for the distribution directory.
        patterns(Iterable):  Sequence of glob-patterns for the
            files you want to copy.

    Returns:
        dict:
    """
    import os
    import glob

    if glob.has_magic(source) or glob.has_magic(target):
        raise ValueError("Magic not allowed in src, target")
    ret = {}
    more = []
    for pattern in patterns:
        pattern = os.path.join(source, pattern)
        for filename in glob.glob(pattern):
            if os.path.isfile(filename):
                targetpath = os.path.join(target, os.path.relpath(filename, source))
                path = os.path.dirname(targetpath)
                ret.setdefault(path, []).append(filename)
            elif os.path.isdir(filename):
                more.extend(find_data_files(filename, filename, '*'))
    ret = sorted(ret.items())
    ret.extend(more)
    return ret

""" Platform specific options """
if sys.platform == 'darwin':
    class SklearnRecipe(object):
        """ Recipe for using sklearn in py2app """
        def check(self, dist, mf):
            m = mf.findNode('sklearn')
            if m is None:
                return None
            # Don't put sklearn in the site-packages.zip file
            return dict(
                packages=['sklearn']
            )
    import py2app.recipes
    py2app.recipes.sklearn = SklearnRecipe()

    """ Patch gevent implicit loader """
    patched = False
    with open("../lib/python2.7/site-packages/gevent/os.py", "r+") as f:
        patch = "\n# make os.path available here\nmy_os = __import__('os')\npath = my_os.path\n"
        for line in f.readlines():
            if line == "# make os.path available here":
                patched = True
        if not patched:
            f.write(patch)

    """ Setup Esky Executable """
    exe = Executable("run.py",
        description="Souma App",
        gui_only=True,
        icon=DARWIN_OPTIONS["iconfile"],
        name="run")

    extra_options = dict(
        setup_requires=['py2app'],
        app=['run.py'],
        options=dict(
            bdist_esky=dict(
                freezer_module="py2app",
                freezer_options=DARWIN_OPTIONS
            )
        ),
        scripts=[exe, ]
    )

    install_requires = open('requirements_osx.txt').read()

elif sys.platform == 'win32':
    extra_options = dict(
        setup_requires=['py2exe'],
        console=[{'script': "run.py"}],
        options=dict(
            bdist_esky=dict(
                freezer_module="py2exe",
                freezer_options=WIN_OPTIONS
            ),
            # Removed because py2exe is called from bdist_esky
            # py2exe=WIN_OPTIONS
        ),
        scripts=APP,
        zipfile=None
    )

    #Some little hacks for making py2exe work
    #Create empty __init__.py in flaskext directory
    #so py2exe recognizes it as module
    import flaskext
    try:
        flaskext.__file__
    except:
        flaskext_init = open(flaskext.__path__[0] + '\\__init__.py', 'w')
        flaskext_init.close()

    install_requires = open('requirements_win.txt').read()

    data_files_tmp = DATA_FILES
    DATA_FILES = []
    for data_file in data_files_tmp:
        DATA_FILES.extend(find_data_files(data_file, data_file, '*'))
else:
    extra_options = dict(
        scripts=APP)

    install_requires = open('requirements_osx.txt').read()

setup(
    name="Souma",
    version=VERSION,
    author="Cognitive Networks Group",
    author_email="team@souma.io",
    url="https://github.com/ciex/souma/",
    packages=["nucleus", "web_ui", "synapse", "astrolab"],
    data_files=DATA_FILES,
    license="Apache License 2.0",
    description="A Cognitive Network for Groups",
    long_description=open("README.md").read(),
    install_requires=install_requires,
    **extra_options
)
<commit_msg>Esky Windows setup<commit_after>"""
Script to install Souma on OsX, Windows, and Unix

Usage:
    python package.py py2app
    python package.py py2exe
"""
import ez_setup
import numpy  # important for py2exe to work
ez_setup.use_setuptools()

import sys
import os
from esky.bdist_esky import Executable
from setuptools import setup

if sys.platform == 'win32':
    import py2exe

APP = ['run.py', ]

""" Read current version identifier as recorded in `souma/__init__.py` """
with open("__init__.py", 'rb') as f:
    VERSION = f.readline().split("=")[1].strip().replace('"', '')

""" Compile .less files """
filenames = ["main", ]
for fn in filenames:
    rv = os.system("touch ./static/css/{}.css".format(fn))
    rv += os.system("lesscpy ./static/css/{fn}.less > ./static/css/{fn}.css".format(fn=fn))

""" Compile list of static files """
with open(".gitignore") as f:
    ignorefiles = [l.strip() for l in f.readlines()]

DATA_FILES = [('', ['__init__.py'])]
for datadir in ['templates', 'static']:
    for root, dirs, files in os.walk(datadir):
        DATA_FILES.append((root, [os.path.join(root, fn) for fn in files if fn not in ignorefiles]))

""" Modules imported using import() need to be manually specified here """
INCLUDES = [
    "web_ui",
    "jinja2.ext",
    "sklearn",
    "sklearn.utils",
    "sqlalchemy.dialects.sqlite",
    "sqlalchemy.ext.declarative",
    "wtforms.ext",
    "wtforms.ext.csrf",
    "flask",
    "flask_sqlalchemy",
    "flask.views",
    "flask.signals",
    "flask.helpers",
    "flask.ext",
    "flaskext",
    "flaskext.uploads",
    "flask_wtf",
    "sqlalchemy.orm",
    "sqlalchemy.event",
    "sqlalchemy.ext.declarative",
    "sqlalchemy.engine.url",
    "sqlalchemy.connectors.mxodbc",
    "sqlalchemy.connectors.mysqldb",
    "sqlalchemy.connectors.zxJDBC",
    "sqlalchemy.dialects.sqlite.base",
    "sqlalchemy.dialects.sybase.base",
    "sqlalchemy.dialects.sybase.mxodbc",
    "sqlalchemy.engine.base",
    "sqlalchemy.engine.default",
    "sqlalchemy.engine.interfaces",
    "sqlalchemy.engine.reflection",
    "sqlalchemy.engine.result",
    "sqlalchemy.engine.strategies",
    "sqlalchemy.engine.threadlocal",
    "sqlalchemy.engine.url",
    "sqlalchemy.engine.util",
    "sqlalchemy.event.api",
    "sqlalchemy.event.attr",
    "sqlalchemy.event.base",
    "sqlalchemy.event.legacy",
    "sqlalchemy.event.registry",
    "sqlalchemy.events",
    "sqlalchemy.exc",
    "sqlalchemy.ext.associationproxy",
    "sqlalchemy.ext.automap",
    "sqlalchemy.ext.compiler",
    "sqlalchemy.ext.declarative.api",
    "sqlalchemy.ext.declarative.base",
    "sqlalchemy.ext.declarative.clsregistry",
    "sqlalchemy.ext.horizontal_shard",
    "sqlalchemy.ext.hybrid",
    "sqlalchemy.ext.instrumentation",
    "sqlalchemy.ext.mutable",
    "sqlalchemy.ext.orderinglist",
    "sqlalchemy.ext.serializer",
    "sqlalchemy.inspection",
    "sqlalchemy.interfaces",
    "sqlalchemy.log",
    "sqlalchemy.orm.attributes",
    "sqlalchemy.orm.base",
    "sqlalchemy.orm.collections",
    "sqlalchemy.orm.dependency",
    "sqlalchemy.orm.deprecated_interfaces",
    "sqlalchemy.orm.descriptor_props",
    "sqlalchemy.orm.dynamic",
    "sqlalchemy.orm.evaluator",
    "sqlalchemy.orm.events",
    "sqlalchemy.orm.exc",
    "sqlalchemy.orm.identity",
    "sqlalchemy.orm.instrumentation",
    "sqlalchemy.orm.interfaces",
    "sqlalchemy.orm.loading",
    "sqlalchemy.orm.mapper",
    "sqlalchemy.orm.path_registry",
    "sqlalchemy.orm.persistence",
    "sqlalchemy.orm.properties",
    "sqlalchemy.orm.query",
    "sqlalchemy.orm.relationships",
    "sqlalchemy.orm.scoping",
    "sqlalchemy.orm.session",
    "sqlalchemy.orm.state",
    "sqlalchemy.orm.strategies",
    "sqlalchemy.orm.strategy_options",
    "sqlalchemy.orm.sync",
    "sqlalchemy.orm.unitofwork",
    "sqlalchemy.orm.util",
    "sqlalchemy.pool",
    "sqlalchemy.processors",
    "sqlalchemy.schema",
    "sqlalchemy.sql.annotation",
    "sqlalchemy.sql.base",
    "sqlalchemy.sql.compiler",
    "sqlalchemy.sql.ddl",
    "sqlalchemy.sql.default_comparator",
    "sqlalchemy.sql.dml",
    "sqlalchemy.sql.elements",
    "sqlalchemy.sql.expression",
    "sqlalchemy.sql.functions",
    "sqlalchemy.sql.naming",
    "sqlalchemy.sql.operators",
    "sqlalchemy.sql.schema",
    "sqlalchemy.sql.selectable",
    "sqlalchemy.sql.sqltypes",
    "sqlalchemy.sql.type_api",
    "sqlalchemy.sql.util",
    "sqlalchemy.sql.visitors",
    "sqlalchemy.types",
    "sqlalchemy.util._collections",
    "sqlalchemy.util.compat",
    "sqlalchemy.util.deprecations",
    "sqlalchemy.util.langhelpers",
    "sqlalchemy.util.queue",
    "sqlalchemy.util.topological",
    "flask_sqlalchemy._compat",
    "lxml._elementpath",
    "lxml.etree",
    "scipy.sparse.csgraph._validation",
    "gzip",
    "scipy.special._ufuncs_cxx",
    "sklearn.utils.sparsetools._graph_validation",
    "gevent",
    "gevent.core",
    "logging",
    "Crypto",
    "Crypto.Hash"
]

# might need to explicitly include dll:
# data_files=[('.', 'libmmd.dll')
# also:
# http://stackoverflow.com/questions/10060765/create-python-exe-without-msvcp90-dll

WIN_OPTIONS = {
    "dist_dir": "../dist",
    "includes": INCLUDES,
    "iconfile": "static/images/icon_win.ico",
    "packages": ["nucleus", "web_ui", "synapse", "astrolab"],
    "dll_excludes": [],
    'bundle_files': 1
}

DARWIN_OPTIONS = {
    "argv_emulation": True,
    "bdist_base": "../build",
    "dist_dir": "../dist",
    "iconfile": "static/images/icon_osx.icns",
    "includes": INCLUDES,
    "packages": ["nucleus", "web_ui", "synapse", "astrolab"],
    "site_packages": True,
    "plist": {
        "CFBundleVersion": VERSION,
        "LSBackgroundOnly": True,
        "LSUIElement": True
    },
}

def find_data_files(source, target, patterns):
    """Locates the specified data-files and returns the matches
    in a data_files compatible format.

    Parameters:
        source(String): Root of the source data tree.
            Use '' or '.' for current directory.
        target(String): Root of the target data tree.
            Use '' or '.' for the distribution directory.
        patterns(Iterable):  Sequence of glob-patterns for the
            files you want to copy.

    Returns:
        dict:
    """
    import os
    import glob

    if glob.has_magic(source) or glob.has_magic(target):
        raise ValueError("Magic not allowed in src, target")
    ret = {}
    more = []
    for pattern in patterns:
        pattern = os.path.join(source, pattern)
        for filename in glob.glob(pattern):
            if os.path.isfile(filename):
                targetpath = os.path.join(target, os.path.relpath(filename, source))
                path = os.path.dirname(targetpath)
                ret.setdefault(path, []).append(filename)
            elif os.path.isdir(filename):
                more.extend(find_data_files(filename, filename, '*'))
    ret = sorted(ret.items())
    ret.extend(more)
    return ret

""" Platform specific options """
if sys.platform == 'darwin':
    class SklearnRecipe(object):
        """ Recipe for using sklearn in py2app """
        def check(self, dist, mf):
            m = mf.findNode('sklearn')
            if m is None:
                return None
            # Don't put sklearn in the site-packages.zip file
            return dict(
                packages=['sklearn']
            )
    import py2app.recipes
    py2app.recipes.sklearn = SklearnRecipe()

    """ Patch gevent implicit loader """
    patched = False
    with open("../lib/python2.7/site-packages/gevent/os.py", "r+") as f:
        patch = "\n# make os.path available here\nmy_os = __import__('os')\npath = my_os.path\n"
        for line in f.readlines():
            if line == "# make os.path available here":
                patched = True
        if not patched:
            f.write(patch)

    """ Setup Esky Executable """
    exe = Executable("run.py",
        description="Souma App",
        gui_only=True,
        icon=DARWIN_OPTIONS["iconfile"],
        name="run")

    extra_options = dict(
        setup_requires=['py2app'],
        app=['run.py'],
        options=dict(
            bdist_esky=dict(
                freezer_module="py2app",
                freezer_options=DARWIN_OPTIONS
            )
        ),
        scripts=[exe, ]
    )

    install_requires = open('requirements_osx.txt').read()

elif sys.platform == 'win32':
    """ Setup Esky Executable """
    exe = Executable("run.py",
        description="Souma App",
        gui_only=True,
        icon=WIN_OPTIONS["iconfile"],
        name="run")

    extra_options = dict(
        setup_requires=['py2exe'],
        console=[{'script': "run.py"}],
        options=dict(
            bdist_esky=dict(
                freezer_module="py2exe",
                freezer_options=WIN_OPTIONS
            )
        ),
        scripts=APP,
        zipfile=None
    )

    #Some little hacks for making py2exe work
    #Create empty __init__.py in flaskext directory
    #so py2exe recognizes it as module
    import flaskext
    try:
        flaskext.__file__
    except:
        flaskext_init = open(flaskext.__path__[0] + '\\__init__.py', 'w')
        flaskext_init.close()

    install_requires = open('requirements_win.txt').read()

    data_files_tmp = DATA_FILES
    DATA_FILES = []
    for data_file in data_files_tmp:
        DATA_FILES.extend(find_data_files(data_file, data_file, '*'))
else:
    extra_options = dict(
        scripts=APP)

    install_requires = open('requirements_osx.txt').read()

setup(
    name="Souma",
    version=VERSION,
    author="Cognitive Networks Group",
    author_email="team@souma.io",
    url="https://github.com/ciex/souma/",
    packages=["nucleus", "web_ui", "synapse", "astrolab"],
    data_files=DATA_FILES,
    license="Apache License 2.0",
    description="A Cognitive Network for Groups",
    long_description=open("README.md").read(),
    install_requires=install_requires,
    **extra_options
)
<|endoftext|>


<commit_before>#!/usr/bin/python3
# Flictionary Learning: Rank-1 Dictionary Learning in Flink!
# Probably PEP 8 compliant.

import argparse
import math
import os
import random
import sys

import numpy as np

from flink.functions.Aggregation import Sum
from flink.functions.FlatMapFunction import FlatMapFunction
from flink.functions.GroupReduceFunction import GroupReduceFunction
from flink.plan.Constants import Order, WriteMode, INT, FLOAT
from flink.plan.Environment import get_environment

###################################
# Flink helper functions
###################################

def parse_and_normalize(line):
    """
    Utility function. Parses a line of text into a floating point array, then
    whitens the array.
    """
    return tuple(map(float, line.strip().split()))

class SExploderFlatMapper(FlatMapFunction):
    def flat_map(self, x, collector):
        data = enumerate(parse_and_normalize(x[1]))
        for y in data:
            collector.collect((x[0], y[0], y[1]))

def get_top_v(r, u, s):
    """
    Calculates v, and then returns the top R elements of v
    :param r: R
    :param u: Old u
    :param s: S matrix
    :return: Top R elements of v
    """
    v_ = vector_matrix(u, s)

    # sort v by using sort_group after grouping on a dummy field
    v_ = v_.map(lambda x: (x[0], x[1], 0)).name('VPreSorter') \
        .set_parallelism(parallelism)
    v_ = v_ \
        .group_by(2).sort_group(1, Order.DESCENDING) \
        .reduce_group(lambda i, c: [(i_[0], i_[1]) for i_ in i], combinable=True) \
        .set_parallelism(parallelism) \
        .name('VSorter')

    v_top_R_ = v_.first(r).set_parallelism(parallelism).name('VFirst')
    return v_top_R_

def vector_matrix(u__, S_):
    """
    Generates v using u.
    """
    v_ = u__ \
        .join_with_huge(S_).where(0).equal_to(0) \
        .using(lambda u_el, s_el: (s_el[1], s_el[2] * u_el[1])) \
        .set_parallelism(parallelism) \
        .name('VectorMatrix')

    v_ = v_ \
        .group_by(0) \
        .aggregate(Sum, 1) \
        .set_parallelism(parallelism) \
        .name('VectorMatrixPost')

    return v_

def random_vector(environment, num_elements, rng=random):
    """
    Generates a vector of random numbers in in [0.0, 1.0). Does NOT normalize.
    """
    dataset = environment.generate_sequence(1, num_elements) \
        .set_parallelism(1)
    dataset = dataset.map(lambda x: (x, rng.random())).set_parallelism(1)
    return dataset

def zero_vector(environment, num_elements):
    """
    Generates a vector of zeroes.
    """
    dataset = environment.generate_sequence(1, num_elements)
    dataset = dataset.map(lambda x: (x, 0.0))
    return dataset

class NormalizeVectorGroupReducer(GroupReduceFunction):
    """
    Normalizes a vector in (index, value) format.
    """

    def reduce(self, iterator, collector):
        data = list(iterator)
        mean = 0.0
        mag = 0.0
        length = len(data)

        for val in data:
            mean += val[1]
            mag += (val[1]) ** 2

        mean /= length
        mag = math.sqrt(mag)

        for val in data:
            new_val = val[1] - mean
            new_val /= mag
            collector.collect((val[0], new_val))

def mean(d, l):
    return d.aggregate(Sum, 1).map(lambda x: (0, x[1] / l))

def magnitude(d):
    return d.map(lambda x: (x[0], x[1] ** 2)).set_parallelism(parallelism) \
        .aggregate(Sum, 1).set_parallelism(parallelism) \
        .map(lambda x: (0, math.sqrt(x[1])))

class MagnitudeGroupReducer(GroupReduceFunction):
    """
    Calculates the magnitude of a vector.
    """

    def reduce(self, iterator, collector):
        mag = 0
        for val in iterator:
            mag += (val[1]) ** 2
        mag = math.sqrt(mag)
        collector.collect((0, mag))

class RngWrapper(object):
    """Compatibility wrapper for different RNG methods."""

    def random(self):
        """Assign something to me."""
        pass

def initialize_rng(seed=None, java=False):
    wrapper = RngWrapper()
    if java:
        import javarandom
        r = javarandom.Random(seed)
        wrapper.random = r.nextDouble
    else:
        r = np.random.RandomState(seed)
        wrapper.random = r.random_sample
    return wrapper

def get_temporary_S_path(m):
    return os.path.join(temporary_directory, 'temp_S.' + str(m))

def normalize(v, l):
    v_mean = mean(v, l)
    v_mag = magnitude(v)
    v = v.cross_with_tiny(v_mean) \
        .using(lambda element, mean: (element[0], element[1] - mean[1])) \
        .set_parallelism(parallelism)
    v = v.cross_with_tiny(v_mag) \
        .using(lambda element, mag: (element[0], element[1] / mag[1])) \
        .set_parallelism(parallelism)
    return v

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Flictionary Learning',
                                     add_help='How to use',
                                     prog='.../pyflink2.sh R1DL_Flink.py -')

    parser.add_argument("-q", "--quiet", action='store_true',
                        help="Don't print out any verbose information. "
                             "Note that this doesn't cover the output from "
                             "Flink itself. (optional)")
    parser.add_argument("-l", "--local", action='store_true',
                        help="Run script on the local machine "
                             "i.e. NOT a cluster. (optional)")
    parser.add_argument("-j", "--javarand", action='store_true',
                        help="Use an implementation of the Java RNG. "
                             "This option allows you to directly compare "
                             "results between the Java and Python "
                             "implementations. Requires java-random package. "
                             "(optional)")

    # Inputs.
    parser.add_argument("-i", "--input", required=True,
                        help="Input file containing the matrix S.")
    parser.add_argument("-t", "--rows", type=int, required=True,
                        help="Number of rows (observations) in the "
                             "input matrix S.")
    parser.add_argument("-p", "--cols", type=int, required=True,
                        help="Number of columns (features) in the "
                             "input matrix S.")
    parser.add_argument("-r", "--pnonzero", type=float, required=True,
                        help="Percentage of non-zero elements.")
    parser.add_argument("-m", "--mdicatom", type=int, required=True,
                        help="Number of the dictionary atoms.")
    parser.add_argument("-e", "--epsilon", type=float, required=True,
                        help="The value of epsilon.")
    parser.add_argument("-a", "--parallelism", type=int, required=False,
                        help="Parallelism to use.")
    parser.add_argument("-z", "--seed", type=int, required=False,
                        help="Random seed. (optional)")

    # Outputs.
    parser.add_argument("-d", "--dictionary", required=True,
                        help="Output path to dictionary file. (file_D)")
    parser.add_argument("-o", "--output", required=True,
                        help="Output path to z matrix. (file_z)")
    parser.add_argument("-x", "--prefix", required=True,
                        help="Prefix strings to the output files")
    parser.add_argument("-y", "--temporary", required=True,
                        help="Temporary directory (on HDFS if you're using a "
                             "cluster) for storing intermediate files. A suffix"
                             "will be added to the end of this path.")

    args = vars(parser.parse_args())

    ##################################################################
    # Here's where the real fun begins.
    #

    # First, we're going to initialize some variables we'll need for the
    # following operations. Next, we'll start the optimization loops. Finally,
    # we'll perform the stepping and deflation operations until convergence.
    #
    # Sound like fun?
    ##################################################################

    # parallelism to use. Python API has some quirks with parallelism, like not
    # picking up on the default parallelism sometimes, or not using a specified
    # parallelism; therefore we use this parameter.
    parallelism = args['parallelism']
    T = args['rows']
    P = args['cols']
    epsilon = args['epsilon']  # convergence stopping criterion
    M = args['mdicatom']  # dimensionality of the learned dictionary
    R = int(round(args['pnonzero'] * P))  # enforces sparsity

    # seed random number generator
    rng = initialize_rng(args['seed'], args['javarand'])

    # Print out some useful information for the user
    if not args['quiet']:
        print('====================')
        print('Flictionary learning: R1DL in Flink!')
        print('Local mode: {local}'.format(**args))
        print('Input has {rows} rows and {cols} cols.'.format(**args))
        args['R'] = R
        print('epsilon = {epsilon}, M = {mdicatom}, R = {R}'.format(**args))

        if args['javarand']:
            print('Using java-random.')
        if args['seed'] is not None:
            print('Using random seed {seed}.'.format(**args))

        print('====================')
        sys.stdout.flush()

    ##################################################################
    # Apparently Flink doesn't like np.arrays. So for vectors, we use Flink
    # DataSets with indexes corresponding to their position in the vector.
    ##################################################################

    # Information we need.
    max_iterations = int(P * 10)
    file_D = os.path.join(args['dictionary'], "{prefix}_D.txt".format(**args))
    file_z = os.path.join(args['output'], "{prefix}_z.txt".format(**args))

    r1dl_id = '{0:.8f}'.format(random.random())  # used for temporary files
    temporary_directory = os.path.join(args['temporary'],
                                       'r1dl_data_' + r1dl_id)

    # Initialize the Flink environment.
    env_first = get_environment()
    env_first.set_parallelism(parallelism)

    # Read the data and convert it into a data set of lines
    raw_data = env_first.read_text(args['input'])

    # Convert each line to a tuple: (row number, vec pos, value)
    raw_data \
        .zip_with_index() \
        .flat_map(SExploderFlatMapper()) \
        .name('SExploderFlatMapper') \
        .write_csv(get_temporary_S_path(0), write_mode=WriteMode.OVERWRITE)

    env_first.execute(local=args['local'])

    # Start the loop!
    for m in range(M):
        # If seed is set, the RNG will be in the same state with every generated
        # u vector, because the Python API spawns new Python processes for each
        # operation. Thus, if seed is set, skip the number of random numbers
        # that would have generated if the operations had taken place in the
        # same Python process.
        # This might take up some performance time, but is included only when
        # the seed is specified for reproductivity purposes.
        if args['seed']:
            for z in range(m * T):
                rng.random()

        env = get_environment()
        # needed to prevent normalization etc. group reduce functions from
        # improperly running in parallel
        env.set_parallelism(1)

        S = env.read_csv(get_temporary_S_path(m), (INT, INT, FLOAT)) \
            .set_parallelism(parallelism)

        # Generate a random vector, subtract off its mean, and normalize it.
        u_old = random_vector(env, T, rng)
        u_old = normalize(u_old, T)
        u_old_it = u_old.iterate(max_iterations)

        # Start the inner loop: this learns a single atom.
        # P2: Vector-matrix multiplication step. Computes v.

        # We can't broadcast partial iteration results, so
        # the Flink implementation is different from Spark.
        v = get_top_v(R, u_old_it, S)

        # P1: Matrix-vector multiplication step. Computes u.

        # We can use a join to do a dot product.
        # (S * v by row-wise multiplication)
        # This replaces matrix_vector in the original Spark implementation.
        # Multiply each corresponding element
        u_new = v \
            .join_with_huge(S).where(0).equal_to(1) \
            .using(lambda v_el, s_el: (s_el[0], s_el[1], s_el[2] * v_el[1])) \
            .set_parallelism(parallelism) \
            .name('MatrixVector')

        # Now, add up all the products to get the dot product result,
        # and remove the vector position field (second field from the left)
        u_new = u_new.group_by(0) \
            .aggregate(Sum, 2) \
            .set_parallelism(parallelism) \
            .project(0, 2) \
            .set_parallelism(parallelism)
        ################################################################

        u_new = normalize(u_new, T)

        # Update for the next iteration
        delta = u_new.join_with_huge(u_old_it).where(0).equal_to(0) \
            .using(lambda new, old: (new[0], old[1] - new[1])) \
            .set_parallelism(parallelism)
        delta = magnitude(delta)
        delta = delta.filter(lambda d: d[1] > epsilon).name('Delta')

        u_new_final = u_old_it.close_with(u_new, delta) \
            .set_parallelism(parallelism)

        # Save the newly-computed u and v to the output files;
        u_new_final.write_csv(file_D + "." + str(m),
                              write_mode=WriteMode.OVERWRITE) \
            .set_parallelism(parallelism)

        # Compute new v from final u
        v_final = get_top_v(R, u_new_final, S)

        # Fill in missing spots with zeroes
        # Fill with 0.0, not 0 or else Flink thinks these are
        # LONGS and NOT doubles!
        v_zeroes = zero_vector(env, P)
        v_final = v_final.union(v_zeroes)
        v_final = v_final.group_by(0).aggregate(Sum, 1) \
            .set_parallelism(parallelism)
        v_final.write_csv(file_z + "." + str(m), write_mode=WriteMode.OVERWRITE) \
            .set_parallelism(parallelism)

        # P4: Deflation step. Update the primary data matrix S.
        # This replaces deflate in the original Spark implementation.
        # We want k, vector - (u[k] * v) for each vector in the original data
        # Our original data is formatted in tuples of (k, pos, value)
        # First, we add u[k] to each tuple
        S_temp = S.join_with_tiny(u_new_final).where(0).equal_to(0) \
            .project_first(0, 1, 2).project_second(1) \
            .set_parallelism(parallelism)

        # Now, we multiply u[k] by v[pos] and get
        # val - (u[r] * v[pos]) for each tuple
        S_temp = S_temp.join_with_tiny(v_final).where(1).equal_to(0) \
            .using(lambda s_el, v_el: (s_el[0],
                                       s_el[1],
                                       s_el[2] - (s_el[3] * v_el[1]))) \
            .set_parallelism(parallelism)

        # Finally, add up everything.
        # Write the new S matrix to a temporary file
        # for use in the next dict atom
        S = S_temp.group_by(0, 1).aggregate(Sum, 1) \
            .set_parallelism(parallelism) \
            .write_csv(get_temporary_S_path(m + 1),
                       write_mode=WriteMode.OVERWRITE) \
            .set_parallelism(parallelism)

        env.execute(local=args['local'])

        # All done! Write out the matrices as text files.
<commit_msg>Make life easier by logging run times<commit_after>#!/usr/bin/python3
# Flictionary Learning: Rank-1 Dictionary Learning in Flink!
# Probably PEP 8 compliant.

import argparse
import math
import os
import random
import sys

import numpy as np

from flink.functions.Aggregation import Sum
from flink.functions.FlatMapFunction import FlatMapFunction
from flink.functions.GroupReduceFunction import GroupReduceFunction
from flink.plan.Constants import Order, WriteMode, INT, FLOAT
from flink.plan.Environment import get_environment

###################################
# Flink helper functions
###################################

def parse_and_normalize(line):
    """
    Utility function. Parses a line of text into a floating point array, then
    whitens the array.
    """
    return tuple(map(float, line.strip().split()))

class SExploderFlatMapper(FlatMapFunction):
    def flat_map(self, x, collector):
        data = enumerate(parse_and_normalize(x[1]))
        for y in data:
            collector.collect((x[0], y[0], y[1]))

def get_top_v(r, u, s):
    """
    Calculates v, and then returns the top R elements of v
    :param r: R
    :param u: Old u
    :param s: S matrix
    :return: Top R elements of v
    """
    v_ = vector_matrix(u, s)

    # sort v by using sort_group after grouping on a dummy field
    v_ = v_.map(lambda x: (x[0], x[1], 0)).name('VPreSorter') \
        .set_parallelism(parallelism)
    v_ = v_ \
        .group_by(2).sort_group(1, Order.DESCENDING) \
        .reduce_group(lambda i, c: [(i_[0], i_[1]) for i_ in i], combinable=True) \
        .set_parallelism(parallelism) \
        .name('VSorter')

    v_top_R_ = v_.first(r).set_parallelism(parallelism).name('VFirst')
    return v_top_R_

def vector_matrix(u__, S_):
    """
    Generates v using u.
    """
    v_ = u__ \
        .join_with_huge(S_).where(0).equal_to(0) \
        .using(lambda u_el, s_el: (s_el[1], s_el[2] * u_el[1])) \
        .set_parallelism(parallelism) \
        .name('VectorMatrix')

    v_ = v_ \
        .group_by(0) \
        .aggregate(Sum, 1) \
        .set_parallelism(parallelism) \
        .name('VectorMatrixPost')

    return v_

def random_vector(environment, num_elements, rng=random):
    """
    Generates a vector of random numbers in in [0.0, 1.0). Does NOT normalize.
    """
    dataset = environment.generate_sequence(1, num_elements) \
        .set_parallelism(1)
    dataset = dataset.map(lambda x: (x, rng.random())).set_parallelism(1)
    return dataset

def zero_vector(environment, num_elements):
    """
    Generates a vector of zeroes.
    """
    dataset = environment.generate_sequence(1, num_elements)
    dataset = dataset.map(lambda x: (x, 0.0))
    return dataset

class NormalizeVectorGroupReducer(GroupReduceFunction):
    """
    Normalizes a vector in (index, value) format.
    """

    def reduce(self, iterator, collector):
        data = list(iterator)
        mean = 0.0
        mag = 0.0
        length = len(data)

        for val in data:
            mean += val[1]
            mag += (val[1]) ** 2

        mean /= length
        mag = math.sqrt(mag)

        for val in data:
            new_val = val[1] - mean
            new_val /= mag
            collector.collect((val[0], new_val))

def mean(d, l):
    return d.aggregate(Sum, 1).map(lambda x: (0, x[1] / l))

def magnitude(d):
    return d.map(lambda x: (x[0], x[1] ** 2)).set_parallelism(parallelism) \
        .aggregate(Sum, 1).set_parallelism(parallelism) \
        .map(lambda x: (0, math.sqrt(x[1])))

class MagnitudeGroupReducer(GroupReduceFunction):
    """
    Calculates the magnitude of a vector.
    """

    def reduce(self, iterator, collector):
        mag = 0
        for val in iterator:
            mag += (val[1]) ** 2
        mag = math.sqrt(mag)
        collector.collect((0, mag))

class RngWrapper(object):
    """Compatibility wrapper for different RNG methods."""

    def random(self):
        """Assign something to me."""
        pass

def initialize_rng(seed=None, java=False):
    wrapper = RngWrapper()
    if java:
        import javarandom
        r = javarandom.Random(seed)
        wrapper.random = r.nextDouble
    else:
        r = np.random.RandomState(seed)
        wrapper.random = r.random_sample
    return wrapper

def get_temporary_S_path(m):
    return os.path.join(temporary_directory, 'temp_S.' + str(m))

def normalize(v, l):
    v_mean = mean(v, l)
    v_mag = magnitude(v)
    v = v.cross_with_tiny(v_mean) \
        .using(lambda element, mean: (element[0], element[1] - mean[1])) \
        .set_parallelism(parallelism)
    v = v.cross_with_tiny(v_mag) \
        .using(lambda element, mag: (element[0], element[1] / mag[1])) \
        .set_parallelism(parallelism)
    return v

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Flictionary Learning',
                                     add_help='How to use',
                                     prog='.../pyflink2.sh R1DL_Flink.py -')

    parser.add_argument("-q", "--quiet", action='store_true',
                        help="Don't print out any verbose information. "
                             "Note that this doesn't cover the output from "
                             "Flink itself. (optional)")
    parser.add_argument("-l", "--local", action='store_true',
                        help="Run script on the local machine "
                             "i.e. NOT a cluster. (optional)")
    parser.add_argument("-j", "--javarand", action='store_true',
                        help="Use an implementation of the Java RNG. "
                             "This option allows you to directly compare "
                             "results between the Java and Python "
                             "implementations. Requires java-random package. "
                             "(optional)")

    # Inputs.
    parser.add_argument("-i", "--input", required=True,
                        help="Input file containing the matrix S.")
    parser.add_argument("-t", "--rows", type=int, required=True,
                        help="Number of rows (observations) in the "
                             "input matrix S.")
    parser.add_argument("-p", "--cols", type=int, required=True,
                        help="Number of columns (features) in the "
                             "input matrix S.")
    parser.add_argument("-r", "--pnonzero", type=float, required=True,
                        help="Percentage of non-zero elements.")
    parser.add_argument("-m", "--mdicatom", type=int, required=True,
                        help="Number of the dictionary atoms.")
    parser.add_argument("-e", "--epsilon", type=float, required=True,
                        help="The value of epsilon.")
    parser.add_argument("-a", "--parallelism", type=int, required=False,
                        help="Parallelism to use.")
    parser.add_argument("-z", "--seed", type=int, required=False,
                        help="Random seed. (optional)")

    # Outputs.
    parser.add_argument("-d", "--dictionary", required=True,
                        help="Output path to dictionary file. (file_D)")
    parser.add_argument("-o", "--output", required=True,
                        help="Output path to z matrix. (file_z)")
    parser.add_argument("-x", "--prefix", required=True,
                        help="Prefix strings to the output files")
    parser.add_argument("-y", "--temporary", required=True,
                        help="Temporary directory (on HDFS if you're using a "
                             "cluster) for storing intermediate files. A suffix"
                             "will be added to the end of this path.")

    args = vars(parser.parse_args())

    ##################################################################
    # Here's where the real fun begins.
    #

    # First, we're going to initialize some variables we'll need for the
    # following operations. Next, we'll start the optimization loops. Finally,
    # we'll perform the stepping and deflation operations until convergence.
    #
    # Sound like fun?
    ##################################################################

    # parallelism to use. Python API has some quirks with parallelism, like not
    # picking up on the default parallelism sometimes, or not using a specified
    # parallelism; therefore we use this parameter.
    parallelism = args['parallelism']
    T = args['rows']
    P = args['cols']
    epsilon = args['epsilon']  # convergence stopping criterion
    M = args['mdicatom']  # dimensionality of the learned dictionary
    R = int(round(args['pnonzero'] * P))  # enforces sparsity

    # seed random number generator
    rng = initialize_rng(args['seed'], args['javarand'])

    # Print out some useful information for the user
    if not args['quiet']:
        print('====================')
        print('Flictionary learning: R1DL in Flink!')
        print('Local mode: {local}'.format(**args))
        print('Input has {rows} rows and {cols} cols.'.format(**args))
        args['R'] = R
        print('epsilon = {epsilon}, M = {mdicatom}, R = {R}'.format(**args))

        if args['javarand']:
            print('Using java-random.')
        if args['seed'] is not None:
            print('Using random seed {seed}.'.format(**args))

        print('====================')
        sys.stdout.flush()

    ##################################################################
    # Apparently Flink doesn't like np.arrays. So for vectors, we use Flink
    # DataSets with indexes corresponding to their position in the vector.
    ##################################################################

    # Information we need.
    max_iterations = int(P * 10)
    file_D = os.path.join(args['dictionary'], "{prefix}_D.txt".format(**args))
    file_z = os.path.join(args['output'], "{prefix}_z.txt".format(**args))

    r1dl_id = '{0:.8f}'.format(random.random())  # used for temporary files
    temporary_directory = os.path.join(args['temporary'],
                                       'r1dl_data_' + r1dl_id)

    run_times = []

    # Initialize the Flink environment.
    env_first = get_environment()
    env_first.set_parallelism(parallelism)

    # Read the data and convert it into a data set of lines
    raw_data = env_first.read_text(args['input'])

    # Convert each line to a tuple: (row number, vec pos, value)
    raw_data \
        .zip_with_index() \
        .flat_map(SExploderFlatMapper()) \
        .name('SExploderFlatMapper') \
        .write_csv(get_temporary_S_path(0), write_mode=WriteMode.OVERWRITE)

    env_first_res = env_first.execute(local=args['local'])
    if env_first_res is not None:
        run_times.append((-1, env_first_res.get_net_runtime()))

    # Start the loop!
    for m in range(M):
        # If seed is set, the RNG will be in the same state with every generated
        # u vector, because the Python API spawns new Python processes for each
        # operation. Thus, if seed is set, skip the number of random numbers
        # that would have generated if the operations had taken place in the
        # same Python process.
        # This might take up some performance time, but is included only when
        # the seed is specified for reproductivity purposes.
        if args['seed']:
            for z in range(m * T):
                rng.random()

        env = get_environment()
        # needed to prevent normalization etc. group reduce functions from
        # improperly running in parallel
        env.set_parallelism(1)

        S = env.read_csv(get_temporary_S_path(m), (INT, INT, FLOAT)) \
            .set_parallelism(parallelism)

        # Generate a random vector, subtract off its mean, and normalize it.
        u_old = random_vector(env, T, rng)
        u_old = normalize(u_old, T)
        u_old_it = u_old.iterate(max_iterations)

        # Start the inner loop: this learns a single atom.
        # P2: Vector-matrix multiplication step. Computes v.

        # We can't broadcast partial iteration results, so
        # the Flink implementation is different from Spark.
        v = get_top_v(R, u_old_it, S)

        # P1: Matrix-vector multiplication step. Computes u.

        # We can use a join to do a dot product.
        # (S * v by row-wise multiplication)
        # This replaces matrix_vector in the original Spark implementation.
        # Multiply each corresponding element
        u_new = v \
            .join_with_huge(S).where(0).equal_to(1) \
            .using(lambda v_el, s_el: (s_el[0], s_el[1], s_el[2] * v_el[1])) \
            .set_parallelism(parallelism) \
            .name('MatrixVector')

        # Now, add up all the products to get the dot product result,
        # and remove the vector position field (second field from the left)
        u_new = u_new.group_by(0) \
            .aggregate(Sum, 2) \
            .set_parallelism(parallelism) \
            .project(0, 2) \
            .set_parallelism(parallelism)
        ################################################################

        u_new = normalize(u_new, T)

        # Update for the next iteration
        delta = u_new.join_with_huge(u_old_it).where(0).equal_to(0) \
            .using(lambda new, old: (new[0], old[1] - new[1])) \
            .set_parallelism(parallelism)
        delta = magnitude(delta)
        delta = delta.filter(lambda d: d[1] > epsilon).name('Delta')

        u_new_final = u_old_it.close_with(u_new, delta) \
            .set_parallelism(parallelism)

        # Save the newly-computed u and v to the output files;
        u_new_final.write_csv(file_D + "." + str(m),
                              write_mode=WriteMode.OVERWRITE) \
            .set_parallelism(parallelism)

        # Compute new v from final u
        v_final = get_top_v(R, u_new_final, S)

        # Fill in missing spots with zeroes
        # Fill with 0.0, not 0 or else Flink thinks these are
        # LONGS and NOT doubles!
        v_zeroes = zero_vector(env, P)
        v_final = v_final.union(v_zeroes)
        v_final = v_final.group_by(0).aggregate(Sum, 1) \
            .set_parallelism(parallelism)
        v_final.write_csv(file_z + "." + str(m), write_mode=WriteMode.OVERWRITE) \
            .set_parallelism(parallelism)

        # P4: Deflation step. Update the primary data matrix S.
        # This replaces deflate in the original Spark implementation.
        # We want k, vector - (u[k] * v) for each vector in the original data
        # Our original data is formatted in tuples of (k, pos, value)
        # First, we add u[k] to each tuple
        S_temp = S.join_with_tiny(u_new_final).where(0).equal_to(0) \
            .project_first(0, 1, 2).project_second(1) \
            .set_parallelism(parallelism)

        # Now, we multiply u[k] by v[pos] and get
        # val - (u[r] * v[pos]) for each tuple
        S_temp = S_temp.join_with_tiny(v_final).where(1).equal_to(0) \
            .using(lambda s_el, v_el: (s_el[0],
                                       s_el[1],
                                       s_el[2] - (s_el[3] * v_el[1]))) \
            .set_parallelism(parallelism)

        # Finally, add up everything.
        # Write the new S matrix to a temporary file
        # for use in the next dict atom
        S = S_temp.group_by(0, 1).aggregate(Sum, 1) \
            .set_parallelism(parallelism) \
            .write_csv(get_temporary_S_path(m + 1),
                       write_mode=WriteMode.OVERWRITE) \
            .set_parallelism(parallelism)

        env_res = env.execute(local=args['local'])
        if env_res is not None:
            run_times.append((m, env_res.get_net_runtime()))

        # All done! Write out the matrices as text files.

    env_run_times = get_environment()
    env_run_times.set_parallelism(1)
    rt = env_run_times.from_elements(*run_times)
    rt.write_csv(file_z + ".run_times", write_mode=WriteMode.OVERWRITE)
    env_run_times.execute(local=args['local'])
<|endoftext|>


<commit_before>from compsoc.cms.models import *
from django.contrib.admin.views.decorators import staff_member_required
from django import forms
from django.shortcuts import render_to_response,get_object_or_404
from datetime import datetime
from django.http import HttpResponseRedirect

class PageForm(forms.Form):
    slug = forms.CharField(max_length=100)
    title = forms.CharField(max_length=100)
    comment = forms.CharField(max_length=100)
    text = forms.CharField(widget=forms.Textarea)
    login = forms.BooleanField(required=False)

@staff_member_required
def revision(request,rev_id):
    rev = get_object_or_404(PageRevision,id=rev_id)
    if request.method == 'POST':
        # clone
        rev.id = None
        rev.date_written = datetime.now()
        rev.comment = request.POST['comment']
        rev.save()
        return HttpResponseRedirect('/admin/cms/page/'+str(rev.page.id))

    else:
        return render_to_response('cms/admin/view_revision.html', {
            'revision':rev,
            'user':request.user,
        })

@staff_member_required
def add_edit(request,page_id=None):
    if request.method == 'POST':
        form = PageForm(request.POST)
        if form.is_valid():
            try:
                page = Page.objects.get(id=page_id)
                page.slug = form.cleaned_data['slug']
            except Page.DoesNotExist:
                page = Page(slug=form.cleaned_data['slug'])

            page.save()
            rev = PageRevision(page=page,
                title = form.cleaned_data['title'],
                text = form.cleaned_data['text'],
                comment = form.cleaned_data['comment'],
                login = form.cleaned_data['login'],
                date_written = datetime.now(),
            )
            rev.save()
            return HttpResponseRedirect('/admin/cms/page/'+str(page.id))

    elif page_id != None:
        page = get_object_or_404(Page,id=page_id)
        rev = page.get_data()
        data = {
            'slug':page.slug,
            'title':rev.title,
            'comment':rev.comment,
            'text':rev.text,
            'login':rev.login,
        }
        form = PageForm(data)
        comments = map(lambda rev: (rev.comment,rev.id),page.pagerevision_set.order_by('-date_written'))
    else:
        form = PageForm()
        comments = []

    return render_to_response('cms/admin/addedit.html', {
        'form': form,
        'user': request.user,
        'id': page_id,
        'comments':comments,
    })
<commit_msg>Fixes an error if we omit a comment when creating (or, possibly, modifying) a page in the CMS.<commit_after>from compsoc.cms.models import *
from django.contrib.admin.views.decorators import staff_member_required
from django import forms
from django.shortcuts import render_to_response,get_object_or_404
from datetime import datetime
from django.http import HttpResponseRedirect

class PageForm(forms.Form):
    slug = forms.CharField(max_length=100)
    title = forms.CharField(max_length=100)
    comment = forms.CharField(max_length=100)
    text = forms.CharField(widget=forms.Textarea)
    login = forms.BooleanField(required=False)

@staff_member_required
def revision(request,rev_id):
    rev = get_object_or_404(PageRevision,id=rev_id)
    if request.method == 'POST':
        # clone
        rev.id = None
        rev.date_written = datetime.now()
        rev.comment = request.POST['comment']
        rev.save()
        return HttpResponseRedirect('/admin/cms/page/'+str(rev.page.id))

    else:
        return render_to_response('cms/admin/view_revision.html', {
            'revision':rev,
            'user':request.user,
        })

@staff_member_required
def add_edit(request,page_id=None):
    if request.method == 'POST':
        form = PageForm(request.POST)
        if form.is_valid():
            try:
                page = Page.objects.get(id=page_id)
                page.slug = form.cleaned_data['slug']
            except Page.DoesNotExist:
                page = Page(slug=form.cleaned_data['slug'])

            page.save()
            rev = PageRevision(page=page,
                title = form.cleaned_data['title'],
                text = form.cleaned_data['text'],
                comment = form.cleaned_data['comment'],
                login = form.cleaned_data['login'],
                date_written = datetime.now(),
            )
            rev.save()
            return HttpResponseRedirect('/admin/cms/page/'+str(page.id))
        comments = []
    elif page_id != None:
        page = get_object_or_404(Page,id=page_id)
        rev = page.get_data()
        data = {
            'slug':page.slug,
            'title':rev.title,
            'comment':rev.comment,
            'text':rev.text,
            'login':rev.login,
        }
        form = PageForm(data)
        comments = map(lambda rev: (rev.comment,rev.id),page.pagerevision_set.order_by('-date_written'))
    else:
        form = PageForm()
        comments = []

    return render_to_response('cms/admin/addedit.html', {
        'form': form,
        'user': request.user,
        'id': page_id,
        'comments':comments,
    })
<|endoftext|>


<commit_before>import argparse
import os
import numpy as np

from PyPi.algorithms.dqn import DQN
from PyPi.approximators import Regressor
from PyPi.core.core import Core
from PyPi.environments import *
from PyPi.policy import EpsGreedy
from PyPi.utils.dataset import compute_scores
from PyPi.utils.parameters import LinearDecayParameter, Parameter
from PyPi.utils.preprocessor import Scaler
from convnet import ConvNet
from rmspropgraves import RMSpropGraves

# Disable tf cpp warnings
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"

def print_epoch(epoch):
    print '################################################################'
    print 'epoch: ', epoch
    print '----------------------------------------------------------------'

def experiment():
    np.random.seed()

    parser = argparse.ArgumentParser()

    arg_game = parser.add_argument_group('Game')
    arg_game.add_argument("--name",
                          type=str,
                          default='BreakoutDeterministic-v4')
    arg_game.add_argument("--screen-width", type=int, default=84)
    arg_game.add_argument("--screen-height", type=int, default=84)

    arg_mem = parser.add_argument_group('Replay Memory')
    arg_mem.add_argument("--initial-replay-size", type=int, default=50000)
    arg_mem.add_argument("--max-replay-size", type=int, default=100000)

    arg_net = parser.add_argument_group('Deep Q-Network')
    arg_net.add_argument("--optimizer",
                         choices=['adam', 'rmsprop', RMSpropGraves()],
                         default=RMSpropGraves())

    arg_alg = parser.add_argument_group('Algorithm')
    arg_alg.add_argument("--batch-size", type=int, default=32)
    arg_alg.add_argument("--history-length", type=int, default=4)
    arg_alg.add_argument("--target-update-frequency", type=int, default=10000)
    arg_alg.add_argument("--evaluation-frequency", type=int, default=250000)
    arg_alg.add_argument("--train-frequency", type=int, default=4)
    arg_alg.add_argument("--fit-steps", type=int, default=1)
    arg_alg.add_argument("--max-steps", type=int, default=50000000)
    arg_alg.add_argument("--final-exploration-frame", type=int, default=1000000)
    arg_alg.add_argument("--initial-exploration-rate", type=float, default=1)
    arg_alg.add_argument("--final-exploration-rate", type=float, default=.1)
    arg_alg.add_argument("--test-exploration-rate", type=float, default=.05)
    arg_alg.add_argument("--test-samples", type=int, default=125000)
    arg_alg.add_argument("--max-no-op-actions", type=int, default=30)
    arg_alg.add_argument("--no-op-action-value", type=int, default=0)

    arg_utils = parser.add_argument_group('Utils')
    arg_utils.add_argument('--render', type=bool, default=False)
    arg_utils.add_argument('--quiet', type=bool, default=False)
    arg_utils.add_argument('--debug', action='store_true')

    args = parser.parse_args()

    # DQN settings
    if args.debug:
        initial_replay_size = 50
        max_replay_size = 500
        train_frequency = 5
        target_update_frequency = 10
        test_samples = 20
        evaluation_frequency = 10
        max_steps = 1000
    else:
        initial_replay_size = args.initial_replay_size
        max_replay_size = args.max_replay_size
        train_frequency = args.train_frequency
        target_update_frequency = args.target_update_frequency
        test_samples = args.test_samples
        evaluation_frequency = args.evaluation_frequency
        max_steps = args.max_steps

    # MDP train
    mdp = Atari(args.name, ends_at_life=True)

    # Policy
    epsilon = LinearDecayParameter(value=args.initial_exploration_rate,
                                   min_value=args.final_exploration_rate,
                                   n=args.final_exploration_frame)
    epsilon_test = Parameter(value=args.test_exploration_rate)
    epsilon_random = Parameter(value=1)
    pi = EpsGreedy(epsilon=epsilon_random,
                   observation_space=mdp.observation_space,
                   action_space=mdp.action_space)

    # Approximator
    approximator_params = dict(n_actions=mdp.action_space.n,
                               optimizer=args.optimizer)
    approximator = Regressor(ConvNet,
                             fit_action=False,
                             preprocessor=[Scaler(mdp.observation_space.high)],
                             **approximator_params)

    # target approximator
    target_approximator = Regressor(
        ConvNet,
        fit_action=False,
        preprocessor=[Scaler(mdp.observation_space.high)],
        **approximator_params)
    target_approximator.model.set_weights(approximator.model.get_weights())

    # Agent
    algorithm_params = dict(
        batch_size=args.batch_size,
        target_approximator=target_approximator,
        initial_replay_size=initial_replay_size,
        max_replay_size=max_replay_size,
        history_length=args.history_length,
        train_frequency=train_frequency,
        target_update_frequency=target_update_frequency,
        max_no_op_actions=args.max_no_op_actions,
        no_op_action_value=args.no_op_action_value
    )
    fit_params = dict()
    agent_params = {'algorithm_params': algorithm_params,
                    'fit_params': fit_params}
    agent = DQN(approximator, pi, **agent_params)

    # Algorithm
    core = Core(agent, mdp)
    core_test = Core(agent, mdp)

    # DQN

    # fill replay memory with random dataset
    print_epoch(0)
    core.learn(n_iterations=1, how_many=initial_replay_size,
               n_fit_steps=0, iterate_over='samples', quiet=args.quiet)

    # evaluate initial policy
    pi.set_epsilon(epsilon_test)
    mdp.set_episode_end(ends_at_life=False)
    dataset = core_test.evaluate(how_many=test_samples,
                                 iterate_over='samples',
                                 render=args.render,
                                 quiet=args.quiet)
    score = compute_scores(dataset)
    print('min_reward: %f, max_reward: %f, mean_reward: %f,'
          ' games_completed: %d' % score)
    for i in xrange(max_steps - evaluation_frequency):
        print_epoch(i + 1)
        print '- Learning:'
        # learning step
        pi.set_epsilon(epsilon)
        mdp.set_episode_end(ends_at_life=True)
        core.learn(n_iterations=evaluation_frequency / train_frequency,
                   how_many=train_frequency,
                   n_fit_steps=args.fit_steps,
                   iterate_over='samples',
                   quiet=args.quiet)
        print '- Evaluation:'
        # evaluation step
        pi.set_epsilon(epsilon_test)
        mdp.set_episode_end(ends_at_life=False)
        core_test.reset()
        dataset = core_test.evaluate(how_many=test_samples,
                                     iterate_over='samples',
                                     render=args.render,
                                     quiet=args.quiet)
        score = compute_scores(dataset)
        print('min_reward: %f, max_reward: %f, mean_reward: %f,'
              ' games_completed: %d' % score)

if __name__ == '__main__':
    experiment()
<commit_msg>minor fix<commit_after>import argparse
import os
import numpy as np

from PyPi.algorithms.dqn import DQN
from PyPi.approximators import Regressor
from PyPi.core.core import Core
from PyPi.environments import *
from PyPi.policy import EpsGreedy
from PyPi.utils.dataset import compute_scores
from PyPi.utils.parameters import LinearDecayParameter, Parameter
from PyPi.utils.preprocessor import Scaler
from convnet import ConvNet
from rmspropgraves import RMSpropGraves

# Disable tf cpp warnings
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"

def print_epoch(epoch):
    print '################################################################'
    print 'epoch: ', epoch
    print '----------------------------------------------------------------'

def experiment():
    np.random.seed()

    parser = argparse.ArgumentParser()

    arg_game = parser.add_argument_group('Game')
    arg_game.add_argument("--name",
                          type=str,
                          default='BreakoutDeterministic-v4')
    arg_game.add_argument("--screen-width", type=int, default=84)
    arg_game.add_argument("--screen-height", type=int, default=84)

    arg_mem = parser.add_argument_group('Replay Memory')
    arg_mem.add_argument("--initial-replay-size", type=int, default=50000)
    arg_mem.add_argument("--max-replay-size", type=int, default=100000)

    arg_net = parser.add_argument_group('Deep Q-Network')
    arg_net.add_argument("--optimizer",
                         choices=['adam', 'rmsprop', RMSpropGraves()],
                         default=RMSpropGraves())

    arg_alg = parser.add_argument_group('Algorithm')
    arg_alg.add_argument("--batch-size", type=int, default=32)
    arg_alg.add_argument("--history-length", type=int, default=4)
    arg_alg.add_argument("--target-update-frequency", type=int, default=10000)
    arg_alg.add_argument("--evaluation-frequency", type=int, default=250000)
    arg_alg.add_argument("--train-frequency", type=int, default=4)
    arg_alg.add_argument("--fit-steps", type=int, default=1)
    arg_alg.add_argument("--max-steps", type=int, default=50000000)
    arg_alg.add_argument("--final-exploration-frame", type=int, default=1000000)
    arg_alg.add_argument("--initial-exploration-rate", type=float, default=1)
    arg_alg.add_argument("--final-exploration-rate", type=float, default=.1)
    arg_alg.add_argument("--test-exploration-rate", type=float, default=.05)
    arg_alg.add_argument("--test-samples", type=int, default=125000)
    arg_alg.add_argument("--max-no-op-actions", type=int, default=30)
    arg_alg.add_argument("--no-op-action-value", type=int, default=0)

    arg_utils = parser.add_argument_group('Utils')
    arg_utils.add_argument('--render', action='store_true')
    arg_utils.add_argument('--quiet', action='store_true')
    arg_utils.add_argument('--debug', action='store_true')

    args = parser.parse_args()

    # DQN settings
    if args.debug:
        initial_replay_size = 50
        max_replay_size = 500
        train_frequency = 5
        target_update_frequency = 10
        test_samples = 20
        evaluation_frequency = 10
        max_steps = 1000
    else:
        initial_replay_size = args.initial_replay_size
        max_replay_size = args.max_replay_size
        train_frequency = args.train_frequency
        target_update_frequency = args.target_update_frequency
        test_samples = args.test_samples
        evaluation_frequency = args.evaluation_frequency
        max_steps = args.max_steps

    # MDP train
    mdp = Atari(args.name, ends_at_life=True)

    # Policy
    epsilon = LinearDecayParameter(value=args.initial_exploration_rate,
                                   min_value=args.final_exploration_rate,
                                   n=args.final_exploration_frame)
    epsilon_test = Parameter(value=args.test_exploration_rate)
    epsilon_random = Parameter(value=1)
    pi = EpsGreedy(epsilon=epsilon_random,
                   observation_space=mdp.observation_space,
                   action_space=mdp.action_space)

    # Approximator
    approximator_params = dict(n_actions=mdp.action_space.n,
                               optimizer=args.optimizer)
    approximator = Regressor(ConvNet,
                             fit_action=False,
                             preprocessor=[Scaler(mdp.observation_space.high)],
                             **approximator_params)

    # target approximator
    target_approximator = Regressor(
        ConvNet,
        fit_action=False,
        preprocessor=[Scaler(mdp.observation_space.high)],
        **approximator_params)
    target_approximator.model.set_weights(approximator.model.get_weights())

    # Agent
    algorithm_params = dict(
        batch_size=args.batch_size,
        target_approximator=target_approximator,
        initial_replay_size=initial_replay_size,
        max_replay_size=max_replay_size,
        history_length=args.history_length,
        train_frequency=train_frequency,
        target_update_frequency=target_update_frequency,
        max_no_op_actions=args.max_no_op_actions,
        no_op_action_value=args.no_op_action_value
    )
    fit_params = dict()
    agent_params = {'algorithm_params': algorithm_params,
                    'fit_params': fit_params}
    agent = DQN(approximator, pi, **agent_params)

    # Algorithm
    core = Core(agent, mdp)
    core_test = Core(agent, mdp)

    # DQN

    # fill replay memory with random dataset
    print_epoch(0)
    core.learn(n_iterations=1, how_many=initial_replay_size,
               n_fit_steps=0, iterate_over='samples', quiet=args.quiet)

    # evaluate initial policy
    pi.set_epsilon(epsilon_test)
    mdp.set_episode_end(ends_at_life=False)
    dataset = core_test.evaluate(how_many=test_samples,
                                 iterate_over='samples',
                                 render=args.render,
                                 quiet=args.quiet)
    score = compute_scores(dataset)
    print('min_reward: %f, max_reward: %f, mean_reward: %f,'
          ' games_completed: %d' % score)
    for i in xrange(max_steps - evaluation_frequency):
        print_epoch(i + 1)
        print '- Learning:'
        # learning step
        pi.set_epsilon(epsilon)
        mdp.set_episode_end(ends_at_life=True)
        core.learn(n_iterations=evaluation_frequency / train_frequency,
                   how_many=train_frequency,
                   n_fit_steps=args.fit_steps,
                   iterate_over='samples',
                   quiet=args.quiet)
        print '- Evaluation:'
        # evaluation step
        pi.set_epsilon(epsilon_test)
        mdp.set_episode_end(ends_at_life=False)
        core_test.reset()
        dataset = core_test.evaluate(how_many=test_samples,
                                     iterate_over='samples',
                                     render=args.render,
                                     quiet=args.quiet)
        score = compute_scores(dataset)
        print('min_reward: %f, max_reward: %f, mean_reward: %f,'
              ' games_completed: %d' % score)

if __name__ == '__main__':
    experiment()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding=utf-8
###############################################################################

import os
import sys
from cro import *

def test_croCreation():
    """
    Test that the REEF has a proper shape
    """
    L = 8
    N = 2
    M = 2
    fitness_coral = lambda coral: 1 # Dummy fitness
    cro = CRO(Ngen=10, N=N, M=M, Fb=0.7, Fa=.1, Fd=.1, r0=.6, k=3, Pd=.1,
              fitness_coral=fitness_coral, opt='max', L=L)
    (REEF, REEFpob) = cro.reefinitialization()
    assert REEFpob.shape == (N*M, L)

def test_croInit():
    """
    Test that the number of corals in the reef is greater than 0
    """
    L = 8
    N = 2
    M = 2
    fitness_coral = lambda coral: 1 # Dummy fitness
    cro = CRO(Ngen=10, N=N, M=M, Fb=0.7, Fa=.1, Fd=.1, r0=.6, k=3, Pd=.1,
              fitness_coral=fitness_coral, opt='max', L=L)
    (REEF, REEFpob) = cro.reefinitialization()
    assert len(np.where(REEF!=0)[0]) > 0
<commit_msg>[ADD] test test_reefinitializationDisc<commit_after>#!/usr/bin/env python
# coding=utf-8
###############################################################################

import os
import sys
from cro import *

def test_croCreation():
    """
    Test that the REEF has a proper shape
    """
    L = 8
    N = 2
    M = 2
    fitness_coral = lambda coral: 1 # Dummy fitness
    cro = CRO(Ngen=10, N=N, M=M, Fb=0.7, Fa=.1, Fd=.1, r0=.6, k=3, Pd=.1,
              fitness_coral=fitness_coral, opt='max', L=L)
    (REEF, REEFpob) = cro.reefinitialization()
    assert REEFpob.shape == (N*M, L)

def test_croInit():
    """
    Test that the number of corals in the reef is greater than 0
    """
    L = 8
    N = 2
    M = 2
    fitness_coral = lambda coral: 1 # Dummy fitness
    cro = CRO(Ngen=10, N=N, M=M, Fb=0.7, Fa=.1, Fd=.1, r0=.6, k=3, Pd=.1,
              fitness_coral=fitness_coral, opt='max', L=L)
    (REEF, REEFpob) = cro.reefinitialization()
    assert len(np.where(REEF!=0)[0]) > 0

def test_reefinitializationDisc():
    """
    Test that reefinitialization function works in a discrete mode option, any coral lower than required min value and
    any higher than max
    """
    L = 8
    N = 2
    M = 2
    grid = {'x': [1, 10]}      # Discrete values between 2 and 10
    fitness_coral = lambda coral: 1 # Dummy fitness

    cro = CRO(Ngen=10, N=N, M=M, Fb=0.7, Fa=.1, Fd=.1, r0=.6, k=3, Pd=.1,
              fitness_coral=fitness_coral, opt='max', L=L, mode='disc', param_grid=grid)
    (REEF, REEFpob) = cro.reefinitialization()
    p = sum(REEFpob[np.where(REEFpob!=0)]<grid['x'][0]) + sum(REEFpob[np.where(REEFpob!=0)]>grid['x'][1])
    assert p == 0

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import urllib
from cms.exceptions import LanguageError
from cms.utils.i18n import get_language_objects, get_language_object
from cms.toolbar.items import Item, List, Break, Switch
from django.contrib.sites.models import Site
from cms.utils import get_language_from_request, get_cms_setting

from cms.toolbar_base import CMSToolbar
from cms.toolbar_pool import toolbar_pool
from cms.utils.permissions import has_page_change_permission
from django.conf import settings
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext_lazy as _
from menus.utils import DefaultLanguageChanger

class PageToolbar(CMSToolbar):
    def insert_items(self, items, toolbar, request, is_app):
        self.is_app = is_app
        self.request = request
        self.toolbar = toolbar
        self.can_change = (hasattr(self.request.current_page, 'has_change_permission') and
                           self.request.current_page.has_change_permission(self.request))
        toolbar.can_change = self.can_change
        if toolbar.is_staff:
            # The 'Admin' Menu
            items.append(self.get_admin_menu())
            if toolbar.request.current_page and is_app:
                has_global_current_page_change_permission = False
                if get_cms_setting('PERMISSION'):
                    has_global_current_page_change_permission = has_page_change_permission(self.request)
                has_current_page_change_permission = self.request.current_page.has_change_permission(self.request)
                if has_global_current_page_change_permission or has_current_page_change_permission:
                    # The 'page' Menu
                    items.append(self.get_page_menu(self.request.current_page))
                    # The 'templates' Menu
                    items.append(self.get_template_menu())
                    # Publish Menu
                    items.append(self.get_history_menu())
                    if self.toolbar.edit_mode:
                        if self.request.current_page.has_publish_permission(self.request):
                            items.append(self.get_publish_menu())
                        items.append(self.get_mode_switchers())
            items.append(self.get_language_menu())
        return items

    def get_language_menu(self):
        site = Site.objects.get_current()
        try:
            current_lang = get_language_object(get_language_from_request(self.request), site.pk)
        except LanguageError:
            current_lang = None
        menu_items = List("#", _("Language"))
        for lang in get_language_objects(site.pk):
            if hasattr(self.request, "_language_changer"):
                url = self.request._language_changer(lang['code'])
            else:
                url = DefaultLanguageChanger(self.request)(lang['code'])
            menu_items.items.append(
                Item(url, lang['name'], active=current_lang and lang['code'] == current_lang['code'], load_modal=False))
        return menu_items

    def get_template_menu(self):
        menu_items = List("#", _("Template"))
        url = reverse('admin:cms_page_change_template', args=(self.request.current_page.pk,))
        for path, name in get_cms_setting('TEMPLATES'):
            args = urllib.urlencode({'template': path})
            active = False
            if self.request.current_page.get_template() == path:
                active = True
            if path == "INHERIT":
                menu_items.items.append(Break())
            menu_items.items.append(
                Item('%s?%s' % (url, args), name, ajax=True, active=active),
            )
        return menu_items

    def get_page_menu(self, page):
        """
        Builds the 'page menu'
        """
        if not self.request.current_page.pk:
            return []

        menu_items = List(reverse("admin:cms_page_change", args=[page.pk]), _("Page"))
        menu_items.items.append(Item("?edit", _('Edit Page'), disabled=self.toolbar.edit_mode))
        menu_items.items.append(
            Item(reverse('admin:cms_page_change', args=[page.pk]), _('Settings'), load_side_frame=True))
        menu_items.items.append(Break())
        menu_items.items.append(Item(reverse('admin:cms_page_changelist'), _('Move/add Pages'), load_side_frame=True))
        menu_items.items.append(Item(_get_add_child_url(self.toolbar), _('Add child page'), load_side_frame=True))
        menu_items.items.append(Item(_get_add_sibling_url(self.toolbar), _('Add sibling page'), load_side_frame=True))
        menu_items.items.append(Break())
        menu_items.items.append(Item(_get_delete_url(self.toolbar), _('Delete Page'), load_side_frame=True))
        if 'reversion' in settings.INSTALLED_APPS:
            menu_items.items.append(Item(_get_page_history_url(self.toolbar), _('View History'), load_side_frame=True))
        return menu_items

    def get_history_menu(self):
        page = self.request.current_page
        if page.publisher_is_draft:
            pk = page.pk
        else:
            pk = page.publisher_draft.pk
        dirty = page.is_dirty()
        menu_items = List('', _("History"))
        menu_items.items.append(Item(reverse('admin:cms_page_revert_page',
                                             args=[pk]), _('Revert to live'), ajax=True,
                                     question=_("Are you sure you want to revert to live?"), disabled=not dirty))

    def get_publish_menu(self):
        page = self.request.current_page
        if page.publisher_is_draft:
            pk = page.pk
        else:
            pk = page.publisher_draft.pk
        dirty = page.is_dirty()

        switch = Switch(right=True)
        switch.addItem(_("Publish now"), reverse('admin:cms_page_publish_page', args=[pk]), True)
        return switch

    def get_admin_menu(self):
        """
        Builds the 'admin menu' (the one with the cogwheel)
        """
        admin_items = List(reverse("admin:index"), _("Admin"))
        if self.can_change:
            admin_items.items.append(Item(reverse("admin:cms_page_changelist"), _('Pages'), load_side_frame=True))
        if self.request.user.has_perm('user.change_user'):
            admin_items.items.append(Item(reverse("admin:auth_user_changelist"), _('Users'), load_side_frame=True))
        admin_items.items.append(Item(reverse('admin:index'), _('Administration'), load_side_frame=True))
        admin_items.items.append(Break())
        admin_items.items.append(Item(reverse('admin:cms_usersettings_change'), _('Settings'), load_side_frame=True))
        admin_items.items.append(Break())
        admin_items.items.append(Item(reverse("admin:logout"), _('Logout'), ajax=True, active=True))
        return admin_items

    def get_mode_switchers(self):
        switch = Switch(right=True)
        switch.addItem(_("Edit"), "?edit", self.toolbar.edit_mode)
        switch.addItem(_("Build"), "?build", not self.toolbar.edit_mode)
        return switch

toolbar_pool.register(PageToolbar)

def _get_page_admin_url(toolbar):
    return reverse('admin:cms_page_change', args=(toolbar.request.current_page.pk,))

def _get_page_history_url(toolbar):
    return reverse('admin:cms_page_history', args=(toolbar.request.current_page.pk,))

def _get_add_child_url(toolbar):
    data = {
        'position': 'last-child',
        'target': toolbar.request.current_page.pk,
    }
    args = urllib.urlencode(data)
    return '%s?%s' % (reverse('admin:cms_page_add'), args)

def _get_add_sibling_url(toolbar):
    data = {
        'position': 'last-child',
    }
    if toolbar.request.current_page.parent_id:
        data['target'] = toolbar.request.current_page.parent_id
    args = urllib.urlencode(data)
    return '%s?%s' % (reverse('admin:cms_page_add'), args)

def _get_delete_url(toolbar):
    return reverse('admin:cms_page_delete', args=(toolbar.request.current_page.pk,))

def _get_approve_url(toolbar):
    return reverse('admin:cms_page_approve_page', args=(toolbar.request.current_page.pk,))

def _get_publish_url(toolbar):
    return reverse('admin:cms_page_publish_page', args=(toolbar.request.current_page.pk,))
<commit_msg>sites menu added<commit_after># -*- coding: utf-8 -*-
import urllib
from cms.exceptions import LanguageError
from cms.utils.i18n import get_language_objects, get_language_object
from cms.toolbar.items import Item, List, Break, Switch
from django.contrib.sites.models import Site
from cms.utils import get_language_from_request, get_cms_setting

from cms.toolbar_base import CMSToolbar
from cms.toolbar_pool import toolbar_pool
from cms.utils.permissions import has_page_change_permission, get_user_sites_queryset
from django.conf import settings
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext_lazy as _
from menus.utils import DefaultLanguageChanger

class PageToolbar(CMSToolbar):
    def insert_items(self, items, toolbar, request, is_app):
        self.is_app = is_app
        self.request = request
        self.toolbar = toolbar
        self.can_change = (hasattr(self.request.current_page, 'has_change_permission') and
                           self.request.current_page.has_change_permission(self.request))
        toolbar.can_change = self.can_change
        if toolbar.is_staff:
            # The 'Admin' Menu
            items.append(self.get_admin_menu())
            self.get_sites_menu(items)
            if toolbar.request.current_page and is_app:
                has_global_current_page_change_permission = False
                if get_cms_setting('PERMISSION'):
                    has_global_current_page_change_permission = has_page_change_permission(self.request)
                has_current_page_change_permission = self.request.current_page.has_change_permission(self.request)
                if has_global_current_page_change_permission or has_current_page_change_permission:
                    # The 'page' Menu
                    items.append(self.get_page_menu(self.request.current_page))
                    # The 'templates' Menu
                    items.append(self.get_template_menu())
                    # Publish Menu
                    items.append(self.get_history_menu())
                    if self.toolbar.edit_mode:
                        if self.request.current_page.has_publish_permission(self.request):
                            items.append(self.get_publish_menu())
                        items.append(self.get_mode_switchers())
            items.append(self.get_language_menu())
        return items

    def get_sites_menu(self, items):
        if get_cms_setting('PERMISSION'):
            sites = get_user_sites_queryset(self.request.user)
        else:
            sites = Site.objects.all()
        if len(sites) > 1:
            menu_items = List("#", _("Sites"))
            menu_items.items.append(
                Item(reverse("admin:sites_site_changelist"), _("Admin Sites"), load_side_frame=True))
            menu_items.items.append(Break())
            items.append(menu_items)
            current_site = Site.objects.get_current()
            for site in sites:
                menu_items.items.append(
                    Item("http://%s" % site.domain, site.name, load_modal=False,
                         active=site.pk == current_site.pk))

    def get_language_menu(self):
        site = Site.objects.get_current()
        try:
            current_lang = get_language_object(get_language_from_request(self.request), site.pk)
        except LanguageError:
            current_lang = None
        menu_items = List("#", _("Language"))
        for lang in get_language_objects(site.pk):
            if hasattr(self.request, "_language_changer"):
                url = self.request._language_changer(lang['code'])
            else:
                url = DefaultLanguageChanger(self.request)(lang['code'])
            menu_items.items.append(
                Item(url, lang['name'], active=current_lang and lang['code'] == current_lang['code'], load_modal=False))
        return menu_items

    def get_template_menu(self):
        menu_items = List("#", _("Template"))
        url = reverse('admin:cms_page_change_template', args=(self.request.current_page.pk,))
        for path, name in get_cms_setting('TEMPLATES'):
            args = urllib.urlencode({'template': path})
            active = False
            if self.request.current_page.get_template() == path:
                active = True
            if path == "INHERIT":
                menu_items.items.append(Break())
            menu_items.items.append(
                Item('%s?%s' % (url, args), name, ajax=True, active=active),
            )
        return menu_items

    def get_page_menu(self, page):
        """
        Builds the 'page menu'
        """
        if not self.request.current_page.pk:
            return []

        menu_items = List(reverse("admin:cms_page_change", args=[page.pk]), _("Page"))
        menu_items.items.append(Item("?edit", _('Edit Page'), disabled=self.toolbar.edit_mode, load_modal=False))
        menu_items.items.append(
            Item(reverse('admin:cms_page_change', args=[page.pk]), _('Settings'), load_side_frame=True))
        menu_items.items.append(Break())
        menu_items.items.append(Item(reverse('admin:cms_page_changelist'), _('Move/add Pages'), load_side_frame=True))
        menu_items.items.append(Item(_get_add_child_url(self.toolbar), _('Add child page'), load_side_frame=True))
        menu_items.items.append(Item(_get_add_sibling_url(self.toolbar), _('Add sibling page'), load_side_frame=True))
        menu_items.items.append(Break())
        menu_items.items.append(Item(_get_delete_url(self.toolbar), _('Delete Page'), load_side_frame=True))
        if 'reversion' in settings.INSTALLED_APPS:
            menu_items.items.append(Item(_get_page_history_url(self.toolbar), _('View History'), load_side_frame=True))
        return menu_items

    def get_history_menu(self):
        page = self.request.current_page
        if page.publisher_is_draft:
            pk = page.pk
        else:
            pk = page.publisher_draft.pk
        dirty = page.is_dirty()
        menu_items = List('', _("History"))
        menu_items.items.append(Item(reverse('admin:cms_page_revert_page',
                                             args=[pk]), _('Revert to live'), ajax=True,
                                     question=_("Are you sure you want to revert to live?"), disabled=not dirty))

    def get_publish_menu(self):
        page = self.request.current_page
        if page.publisher_is_draft:
            pk = page.pk
        else:
            pk = page.publisher_draft.pk
        dirty = page.is_dirty()

        switch = Switch(right=True)
        switch.addItem(_("Publish now"), reverse('admin:cms_page_publish_page', args=[pk]), True)
        return switch

    def get_admin_menu(self):
        """
        Builds the 'admin menu' (the one with the cogwheel)
        """
        admin_items = List(reverse("admin:index"), _("Admin"))
        if self.can_change:
            admin_items.items.append(Item(reverse("admin:cms_page_changelist"), _('Pages'), load_side_frame=True))
        if self.request.user.has_perm('user.change_user'):
            admin_items.items.append(Item(reverse("admin:auth_user_changelist"), _('Users'), load_side_frame=True))
        admin_items.items.append(Item(reverse('admin:index'), _('Administration'), load_side_frame=True))
        admin_items.items.append(Break())
        admin_items.items.append(Item(reverse('admin:cms_usersettings_change'), _('Settings'), load_side_frame=True))
        admin_items.items.append(Break())
        admin_items.items.append(Item(reverse("admin:logout"), _('Logout'), ajax=True, active=True))
        return admin_items

    def get_mode_switchers(self):
        switch = Switch(right=True)
        switch.addItem(_("Edit"), "?edit", self.toolbar.edit_mode)
        switch.addItem(_("Build"), "?build", not self.toolbar.edit_mode)
        return switch

toolbar_pool.register(PageToolbar)

def _get_page_admin_url(toolbar):
    return reverse('admin:cms_page_change', args=(toolbar.request.current_page.pk,))

def _get_page_history_url(toolbar):
    return reverse('admin:cms_page_history', args=(toolbar.request.current_page.pk,))

def _get_add_child_url(toolbar):
    data = {
        'position': 'last-child',
        'target': toolbar.request.current_page.pk,
    }
    args = urllib.urlencode(data)
    return '%s?%s' % (reverse('admin:cms_page_add'), args)

def _get_add_sibling_url(toolbar):
    data = {
        'position': 'last-child',
    }
    if toolbar.request.current_page.parent_id:
        data['target'] = toolbar.request.current_page.parent_id
    args = urllib.urlencode(data)
    return '%s?%s' % (reverse('admin:cms_page_add'), args)

def _get_delete_url(toolbar):
    return reverse('admin:cms_page_delete', args=(toolbar.request.current_page.pk,))

def _get_approve_url(toolbar):
    return reverse('admin:cms_page_approve_page', args=(toolbar.request.current_page.pk,))

def _get_publish_url(toolbar):
    return reverse('admin:cms_page_publish_page', args=(toolbar.request.current_page.pk,))
<|endoftext|>


<commit_before>import yt
from galaxy_analysis.analysis import Galaxy
import numpy as np
from galaxy_analysis.static_data import ISM
from galaxy_analysis.plot.plot_styles import *
from scipy.stats import binned_statistic
import matplotlib.pyplot as plt
import sys

if len(sys.argv) > 1:
    dsname = str(sys.argv[1])
else:
    dsname = 'DD0401'

gal = Galaxy(dsname)
#
#
#
#scipy.stats.binned_statistic(x, values, statistic='mean', bins=10, range=None)[source]
#

n = gal.disk['number_density']
M = gal.disk['cell_mass'].convert_to_units('Msun')
M_tot = np.sum(M)

#
n_bins = np.arange(-6, 3.01, 0.05) # 0.05 dex bins in n

def compute_pdf(n, M, norm = None):
    stat, bin_edges, dummy = binned_statistic(np.log10(n), M, statistic = 'sum', bins = n_bins)
    bin_sizes = 0.5 * (10.0**n_bins[1:] + 10.0**(n_bins[:-1]))

#    bin_sizes = 0.5 * (bins

    if norm is None:
        norm = 1.0 * np.sum(M)

    frac = stat / (norm)
    PDF  = frac / bin_sizes
    return PDF

fig, ax = plt.subplots()
fig.set_size_inches(8,8)

PDF = compute_pdf(n,M)
plot_histogram(ax, n_bins, PDF, lw = 3, color = 'black', label = 'Total')

for field in ['CNM','WNM','WIM','HIM']:
    n = gal.disk.cut_region(ISM[field])['number_density']
    M = gal.disk.cut_region(ISM[field])['cell_mass'].convert_to_units('Msun')

    PDF = compute_pdf(n,M, norm = M_tot)
    plot_histogram(ax, n_bins, PDF, lw = 3, color = color_dict[field], label = field)

ax.set_xlabel(r'n (cm$^{-3}$)')
ax.set_ylabel(r'PDF')
ax.set_xlim(-6,3)
ax.set_ylim(4.0E-6, 11.0)
ax.legend(loc='best')
ax.semilogy()
plt.tight_layout()
fig.savefig(dsname + '_density_PDF.png')

fig, ax = plt.subplots()
fig.set_size_inches(8,8)

n = gal.disk['number_density']
V = gal.disk['cell_volume']
V_tot = np.sum(V) * 1.0

PDF = compute_pdf(n,V, norm = V_tot)
plot_histogram(ax, n_bins, PDF, lw = 3, color = 'black', label = 'Total')

for field in ['CNM','WNM','WIM','HIM']:
    n = gal.disk.cut_region(ISM[field])['number_density']
    V = gal.disk.cut_region(ISM[field])['cell_volume']

    PDF = compute_pdf(n,V, norm = V_tot)
    plot_histogram(ax, n_bins, PDF, lw = 3, color = color_dict[field], label = field)

ax.set_xlabel(r'n (cm$^{-3}$)')
ax.set_ylabel(r'PDF')
ax.set_xlim(-6,3)
ax.set_ylim(1.0E-8, 1.0E4)
ax.legend(loc='best')
ax.semilogy()
plt.tight_layout()
fig.savefig(dsname + '_density_PDF_volume_weighted.png')

<commit_msg>fixing definition of PDF to dlogn instead of dn<commit_after>import yt
from galaxy_analysis.analysis import Galaxy
import numpy as np
from galaxy_analysis.static_data import ISM
from galaxy_analysis.plot.plot_styles import *
from scipy.stats import binned_statistic
import matplotlib.pyplot as plt
import sys

if len(sys.argv) > 1:
    dsname = str(sys.argv[1])
else:
    dsname = 'DD0401'

gal = Galaxy(dsname)
#
#
#
#scipy.stats.binned_statistic(x, values, statistic='mean', bins=10, range=None)[source]
#

n = gal.disk['number_density']
M = gal.disk['cell_mass'].convert_to_units('Msun')
M_tot = np.sum(M)

#
n_bins = np.arange(-6, 3.01, 0.05) # 0.05 dex bins in n

def compute_pdf(n, M, norm = None):
    stat, bin_edges, dummy = binned_statistic(np.log10(n), M, statistic = 'sum', bins = n_bins)
    bin_sizes = (10.0**n_bins[1:] - 10.0**(n_bins[:-1]))
    bin_sizes = (n_bins[1:] - n_bins[:-1])
#    bin_sizes = 0.5 * (bins

    if norm is None:
        norm = 1.0 * np.sum(M)

    frac = stat / (norm)
    PDF  = frac / bin_sizes
    return PDF

fig, ax = plt.subplots()
fig.set_size_inches(8,8)

PDF = compute_pdf(n,M)
plot_histogram(ax, n_bins, PDF, lw = 3, color = 'black', label = 'Total')

for field in ['CNM','WNM','WIM','HIM']:
    n = gal.disk.cut_region(ISM[field])['number_density']
    M = gal.disk.cut_region(ISM[field])['cell_mass'].convert_to_units('Msun')

    PDF = compute_pdf(n,M, norm = M_tot)
    plot_histogram(ax, n_bins, PDF, lw = 3, color = color_dict[field], label = field)

ax.set_xlabel(r'n (cm$^{-3}$)')
ax.set_ylabel(r'PDF')
ax.set_xlim(-6,3)
ax.set_ylim(1.0E-7, 8.0)
ax.legend(loc='best')
ax.semilogy()
plt.tight_layout()
fig.savefig(dsname + '_density_PDF.png')

fig, ax = plt.subplots()
fig.set_size_inches(8,8)

n = gal.disk['number_density']
V = gal.disk['cell_volume']
V_tot = np.sum(V) * 1.0

PDF = compute_pdf(n,V, norm = V_tot)
plot_histogram(ax, n_bins, PDF, lw = 3, color = 'black', label = 'Total')

for field in ['CNM','WNM','WIM','HIM']:
    n = gal.disk.cut_region(ISM[field])['number_density']
    V = gal.disk.cut_region(ISM[field])['cell_volume']

    PDF = compute_pdf(n,V, norm = V_tot)
    plot_histogram(ax, n_bins, PDF, lw = 3, color = color_dict[field], label = field)

ax.set_xlabel(r'n (cm$^{-3}$)')
ax.set_ylabel(r'PDF')
ax.set_xlim(-6,3)
ax.set_ylim(1.0E-7, 8.0)
ax.legend(loc='best')
ax.semilogy()
plt.tight_layout()
fig.savefig(dsname + '_density_PDF_volume_weighted.png')

<|endoftext|>