<commit_before>"""
What's the big idea?

An endpoint that traverses all restful endpoints producing a swagger 2.0 schema
If a swagger yaml description is found in the docstrings for an endpoint
we add the endpoint to swagger specification output

Taken from https://github.com/gangverk/flask-swagger
Under MIT License

"""
import inspect
from django.http.response import JsonResponse
import yaml
import re

from collections import defaultdict
import yunity

def _sanitize(comment):
    return comment.replace('\n', '
') if comment else comment

def _parse_docstring(obj, process_doc):
    first_line, other_lines, swag = None, None, None
    full_doc = inspect.getdoc(obj)
    if full_doc:
        line_feed = full_doc.find('\n')
        if line_feed != -1:
            first_line = process_doc(full_doc[:line_feed])
            yaml_sep = full_doc[line_feed+1:].find('---')
            if yaml_sep != -1:
                other_lines = process_doc(full_doc[line_feed+1:line_feed+yaml_sep])
                separator = full_doc[line_feed+yaml_sep+3:].find('...')
                if separator > 0:
                    yaml_doc = full_doc[line_feed+yaml_sep:line_feed+yaml_sep+separator+3]
                else:
                    yaml_doc = full_doc[line_feed+yaml_sep:]

                swag = yaml.safe_load(yaml_doc)
            else:
                other_lines = process_doc(full_doc[line_feed+1:])
        else:
            first_line = full_doc
    return first_line, other_lines, swag

def _extract_definitions(alist, level=None):
    """
    Since we couldn't be bothered to register models elsewhere
    our definitions need to be extracted from the parameters.
    We require an 'id' field for the schema to be correctly
    added to the definitions list.
    """

    def _extract_array_defs(source):
        # extract any definitions that are within arrays
        # this occurs recursively
        ret = []
        items = source.get('items')
        if items is not None and 'schema' in items:
            ret += _extract_definitions([items], level+1)
        return ret

    # for tracking level of recursion
    if level is None:
        level = 0

    defs = list()
    if alist is not None:
        for item in alist:
            schema = item.get("schema")
            if schema is not None:
                schema_id = schema.get("id")
                if schema_id is not None:
                    defs.append(schema)
                    ref = {"$ref": "#/definitions/{}".format(schema_id)}

                    # only add the reference as a schema if we are in a response or
                    # a parameter i.e. at the top level
                    # directly ref if a definition is used within another definition
                    if level == 0:
                        item['schema'] = ref
                    else:
                        item.update(ref)
                        del item['schema']

                # extract any definitions that are within properties
                # this occurs recursively
                properties = schema.get('properties')
                if properties is not None:
                    defs += _extract_definitions(properties.values(), level+1)

                defs += _extract_array_defs(schema)

            defs += _extract_array_defs(item)

    return defs

def get_rules(urlpattern, rule_base = ''):
    """
    :param urlpattern: base urlpatterns of the application to document
    :return: list of dictionaries of all rules and methodname/method name pairs for each rule
    """
    urls = []
    for pattern in urlpattern:
        pattern_rule = rule_base + pattern.regex.pattern
        if hasattr(pattern, 'url_patterns'):
            urls.extend(get_rules(pattern.url_patterns, pattern_rule))
        else:
            possible_methods = pattern.callback.view_class.http_method_names
            method_names = list(filter(lambda x: hasattr(pattern.callback.view_class, x), possible_methods))
            methods = list(map(lambda m: getattr(pattern.callback.view_class, m), method_names))
            verb_method = zip(method_names, methods)
            urls.append({'rule' : pattern_rule, 'methods': list(verb_method)})
    return urls

def swagger(app, process_doc=_sanitize, base=''):
    """
    Call this from an @app.route method like this
    @app.route('/spec.json')
    def spec():
       return jsonify(swagger(app))

    We go through all endpoints of the app searching for swagger endpoints
    We provide the minimum required data according to swagger specs
    Callers can and should add and override at will

    Arguments:
    app -- the flask app to inspect

    Keyword arguments:
    process_doc -- text sanitization method, the default simply replaces \n with

    """

    output = {
        "swagger": "2.0",
        "info": {
            "version": "0.0.0",
            "title": "Yunity",
        },
        "paths": defaultdict(dict),
        "definitions": defaultdict(dict)
    }

    paths = output['paths']
    definitions = output['definitions']
    ignore_verbs = {"HEAD", "OPTIONS"}
    # technically only responses is non-optional
    optional_fields = ['tags', 'consumes', 'produces', 'schemes', 'security',
                       'deprecated', 'operationId', 'externalDocs']

    rules = get_rules(app.urls.urlpatterns, base)
    for rule in rules:
        operations = dict()
        for verb, method in rule['methods']:
            summary, description, swag = _parse_docstring(method, process_doc)
            if swag is not None:  # we only add endpoints with swagger data in the docstrings
                params = swag.get('parameters', [])
                defs = _extract_definitions(params)
                responses = swag.get('responses', {})
                responses = {
                    str(key): value
                    for key, value in responses.items()
                }
                if responses is not None:
                    defs = defs + _extract_definitions(responses.values())
                for definition in defs:
                    def_id = definition.pop('id')
                    if def_id is not None:
                        definitions[def_id].update(definition)
                operation = dict(
                    summary=summary,
                    description=description,
                    responses=responses
                )
                # parameters - swagger ui dislikes empty parameter lists
                if len(params) > 0:
                    operation['parameters'] = params
                # other optionals
                for key in optional_fields:
                    if key in swag:
                        operation[key] = swag.get(key)
                operations[verb] = operation

        if len(operations):
            rule = rule['rule']
            for arg in re.findall('(<(.*?\:)?(.*?)>)', rule):
                rule = rule.replace(arg[0], '{%s}' % arg[2])
            rule = rule.translate(str.maketrans('','','^$'))
            paths[rule].update(operations)
    return output

def doc(request):
    return JsonResponse(swagger(yunity, base='/api/'))

<commit_msg>Extent Swagger regex rule to strip whole regex parameter<commit_after>"""
What's the big idea?

An endpoint that traverses all restful endpoints producing a swagger 2.0 schema
If a swagger yaml description is found in the docstrings for an endpoint
we add the endpoint to swagger specification output

Taken from https://github.com/gangverk/flask-swagger
Under MIT License

"""
import inspect
from django.http.response import JsonResponse
import yaml
import re

from collections import defaultdict
import yunity

def _sanitize(comment):
    return comment.replace('\n', '
') if comment else comment

def _parse_docstring(obj, process_doc):
    first_line, other_lines, swag = None, None, None
    full_doc = inspect.getdoc(obj)
    if full_doc:
        line_feed = full_doc.find('\n')
        if line_feed != -1:
            first_line = process_doc(full_doc[:line_feed])
            yaml_sep = full_doc[line_feed+1:].find('---')
            if yaml_sep != -1:
                other_lines = process_doc(full_doc[line_feed+1:line_feed+yaml_sep])
                separator = full_doc[line_feed+yaml_sep+3:].find('...')
                if separator > 0:
                    yaml_doc = full_doc[line_feed+yaml_sep:line_feed+yaml_sep+separator+3]
                else:
                    yaml_doc = full_doc[line_feed+yaml_sep:]

                swag = yaml.safe_load(yaml_doc)
            else:
                other_lines = process_doc(full_doc[line_feed+1:])
        else:
            first_line = full_doc
    return first_line, other_lines, swag

def _extract_definitions(alist, level=None):
    """
    Since we couldn't be bothered to register models elsewhere
    our definitions need to be extracted from the parameters.
    We require an 'id' field for the schema to be correctly
    added to the definitions list.
    """

    def _extract_array_defs(source):
        # extract any definitions that are within arrays
        # this occurs recursively
        ret = []
        items = source.get('items')
        if items is not None and 'schema' in items:
            ret += _extract_definitions([items], level+1)
        return ret

    # for tracking level of recursion
    if level is None:
        level = 0

    defs = list()
    if alist is not None:
        for item in alist:
            schema = item.get("schema")
            if schema is not None:
                schema_id = schema.get("id")
                if schema_id is not None:
                    defs.append(schema)
                    ref = {"$ref": "#/definitions/{}".format(schema_id)}

                    # only add the reference as a schema if we are in a response or
                    # a parameter i.e. at the top level
                    # directly ref if a definition is used within another definition
                    if level == 0:
                        item['schema'] = ref
                    else:
                        item.update(ref)
                        del item['schema']

                # extract any definitions that are within properties
                # this occurs recursively
                properties = schema.get('properties')
                if properties is not None:
                    defs += _extract_definitions(properties.values(), level+1)

                defs += _extract_array_defs(schema)

            defs += _extract_array_defs(item)

    return defs

def get_rules(urlpattern, rule_base = ''):
    """
    :param urlpattern: base urlpatterns of the application to document
    :return: list of dictionaries of all rules and methodname/method name pairs for each rule
    """
    urls = []
    for pattern in urlpattern:
        pattern_rule = rule_base + pattern.regex.pattern
        if hasattr(pattern, 'url_patterns'):
            urls.extend(get_rules(pattern.url_patterns, pattern_rule))
        else:
            possible_methods = pattern.callback.view_class.http_method_names
            method_names = list(filter(lambda x: hasattr(pattern.callback.view_class, x), possible_methods))
            methods = list(map(lambda m: getattr(pattern.callback.view_class, m), method_names))
            verb_method = zip(method_names, methods)
            urls.append({'rule' : pattern_rule, 'methods': list(verb_method)})
    return urls

def swagger(app, process_doc=_sanitize, base=''):
    """
    Call this from an @app.route method like this
    @app.route('/spec.json')
    def spec():
       return jsonify(swagger(app))

    We go through all endpoints of the app searching for swagger endpoints
    We provide the minimum required data according to swagger specs
    Callers can and should add and override at will

    Arguments:
    app -- the flask app to inspect

    Keyword arguments:
    process_doc -- text sanitization method, the default simply replaces \n with

    """

    output = {
        "swagger": "2.0",
        "info": {
            "version": "0.0.0",
            "title": "Yunity",
        },
        "paths": defaultdict(dict),
        "definitions": defaultdict(dict)
    }

    paths = output['paths']
    definitions = output['definitions']
    ignore_verbs = {"HEAD", "OPTIONS"}
    # technically only responses is non-optional
    optional_fields = ['tags', 'consumes', 'produces', 'schemes', 'security',
                       'deprecated', 'operationId', 'externalDocs']

    rules = get_rules(app.urls.urlpatterns, base)
    for rule in rules:
        operations = dict()
        for verb, method in rule['methods']:
            summary, description, swag = _parse_docstring(method, process_doc)
            if swag is not None:  # we only add endpoints with swagger data in the docstrings
                params = swag.get('parameters', [])
                defs = _extract_definitions(params)
                responses = swag.get('responses', {})
                responses = {
                    str(key): value
                    for key, value in responses.items()
                }
                if responses is not None:
                    defs = defs + _extract_definitions(responses.values())
                for definition in defs:
                    def_id = definition.pop('id')
                    if def_id is not None:
                        definitions[def_id].update(definition)
                operation = dict(
                    summary=summary,
                    description=description,
                    responses=responses
                )
                # parameters - swagger ui dislikes empty parameter lists
                if len(params) > 0:
                    operation['parameters'] = params
                # other optionals
                for key in optional_fields:
                    if key in swag:
                        operation[key] = swag.get(key)
                operations[verb] = operation

        if len(operations):
            rule = rule['rule']
            for arg in re.findall('(\(\?P<(.*?\:)?(.*?)>.*\))', rule):
                rule = rule.replace(arg[0], '{%s}' % arg[2])
            rule = rule.translate(str.maketrans('','','^$'))
            paths[rule].update(operations)
    return output

def doc(request):
    return JsonResponse(swagger(yunity, base='/api/'))

<|endoftext|>


<commit_before># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyShroud(PythonPackage):
    """Create Fortran wrappers for a C++ library."""

    homepage = "https://github.com/LLNL/shroud"
    git      = "https://github.com/LLNL/shroud.git"

    version('develop', branch='develop')
    version('master',  branch='master')
    version('0.11.0', tag='v0.11.0')
    version('0.10.1', tag='v0.10.1')
    version('0.9.0', tag='v0.9.0')
    version('0.8.0', tag='v0.8.0')

    depends_on("py-setuptools", type='build')
    depends_on("py-pyyaml@4.2b1:", type=('build', 'run'))
<commit_msg>py-shroud: added v0.12.1 (#17332)<commit_after># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyShroud(PythonPackage):
    """Create Fortran wrappers for a C++ library."""

    homepage = "https://github.com/LLNL/shroud"
    git      = "https://github.com/LLNL/shroud.git"

    version('develop', branch='develop')
    version('master',  branch='master')
    version('0.12.1', tag='v0.12.1')
    version('0.11.0', tag='v0.11.0')
    version('0.10.1', tag='v0.10.1')
    version('0.9.0', tag='v0.9.0')
    version('0.8.0', tag='v0.8.0')

    depends_on("py-setuptools", type=('build', 'run'))
    depends_on("py-pyyaml@4.2b1:", type=('build', 'run'))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import beer

def before_all(context):
    beer.app.config['TESTING'] = True
    context.client = beer.app.test_client()

def before_tag(context, tag):
    if 'beerdesc' in tag:
        beer.app.config['BEER_JSON'] = 'tests_json/test_beer_shortage.json'
    if 'encode' in tag:
        beer.app.config['BEER_JSON'] = 'tests_json/test_beer_encode.json'

def after_tag(context, tag):
    if 'beerdesc' or 'encode' in tag:
        beer.app.config['BEER_JSON'] = 'beerpoint.json'
    if 'encode' in tag:
        beer.app.config['BEER_JSON'] = 'beerpoint.json'

def after_all(context):
    pass
<commit_msg>cleaning features/environment in beer_flask<commit_after># -*- coding: utf-8 -*-
import beer

def before_all(context):
    beer.app.config['TESTING'] = True
    context.client = beer.app.test_client()

def before_tag(context, tag):
    context.beer_json = beer.app.config['BEER_JSON']
    if 'beerdesc' in tag:
        beer.app.config['BEER_JSON'] = 'tests_json/test_beer_shortage.json'
    if 'encode' in tag:
        beer.app.config['BEER_JSON'] = 'tests_json/test_beer_encode.json'

def after_tag(context, tag):
    beer.app.config['BEER_JSON'] = context.beer_json

def after_all(context):
    pass
<|endoftext|>


<commit_before>import os
import time
import numpy as np
import skimage.transform
from sklearn.cluster import spectral_clustering
import BerkeleyImgSegmentationPerfEvaluator.RI_VOI_computer

import utils
import dominant_sets

root = "..//batches"
main_dir = root + "//batch0"
img_dir_path = main_dir + "//original"
gt_dir_path = main_dir + "//groundTruth"
res_dir_path = main_dir + "//results"

clustering_alg = "DS"
n_clusters = np.empty((0, 1))
resize_factor = 0.25

if __name__ == "__main__":
    is_colored = False
    is_weighted = True

    if not os.path.exists(os.path.normpath(res_dir_path)):
        os.mkdir(os.path.normpath(res_dir_path))

    res_dir_path += "//resize_factor_" + str.replace(str(resize_factor), ".", "_")
    if not os.path.exists(os.path.normpath(res_dir_path)):
        os.mkdir(os.path.normpath(res_dir_path))

    res_dir_path += "//" + clustering_alg
    if not os.path.exists(os.path.normpath(res_dir_path)):
        os.mkdir(os.path.normpath(res_dir_path))

    res_dir_path += "//original"
    if not os.path.exists(os.path.normpath(res_dir_path)):
        os.mkdir(os.path.normpath(res_dir_path))

    log = open(os.path.normpath(res_dir_path + "//one_stage.log"), "w")
    log.truncate()
    data = open(os.path.normpath(res_dir_path + "//" + clustering_alg + ".csv"), "w")
    data.truncate()
    data.write("dens,d,cf,t,PRI,VOI" + "\n")
    RI_VOI_comp = BerkeleyImgSegmentationPerfEvaluator.RI_VOI_computer.RIVOIComputer(1)

    for img_fname in sorted(os.listdir(os.path.normpath(img_dir_path))):
        print("processing image " + img_fname)

        img_pname = os.path.normpath(img_dir_path + "//" + img_fname)
        img_name = os.path.splitext(img_fname)[0]
        curr_res_dir_path = os.path.normpath(res_dir_path + "//" + img_name)
        if not os.path.exists(os.path.normpath(curr_res_dir_path)):
            os.mkdir(os.path.normpath(curr_res_dir_path))

        graph_mat, original_img_shape, resized_img_shape = utils.from_img_to_adj_mat(img_pname, 1.0, is_colored,
                                                                                is_weighted, 1.0, resize_factor)

        log.write("image " + img_fname + "\n")

        start_time = time.time()
        if clustering_alg == "SC":
            results = spectral_clustering(graph_mat, 4) + 1
        elif clustering_alg == "DS":
            results = dominant_sets.dominant_sets(graph_mat) + 1
        else:
            raise ValueError("Incorrect clustering algorithm")

        results = results.reshape(resized_img_shape)
        clustering_time = time.time() - start_time

        clusters_found = int(np.max(results))
        n_clusters = np.vstack((n_clusters, np.array([clusters_found])))

        results = skimage.transform.resize(results, original_img_shape, 0, preserve_range=True).astype(int)
        RI_VOI_comp.set_seg(results)

        gts, _ = utils.read_gts(gt_dir_path, img_name + ".mat")

        for gt in gts:
            RI_VOI_comp.set_gt(gt)
            RI_VOI_comp.compute_RI_and_VOI_sums()
        RI_VOI_comp.update_partial_values(0, len(gts))

        print("finished. Saving data")
        log.write("  clusters found = " + str(int(clusters_found)).zfill(2) +
                  ". time = " + str(clustering_time) + "\n")
        data.write(img_name.zfill(6) + "," +
                   str(utils.graph_density(graph_mat)) + "," +
                   str(utils.graph_avg_degree(graph_mat)) + "," +
                   str(int(clusters_found)).zfill(2) + "," +
                   format(clustering_time, '.4f').zfill(8) + "," +
                   format(RI_VOI_comp.avgRI, '.4f') + "," +
                   format(RI_VOI_comp.avgVOI, '.4f') + "\n")
        np.save(os.path.normpath(curr_res_dir_path + "//" + img_name + "_cl_" + str(int(clusters_found)) + ".npy"), results)
        del graph_mat
    np.save(os.path.normpath(res_dir_path + "//n_clusters.npy"), n_clusters)
<commit_msg>Initial Commit<commit_after>import os
import time
import numpy as np
import skimage.transform
from sklearn.cluster import spectral_clustering
import BerkeleyImgSegmentationPerfEvaluator.RI_VOI_computer

import utils
import dominant_sets

root = "..//batches"
main_dir = root + "//batch0"
img_dir_path = main_dir + "//original"
gt_dir_path = main_dir + "//groundTruth"
res_dir_path = main_dir + "//results"

clustering_alg = "DS"
n_clusters = np.empty((0, 1))
resize_factor = 0.25

if __name__ == "__main__":
    is_colored = False
    is_weighted = True

    if not os.path.exists(os.path.normpath(res_dir_path)):
        os.mkdir(os.path.normpath(res_dir_path))

    res_dir_path += "//resize_factor_" + str.replace(str(resize_factor), ".", "_")
    if not os.path.exists(os.path.normpath(res_dir_path)):
        os.mkdir(os.path.normpath(res_dir_path))

    res_dir_path += "//" + clustering_alg
    if not os.path.exists(os.path.normpath(res_dir_path)):
        os.mkdir(os.path.normpath(res_dir_path))

    res_dir_path += "//original"
    if not os.path.exists(os.path.normpath(res_dir_path)):
        os.mkdir(os.path.normpath(res_dir_path))

    log = open(os.path.normpath(res_dir_path + "//one_stage.log"), "w")
    log.truncate()
    data = open(os.path.normpath(res_dir_path + "//" + clustering_alg + ".csv"), "w")
    data.truncate()
    data.write("iid,dens,d,cf,t,PRI,VOI" + "\n")
    RI_VOI_comp = BerkeleyImgSegmentationPerfEvaluator.RI_VOI_computer.RIVOIComputer(1)

    for img_fname in sorted(os.listdir(os.path.normpath(img_dir_path))):
        print("processing image " + img_fname)

        img_pname = os.path.normpath(img_dir_path + "//" + img_fname)
        img_name = os.path.splitext(img_fname)[0]
        curr_res_dir_path = os.path.normpath(res_dir_path + "//" + img_name)
        if not os.path.exists(os.path.normpath(curr_res_dir_path)):
            os.mkdir(os.path.normpath(curr_res_dir_path))

        graph_mat, original_img_shape, resized_img_shape = utils.from_img_to_adj_mat(img_pname, 1.0, is_colored,
                                                                                is_weighted, 1.0, resize_factor)

        log.write("image " + img_fname + "\n")

        start_time = time.time()
        if clustering_alg == "SC":
            results = spectral_clustering(graph_mat, 4) + 1
        elif clustering_alg == "DS":
            results = dominant_sets.dominant_sets(graph_mat) + 1
        else:
            raise ValueError("Incorrect clustering algorithm")

        results = results.reshape(resized_img_shape)
        clustering_time = time.time() - start_time

        clusters_found = int(np.max(results))
        n_clusters = np.vstack((n_clusters, np.array([clusters_found])))

        results = skimage.transform.resize(results, original_img_shape, 0, preserve_range=True).astype(int)
        RI_VOI_comp.set_seg(results)

        gts, _ = utils.read_gts(gt_dir_path, img_name + ".mat")

        for gt in gts:
            RI_VOI_comp.set_gt(gt)
            RI_VOI_comp.compute_RI_and_VOI_sums()
        RI_VOI_comp.update_partial_values(0, len(gts))

        print("finished. Saving data")
        log.write("  clusters found = " + str(int(clusters_found)).zfill(2) +
                  ". time = " + str(clustering_time) + "\n")
        data.write(img_name.zfill(6) + "," +
                   str(utils.graph_density(graph_mat)) + "," +
                   str(utils.graph_avg_degree(graph_mat)) + "," +
                   str(int(clusters_found)).zfill(2) + "," +
                   format(clustering_time, '.4f').zfill(8) + "," +
                   format(RI_VOI_comp.avgRI, '.4f') + "," +
                   format(RI_VOI_comp.avgVOI, '.4f') + "\n")
        np.save(os.path.normpath(curr_res_dir_path + "//" + img_name + "_cl_" + str(int(clusters_found)) + ".npy"), results)
        del graph_mat
    np.save(os.path.normpath(res_dir_path + "//n_clusters.npy"), n_clusters)
<|endoftext|>


<commit_before><commit_msg>FEM: utils, fix working dir<commit_after><|endoftext|>


<commit_before>version = (0, 8, 0)
version_info = '.'.join(str(c) for c in version)
<commit_msg>bump version<commit_after>version = (0, 9, 1)
version_info = '.'.join(str(c) for c in version)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#ckwg +5
# Copyright 2011 by Kitware, Inc. All Rights Reserved. Please refer to
# KITWARE_LICENSE.TXT for licensing information, or contact General Counsel,
# Kitware, Inc., 28 Corporate Drive, Clifton Park, NY 12065.

def log(msg):
    import sys
    sys.stderr.write("%s\n" % msg)

def ensure_exception(action, func, *args):
    got_exception = False

    try:
        func(*args)
    except:
        got_exception = True

    if not got_exception:
        log("Error: Did not get exception when %s" % action)

def test_import():
    try:
        import vistk.pipeline.process_registry
    except:
        log("Error: Failed to import the process_registry module")

def test_create():
    from vistk.pipeline import process_registry

    process_registry.ProcessRegistry.self()
    process_registry.ProcessType()
    process_registry.ProcessTypes()
    process_registry.ProcessDescription()

def test_api_calls():
    from vistk.pipeline import config
    from vistk.pipeline import modules
    from vistk.pipeline import process_registry

    modules.load_known_modules()

    reg = process_registry.ProcessRegistry.self()

    proc_type = 'orphan'
    c = config.empty_config()

    reg.create_process(proc_type, c)
    reg.types()
    reg.description(proc_type)

    process_registry.Process.constraint_no_threads
    process_registry.Process.constraint_no_reentrancy
    process_registry.Process.constraint_unsync_input
    process_registry.Process.constraint_unsync_output
    process_registry.Process.port_heartbeat
    process_registry.Process.config_name
    process_registry.Process.config_type
    process_registry.Process.type_any
    process_registry.Process.type_none
    process_registry.Process.type_data_dependent
    process_registry.Process.type_flow_dependent
    process_registry.Process.flag_output_const
    process_registry.Process.flag_input_mutable
    process_registry.Process.flag_input_nodep
    process_registry.Process.flag_required

def example_process():
    from vistk.pipeline import process

    class PythonExample(process.PythonProcess):
        def __init__(self, conf):
            process.PythonProcess.__init__(self, conf)

            self.ran_configure = False
            self.ran_init = False
            self.ran_step = False
            self.ran_constraints = False
            self.ran_connect_input_port = False
            self.ran_connect_output_port = False
            self.ran_input_ports = False
            self.ran_output_ports = False
            self.ran_input_port_info = False
            self.ran_output_port_info = False
            self.ran_set_input_port_type = False
            self.ran_set_output_port_type = False
            self.ran_available_config = False
            self.ran_conf_info = False

        def _configure(self):
            self.ran_configure = True

            self._base_configure()

        def _init(self):
            self.ran_init = True

            self._base_init()

        def _step(self):
            self.ran_step = True

            self._base_step()

        def _constraints(self):
            self.ran_constraints = True

            return self._base_constraints()

        def _connect_input_port(self, port, edge):
            self.ran_connect_input_port = True

            self._base_connect_input_port(port, edge)

        def _connect_output_port(self, port, edge):
            self.ran_connect_output_port = True

            self._base_connect_output_port(port, edge)

        def _input_ports(self):
            self.ran_input_ports = True

            return self._base_input_ports()

        def _output_ports(self):
            self.ran_output_ports = True

            return self._base_output_ports()

        def _input_port_info(self, port):
            self.ran_input_port_info = True

            return self._base_input_port_info(port)

        def _output_port_info(self, port):
            self.ran_output_port_info = True

            return self._base_output_port_info(port)

        def _set_input_port_type(self, port, type):
            self.ran_set_input_port_type = True

            return self._base_set_input_port_type(port, type)

        def _set_output_port_type(self, port, type):
            self.ran_set_output_port_type = True

            return self._base_set_output_port_type(port, type)

        def _available_config(self):
            self.ran_available_config = True

            return self._base_available_config()

        def _config_info(self, key):
            self.ran_conf_info = True

            return self._base_conf_info(key)

        def check(self):
            if not self.ran_configure:
                log("Error: _configure override was not called")
            if not self.ran_init:
                log("Error: _init override was not called")
            if not self.ran_step:
                log("Error: _step override was not called")
            if not self.ran_constraints:
                log("Error: _constraints override was not called")
            if not self.ran_connect_input_port:
                log("Error: _connect_input_port override was not called")
            if not self.ran_connect_output_port:
                log("Error: _connect_output_port override was not called")
            if not self.ran_input_ports:
                log("Error: _input_ports override was not called")
            if not self.ran_output_ports:
                log("Error: _output_ports override was not called")
            if not self.ran_input_port_info:
                log("Error: _input_port_info override was not called")
            if not self.ran_output_port_info:
                log("Error: _output_port_info override was not called")
            if not self.ran_set_input_port_type:
                log("Error: _set_input_port_type override was not called")
            if not self.ran_set_output_port_type:
                log("Error: _set_output_port_type override was not called")
            if not self.ran_available_config:
                log("Error: _available_config override was not called")
            if not self.ran_conf_info:
                log("Error: _conf_info override was not called")

    return PythonExample

def base_example_process():
    from vistk.pipeline import process

    class PythonBaseExample(process.PythonProcess):
        def __init__(self, conf):
            process.PythonProcess.__init__(self, conf)

        def check(self):
            pass

    return PythonBaseExample

def test_register():
    from vistk.pipeline import config
    from vistk.pipeline import process
    from vistk.pipeline import process_registry

    proc_type = 'python_example'
    proc_desc = 'simple description'

    reg = process_registry.ProcessRegistry.self()

    reg.register_process(proc_type, proc_desc, example_process())

    if not proc_desc == reg.description(proc_type):
        log("Error: Description was not preserved when registering")

    c = config.empty_config()

    try:
        p = reg.create_process(proc_type, c)
        if p is None:
            raise Exception()
    except:
        log("Error: Could not create newly registered process type")

def test_wrapper_api():
    from vistk.pipeline import config
    from vistk.pipeline import edge
    from vistk.pipeline import process
    from vistk.pipeline import process_registry

    proc_type = 'python_example'
    proc_desc = 'simple description'

    proc_base_type = 'python_base_example'
    proc_base_desc = 'simple base description'

    iport = 'no_such_iport'
    oport = 'no_such_oport'
    key = 'no_such_key'
    ptype = 'no_type'

    reg = process_registry.ProcessRegistry.self()

    reg.register_process(proc_type, proc_desc, example_process())
    reg.register_process(proc_base_type, proc_base_desc, base_example_process())

    c = config.empty_config()

    def check_process(p):
        if p is None:
            log("Error: Got a 'None' process")
            return

        p.constraints()

        p.input_ports()
        p.output_ports()
        ensure_exception("asking for info on a non-existant input port",
                         p.input_port_info, iport)
        ensure_exception("asking for info on a non-existant output port",
                         p.output_port_info, oport)

        e = edge.Edge(c)

        ensure_exception("connecting to a non-existant input port",
                         p.connect_input_port, iport, e)
        ensure_exception("connecting to a non-existant output port",
                         p.connect_output_port, oport, e)

        p.available_config()
        ensure_exception("asking for info on a non-existant config key",
                         p.config_info, key)

        ensure_exception("setting a type on a non-existent input port",
                         p.set_input_port_type, iport, ptype)
        ensure_exception("setting a type on a non-existent output port",
                         p.set_output_port_type, oport, ptype)

        p.configure()
        p.init()
        p.step()

        p.check()

    p = reg.create_process(proc_type, c)
    check_process(p)

    p = reg.create_process(proc_base_type, c)
    check_process(p)

def main(testname):
    if testname == 'import':
        test_import()
    elif testname == 'create':
        test_create()
    elif testname == 'api_calls':
        test_api_calls()
    elif testname == 'register':
        test_register()
    elif testname == 'wrapper_api':
        test_wrapper_api()
    else:
        log("Error: No such test '%s'" % testname)

if __name__ == '__main__':
    import os
    import sys

    if not len(sys.argv) == 4:
        log("Error: Expected three arguments")
        sys.exit(1)

    testname = sys.argv[1]

    os.chdir(sys.argv[2])

    sys.path.append(sys.argv[3])

    try:
        main(testname)
    except BaseException as e:
        log("Error: Unexpected exception: %s" % str(e))
<commit_msg>Add tests for the Python bindings for reset<commit_after>#!/usr/bin/env python
#ckwg +5
# Copyright 2011 by Kitware, Inc. All Rights Reserved. Please refer to
# KITWARE_LICENSE.TXT for licensing information, or contact General Counsel,
# Kitware, Inc., 28 Corporate Drive, Clifton Park, NY 12065.

def log(msg):
    import sys
    sys.stderr.write("%s\n" % msg)

def ensure_exception(action, func, *args):
    got_exception = False

    try:
        func(*args)
    except:
        got_exception = True

    if not got_exception:
        log("Error: Did not get exception when %s" % action)

def test_import():
    try:
        import vistk.pipeline.process_registry
    except:
        log("Error: Failed to import the process_registry module")

def test_create():
    from vistk.pipeline import process_registry

    process_registry.ProcessRegistry.self()
    process_registry.ProcessType()
    process_registry.ProcessTypes()
    process_registry.ProcessDescription()

def test_api_calls():
    from vistk.pipeline import config
    from vistk.pipeline import modules
    from vistk.pipeline import process_registry

    modules.load_known_modules()

    reg = process_registry.ProcessRegistry.self()

    proc_type = 'orphan'
    c = config.empty_config()

    reg.create_process(proc_type, c)
    reg.types()
    reg.description(proc_type)

    process_registry.Process.constraint_no_threads
    process_registry.Process.constraint_no_reentrancy
    process_registry.Process.constraint_unsync_input
    process_registry.Process.constraint_unsync_output
    process_registry.Process.port_heartbeat
    process_registry.Process.config_name
    process_registry.Process.config_type
    process_registry.Process.type_any
    process_registry.Process.type_none
    process_registry.Process.type_data_dependent
    process_registry.Process.type_flow_dependent
    process_registry.Process.flag_output_const
    process_registry.Process.flag_input_mutable
    process_registry.Process.flag_input_nodep
    process_registry.Process.flag_required

def example_process():
    from vistk.pipeline import process

    class PythonExample(process.PythonProcess):
        def __init__(self, conf):
            process.PythonProcess.__init__(self, conf)

            self.ran_configure = False
            self.ran_init = False
            self.ran_reset = False
            self.ran_step = False
            self.ran_constraints = False
            self.ran_connect_input_port = False
            self.ran_connect_output_port = False
            self.ran_input_ports = False
            self.ran_output_ports = False
            self.ran_input_port_info = False
            self.ran_output_port_info = False
            self.ran_set_input_port_type = False
            self.ran_set_output_port_type = False
            self.ran_available_config = False
            self.ran_conf_info = False

        def _configure(self):
            self.ran_configure = True

            self._base_configure()

        def _init(self):
            self.ran_init = True

            self._base_init()

        def _reset(self):
            self.ran_reset = True

            self._base_reset()

        def _step(self):
            self.ran_step = True

            self._base_step()

        def _constraints(self):
            self.ran_constraints = True

            return self._base_constraints()

        def _connect_input_port(self, port, edge):
            self.ran_connect_input_port = True

            self._base_connect_input_port(port, edge)

        def _connect_output_port(self, port, edge):
            self.ran_connect_output_port = True

            self._base_connect_output_port(port, edge)

        def _input_ports(self):
            self.ran_input_ports = True

            return self._base_input_ports()

        def _output_ports(self):
            self.ran_output_ports = True

            return self._base_output_ports()

        def _input_port_info(self, port):
            self.ran_input_port_info = True

            return self._base_input_port_info(port)

        def _output_port_info(self, port):
            self.ran_output_port_info = True

            return self._base_output_port_info(port)

        def _set_input_port_type(self, port, type):
            self.ran_set_input_port_type = True

            return self._base_set_input_port_type(port, type)

        def _set_output_port_type(self, port, type):
            self.ran_set_output_port_type = True

            return self._base_set_output_port_type(port, type)

        def _available_config(self):
            self.ran_available_config = True

            return self._base_available_config()

        def _config_info(self, key):
            self.ran_conf_info = True

            return self._base_conf_info(key)

        def check(self):
            if not self.ran_configure:
                log("Error: _configure override was not called")
            if not self.ran_init:
                log("Error: _init override was not called")
            if not self.ran_reset:
                log("Error: _reset override was not called")
            if not self.ran_step:
                log("Error: _step override was not called")
            if not self.ran_constraints:
                log("Error: _constraints override was not called")
            if not self.ran_connect_input_port:
                log("Error: _connect_input_port override was not called")
            if not self.ran_connect_output_port:
                log("Error: _connect_output_port override was not called")
            if not self.ran_input_ports:
                log("Error: _input_ports override was not called")
            if not self.ran_output_ports:
                log("Error: _output_ports override was not called")
            if not self.ran_input_port_info:
                log("Error: _input_port_info override was not called")
            if not self.ran_output_port_info:
                log("Error: _output_port_info override was not called")
            if not self.ran_set_input_port_type:
                log("Error: _set_input_port_type override was not called")
            if not self.ran_set_output_port_type:
                log("Error: _set_output_port_type override was not called")
            if not self.ran_available_config:
                log("Error: _available_config override was not called")
            if not self.ran_conf_info:
                log("Error: _conf_info override was not called")

    return PythonExample

def base_example_process():
    from vistk.pipeline import process

    class PythonBaseExample(process.PythonProcess):
        def __init__(self, conf):
            process.PythonProcess.__init__(self, conf)

        def check(self):
            pass

    return PythonBaseExample

def test_register():
    from vistk.pipeline import config
    from vistk.pipeline import process
    from vistk.pipeline import process_registry

    proc_type = 'python_example'
    proc_desc = 'simple description'

    reg = process_registry.ProcessRegistry.self()

    reg.register_process(proc_type, proc_desc, example_process())

    if not proc_desc == reg.description(proc_type):
        log("Error: Description was not preserved when registering")

    c = config.empty_config()

    try:
        p = reg.create_process(proc_type, c)
        if p is None:
            raise Exception()
    except:
        log("Error: Could not create newly registered process type")

def test_wrapper_api():
    from vistk.pipeline import config
    from vistk.pipeline import edge
    from vistk.pipeline import process
    from vistk.pipeline import process_registry

    proc_type = 'python_example'
    proc_desc = 'simple description'

    proc_base_type = 'python_base_example'
    proc_base_desc = 'simple base description'

    iport = 'no_such_iport'
    oport = 'no_such_oport'
    key = 'no_such_key'
    ptype = 'no_type'

    reg = process_registry.ProcessRegistry.self()

    reg.register_process(proc_type, proc_desc, example_process())
    reg.register_process(proc_base_type, proc_base_desc, base_example_process())

    c = config.empty_config()

    def check_process(p):
        if p is None:
            log("Error: Got a 'None' process")
            return

        p.constraints()

        p.input_ports()
        p.output_ports()
        ensure_exception("asking for info on a non-existant input port",
                         p.input_port_info, iport)
        ensure_exception("asking for info on a non-existant output port",
                         p.output_port_info, oport)

        e = edge.Edge(c)

        ensure_exception("connecting to a non-existant input port",
                         p.connect_input_port, iport, e)
        ensure_exception("connecting to a non-existant output port",
                         p.connect_output_port, oport, e)

        p.available_config()
        ensure_exception("asking for info on a non-existant config key",
                         p.config_info, key)

        ensure_exception("setting a type on a non-existent input port",
                         p.set_input_port_type, iport, ptype)
        ensure_exception("setting a type on a non-existent output port",
                         p.set_output_port_type, oport, ptype)

        p.reset()

        p.configure()
        p.init()
        p.step()

        p.check()

    p = reg.create_process(proc_type, c)
    check_process(p)

    p = reg.create_process(proc_base_type, c)
    check_process(p)

def main(testname):
    if testname == 'import':
        test_import()
    elif testname == 'create':
        test_create()
    elif testname == 'api_calls':
        test_api_calls()
    elif testname == 'register':
        test_register()
    elif testname == 'wrapper_api':
        test_wrapper_api()
    else:
        log("Error: No such test '%s'" % testname)

if __name__ == '__main__':
    import os
    import sys

    if not len(sys.argv) == 4:
        log("Error: Expected three arguments")
        sys.exit(1)

    testname = sys.argv[1]

    os.chdir(sys.argv[2])

    sys.path.append(sys.argv[3])

    try:
        main(testname)
    except BaseException as e:
        log("Error: Unexpected exception: %s" % str(e))
<|endoftext|>


<commit_before>from __future__ import absolute_import
from cStringIO import StringIO
from couchdbkit import ResourceNotFound, BadValueError
from django.core.urlresolvers import reverse
from django.http import HttpResponseBadRequest, HttpResponse, Http404
from django.utils.translation import ugettext_lazy
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_GET, require_POST
from django.shortcuts import render
from django.utils.decorators import method_decorator
from corehq.apps.hqwebapp.views import BasePageView
from corehq.apps.style.decorators import use_jquery_ui
from corehq.util.view_utils import json_error
from dimagi.utils.web import json_request, json_response
from dimagi.utils.couch.database import get_db

from corehq.apps.api.models import require_api_user
from corehq.apps.domain.decorators import require_superuser

from .models import CommCareBuild, CommCareBuildConfig, SemanticVersionProperty
from .utils import get_all_versions, extract_build_info_from_filename

import requests
import requests.exceptions

@csrf_exempt  # is used by an API
@json_error
@require_api_user
def post(request):
    artifacts       = request.FILES.get('artifacts')
    build_number    = request.POST.get('build_number')
    version         = request.POST.get('version')
    try:
        build_number = int(build_number)
    except Exception:
        return HttpResponseBadRequest("build_number has to be a base-10 integer")

    if not artifacts:
        CommCareBuild.create_without_artifacts(version, build_number)
    else:
        CommCareBuild.create_from_zip(artifacts, build_number=build_number, version=version)
    return HttpResponse()

@require_GET
def get(request, version, build_number, path):
    build = CommCareBuild.get_build(version, build_number)
    try:
        file = build.fetch_file(path)
    except ResourceNotFound:
        raise Http404()

    response = HttpResponse(file)
    response['Content-Disposition'] = 'attachment; filename="%s"' % path.split("/")[-1]
    return response

@require_GET
@require_superuser
def get_all(request):
    builds = sorted(CommCareBuild.all_builds(), key=lambda build: build.time)
    return render(request, 'builds/all.html', {'builds': builds})

class EditMenuView(BasePageView):
    template_name = "builds/edit_menu.html"
    urlname = 'edit_menu'
    doc_id = "config--commcare-builds"
    page_title = ugettext_lazy("Edit CommCare Builds")

    @method_decorator(require_superuser)
    @use_jquery_ui
    def dispatch(self, *args, **kwargs):
        # different local caches on different workers
        # but this at least makes it so your changes take effect immediately
        # while you're editing the config
        CommCareBuildConfig.clear_local_cache()
        self.doc = CommCareBuildConfig.fetch()
        return super(EditMenuView, self).dispatch(*args, **kwargs)

    def save_doc(self):
        db = get_db()
        return db.save_doc(self.doc)

    @property
    def page_context(self):
        return {
            'doc': self.doc,
            'all_versions': get_all_versions(
                [v['build']['version'] for v in self.doc['menu']]),
            'j2me_enabled_versions': CommCareBuild.j2me_enabled_build_versions()
        }

    @property
    def page_url(self):
        return reverse(self.urlname)

    def post(self, request, *args, **kwargs):
        request_json = json_request(request.POST)
        self.doc = request_json.get('doc')
        self.save_doc()
        return self.get(request, success=True, *args, **kwargs)

KNOWN_BUILD_SERVER_LOGINS = {
    'http://build.dimagi.com:250/': (
        lambda session:
        session.get('http://build.dimagi.com:250/guestLogin.html?guest=1')
    )
}

@require_POST
def import_build(request):
    """
    example POST params:
    source: 'http://build.dimagi.com:250/repository/downloadAll/bt97/14163:id/artifacts.zip'
    version: '2.13.0'
    build_number: 32703

    """
    source = request.POST.get('source')
    version = request.POST.get('version')
    build_number = request.POST.get('build_number')

    try:
        SemanticVersionProperty().validate(version)
    except BadValueError as e:
        return json_response({
            'reason': 'Badly formatted version',
            'info': {
                'error_message': unicode(e),
                'error_type': unicode(type(e))
            }
        }, status_code=400)

    if build_number:
        try:
            build_number = int(build_number)
        except ValueError:
            return json_response({
                'reason': 'build_number must be an int'
            }, status_code=400)

    session = requests.session()

    # log in to the build server if we know how
    for key in KNOWN_BUILD_SERVER_LOGINS:
        if source.startswith(key):
            KNOWN_BUILD_SERVER_LOGINS[key](session)

    if source:
        r = session.get(source)

        try:
            r.raise_for_status()
        except requests.exceptions.HTTPError:
            return json_response({
                'reason': 'Fetching artifacts.zip failed',
                'response': {
                    'status_code': r.status_code,
                    'content': r.content,
                    'headers': r.headers,
                }
            }, status_code=400)
        try:
            _, inferred_build_number = (
                extract_build_info_from_filename(r.headers['content-disposition'])
            )
        except (KeyError, ValueError):  # no header or header doesn't match
            inferred_build_number = None

        if inferred_build_number:
            build_number = inferred_build_number

        if not build_number:
            return json_response({
                'reason': "You didn't give us a build number "
                          "and we couldn't infer it"
            }, status_code=400)

        build = CommCareBuild.create_from_zip(
            StringIO(r.content), version, build_number)

    else:
        build = CommCareBuild.create_without_artifacts(version, build_number)
    return json_response({
        'message': 'New CommCare build added',
        'info': {
            'version': version,
            'build_number': build_number,
            '_id': build.get_id,
        }
    })
<commit_msg>six.text_type<commit_after>from __future__ import absolute_import
from cStringIO import StringIO
from couchdbkit import ResourceNotFound, BadValueError
from django.core.urlresolvers import reverse
from django.http import HttpResponseBadRequest, HttpResponse, Http404
from django.utils.translation import ugettext_lazy
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_GET, require_POST
from django.shortcuts import render
from django.utils.decorators import method_decorator
from corehq.apps.hqwebapp.views import BasePageView
from corehq.apps.style.decorators import use_jquery_ui
from corehq.util.view_utils import json_error
from dimagi.utils.web import json_request, json_response
from dimagi.utils.couch.database import get_db

from corehq.apps.api.models import require_api_user
from corehq.apps.domain.decorators import require_superuser

from .models import CommCareBuild, CommCareBuildConfig, SemanticVersionProperty
from .utils import get_all_versions, extract_build_info_from_filename

import requests
import requests.exceptions
import six

@csrf_exempt  # is used by an API
@json_error
@require_api_user
def post(request):
    artifacts       = request.FILES.get('artifacts')
    build_number    = request.POST.get('build_number')
    version         = request.POST.get('version')
    try:
        build_number = int(build_number)
    except Exception:
        return HttpResponseBadRequest("build_number has to be a base-10 integer")

    if not artifacts:
        CommCareBuild.create_without_artifacts(version, build_number)
    else:
        CommCareBuild.create_from_zip(artifacts, build_number=build_number, version=version)
    return HttpResponse()

@require_GET
def get(request, version, build_number, path):
    build = CommCareBuild.get_build(version, build_number)
    try:
        file = build.fetch_file(path)
    except ResourceNotFound:
        raise Http404()

    response = HttpResponse(file)
    response['Content-Disposition'] = 'attachment; filename="%s"' % path.split("/")[-1]
    return response

@require_GET
@require_superuser
def get_all(request):
    builds = sorted(CommCareBuild.all_builds(), key=lambda build: build.time)
    return render(request, 'builds/all.html', {'builds': builds})

class EditMenuView(BasePageView):
    template_name = "builds/edit_menu.html"
    urlname = 'edit_menu'
    doc_id = "config--commcare-builds"
    page_title = ugettext_lazy("Edit CommCare Builds")

    @method_decorator(require_superuser)
    @use_jquery_ui
    def dispatch(self, *args, **kwargs):
        # different local caches on different workers
        # but this at least makes it so your changes take effect immediately
        # while you're editing the config
        CommCareBuildConfig.clear_local_cache()
        self.doc = CommCareBuildConfig.fetch()
        return super(EditMenuView, self).dispatch(*args, **kwargs)

    def save_doc(self):
        db = get_db()
        return db.save_doc(self.doc)

    @property
    def page_context(self):
        return {
            'doc': self.doc,
            'all_versions': get_all_versions(
                [v['build']['version'] for v in self.doc['menu']]),
            'j2me_enabled_versions': CommCareBuild.j2me_enabled_build_versions()
        }

    @property
    def page_url(self):
        return reverse(self.urlname)

    def post(self, request, *args, **kwargs):
        request_json = json_request(request.POST)
        self.doc = request_json.get('doc')
        self.save_doc()
        return self.get(request, success=True, *args, **kwargs)

KNOWN_BUILD_SERVER_LOGINS = {
    'http://build.dimagi.com:250/': (
        lambda session:
        session.get('http://build.dimagi.com:250/guestLogin.html?guest=1')
    )
}

@require_POST
def import_build(request):
    """
    example POST params:
    source: 'http://build.dimagi.com:250/repository/downloadAll/bt97/14163:id/artifacts.zip'
    version: '2.13.0'
    build_number: 32703

    """
    source = request.POST.get('source')
    version = request.POST.get('version')
    build_number = request.POST.get('build_number')

    try:
        SemanticVersionProperty().validate(version)
    except BadValueError as e:
        return json_response({
            'reason': 'Badly formatted version',
            'info': {
                'error_message': six.text_type(e),
                'error_type': six.text_type(type(e))
            }
        }, status_code=400)

    if build_number:
        try:
            build_number = int(build_number)
        except ValueError:
            return json_response({
                'reason': 'build_number must be an int'
            }, status_code=400)

    session = requests.session()

    # log in to the build server if we know how
    for key in KNOWN_BUILD_SERVER_LOGINS:
        if source.startswith(key):
            KNOWN_BUILD_SERVER_LOGINS[key](session)

    if source:
        r = session.get(source)

        try:
            r.raise_for_status()
        except requests.exceptions.HTTPError:
            return json_response({
                'reason': 'Fetching artifacts.zip failed',
                'response': {
                    'status_code': r.status_code,
                    'content': r.content,
                    'headers': r.headers,
                }
            }, status_code=400)
        try:
            _, inferred_build_number = (
                extract_build_info_from_filename(r.headers['content-disposition'])
            )
        except (KeyError, ValueError):  # no header or header doesn't match
            inferred_build_number = None

        if inferred_build_number:
            build_number = inferred_build_number

        if not build_number:
            return json_response({
                'reason': "You didn't give us a build number "
                          "and we couldn't infer it"
            }, status_code=400)

        build = CommCareBuild.create_from_zip(
            StringIO(r.content), version, build_number)

    else:
        build = CommCareBuild.create_without_artifacts(version, build_number)
    return json_response({
        'message': 'New CommCare build added',
        'info': {
            'version': version,
            'build_number': build_number,
            '_id': build.get_id,
        }
    })
<|endoftext|>


<commit_before>#!/usr/bin/python

import os
import lxml.etree
import re
import json

from esperanto_sort import compare_esperanto_strings

def clean_string(string):
    """Discard newlines, remove multiple spaces and remove leading or
    trailing whitespace.

    >>> clean_string(' \nfoo   bar  \n  ')
    'foo bar'

    """

    return re.sub('[\n\t ]+', ' ', string).strip()

def get_words_from_kap(node):
    """Return a list of all the terms in a kap node. This is not
    necessarily single words, since ReVo includes entries such as
    'brazila nukso'.

    The heavy lifting is done in flatten_kap, all we do here is
    separate out terms and remove extraneous whitespace.

    Possible formats encountered:
    'foo'
    'foo, bar'
    'foo,\n   bar'
    '(n,p)-matrico' (only term in ReVo with an internal comma)

    """
    flat_string = flatten_kap(node)

    if flat_string == '(n,p)-matrico':
        words = ['(n,p)-matrico']
    else:
        words = flat_string.split(',')
    if len(words) > 1:
        for i in range(len(words)):
            # remove trailing/leading space and awkard newlines
            words[i] = clean_string(words[i])

    return words

def flatten_kap(kap):
    """Take everything between <kap> and </kap> and return a naked
    string. This will either be one word or multiple (usually
    separated by commas but I haven't checked whether this is
    universally true).

    For the interested reader, some examples:

    <kap><ofc>*</ofc><tld/>o</kap>
    <kap>brazil<tld/>arbo, <var><kap>brazila <tld/>arbo</kap></var></kap>
    (from nuks.xml)

    <kap><tld lit="A"/>o</kap>
    (from agl.xml)

    <kap><tld/>ino<fnt>Z</fnt></kap>
    (from hom.xml)

    <kap>
      <tld lit="S"/>lando
      <fnt>
        <vrk>Oficiala Informo de AdE</vrk>,
        <lok>numero 12</lok>
      </fnt>
    </kap>
    (from skot.xml)

    """
    assert kap != None
    root = get_word_root(kap)

    flat_string = ""
    if kap.text != None:
        flat_string += kap.text

    # flatten, get all the text, throw away ofc and fnt tags
    # this is not simple, but the xml structure is a pain
    for child in kap.getchildren():
        if child.tag == 'tld':
            """The lit attribute of a tld tag signifies that in this
            particular case the root starts with a different letter
            than normal. For example, 'Aglo' has root 'agl-'. I
            haven't seen this used for anything other than
            capitalisation (both changing to upper case and changing
            to lower case).

            The relevant part of the ReVo documentation is vokoxml.dtd,
            lines 340 to 344.

            """
            if "lit" in child.attrib:
                new_letter = child.attrib['lit']
                flat_string += new_letter + root[1:]
            else:
                flat_string += root

            if child.text != None:
                flat_string += child.text
        elif child.tag == 'fnt':
            # we throw away source of word, not interested right now
            pass
        elif child.tag == 'ofc':
            # also throw away oficialness, not interested
            pass
        elif child.tag == 'var':
            # recurse -- egads! Why isn't the xml simpler?
            child_kap = child.getchildren()[0]
            flat_string += flatten_kap(child_kap)
        else:
            # shouldn't get here
            assert False
        if child.tail != None:
            flat_string += child.tail

    return clean_string(flat_string)

def get_word_root(arbitrary_node):
    """Get the word root corresponding to this word. The XML files are
    grouped such that every word in the same file has the same word
    root. We therefore do not need any specific node for this
    function.

    A minimal example:

    <vortaro>
    <kap><rad>salut</rad></kap>
    </vortaro>

    """
    assert arbitrary_node != None
    tree = arbitrary_node.getroottree()
    return list(tree.iter('rad'))[0].text

def get_tree(xml_file):
    parser = lxml.etree.XMLParser(load_dtd=True)
    return lxml.etree.parse(xml_file, parser)

def get_all_definitions(drv_node):
    # get all definitions for a word
    # this probably has bugs given the complexity of the input

    # some representative examples are:
    # sxilin.xml for subsenses
    # jakobi1.xml only <ref>, no <dif> node
    # frakci.xml only <ref> but huge and complex
    # ad.xml has a load of stuff, some of which is not documented

    definitions = []

    # every word is defined by <dif>, <subsnc> or <ref>
    # (affixes have other stuff)
    for sense in drv_node.findall('snc'):
        for child in sense.getchildren():
            if child.tag == 'subsnc':
                for dif_node in child.findall('dif'):
                    definitions.append(get_definition(dif_node))
            elif child.tag == 'dif':
                definitions.append(get_definition(child))
            elif child.tag == 'ref' and 'tip' in child.attrib and \
                    child.attrib['tip'] == 'dif':
                # this word is undefined and just references another
                definitions.append(get_reference_to_another(child))

    return definitions

def get_definition(dif_node):
    # convert a definition node to a simple unicode string
    # (this requires us to flatten it)

    definition = ""

    if dif_node.text:
        definition += dif_node.text
    for node in dif_node:
        if node.tag == 'ekz':
            # skip examples
            continue
        if node.tag == 'tld':
            definition += get_word_root(node)
        if node.text:
            definition += node.text
        if node.tail:
            definition += node.tail

    return clean_string(definition)

def get_reference_to_another(ref_node):
    # if a word is only defined by a reference to another
    # return a string that describes the reference
    # (note there are other places ref_node are used)
    assert ref_node.attrib['tip'] == 'dif'

    reference = ""

    if ref_node.text:
        reference += ref_node.text
    for node in ref_node:
        if node.tag == 'tld':
            reference += get_word_root(node)
        if node.text:
            reference += node.text
        if node.tail:
            reference += node.tail

    return "Vidu " + reference.strip() + "."

def get_entries(xml_file):
    """Get every entry from a given XML file: the words, their roots
    and their definitions.

    """

    tree = get_tree(xml_file)

    # each word is a drv node
    results = []
    for drv_node in tree.iter('drv'):
        node_words = get_words_from_kap(drv_node.find('kap'))
        root = get_word_root(drv_node)
        definitions = get_all_definitions(drv_node)
        for word in node_words:
            results.append((word, root, definitions))

    return results

def get_all_entries():
    """Extract all dictionary data from every XML file in the ../xml
    directory.

    """

    entries = {}
    """For each root we assign a primary word, which is
    the first word we encounter with this root. This is
    the word we will link to the the morphology parser
    encounters the root.

    """
    roots_seen = {}

    # fetch from xml files
    path = '../xml/'
    for file in [(path + file) for file in os.listdir(path)]:
        print file
        for (word, root, definitions) in get_entries(file):
            if word in entries:
                # we've already got an entry for this word, so add these definitions
                entry = entries[word]
                entry['definitions'] = entry['definitions'] + definitions
            else:
                # new entry
                if root in roots_seen:
                    entries[word] = {"root":root,
                                    "definitions":definitions, "primary":True}
                else:
                    entries[word] = {"root":root,
                                    "definitions":definitions, "primary":False}
                    roots_seen[root] = True

    return entries

if __name__ == '__main__':
    whole_dictionary = get_all_entries()

    output_file = open('dictionary.json', 'w')
    json.dump(whole_dictionary, output_file)
<commit_msg>stoping duplicate word definitions due to multiple very similar references<commit_after>#!/usr/bin/python

import os
import lxml.etree
import re
import json

from esperanto_sort import compare_esperanto_strings

def clean_string(string):
    """Discard newlines, remove multiple spaces and remove leading or
    trailing whitespace.

    >>> clean_string(' \nfoo   bar  \n  ')
    'foo bar'

    """

    return re.sub('[\n\t ]+', ' ', string).strip()

def get_words_from_kap(node):
    """Return a list of all the terms in a kap node. This is not
    necessarily single words, since ReVo includes entries such as
    'brazila nukso'.

    The heavy lifting is done in flatten_kap, all we do here is
    separate out terms and remove extraneous whitespace.

    Possible formats encountered:
    'foo'
    'foo, bar'
    'foo,\n   bar'
    '(n,p)-matrico' (only term in ReVo with an internal comma)

    """
    flat_string = flatten_kap(node)

    if flat_string == '(n,p)-matrico':
        words = ['(n,p)-matrico']
    else:
        words = flat_string.split(',')
    if len(words) > 1:
        for i in range(len(words)):
            # remove trailing/leading space and awkard newlines
            words[i] = clean_string(words[i])

    return words

def flatten_kap(kap):
    """Take everything between <kap> and </kap> and return a naked
    string. This will either be one word or multiple (usually
    separated by commas but I haven't checked whether this is
    universally true).

    For the interested reader, some examples:

    <kap><ofc>*</ofc><tld/>o</kap>
    <kap>brazil<tld/>arbo, <var><kap>brazila <tld/>arbo</kap></var></kap>
    (from nuks.xml)

    <kap><tld lit="A"/>o</kap>
    (from agl.xml)

    <kap><tld/>ino<fnt>Z</fnt></kap>
    (from hom.xml)

    <kap>
      <tld lit="S"/>lando
      <fnt>
        <vrk>Oficiala Informo de AdE</vrk>,
        <lok>numero 12</lok>
      </fnt>
    </kap>
    (from skot.xml)

    """
    assert kap != None
    root = get_word_root(kap)

    flat_string = ""
    if kap.text != None:
        flat_string += kap.text

    # flatten, get all the text, throw away ofc and fnt tags
    # this is not simple, but the xml structure is a pain
    for child in kap.getchildren():
        if child.tag == 'tld':
            """The lit attribute of a tld tag signifies that in this
            particular case the root starts with a different letter
            than normal. For example, 'Aglo' has root 'agl-'. I
            haven't seen this used for anything other than
            capitalisation (both changing to upper case and changing
            to lower case).

            The relevant part of the ReVo documentation is vokoxml.dtd,
            lines 340 to 344.

            """
            if "lit" in child.attrib:
                new_letter = child.attrib['lit']
                flat_string += new_letter + root[1:]
            else:
                flat_string += root

            if child.text != None:
                flat_string += child.text
        elif child.tag == 'fnt':
            # we throw away source of word, not interested right now
            pass
        elif child.tag == 'ofc':
            # also throw away oficialness, not interested
            pass
        elif child.tag == 'var':
            # recurse -- egads! Why isn't the xml simpler?
            child_kap = child.getchildren()[0]
            flat_string += flatten_kap(child_kap)
        else:
            # shouldn't get here
            assert False
        if child.tail != None:
            flat_string += child.tail

    return clean_string(flat_string)

def get_word_root(arbitrary_node):
    """Get the word root corresponding to this word. The XML files are
    grouped such that every word in the same file has the same word
    root. We therefore do not need any specific node for this
    function.

    A minimal example:

    <vortaro>
    <kap><rad>salut</rad></kap>
    </vortaro>

    """
    assert arbitrary_node != None
    tree = arbitrary_node.getroottree()
    return list(tree.iter('rad'))[0].text

def get_tree(xml_file):
    parser = lxml.etree.XMLParser(load_dtd=True)
    return lxml.etree.parse(xml_file, parser)

def get_all_definitions(drv_node):
    # get all definitions for a word
    # this probably has bugs given the complexity of the input

    # some representative examples are:
    # sxilin.xml for subsenses
    # jakobi1.xml only <ref>, no <dif> node
    # frakci.xml only <ref> but huge and complex
    # ad.xml has a load of stuff, some of which is not documented

    definitions = []

    # every word is defined by <dif>, <subsnc> or <ref>
    # (affixes have other stuff)
    for sense in drv_node.findall('snc'):
        for child in sense.getchildren():
            if child.tag == 'subsnc':
                for dif_node in child.findall('dif'):
                    definitions.append(get_definition(dif_node))
            elif child.tag == 'dif':
                definitions.append(get_definition(child))
            elif child.tag == 'ref' and 'tip' in child.attrib and \
                    child.attrib['tip'] == 'dif':
                # this word is undefined and just references another
                definitions.append(get_reference_to_another(child))

    # remove any duplicates (happens with multiple <ref>s e.g. direkt3.xml)
    no_duplicates = []
    for definition in definitions:
        if definition not in no_duplicates:
            no_duplicates.append(definition)

    return no_duplicates

def get_definition(dif_node):
    # convert a definition node to a simple unicode string
    # (this requires us to flatten it)

    definition = ""

    if dif_node.text:
        definition += dif_node.text
    for node in dif_node:
        if node.tag == 'ekz':
            # skip examples
            continue
        if node.tag == 'tld':
            definition += get_word_root(node)
        if node.text:
            definition += node.text
        if node.tail:
            definition += node.tail

    return clean_string(definition)

def get_reference_to_another(ref_node):
    # if a word is only defined by a reference to another
    # return a string that describes the reference
    # (note there are other places ref_node are used)
    assert ref_node.attrib['tip'] == 'dif'

    reference = ""

    if ref_node.text:
        reference += ref_node.text
    for node in ref_node:
        if node.tag == 'tld':
            reference += get_word_root(node)
        if node.text:
            reference += node.text
        if node.tail:
            reference += node.tail

    return "Vidu " + reference.strip() + "."

def get_entries(xml_file):
    """Get every entry from a given XML file: the words, their roots
    and their definitions.

    """

    tree = get_tree(xml_file)

    # each word is a drv node
    results = []
    for drv_node in tree.iter('drv'):
        node_words = get_words_from_kap(drv_node.find('kap'))
        root = get_word_root(drv_node)
        definitions = get_all_definitions(drv_node)
        for word in node_words:
            results.append((word, root, definitions))

    return results

def get_all_entries():
    """Extract all dictionary data from every XML file in the ../xml
    directory.

    """

    entries = {}
    """For each root we assign a primary word, which is
    the first word we encounter with this root. This is
    the word we will link to the the morphology parser
    encounters the root.

    """
    roots_seen = {}

    # fetch from xml files
    path = '../xml/'
    for file in [(path + file) for file in os.listdir(path)]:
        for (word, root, definitions) in get_entries(file):
            if word in entries:
                # we've already got an entry for this word, so add these definitions
                entry = entries[word]
                entry['definitions'] = entry['definitions'] + definitions
            else:
                # new entry
                if root in roots_seen:
                    entries[word] = {"root":root,
                                    "definitions":definitions, "primary":True}
                else:
                    entries[word] = {"root":root,
                                    "definitions":definitions, "primary":False}
                    roots_seen[root] = True

    return entries

if __name__ == '__main__':
    whole_dictionary = get_all_entries()

    output_file = open('dictionary.json', 'w')
    json.dump(whole_dictionary, output_file)
<|endoftext|>


<commit_before>
import numpy as np
from math import *
import random

import sys
import os
sys.path.insert(0, sys.path[0] + '/../..')
import neupy
print neupy.__version__

import theano
print theano.config.base_compiledir
# theano.config.exception_verbosity = 'high'
import theano.tensor as T
import itertools

tdr = os.environ['TMPDIR']
# os.environ['THEANO_FLAGS'] = 'base_compiledir=' + tdr

class Cluster(object):
  def __init__(self, n):
    self.n = n
    self.atoms = np.zeros((n, 3))
    self.elems = [''] * n
    self.energy = 0.0
    self.mat_f = []
    self.mat_x = []
    self.mat_ll = []
    ee = np.eye(n, dtype=int)
    for i in range(1, n + 1):
      le = [list(g) for g in itertools.combinations(ee, i)]
      lx = [list(g) for g in itertools.combinations(range(0, n), i)]
      le = np.asarray(le)
      lx = np.asarray(lx)
      self.mat_f.append(lambda x,le=le: np.dot(le, x))
      self.mat_x.append(lx)
    ll = np.zeros((self.n, self.n))
    for i in range(0, self.n):
      for j in range(0, i):
        ll[i, j] = np.linalg.norm(self.atoms[i] - self.atoms[j])
    self.mat_ll = ll

  def get_center(self):
    return np.average(self.atoms, axis = 0)

  def get_flat(self):
    return self.atoms.reshape(self.n * 3)

  def get_lengths_x(self, num=None):
    if num is None: num = self.n
    xx = self.mat_x[num - 1]
    ll = self.mat_ll
    r = []
    for k in range(0, xx.shape[0]):
      r.append([])
      for i in range(0, xx.shape[1]):
        for j in range(0, i):
          r[k].append(ll[xx[k, i], xx[k, j]])
    return np.asarray(r)

  def get_lengths_sp(self, num=None):
    if num is None: num = self.n
    ll = self.mat_ll
    xx = self.mat_x[num - 1]
    r = []
    for l in range(0, xx.shape[0]):
      r.append([])
      for i in range(0, xx.shape[1]):
        for j in range(0, i):
          lij = ll[xx[l, i], xx[l, j]]
          for k in range(0, j):
            lik = ll[xx[l, i], xx[l, k]]
            ljk = ll[xx[l, j], xx[l, k]]
            r[l] += [lij + lik, lij + ljk, lik + ljk,
              lij * lik, lij * ljk, lik * ljk]
    return np.asarray(r)

  def get_lengths(self):
    return self.get_lengths_x()

  def center(self):
    self.atoms = self.atoms - self.get_center()

  def shuffle(self):
    np.random.shuffle(self.atoms)

  def rotate(self):
    theta = np.random.random() * 2.0 * pi
    the = np.random.random() * pi
    phi = np.random.random() * 2.0 * pi
    u = [sin(the) * cos(phi), sin(the) * sin(phi), cos(the)]
    r = np.array([[cos(theta) + u[0]**2 * (1 - cos(theta)),
      u[0]*u[1] * (1 - cos(theta)) - u[2] * sin(theta),
      u[0]*u[2] * (1 - cos(theta)) + u[1] * sin(theta)],
      [u[1]*u[0] * (1 - cos(theta)) + u[2] * sin(theta),
      cos(theta) + u[1]**2 * (1 - cos(theta)),
      u[1]*u[2] * (1 - cos(theta)) - u[0] * sin(theta)],
      [u[2]*u[0] * (1 - cos(theta)) - u[1] * sin(theta),
      u[2]*u[1] * (1 - cos(theta)) + u[0] * sin(theta),
      cos(theta) + u[2]**2 * (1 - cos(theta))]])
    self.atoms = self.atoms.dot(r.T)

def read_cluster(ener, xyz):
  f = open(ener, 'r')
  fs = f.readlines()
  f.close()
  lf = []
  for f in fs:
    g = f.replace('\n', '').split(' ')
    lf += [[g[0], float(g[1])]]
  clul = []
  for l in lf:
    fn = xyz.replace('#', l[0])
    f = open(fn, 'r')
    fs = f.readlines()
    f.close()
    fs = [[g for g in f.replace('\n', '').split(' ') if len(g) != 0] for f in fs]
    clu = Cluster(int(fs[0][0]))
    i = 0
    clu.energy = l[1]
    for f in fs[2:]:
      clu.elems[i] = f[0]
      ar = np.asarray([float(g) for g in f[1:]])
      clu.atoms[i] = ar
      i = i + 1
    clu.center()
    if clu.energy < -954.46:
      clul.append(clu)
  print 'struct loaded: ', len(clul)
  return clul

def load_data(clus, n, num=None):
  x_d = []
  y_d = []
  for i in xrange(0, n):
    if i % (n / 100) == 0: print '{0} %'.format(i / (n / 100))
    idx = np.random.randint(len(clus))
    clu = clus[idx]
    clu.shuffle()
    x_d.append(clu.get_lengths_sp(num))
    y_d.append(clu.energy)
  y_d = np.array(y_d)
  x_d = np.asarray(x_d)
  return x_d, y_d

def find_max_min(clus):
  ratio = 0.1
  x_train = np.asarray([x.atoms for x in clus])
  y_train = np.asarray([x.energy for x in clus])
  dmax = [np.ma.max(x_train), np.ma.max(y_train)]
  dmin = [np.ma.min(x_train), np.ma.min(y_train)]
  for i in range(0, 2):
    dmm = dmax[i] - dmin[i]
    dmin[i] -= dmm * ratio
    dmax[i] += dmm * ratio
  return dmax, dmin

def trans_forward(clus, dmax, dmin):
  for c in clus:
    c.atoms = (c.atoms - dmin[0]) / (dmax[0] - dmin[0])
    c.energy = (c.energy - dmin[1]) / (dmax[1] - dmin[1])

def trans_backward(clus, dmax, dmin):
  for c in clus:
    c.atoms = c.atoms * (dmax[0] - dmin[0]) + dmin[0]
    c.energy = c.energy * (dmax[1] - dmin[1]) + dmin[1]

def trans_backward_y(y, dmax, dmin):
  return y * (dmax[1] - dmin[1]) + dmin[1]

def new_file_name(x):
  i = 0
  y = x + '.' + str(i)
  while os.path.isfile(y):
    i += 1
    y = x + '.' + str(i)
  return y

import dill
def store(network):
  c = new_file_name('data/network-storage.dill')
  print 'dump at ' + c
  with open(c, 'wb') as f:
    dill.dump(network, f)

def load(i):
  print 'load at ' + 'data/network-storage.dill.' + str(i)
  with open('data/network-storage.dill.' + str(i), 'rb') as f:
    return dill.load(f)

print 'load data ...'
clus = read_cluster('./data/energyx.txt', './data/ck_structs/pos_#.xyz')
dmax, dmin = find_max_min(clus)
trans_forward(clus, dmax, dmin)
random.shuffle(clus)
ratio = 9.0 / 10.0
x_train, y_train = load_data(clus[1:int(len(clus)*ratio)], 60000, num=5)
x_test, y_test = load_data(clus[int(len(clus)*ratio):], 10000, num=5)

from neupy import algorithms, layers, __version__

from neupy import environment
environment.reproducible()

print __version__
print os.getcwd()

class ACT(layers.ActivationLayer):
    # activation_function = (lambda x:T.nnet.relu(x) * 2 - 1)
    # activation_function = (lambda x:T.nnet.sigmoid(x) * 2 - 1)
    # activation_function = (lambda x:T.tanh(x/2) + T.nnet.sigmoid(x))
    # activation_function = (lambda x:T.nnet.sigmoid(x))
    # activation_function = T.tanh
    activation_function = T.nnet.relu
    # activation_function = (lambda x: (x/5)**2)
    # activation_function = (lambda x: T.nnet.relu(x) + 2*T.nnet.sigmoid(x*5))

load_i = -1
print 'construct network ...'
network = algorithms.Momentum(
  [
    ACT(x_train.shape[-1], ndim=3), # 28 x 1 -> 28 x 50
    ACT(100), # 28 x 50 -> 28 x 1
    ACT(50), # 28 x 50 -> 28 x 1
    layers.Softplus(6),
    layers.Reshape(presize=2), # 28 x 1 -> 28
    layers.Average(), # 28 -> 1
    layers.Output(1),
  ],
  # error='binary_crossentropy',
  error='mse',
  step=0.15,
  verbose=True,
  batch_size = 20,
  # mu=0.1,
  # mu_update_factor = 1,
  nesterov = True,
  momentum = 0.8,
  shuffle_data=True,
  show_epoch = 10
) if load_i == -1 else load(load_i)

print network
print 'train ...'
# network.train(x_train, y_train, x_test, y_test, epochs=1000)

y_pre = network.predict(x_test)

y_pre = trans_backward_y(y_pre, dmax, dmin)
y_test = trans_backward_y(y_test, dmax, dmin)

import math
httoev = 27.21138505
res = 0.0
ff = open('data/fitenergy.txt', 'w')
for x, y in zip(y_test, y_pre):
  # print x, y
  ff.write('%15.6f %15.6f\n' % (x, y[0]))
  res += (x - y)**2
ff.close()
print math.sqrt(res/len(y_test)) * httoev

store(network)
<commit_msg>gen ll<commit_after>
import numpy as np
from math import *
import random

import sys
import os
sys.path.insert(0, sys.path[0] + '/../..')
import neupy
print neupy.__version__

import theano
print theano.config.base_compiledir
# theano.config.exception_verbosity = 'high'
import theano.tensor as T
import itertools

tdr = os.environ['TMPDIR']
# os.environ['THEANO_FLAGS'] = 'base_compiledir=' + tdr

class Cluster(object):
  def __init__(self, n):
    self.n = n
    self.atoms = np.zeros((n, 3))
    self.elems = [''] * n
    self.energy = 0.0
    self.mat_f = []
    self.mat_x = []
    self.mat_ll = []
    ee = np.eye(n, dtype=int)
    for i in range(1, n + 1):
      le = [list(g) for g in itertools.combinations(ee, i)]
      lx = [list(g) for g in itertools.combinations(range(0, n), i)]
      le = np.asarray(le)
      lx = np.asarray(lx)
      self.mat_f.append(lambda x,le=le: np.dot(le, x))
      self.mat_x.append(lx)

  def gen_ll():
    ll = np.zeros((self.n, self.n))
    for i in range(0, self.n):
      for j in range(0, i):
        ll[i, j] = np.linalg.norm(self.atoms[i] - self.atoms[j])
    self.mat_ll = ll

  def get_center(self):
    return np.average(self.atoms, axis = 0)

  def get_flat(self):
    return self.atoms.reshape(self.n * 3)

  def get_lengths_x(self, num=None):
    if num is None: num = self.n
    xx = self.mat_x[num - 1]
    ll = self.mat_ll
    r = []
    for k in range(0, xx.shape[0]):
      r.append([])
      for i in range(0, xx.shape[1]):
        for j in range(0, i):
          r[k].append(ll[xx[k, i], xx[k, j]])
    return np.asarray(r)

  def get_lengths_sp(self, num=None):
    if num is None: num = self.n
    ll = self.mat_ll
    xx = self.mat_x[num - 1]
    r = []
    for l in range(0, xx.shape[0]):
      r.append([])
      for i in range(0, xx.shape[1]):
        for j in range(0, i):
          lij = ll[xx[l, i], xx[l, j]]
          for k in range(0, j):
            lik = ll[xx[l, i], xx[l, k]]
            ljk = ll[xx[l, j], xx[l, k]]
            r[l] += [lij + lik, lij + ljk, lik + ljk,
              lij * lik, lij * ljk, lik * ljk]
    return np.asarray(r)

  def get_lengths(self):
    return self.get_lengths_x()

  def center(self):
    self.atoms = self.atoms - self.get_center()

  def shuffle(self):
    np.random.shuffle(self.atoms)

  def rotate(self):
    theta = np.random.random() * 2.0 * pi
    the = np.random.random() * pi
    phi = np.random.random() * 2.0 * pi
    u = [sin(the) * cos(phi), sin(the) * sin(phi), cos(the)]
    r = np.array([[cos(theta) + u[0]**2 * (1 - cos(theta)),
      u[0]*u[1] * (1 - cos(theta)) - u[2] * sin(theta),
      u[0]*u[2] * (1 - cos(theta)) + u[1] * sin(theta)],
      [u[1]*u[0] * (1 - cos(theta)) + u[2] * sin(theta),
      cos(theta) + u[1]**2 * (1 - cos(theta)),
      u[1]*u[2] * (1 - cos(theta)) - u[0] * sin(theta)],
      [u[2]*u[0] * (1 - cos(theta)) - u[1] * sin(theta),
      u[2]*u[1] * (1 - cos(theta)) + u[0] * sin(theta),
      cos(theta) + u[2]**2 * (1 - cos(theta))]])
    self.atoms = self.atoms.dot(r.T)

def read_cluster(ener, xyz):
  f = open(ener, 'r')
  fs = f.readlines()
  f.close()
  lf = []
  for f in fs:
    g = f.replace('\n', '').split(' ')
    lf += [[g[0], float(g[1])]]
  clul = []
  for l in lf:
    fn = xyz.replace('#', l[0])
    f = open(fn, 'r')
    fs = f.readlines()
    f.close()
    fs = [[g for g in f.replace('\n', '').split(' ') if len(g) != 0] for f in fs]
    clu = Cluster(int(fs[0][0]))
    i = 0
    clu.energy = l[1]
    for f in fs[2:]:
      clu.elems[i] = f[0]
      ar = np.asarray([float(g) for g in f[1:]])
      clu.atoms[i] = ar
      i = i + 1
    clu.center()
    if clu.energy < -954.46:
      clul.append(clu)
  print 'struct loaded: ', len(clul)
  return clul

def load_data(clus, n, num=None):
  x_d = []
  y_d = []
  for i in xrange(0, n):
    if i % (n / 100) == 0: print '{0} %'.format(i / (n / 100))
    idx = np.random.randint(len(clus))
    clu = clus[idx]
    clu.shuffle()
    x_d.append(clu.get_lengths_x(num))
    y_d.append(clu.energy)
  y_d = np.array(y_d)
  x_d = np.asarray(x_d)
  return x_d, y_d

def find_max_min(clus):
  ratio = 0.1
  x_train = np.asarray([x.atoms for x in clus])
  y_train = np.asarray([x.energy for x in clus])
  dmax = [np.ma.max(x_train), np.ma.max(y_train)]
  dmin = [np.ma.min(x_train), np.ma.min(y_train)]
  for i in range(0, 2):
    dmm = dmax[i] - dmin[i]
    dmin[i] -= dmm * ratio
    dmax[i] += dmm * ratio
  return dmax, dmin

def trans_forward(clus, dmax, dmin):
  for c in clus:
    c.atoms = (c.atoms - dmin[0]) / (dmax[0] - dmin[0])
    c.energy = (c.energy - dmin[1]) / (dmax[1] - dmin[1])

def trans_backward(clus, dmax, dmin):
  for c in clus:
    c.atoms = c.atoms * (dmax[0] - dmin[0]) + dmin[0]
    c.energy = c.energy * (dmax[1] - dmin[1]) + dmin[1]

def trans_backward_y(y, dmax, dmin):
  return y * (dmax[1] - dmin[1]) + dmin[1]

def new_file_name(x):
  i = 0
  y = x + '.' + str(i)
  while os.path.isfile(y):
    i += 1
    y = x + '.' + str(i)
  return y

import dill
def store(network):
  c = new_file_name('data/network-storage.dill')
  print 'dump at ' + c
  with open(c, 'wb') as f:
    dill.dump(network, f)

def load(i):
  print 'load at ' + 'data/network-storage.dill.' + str(i)
  with open('data/network-storage.dill.' + str(i), 'rb') as f:
    return dill.load(f)

print 'load data ...'
clus = read_cluster('./data/energyx.txt', './data/ck_structs/pos_#.xyz')
dmax, dmin = find_max_min(clus)
trans_forward(clus, dmax, dmin)
random.shuffle(clus)
for c in clus: c.gen_ll()
ratio = 9.0 / 10.0
x_train, y_train = load_data(clus[1:int(len(clus)*ratio)], 10000, num=5)
x_test, y_test = load_data(clus[int(len(clus)*ratio):], 2000, num=5)

from neupy import algorithms, layers, __version__

from neupy import environment
environment.reproducible()

print __version__
print os.getcwd()

class ACT(layers.ActivationLayer):
    # activation_function = (lambda x:T.nnet.relu(x) * 2 - 1)
    # activation_function = (lambda x:T.nnet.sigmoid(x) * 2 - 1)
    # activation_function = (lambda x:T.tanh(x/2) + T.nnet.sigmoid(x))
    # activation_function = (lambda x:T.nnet.sigmoid(x))
    # activation_function = T.tanh
    activation_function = T.nnet.relu
    # activation_function = (lambda x: (x/5)**2)
    # activation_function = (lambda x: T.nnet.relu(x) + 2*T.nnet.sigmoid(x*5))

load_i = -1
print 'construct network ...'
network = algorithms.Momentum(
  [
    ACT(x_train.shape[-1], ndim=3), # 28 x 1 -> 28 x 50
    ACT(100), # 28 x 50 -> 28 x 1
    ACT(50), # 28 x 50 -> 28 x 1
    layers.Softplus(6),
    layers.Reshape(presize=2), # 28 x 1 -> 28
    layers.Average(), # 28 -> 1
    layers.Output(1),
  ],
  # error='binary_crossentropy',
  error='mse',
  step=0.15,
  verbose=True,
  batch_size = 20,
  # mu=0.1,
  # mu_update_factor = 1,
  nesterov = True,
  momentum = 0.8,
  shuffle_data=True,
  show_epoch = 10
) if load_i == -1 else load(load_i)

print network
print 'train ...'
print x_train[0], y_train[0]
network.train(x_train, y_train, x_test, y_test, epochs=1000)

y_pre = network.predict(x_test)

y_pre = trans_backward_y(y_pre, dmax, dmin)
y_test = trans_backward_y(y_test, dmax, dmin)

import math
httoev = 27.21138505
res = 0.0
ff = open('data/fitenergy.txt', 'w')
for x, y in zip(y_test, y_pre):
  # print x, y
  ff.write('%15.6f %15.6f\n' % (x, y[0]))
  res += (x - y)**2
ff.close()
print math.sqrt(res/len(y_test)) * httoev

store(network)
<|endoftext|>