<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
###############################################################################
#
#  $Id$
#  Project:  RX Cadre Data Visualization
#  Purpose:  Import csv data into tables
#  Author:   Kyle Shannon <kyle@pobox.com>
#            Kegan Rabil <krabil@yourdatasmarter.com>
#
###############################################################################
#
#  This is free and unencumbered software released into the public domain.
#
#  Anyone is free to copy, modify, publish, use, compile, sell, or
#  distribute this software, either in source code form or as a compiled
#  binary, for any purpose, commercial or non-commercial, and by any
#  means.
#
#  In jurisdictions that recognize copyright laws, the author or authors
#  of this software dedicate any and all copyright interest in the
#  software to the public domain. We make this dedication for the benefit
#  of the public at large and to the detriment of our heirs and
#  successors. We intend this dedication to be an overt act of
#  relinquishment in perpetuity of all present and future rights to this
#  software under copyright law.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
#  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
#  ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
#  OTHER DEALINGS IN THE SOFTWARE.
#
#  For more information, please refer to <http://unlicense.org/>
#
###############################################################################
"""
Module for importing spatial time-series data into a searchable format for
subsetting in space and time.  Typical import format would be csv files.  The
values are imported into a sqlite3 database.  'Meta' tables store information
about what to display.
"""

import csv
import os
import sqlite3
import json

class RxCadreIOError(Exception):pass
class RxCadreInvalidDbError(Exception):pass

class RxCadre:
    """
    Main interface for RX Cadre data.
    """

    def init_new_db(self, filename):
        """
        Create a new, empty database with appropriate metatables.  If the file
        previously exists, we fail before connecting using sqlite.  It must be
        a new file.
        """
        if os.path.exists(filename):
            raise RxCadreIOError("Database file already exists")
        db = sqlite3.connect(filename)
        cursor = db.cursor()
        sql = """CREATE TABLE plot_location(plot_id TEXT NOT NULL PRIMARY KEY,
                                            geometry TEXT, plot_type TEXT)"""
        cursor.execute(sql)
        sql = """CREATE TABLE event(project_name TEXT,
                                    event_name TEXT NOT NULL,
                                    event_start TEXT NOT NULL,
                                    event_end TEXT NOT NULL,
                                    PRIMARY KEY(project_name, event_name))"""
        cursor.execute(sql)
        sql = """CREATE TABLE obs_table(obs_table_name TEXT NOT NULL,
                                        geometry_column TEXT NOT NULL,
                                        obs_cols TEXT NOT NULL,
                                        obs_col_names TEXT)"""
        cursor.execute(sql)
        db.commit()
        valid = self.check_valid_db(db)
        db.close()
        if not valid:
            e = "Failed to create a valid database."
            raise RxCadreInvalidDbError(e)

    def check_valid_db(self, db):
        """
        Check the schema of an existing db.  This involves checking the
        metatables, and checking to make sure tables registered in obs_tables
        exist.
        """
        con = sqlite3.connect(db)
        cursor = con.cursor()
        cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
        cursor_fetch = cursor.fetchall()
        if (u'plot_location',) in cursor_fetch and (u'event',) in cursor_fetch and (u'obs_table',) in cursor_fetch
            cursor.execute("select obs_table_name from obs_table")
            z = cursor.fetchall()
            int table_number = len(z)
            if table_number != 0
                bool name_in = true
                for i in 0:table_number
                    if not z[i] in cursor_fetch
                        name_in = false
                if name_in == true
                    print "Success"
                    #Don't actually print that

        else

            raise RxCadreInvalidDbError("Database is invalid")

    def import_data(self, db, file_path, import_fx=None):
        """
        Import a new data set into the database.  An obs table should be
        created and registered with obs_tables.  It also should populate the
        plot_location data if possible.  The import_fx is an optional function
        that does the importing.
        """
        con = sqlite3.connect(db)

        data_file = open(file_path,"r")
        header = data_file.readline().split(",")

        cursor = con.cursor()
        hold_name = header[0]
        cursor.execute("CREAT TABLE " + hold_name + "(" + header[0] + ")")
        for i in range(1,len(header)):
            cursor.execute("ALTER TABLE " + hold_name + " ADD COLUMN " + json.dumps(header[i]))

        #The line below is what I'm trying to fix
        cursor.execute("insert into hold_name values (?)", data_file.readline().split(","))
        cursor.execute("insert into obs_table values(header[0],header[1],header[2],header[3])")

        raise NotImplementedError

<commit_msg>add code back after commit<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
###############################################################################
#
#  $Id$
#  Project:  RX Cadre Data Visualization
#  Purpose:  Import csv data into tables
#  Author:   Kyle Shannon <kyle@pobox.com>
#            Kegan Rabil <krabil@yourdatasmarter.com>
#
###############################################################################
#
#  This is free and unencumbered software released into the public domain.
#
#  Anyone is free to copy, modify, publish, use, compile, sell, or
#  distribute this software, either in source code form or as a compiled
#  binary, for any purpose, commercial or non-commercial, and by any
#  means.
#
#  In jurisdictions that recognize copyright laws, the author or authors
#  of this software dedicate any and all copyright interest in the
#  software to the public domain. We make this dedication for the benefit
#  of the public at large and to the detriment of our heirs and
#  successors. We intend this dedication to be an overt act of
#  relinquishment in perpetuity of all present and future rights to this
#  software under copyright law.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
#  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
#  ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
#  OTHER DEALINGS IN THE SOFTWARE.
#
#  For more information, please refer to <http://unlicense.org/>
#
###############################################################################
"""
Module for importing spatial time-series data into a searchable format for
subsetting in space and time.  Typical import format would be csv files.  The
values are imported into a sqlite3 database.  'Meta' tables store information
about what to display.
"""

import csv
import os
import sqlite3
import json

class RxCadreIOError(Exception):pass
class RxCadreInvalidDbError(Exception):pass

class RxCadre:
    """
    Main interface for RX Cadre data.
    """

    def init_new_db(self, filename):
        """
        Create a new, empty database with appropriate metatables.  If the file
        previously exists, we fail before connecting using sqlite.  It must be
        a new file.
        """
        if os.path.exists(filename):
            raise RxCadreIOError("Database file already exists")
        db = sqlite3.connect(filename)
        cursor = db.cursor()
        sql = """CREATE TABLE plot_location(plot_id TEXT NOT NULL PRIMARY KEY,
                                            geometry TEXT, plot_type TEXT)"""
        cursor.execute(sql)
        sql = """CREATE TABLE event(project_name TEXT,
                                    event_name TEXT NOT NULL,
                                    event_start TEXT NOT NULL,
                                    event_end TEXT NOT NULL,
                                    PRIMARY KEY(project_name, event_name))"""
        cursor.execute(sql)
        sql = """CREATE TABLE obs_table(obs_table_name TEXT NOT NULL,
                                        geometry_column TEXT NOT NULL,
                                        obs_cols TEXT NOT NULL,
                                        obs_col_names TEXT)"""
        cursor.execute(sql)
        db.commit()
        valid = self.check_valid_db(db)
        db.close()
        if not valid:
            e = "Failed to create a valid database."
            raise RxCadreInvalidDbError(e)

    def check_valid_db(self, db):
        """
        Check the schema of an existing db.  This involves checking the
        metatables, and checking to make sure tables registered in obs_tables
        exist.
        """

        cursor = db.cursor()
        required_tables = set([u'plot_location', u'event', u'obs_table'])
        cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
        table_names = [t[0] for t in cursor.fetchall()]
        if not set(table_names) <= required_tables:
            e = "Database does not contain the required tables, missing:"
            e += ",".join(required_tables.difference(table_names))
            return False
        cursor.execute("select obs_table_name from obs_table")
        obs_names = cursor.fetchall()
        for name in obs_names:
            if n not in table_names:
                e = "Database is invalid, missing table: %s" % n
                return False
        return True

    def import_data(self, db, file_path, import_fx=None):
        """
        Import a new data set into the database.  An obs table should be
        created and registered with obs_tables.  It also should populate the
        plot_location data if possible.  The import_fx is an optional function
        that does the importing.
        """
        con = sqlite3.connect(db)

        data_file = open(file_path,"r")
        header = data_file.readline().split(",")

        cursor = con.cursor()
        hold_name = header[0]
        cursor.execute("CREAT TABLE " + hold_name + "(" + header[0] + ")")
        for i in range(1,len(header)):
            cursor.execute("ALTER TABLE " + hold_name + " ADD COLUMN " + json.dumps(header[i]))

        #The line below is what I'm trying to fix
        cursor.execute("insert into hold_name values (?)", data_file.readline().split(","))
        cursor.execute("insert into obs_table values(header[0],header[1],header[2],header[3])")

<|endoftext|>


<commit_before>from collections import OrderedDict
import logging
import json
import re
import itertools

import sublime
import sublime_plugin

from ..lib import sorted_completions
from .commandinfo import (
    get_command_name,
    get_builtin_command_meta_data,
    get_builtin_commands,
    iter_python_command_classes,
    get_args_from_command_name
)

__all__ = (
    "SublimeTextCommandCompletionPythonListener",
    "SublimeTextCommandArgsCompletionListener",
    "SublimeTextCommandArgsCompletionPythonListener",
    "SublimeTextCommandCompletionListener",
)

l = logging.getLogger(__name__)

def _escape_in_snippet(v):
    return v.replace("}", "\\}").replace("$", "\\$")

def create_args_snippet_from_command_args(command_args, for_json=True):
    """Create an argument snippet to insert from the arguments to run a command.

    Parameters:
        command_args (dict)
            The arguments with their default value.
        for_json (bool)
            Whether it should be done for a json or a python file.

    Returns (str)
        The formatted entry to insert into the sublime text package
        file.
    """
    counter = itertools.count(1)

    def make_snippet_item(k, v):
        if v is not None:
            if isinstance(v, str):
                v = '"${{{i}:{v}}}"'.format(i=next(counter), v=_escape_in_snippet(v))
            else:
                if for_json:
                    dumps = json.dumps(v)
                else:  # python
                    dumps = repr(v)
                v = '${{{i}:{v}}}'.format(i=next(counter), v=_escape_in_snippet(dumps))
        else:
            v = '${i}'.format(i=next(counter))
        return '"{k}": {v}'.format(k=k, v=v)

    keys = iter(command_args)
    if not isinstance(command_args, OrderedDict):
        keys = sorted(keys)
    snippet_items = (make_snippet_item(k, command_args[k]) for k in keys)
    if for_json:
        args_content = ",\n\t".join(snippet_items)
        args_snippet = '"args": {{\n\t{0}\n}},$0'.format(args_content)
    else:
        args_content = ", ".join(snippet_items)
        args_snippet = '{{{0}}}'.format(args_content)
    return args_snippet

class SublimeTextCommandCompletionListener(sublime_plugin.EventListener):

    @staticmethod
    def _create_completion(c):
        name = get_command_name(c)
        module = c.__module__
        package = module.split(".")[0]
        show = "{name}\t{package}".format(**locals())
        return show, name

    def on_query_completions(self, view, prefix, locations):
        keymap_scope = "source.json.sublime meta.command-name"
        loc = locations[0]
        if not view.score_selector(loc, keymap_scope):
            return
        command_classes = iter_python_command_classes()

        completions = set()
        completions.update((c + "\tbuilt-in", c) for c in get_builtin_commands())
        completions.update(self._create_completion(c) for c in command_classes)
        return sorted_completions(completions), sublime.INHIBIT_WORD_COMPLETIONS

class SublimeTextCommandCompletionPythonListener(sublime_plugin.EventListener):

    _RE_LINE_BEFORE = re.compile(
        r"(?P<callervar>\w+)\s*\.\s*run_command\s*\("
        r"\s*['\"]\w*$",
        re.MULTILINE
    )

    @staticmethod
    def _create_builtin_completion(c):
        meta = get_builtin_command_meta_data()
        show = ("{c}\t({stype}) built-in"
                .format(c=c, stype=meta[c].get("command_type", " ")[:1].upper()))
        return show, c

    @staticmethod
    def _create_completion(c):
        name = get_command_name(c)
        module = c.__module__
        package = module.split(".")[0]
        if issubclass(c, sublime_plugin.TextCommand):
            stype = "T"
        elif issubclass(c, sublime_plugin.WindowCommand):
            stype = "W"
        elif issubclass(c, sublime_plugin.ApplicationCommand):
            stype = "A"
        else:
            stype = "?"

        show = "{name}\t({stype}) {package}".format(**locals())
        return show, name

    def on_query_completions(self, view, prefix, locations):
        loc = locations[0]
        python_arg_scope = (
            "source.python meta.function-call.python "
            "meta.function-call.arguments.python string.quoted"
        )
        if not view.score_selector(loc, python_arg_scope):
            return
        if sublime.packages_path() not in (view.file_name() or ""):
            return

        before_region = sublime.Region(view.line(loc).a, loc)
        lines = view.line(sublime.Region(view.line(locations[0]).a - 1, loc))
        before_region = sublime.Region(lines.a, loc)
        before = view.substr(before_region)
        m = self._RE_LINE_BEFORE.search(before)
        if not m:
            return
        # get the command type
        caller_var = m.group('callervar')
        if "view" in caller_var or caller_var == "v":
            command_type = 'text'
        elif caller_var == "sublime":
            command_type = 'app'
        else:
            # window.run_command allows all command types
            command_type = ''

        command_classes = iter_python_command_classes(command_type)
        completions = set()
        completions.update(
            self._create_builtin_completion(c)
            for c in get_builtin_commands(command_type)
        )
        completions.update(self._create_completion(c) for c in command_classes)
        return sorted_completions(completions), sublime.INHIBIT_WORD_COMPLETIONS

class SublimeTextCommandArgsCompletionListener(sublime_plugin.EventListener):

    _default_args = [("args\tArguments", '"args": {\n\t"$1": "$2"$0\n},')]
    _st_insert_arg_scope = (
        "("
        "(source.json.sublime.completions, source.json.sublime.keymap, "
        " source.json.sublime.macro, source.json.sublime.mousemap)"
        " & "
        "meta.sequence.json meta.mapping.json"
        ")"
        "- string "
        "- comment "
        "- meta.value.json "
        "- meta.mapping.json meta.mapping.json "
        "- meta.sequence.json meta.sequence.json "
    )
    _RE_COMMAND_SEARCH = re.compile(r'\"command\"\s*\:\s*\"(\w+)\"')

    def on_query_completions(self, view, prefix, locations):
        if not view.score_selector(locations[0], self._st_insert_arg_scope):
            return
        # extract the line and the line above to search for the command
        lines_reg = view.line(sublime.Region(view.line(locations[0]).a - 1, locations[0]))
        lines = view.substr(lines_reg)

        results = self._RE_COMMAND_SEARCH.findall(lines)
        if not results:
            return self._default_args

        command_name = results[-1]
        l.debug("building args completions for command %r", command_name)
        command_args = get_args_from_command_name(command_name)
        if not command_args:
            return self._default_args
        args = create_args_snippet_from_command_args(command_args)

        completions = [("args\tauto-detected arguments", args)]
        return completions

class SublimeTextCommandArgsCompletionPythonListener(sublime_plugin.EventListener):

    _default_args = [("args\targuments", '{"$1": "$2"$0}')]
    _RE_LINE_BEFORE = re.compile(
        r"\w+\s*\.\s*run_command\s*\("
        r"\s*(['\"])(\w+)\1,\s*\w*$"
    )

    def on_query_completions(self, view, prefix, locations):
        loc = locations[0]
        python_arg_scope = "source.python meta.function-call.python"
        if not view.score_selector(loc, python_arg_scope):
            return
        if sublime.packages_path() not in (view.file_name() or ""):
            return

        before_region = sublime.Region(view.line(loc).a, loc)
        before = view.substr(before_region)
        m = self._RE_LINE_BEFORE.search(before)
        if not m:
            return
        command_name = m.group(2)
        l.debug("building args completions for command %r", command_name)

        command_args = get_args_from_command_name(command_name)
        if command_args is None:
            return self._default_args
        args = create_args_snippet_from_command_args(command_args, False)

        completions = [("args\tauto-detected arguments", args)]
        return completions
<commit_msg>Source quote char in completions from command name<commit_after>from collections import OrderedDict
import logging
import json
import re
import itertools

import sublime
import sublime_plugin

from ..lib import sorted_completions
from .commandinfo import (
    get_command_name,
    get_builtin_command_meta_data,
    get_builtin_commands,
    iter_python_command_classes,
    get_args_from_command_name
)

__all__ = (
    "SublimeTextCommandCompletionPythonListener",
    "SublimeTextCommandArgsCompletionListener",
    "SublimeTextCommandArgsCompletionPythonListener",
    "SublimeTextCommandCompletionListener",
)

l = logging.getLogger(__name__)

def _escape_in_snippet(v):
    return v.replace("}", "\\}").replace("$", "\\$")

def create_args_snippet_from_command_args(command_args, quote_char='"', for_json=True):
    """Create an argument snippet to insert from the arguments to run a command.

    Parameters:
        command_args (dict)
            The arguments with their default value.
        quote_char (str)
            Which char should be used for string quoting.
        for_json (bool)
            Whether it should be done for a json or a python file.

    Returns (str)
        The formatted entry to insert into the sublime text package
        file.
    """
    counter = itertools.count(1)

    def make_snippet_item(k, v):
        if v is not None:
            if isinstance(v, str):
                v = '{q}${{{i}:{v}}}{q}'.format(i=next(counter),
                                                v=_escape_in_snippet(v),
                                                q=quote_char)
            else:
                if for_json:
                    dumps = json.dumps(v)
                else:  # python
                    dumps = repr(v)
                v = '${{{i}:{v}}}'.format(i=next(counter), v=_escape_in_snippet(dumps))
        else:
            v = '${i}'.format(i=next(counter))
        return '{q}{k}{q}: {v}'.format(k=k, v=v, q=quote_char)

    keys = iter(command_args)
    if not isinstance(command_args, OrderedDict):
        keys = sorted(keys)
    snippet_items = (make_snippet_item(k, command_args[k]) for k in keys)
    if for_json:
        args_content = ",\n\t".join(snippet_items)
        args_snippet = '"args": {{\n\t{0}\n}},$0'.format(args_content)
    else:
        args_content = ", ".join(snippet_items)
        args_snippet = '{{{0}}}'.format(args_content)
    return args_snippet

class SublimeTextCommandCompletionListener(sublime_plugin.EventListener):

    @staticmethod
    def _create_completion(c):
        name = get_command_name(c)
        module = c.__module__
        package = module.split(".")[0]
        show = "{name}\t{package}".format(**locals())
        return show, name

    def on_query_completions(self, view, prefix, locations):
        keymap_scope = "source.json.sublime meta.command-name"
        loc = locations[0]
        if not view.score_selector(loc, keymap_scope):
            return
        command_classes = iter_python_command_classes()

        completions = set()
        completions.update((c + "\tbuilt-in", c) for c in get_builtin_commands())
        completions.update(self._create_completion(c) for c in command_classes)
        return sorted_completions(completions), sublime.INHIBIT_WORD_COMPLETIONS

class SublimeTextCommandCompletionPythonListener(sublime_plugin.EventListener):

    _RE_LINE_BEFORE = re.compile(
        r"(?P<callervar>\w+)\s*\.\s*run_command\s*\("
        r"\s*['\"]\w*$",
        re.MULTILINE
    )

    @staticmethod
    def _create_builtin_completion(c):
        meta = get_builtin_command_meta_data()
        show = ("{c}\t({stype}) built-in"
                .format(c=c, stype=meta[c].get("command_type", " ")[:1].upper()))
        return show, c

    @staticmethod
    def _create_completion(c):
        name = get_command_name(c)
        module = c.__module__
        package = module.split(".")[0]
        if issubclass(c, sublime_plugin.TextCommand):
            stype = "T"
        elif issubclass(c, sublime_plugin.WindowCommand):
            stype = "W"
        elif issubclass(c, sublime_plugin.ApplicationCommand):
            stype = "A"
        else:
            stype = "?"

        show = "{name}\t({stype}) {package}".format(**locals())
        return show, name

    def on_query_completions(self, view, prefix, locations):
        loc = locations[0]
        python_arg_scope = (
            "source.python meta.function-call.python "
            "meta.function-call.arguments.python string.quoted"
        )
        if not view.score_selector(loc, python_arg_scope):
            return
        if sublime.packages_path() not in (view.file_name() or ""):
            return

        before_region = sublime.Region(view.line(loc).a, loc)
        lines = view.line(sublime.Region(view.line(locations[0]).a - 1, loc))
        before_region = sublime.Region(lines.a, loc)
        before = view.substr(before_region)
        m = self._RE_LINE_BEFORE.search(before)
        if not m:
            return
        # get the command type
        caller_var = m.group('callervar')
        if "view" in caller_var or caller_var == "v":
            command_type = 'text'
        elif caller_var == "sublime":
            command_type = 'app'
        else:
            # window.run_command allows all command types
            command_type = ''

        command_classes = iter_python_command_classes(command_type)
        completions = set()
        completions.update(
            self._create_builtin_completion(c)
            for c in get_builtin_commands(command_type)
        )
        completions.update(self._create_completion(c) for c in command_classes)
        return sorted_completions(completions), sublime.INHIBIT_WORD_COMPLETIONS

class SublimeTextCommandArgsCompletionListener(sublime_plugin.EventListener):

    _default_args = [("args\tArguments", '"args": {\n\t"$1": "$2"$0\n},')]
    _st_insert_arg_scope = (
        "("
        "(source.json.sublime.completions, source.json.sublime.keymap, "
        " source.json.sublime.macro, source.json.sublime.mousemap)"
        " & "
        "meta.sequence.json meta.mapping.json"
        ")"
        "- string "
        "- comment "
        "- meta.value.json "
        "- meta.mapping.json meta.mapping.json "
        "- meta.sequence.json meta.sequence.json "
    )
    _RE_COMMAND_SEARCH = re.compile(r'\"command\"\s*\:\s*\"(\w+)\"')

    def on_query_completions(self, view, prefix, locations):
        if not view.score_selector(locations[0], self._st_insert_arg_scope):
            return
        # extract the line and the line above to search for the command
        lines_reg = view.line(sublime.Region(view.line(locations[0]).a - 1, locations[0]))
        lines = view.substr(lines_reg)

        results = self._RE_COMMAND_SEARCH.findall(lines)
        if not results:
            return self._default_args

        command_name = results[-1]
        l.debug("building args completions for command %r", command_name)
        command_args = get_args_from_command_name(command_name)
        if not command_args:
            return self._default_args
        args = create_args_snippet_from_command_args(command_args, for_json=True)

        completions = [("args\tauto-detected arguments", args)]
        return completions

class SublimeTextCommandArgsCompletionPythonListener(sublime_plugin.EventListener):

    _default_args = [("args\targuments", '{"$1": "$2"$0}')]
    _RE_LINE_BEFORE = re.compile(
        r"\w+\s*\.\s*run_command\s*\("
        r"\s*(['\"])(\w+)\1,\s*\w*$"
    )

    def on_query_completions(self, view, prefix, locations):
        loc = locations[0]
        python_arg_scope = "source.python meta.function-call.python"
        if not view.score_selector(loc, python_arg_scope):
            return
        if sublime.packages_path() not in (view.file_name() or ""):
            return

        before_region = sublime.Region(view.line(loc).a, loc)
        before = view.substr(before_region)
        m = self._RE_LINE_BEFORE.search(before)
        if not m:
            return
        quote_char, command_name = m.groups()
        l.debug("building args completions for command %r", command_name)

        command_args = get_args_from_command_name(command_name)
        if command_args is None:
            return self._default_args
        args = create_args_snippet_from_command_args(command_args, quote_char, for_json=False)

        completions = [("args\tauto-detected arguments", args)]
        return completions
<|endoftext|>


<commit_before>#
# Copyright (C) 2012 - 2015 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
"""
Java properties file support.

- Format to support: Java Properties file, e.g.
  http://docs.oracle.com/javase/1.5.0/docs/api/java/util/Properties.html
- Requirements: pyjavaproperties,
  https://pypi.python.org/pypi/pyjavaproperties
- Limitations: None obvious
- Special options: None obvious
"""
from __future__ import absolute_import

import pyjavaproperties
import anyconfig.backend.base
import anyconfig.compat

class Parser(anyconfig.backend.base.LParser, anyconfig.backend.base.L2Parser,
             anyconfig.backend.base.D2Parser):
    """
    Parser for Java properties files.
    """
    _type = "properties"
    _extensions = ["properties"]

    def load_from_stream(self, stream, **kwargs):
        """
        Load config from given file like object `stream`.

        :param stream: A file or file like object of Java properties files
        :param kwargs: optional keyword parameters (ignored)

        :return: self.container object holding config parameters
        """
        props = pyjavaproperties.Properties()
        props.load(stream)

        return props.getPropertyDict()

    def dump_to_stream(self, cnf, stream, **kwargs):
        """
        Dump config `cnf` to a file or file-like object `stream`.

        :param cnf: Java properties config data to dump :: self.container
        :param stream: Java properties file or file like object
        :param kwargs: backend-specific optional keyword parameters :: dict
        """
        props = pyjavaproperties.Properties()
        for key, val in anyconfig.compat.iteritems(cnf):
            props.setProperty(key, val)

        props.store(stream)

# vim:sw=4:ts=4:et:
<commit_msg>add a note<commit_after>#
# Copyright (C) 2012 - 2015 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
"""
Java properties file support.

.. note::
   Now anyconfig has a native Java properties backend and there is no need to
   install this package. This package will be kept as an reference but
   obsoleted and not updated anymore.

- Format to support: Java Properties file, e.g.
  http://docs.oracle.com/javase/1.5.0/docs/api/java/util/Properties.html
- Requirements: pyjavaproperties,
  https://pypi.python.org/pypi/pyjavaproperties
- Limitations: None obvious
- Special options: None obvious
"""
from __future__ import absolute_import

import pyjavaproperties
import anyconfig.backend.base
import anyconfig.compat

class Parser(anyconfig.backend.base.LParser, anyconfig.backend.base.L2Parser,
             anyconfig.backend.base.D2Parser):
    """
    Parser for Java properties files (sample).
    """
    _type = "properties"
    _extensions = ["properties"]

    def load_from_stream(self, stream, **kwargs):
        """
        Load config from given file like object `stream`.

        :param stream: A file or file like object of Java properties files
        :param kwargs: optional keyword parameters (ignored)

        :return: self.container object holding config parameters
        """
        props = pyjavaproperties.Properties()
        props.load(stream)

        return props.getPropertyDict()

    def dump_to_stream(self, cnf, stream, **kwargs):
        """
        Dump config `cnf` to a file or file-like object `stream`.

        :param cnf: Java properties config data to dump :: self.container
        :param stream: Java properties file or file like object
        :param kwargs: backend-specific optional keyword parameters :: dict
        """
        props = pyjavaproperties.Properties()
        for key, val in anyconfig.compat.iteritems(cnf):
            props.setProperty(key, val)

        props.store(stream)

# vim:sw=4:ts=4:et:
<|endoftext|>


<commit_before>import unittest

import numpy
import six

import chainer
from chainer.backends import cuda
from chainer import gradient_check
from chainer import links
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition
from chainer.utils import type_check

def _batch_normalization(expander, gamma, beta, x, mean, var, eps, test):
    mean = mean[expander]
    if test:
        std = numpy.sqrt(var[expander])
    else:
        std = numpy.sqrt(var[expander] + eps)
    y_expect = gamma * (x - mean) / std + beta
    return y_expect

@testing.parameterize(*(testing.product({
    'test': [True, False],
    'ndim': [0, 1, 2, 3],
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
})))
class BatchNormalizationTest(unittest.TestCase):

    def setUp(self):
        self.expander = (None, Ellipsis) + (None,) * self.ndim
        self.aggr_axes = (0,) + tuple(six.moves.range(2, self.ndim + 2))

        self.link = links.BatchNormalization(3, dtype=self.dtype)
        gamma = self.link.gamma.data
        gamma[...] = numpy.random.uniform(.5, 1, gamma.shape)
        beta = self.link.beta.data
        beta[...] = numpy.random.uniform(-1, 1, beta.shape)
        self.link.cleargrads()

        self.gamma = gamma.copy()[self.expander]  # fixed on CPU
        self.beta = beta.copy()[self.expander]   # fixed on CPU

        shape = (5, 3) + (2,) * self.ndim
        self.x = numpy.random.uniform(-1, 1, shape).astype(self.dtype)
        self.gy = numpy.random.uniform(-1, 1, shape).astype(self.dtype)

        if self.test:
            self.mean = numpy.random.uniform(-1, 1, (3,)).astype(self.dtype)
            self.var = numpy.random.uniform(0.5, 1, (3,)).astype(self.dtype)
            self.link.avg_mean[...] = self.mean
            self.link.avg_var[...] = self.var
        else:
            self.mean = self.x.mean(axis=self.aggr_axes)
            self.var = self.x.var(axis=self.aggr_axes)
        self.check_forward_optionss = {'atol': 1e-4, 'rtol': 1e-3}
        self.check_backward_optionss = {'atol': 1e-4, 'rtol': 1e-3}
        if self.dtype == numpy.float16:
            self.check_forward_optionss = {'atol': 1e-3, 'rtol': 1e-2}
            self.check_backward_optionss = {'atol': 5e-1, 'rtol': 1e-1}

    def check_forward(self, x_data):
        with chainer.using_config('train', not self.test):
            x = chainer.Variable(x_data)
            y = self.link(x)
            self.assertEqual(y.data.dtype, self.dtype)

        y_expect = _batch_normalization(
            self.expander, self.gamma, self.beta, self.x, self.mean,
            self.var, self.link.eps, self.test)

        testing.assert_allclose(
            y_expect, y.data, **self.check_forward_optionss)

    @condition.retry(3)
    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.link.to_gpu()
        self.check_forward(cuda.to_gpu(self.x))

    @attr.cudnn
    def test_forward_gpu_without_cudnn(self):
        with chainer.using_config('use_cudnn', 'never'):
            self.test_forward_gpu()

    @attr.multi_gpu(2)
    @condition.retry(3)
    def test_forward_multi_gpu(self):
        with cuda.get_device_from_id(1):
            self.link.to_gpu()
            x = cuda.to_gpu(self.x)
        with cuda.get_device_from_id(0):
            self.check_forward(x)

    def check_backward(self, x_data, y_grad):
        gradient_check.check_backward(
            self.link, x_data, y_grad, (self.link.gamma, self.link.beta),
            eps=1e-2, **self.check_backward_optionss)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.link.to_gpu()
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))

    @attr.cudnn
    def test_backward_gpu_without_cudnn(self):
        with chainer.using_config('use_cudnn', 'never'):
            self.test_backward_gpu()

@testing.parameterize(
    {'nx': 10, 'ny': 10},
    # TODO(Kenta Oono)
    # Pass the case below (this test does not pass when nx != ny).
    # {'nx': 10, 'ny': 15}
)
class TestPopulationStatistics(unittest.TestCase):

    def setUp(self):
        self.decay = 0.9
        self.size = 3
        self.link = links.BatchNormalization(self.size, self.decay)
        self.x = numpy.random.uniform(
            -1, 1, (self.nx, self.size)).astype(numpy.float32)
        self.y = numpy.random.uniform(
            -1, 1, (self.ny, self.size)).astype(numpy.float32)

    def check_statistics(self, x, y):
        x = chainer.Variable(x)
        self.link(x, finetune=True)
        mean = self.x.mean(axis=0)
        testing.assert_allclose(mean, self.link.avg_mean)
        unbiased_var = self.x.var(axis=0) * self.nx / (self.nx - 1)
        testing.assert_allclose(unbiased_var, self.link.avg_var)

        y = chainer.Variable(y)
        with chainer.using_config('train', False):
            self.link(y, finetune=True)
        testing.assert_allclose(mean, self.link.avg_mean)
        testing.assert_allclose(unbiased_var, self.link.avg_var)

    @condition.retry(3)
    def test_statistics_cpu(self):
        self.check_statistics(self.x, self.y)

    @attr.gpu
    @condition.retry(3)
    def test_statistics_gpu(self):
        self.link.to_gpu()
        self.check_statistics(cuda.to_gpu(self.x), cuda.to_gpu(self.y))

    @attr.cudnn
    def test_statistics_gpu_without_cudnn(self):
        with chainer.using_config('use_cudnn', 'never'):
            self.test_statistics_gpu()

    def check_statistics2(self, x, y):
        x = chainer.Variable(x)
        y = chainer.Variable(y)
        self.link(x, finetune=True)
        self.link(y, finetune=True)
        mean = (self.x.sum(axis=0) + self.y.sum(axis=0)) / (self.nx + self.ny)
        var = (self.x.var(axis=0) * self.nx +
               self.y.var(axis=0) * self.ny) / (self.nx + self.ny)

        # TODO(Kenta Oono)
        # Fix the estimate of the unbiased variance.
        # Unbiased variance should be (nx + ny) / (nx + ny - 1) times of
        # the variance.
        # But the multiplier is ny / (ny - 1) in current implementation
        # these two values are different when nx is not equal to ny.
        unbiased_var = var * self.ny / (self.ny - 1)
        testing.assert_allclose(mean, self.link.avg_mean)
        testing.assert_allclose(unbiased_var, self.link.avg_var)

    @condition.retry(3)
    def test_statistics2_cpu(self):
        self.check_statistics2(self.x, self.y)

    @attr.gpu
    @condition.retry(3)
    def test_statistics2_gpu(self):
        self.link.to_gpu()
        self.check_statistics2(
            cuda.to_gpu(self.x),
            cuda.to_gpu(self.y))

    @attr.cudnn
    def test_statistics2_gpu_without_cudnn(self):
        with chainer.using_config('use_cudnn', 'never'):
            self.test_statistics2_gpu()

@testing.parameterize(*testing.product({
    'test': [True, False],
    'ndim': [0, 1, 2, 3],
}))
class BatchNormalizationTestWithoutGammaAndBeta(unittest.TestCase):

    def setUp(self):
        self.link = links.BatchNormalization(
            3, use_gamma=False, use_beta=False)
        if self.test:
            mean = numpy.random.uniform(-1, 1, (3,)).astype(numpy.float32)
            self.link.avg_mean[...] = mean
            var = numpy.random.uniform(0.5, 1, (3,)).astype(numpy.float32)
            self.link.avg_var[...] = var
        self.link.cleargrads()

        shape = (7, 3) + (2,) * self.ndim
        self.x = numpy.random.uniform(-1, 1, shape).astype(numpy.float32)
        self.gy = numpy.random.uniform(-1, 1, shape).astype(numpy.float32)

        expander = (None, Ellipsis) + (None,) * self.ndim
        gamma = numpy.ones((3,), dtype=numpy.float32)[expander]
        beta = numpy.zeros((3,), dtype=numpy.float32)[expander]
        if self.test:
            mean = self.link.avg_mean
            var = self.link.avg_var
        else:
            aggr_axes = (0,) + tuple(six.moves.range(2, self.ndim + 2))
            mean = self.x.mean(axis=aggr_axes)
            var = self.x.var(axis=aggr_axes)
        self.y_expected = _batch_normalization(
            expander, gamma, beta, self.x, mean, var, self.link.eps, self.test)

    def test_no_gamma_and_beta(self):
        self.assertFalse(hasattr(self.link, 'gamma'))
        self.assertFalse(hasattr(self.link, 'beta'))

    def check_forward(self, x_data):
        x = chainer.Variable(x_data)
        with chainer.using_config('train', not self.test):
            y = self.link(x)
        testing.assert_allclose(self.y_expected, y.data)

    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.gpu
    def test_forward_gpu(self):
        self.link.to_gpu()
        x = cuda.to_gpu(self.x)
        self.check_forward(x)

    @attr.multi_gpu(2)
    def test_forward_gpu_multi(self):
        with cuda.get_device_from_id(0):
            self.link.to_gpu()
            x = cuda.to_gpu(self.x)
        with cuda.get_device_from_id(1):
            self.check_forward(x)

    @attr.cudnn
    def test_forward_gpu_without_cudnn(self):
        with chainer.using_config('use_cudnn', 'never'):
            self.test_forward_gpu()

    def check_backward(self, x_data, y_grad):
        gradient_check.check_backward(self.link, x_data, y_grad,
                                      eps=1e-2, rtol=1e-3, atol=1e-4)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.link.to_gpu()
        x = cuda.to_gpu(self.x)
        gy = cuda.to_gpu(self.gy)
        self.check_backward(x, gy)

    @attr.cudnn
    def test_backward_gpu_without_cudnn(self):
        with chainer.using_config('use_cudnn', 'never'):
            self.test_backward_gpu()

@testing.parameterize(*testing.product({
    'size': [3, (2, 3)],
}))
class TestInitialize(unittest.TestCase):

    def setUp(self):
        self.decay = 0.9
        self.initial_gamma = numpy.random.uniform(-1, 1, self.size)
        self.initial_gamma = self.initial_gamma.astype(numpy.float32)
        self.initial_beta = numpy.random.uniform(-1, 1, self.size)
        self.initial_beta = self.initial_beta.astype(numpy.float32)
        self.link = links.BatchNormalization(self.size, self.decay,
                                             initial_gamma=self.initial_gamma,
                                             initial_beta=self.initial_beta)

    @condition.retry(3)
    def test_initialize_cpu(self):
        testing.assert_allclose(self.initial_gamma, self.link.gamma.data)
        testing.assert_allclose(self.initial_beta, self.link.beta.data)

    @attr.gpu
    @condition.retry(3)
    def test_initialize_gpu(self):
        self.link.to_gpu()
        testing.assert_allclose(self.initial_gamma, self.link.gamma.data)
        testing.assert_allclose(self.initial_beta, self.link.beta.data)

class TestDefaultInitializer(unittest.TestCase):

    def setUp(self):
        self.decay = 0.9
        self.size = 3
        self.link = links.BatchNormalization(self.size, self.decay)

    def test_initialize_cpu(self):
        testing.assert_allclose(numpy.ones(self.size), self.link.gamma.data)
        testing.assert_allclose(numpy.zeros(self.size), self.link.beta.data)

    @attr.gpu
    def test_initialize_gpu(self):
        self.link.to_gpu()
        testing.assert_allclose(numpy.ones(self.size), self.link.gamma.data)
        testing.assert_allclose(numpy.zeros(self.size), self.link.beta.data)

@testing.parameterize(*testing.product({
    'shape': [(2, 4), (2, 5, 3, 4)],
}))
class TestInvalidInput(unittest.TestCase):

    def setUp(self):
        self.link = links.BatchNormalization(3)

    def test_invalid_shape_cpu(self):
        with self.assertRaises(type_check.InvalidType):
            self.link(chainer.Variable(numpy.zeros(self.shape, dtype='f')))

    @attr.gpu
    def test_invalid_shape_gpu(self):
        self.link.to_gpu()
        with self.assertRaises(type_check.InvalidType):
            self.link(chainer.Variable(cuda.cupy.zeros(self.shape, dtype='f')))

class TestInvalidInitialize(unittest.TestCase):

    def test_invalid_type(self):
        with self.assertRaises(TypeError):
            self.link = links.BatchNormalization({})

class TestInvalidArgument(unittest.TestCase):

    def setUp(self):
        self.link = links.BatchNormalization(1)
        self.x = numpy.random.uniform(-1, 1, (3,)).astype('f')

    def test_test_argument(self):
        with self.assertRaises(ValueError):
            self.link(self.x, test=True)

    def test_positional_argument(self):
        # positional argument is prohibited from v2
        with self.assertRaises(TypeError):
            self.link(self.x, True)

    def test_redundant_argument(self):
        with self.assertRaises(TypeError):
            self.link(self.x, unknown_argument=1)

testing.run_module(__name__, __file__)
<commit_msg>test larger eps<commit_after>import unittest

import numpy
import six

import chainer
from chainer.backends import cuda
from chainer import gradient_check
from chainer import links
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition
from chainer.utils import type_check

def _batch_normalization(expander, gamma, beta, x, mean, var, eps, test):
    mean = mean[expander]
    if test:
        std = numpy.sqrt(var[expander])
    else:
        std = numpy.sqrt(var[expander] + eps)
    y_expect = gamma * (x - mean) / std + beta
    return y_expect

@testing.parameterize(*(testing.product({
    'test': [True, False],
    'ndim': [0, 1, 2, 3],
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
})))
class BatchNormalizationTest(unittest.TestCase):

    def setUp(self):
        self.expander = (None, Ellipsis) + (None,) * self.ndim
        self.aggr_axes = (0,) + tuple(six.moves.range(2, self.ndim + 2))

        self.link = links.BatchNormalization(3, dtype=self.dtype)
        gamma = self.link.gamma.data
        gamma[...] = numpy.random.uniform(.5, 1, gamma.shape)
        beta = self.link.beta.data
        beta[...] = numpy.random.uniform(-1, 1, beta.shape)
        self.link.cleargrads()

        self.gamma = gamma.copy()[self.expander]  # fixed on CPU
        self.beta = beta.copy()[self.expander]   # fixed on CPU

        shape = (5, 3) + (2,) * self.ndim
        self.x = numpy.random.uniform(-1, 1, shape).astype(self.dtype)
        self.gy = numpy.random.uniform(-1, 1, shape).astype(self.dtype)

        if self.test:
            self.mean = numpy.random.uniform(-1, 1, (3,)).astype(self.dtype)
            self.var = numpy.random.uniform(0.5, 1, (3,)).astype(self.dtype)
            self.link.avg_mean[...] = self.mean
            self.link.avg_var[...] = self.var
        else:
            self.mean = self.x.mean(axis=self.aggr_axes)
            self.var = self.x.var(axis=self.aggr_axes)
        self.check_forward_optionss = {'atol': 1e-4, 'rtol': 1e-3}
        self.check_backward_optionss = {'atol': 1e-4, 'rtol': 1e-3}
        if self.dtype == numpy.float16:
            self.check_forward_optionss = {'atol': 1e-3, 'rtol': 1e-2}
            self.check_backward_optionss = {'atol': 5e-1, 'rtol': 1e-1}

    def check_forward(self, x_data):
        with chainer.using_config('train', not self.test):
            x = chainer.Variable(x_data)
            y = self.link(x)
            self.assertEqual(y.data.dtype, self.dtype)

        y_expect = _batch_normalization(
            self.expander, self.gamma, self.beta, self.x, self.mean,
            self.var, self.link.eps, self.test)

        testing.assert_allclose(
            y_expect, y.data, **self.check_forward_optionss)

    @condition.retry(3)
    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.link.to_gpu()
        self.check_forward(cuda.to_gpu(self.x))

    @attr.cudnn
    def test_forward_gpu_without_cudnn(self):
        with chainer.using_config('use_cudnn', 'never'):
            self.test_forward_gpu()

    @attr.multi_gpu(2)
    @condition.retry(3)
    def test_forward_multi_gpu(self):
        with cuda.get_device_from_id(1):
            self.link.to_gpu()
            x = cuda.to_gpu(self.x)
        with cuda.get_device_from_id(0):
            self.check_forward(x)

    def check_backward(self, x_data, y_grad):
        gradient_check.check_backward(
            self.link, x_data, y_grad, (self.link.gamma, self.link.beta),
            eps=1e-2, **self.check_backward_optionss)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.link.to_gpu()
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))

    @attr.cudnn
    def test_backward_gpu_without_cudnn(self):
        with chainer.using_config('use_cudnn', 'never'):
            self.test_backward_gpu()

@testing.parameterize(
    {'nx': 10, 'ny': 10, 'eps': 2e-5},
    {'nx': 10, 'ny': 10, 'eps': 1e-1},
    # TODO(Kenta Oono)
    # Pass the case below (this test does not pass when nx != ny).
    # {'nx': 10, 'ny': 15}
)
class TestPopulationStatistics(unittest.TestCase):

    def setUp(self):
        self.decay = 0.9
        self.size = 3
        self.link = links.BatchNormalization(self.size, self.decay, self.eps)
        self.x = numpy.random.uniform(
            -1, 1, (self.nx, self.size)).astype(numpy.float32)
        self.y = numpy.random.uniform(
            -1, 1, (self.ny, self.size)).astype(numpy.float32)

    def check_statistics(self, x, y):
        x = chainer.Variable(x)
        self.link(x, finetune=True)
        mean = self.x.mean(axis=0)
        testing.assert_allclose(mean, self.link.avg_mean)
        unbiased_var = self.x.var(axis=0) * self.nx / (self.nx - 1)
        testing.assert_allclose(unbiased_var, self.link.avg_var)

        y = chainer.Variable(y)
        with chainer.using_config('train', False):
            self.link(y, finetune=True)
        testing.assert_allclose(mean, self.link.avg_mean)
        testing.assert_allclose(unbiased_var, self.link.avg_var)

    @condition.retry(3)
    def test_statistics_cpu(self):
        self.check_statistics(self.x, self.y)

    @attr.gpu
    @condition.retry(3)
    def test_statistics_gpu(self):
        self.link.to_gpu()
        self.check_statistics(cuda.to_gpu(self.x), cuda.to_gpu(self.y))

    @attr.cudnn
    def test_statistics_gpu_without_cudnn(self):
        with chainer.using_config('use_cudnn', 'never'):
            self.test_statistics_gpu()

    def check_statistics2(self, x, y):
        x = chainer.Variable(x)
        y = chainer.Variable(y)
        self.link(x, finetune=True)
        self.link(y, finetune=True)
        mean = (self.x.sum(axis=0) + self.y.sum(axis=0)) / (self.nx + self.ny)
        var = (self.x.var(axis=0) * self.nx +
               self.y.var(axis=0) * self.ny) / (self.nx + self.ny)

        # TODO(Kenta Oono)
        # Fix the estimate of the unbiased variance.
        # Unbiased variance should be (nx + ny) / (nx + ny - 1) times of
        # the variance.
        # But the multiplier is ny / (ny - 1) in current implementation
        # these two values are different when nx is not equal to ny.
        unbiased_var = var * self.ny / (self.ny - 1)
        testing.assert_allclose(mean, self.link.avg_mean)
        testing.assert_allclose(unbiased_var, self.link.avg_var)

    @condition.retry(3)
    def test_statistics2_cpu(self):
        self.check_statistics2(self.x, self.y)

    @attr.gpu
    @condition.retry(3)
    def test_statistics2_gpu(self):
        self.link.to_gpu()
        self.check_statistics2(
            cuda.to_gpu(self.x),
            cuda.to_gpu(self.y))

    @attr.cudnn
    def test_statistics2_gpu_without_cudnn(self):
        with chainer.using_config('use_cudnn', 'never'):
            self.test_statistics2_gpu()

@testing.parameterize(*testing.product({
    'test': [True, False],
    'ndim': [0, 1, 2, 3],
}))
class BatchNormalizationTestWithoutGammaAndBeta(unittest.TestCase):

    def setUp(self):
        self.link = links.BatchNormalization(
            3, use_gamma=False, use_beta=False)
        if self.test:
            mean = numpy.random.uniform(-1, 1, (3,)).astype(numpy.float32)
            self.link.avg_mean[...] = mean
            var = numpy.random.uniform(0.5, 1, (3,)).astype(numpy.float32)
            self.link.avg_var[...] = var
        self.link.cleargrads()

        shape = (7, 3) + (2,) * self.ndim
        self.x = numpy.random.uniform(-1, 1, shape).astype(numpy.float32)
        self.gy = numpy.random.uniform(-1, 1, shape).astype(numpy.float32)

        expander = (None, Ellipsis) + (None,) * self.ndim
        gamma = numpy.ones((3,), dtype=numpy.float32)[expander]
        beta = numpy.zeros((3,), dtype=numpy.float32)[expander]
        if self.test:
            mean = self.link.avg_mean
            var = self.link.avg_var
        else:
            aggr_axes = (0,) + tuple(six.moves.range(2, self.ndim + 2))
            mean = self.x.mean(axis=aggr_axes)
            var = self.x.var(axis=aggr_axes)
        self.y_expected = _batch_normalization(
            expander, gamma, beta, self.x, mean, var, self.link.eps, self.test)

    def test_no_gamma_and_beta(self):
        self.assertFalse(hasattr(self.link, 'gamma'))
        self.assertFalse(hasattr(self.link, 'beta'))

    def check_forward(self, x_data):
        x = chainer.Variable(x_data)
        with chainer.using_config('train', not self.test):
            y = self.link(x)
        testing.assert_allclose(self.y_expected, y.data)

    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.gpu
    def test_forward_gpu(self):
        self.link.to_gpu()
        x = cuda.to_gpu(self.x)
        self.check_forward(x)

    @attr.multi_gpu(2)
    def test_forward_gpu_multi(self):
        with cuda.get_device_from_id(0):
            self.link.to_gpu()
            x = cuda.to_gpu(self.x)
        with cuda.get_device_from_id(1):
            self.check_forward(x)

    @attr.cudnn
    def test_forward_gpu_without_cudnn(self):
        with chainer.using_config('use_cudnn', 'never'):
            self.test_forward_gpu()

    def check_backward(self, x_data, y_grad):
        gradient_check.check_backward(self.link, x_data, y_grad,
                                      eps=1e-2, rtol=1e-3, atol=1e-4)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.link.to_gpu()
        x = cuda.to_gpu(self.x)
        gy = cuda.to_gpu(self.gy)
        self.check_backward(x, gy)

    @attr.cudnn
    def test_backward_gpu_without_cudnn(self):
        with chainer.using_config('use_cudnn', 'never'):
            self.test_backward_gpu()

@testing.parameterize(*testing.product({
    'size': [3, (2, 3)],
}))
class TestInitialize(unittest.TestCase):

    def setUp(self):
        self.decay = 0.9
        self.initial_gamma = numpy.random.uniform(-1, 1, self.size)
        self.initial_gamma = self.initial_gamma.astype(numpy.float32)
        self.initial_beta = numpy.random.uniform(-1, 1, self.size)
        self.initial_beta = self.initial_beta.astype(numpy.float32)
        self.link = links.BatchNormalization(self.size, self.decay,
                                             initial_gamma=self.initial_gamma,
                                             initial_beta=self.initial_beta)

    @condition.retry(3)
    def test_initialize_cpu(self):
        testing.assert_allclose(self.initial_gamma, self.link.gamma.data)
        testing.assert_allclose(self.initial_beta, self.link.beta.data)

    @attr.gpu
    @condition.retry(3)
    def test_initialize_gpu(self):
        self.link.to_gpu()
        testing.assert_allclose(self.initial_gamma, self.link.gamma.data)
        testing.assert_allclose(self.initial_beta, self.link.beta.data)

class TestDefaultInitializer(unittest.TestCase):

    def setUp(self):
        self.decay = 0.9
        self.size = 3
        self.link = links.BatchNormalization(self.size, self.decay)

    def test_initialize_cpu(self):
        testing.assert_allclose(numpy.ones(self.size), self.link.gamma.data)
        testing.assert_allclose(numpy.zeros(self.size), self.link.beta.data)

    @attr.gpu
    def test_initialize_gpu(self):
        self.link.to_gpu()
        testing.assert_allclose(numpy.ones(self.size), self.link.gamma.data)
        testing.assert_allclose(numpy.zeros(self.size), self.link.beta.data)

@testing.parameterize(*testing.product({
    'shape': [(2, 4), (2, 5, 3, 4)],
}))
class TestInvalidInput(unittest.TestCase):

    def setUp(self):
        self.link = links.BatchNormalization(3)

    def test_invalid_shape_cpu(self):
        with self.assertRaises(type_check.InvalidType):
            self.link(chainer.Variable(numpy.zeros(self.shape, dtype='f')))

    @attr.gpu
    def test_invalid_shape_gpu(self):
        self.link.to_gpu()
        with self.assertRaises(type_check.InvalidType):
            self.link(chainer.Variable(cuda.cupy.zeros(self.shape, dtype='f')))

class TestInvalidInitialize(unittest.TestCase):

    def test_invalid_type(self):
        with self.assertRaises(TypeError):
            self.link = links.BatchNormalization({})

class TestInvalidArgument(unittest.TestCase):

    def setUp(self):
        self.link = links.BatchNormalization(1)
        self.x = numpy.random.uniform(-1, 1, (3,)).astype('f')

    def test_test_argument(self):
        with self.assertRaises(ValueError):
            self.link(self.x, test=True)

    def test_positional_argument(self):
        # positional argument is prohibited from v2
        with self.assertRaises(TypeError):
            self.link(self.x, True)

    def test_redundant_argument(self):
        with self.assertRaises(TypeError):
            self.link(self.x, unknown_argument=1)

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before>import os
import shutil
import tempfile
import unittest

import numpy

import chainer
from chainer import configuration
from chainer import functions
from chainer import links
from chainer import testing
from chainer import training
from chainer.training.extensions import computational_graph as c

class Dataset(chainer.dataset.DatasetMixin):
    def __init__(self, values):
        self.values = values

    def __len__(self):
        return len(self.values)

    def get_example(self, i):
        return numpy.array([self.values[i]], numpy.float32), numpy.int32(i % 2)

class Model(chainer.Link):
    def __init__(self):
        super(Model, self).__init__()
        self.flag_history = []
        self.l1 = links.Linear(2)
        self.l2 = links.Linear(2)
        self.i = 0

    def __call__(self, x):
        self.flag_history.append(
            configuration.config.keep_graph_on_report)

        h = self.l1(x)
        if self.i == 0:
            h, = functions.Sigmoid().apply((h,))
        else:
            h = functions.Tanh()(h)
        h = self.l2(h)

        self.i += 1
        return h

class TestGraphBuilderKeepGraphOnReport(unittest.TestCase):

    def _run_test(self, tempdir, initial_flag):
        n_data = 4
        n_epochs = 3
        outdir = os.path.join(tempdir, 'testresult')

        # Prepare
        model = Model()
        classifier = links.Classifier(model)
        optimizer = chainer.optimizers.Adam()
        optimizer.setup(classifier)

        dataset = Dataset([i for i in range(n_data)])
        iterator = chainer.iterators.SerialIterator(dataset, 1, shuffle=False)
        updater = training.StandardUpdater(iterator, optimizer)
        trainer = training.Trainer(updater, (n_epochs, 'epoch'), out=outdir)

        extension = c.dump_graph('main/loss', out_name='test.dot')
        trainer.extend(extension)

        # Run
        with chainer.using_config('keep_graph_on_report', initial_flag):
            trainer.run()

        # Check flag history
        self.assertEqual(model.flag_history,
                         [True] + [initial_flag] * (n_data * n_epochs - 1))

        # Check the dumped graph
        graph_path = os.path.join(outdir, 'test.dot')
        with open(graph_path) as f:
            graph_dot = f.read()

        # Check that only the first iteration is dumped
        self.assertIn('Sigmoid', graph_dot)
        self.assertNotIn('Tanh', graph_dot)

    def _check(self, initial_flag):
        tempdir = tempfile.mkdtemp()
        try:
            self._run_test(tempdir, initial_flag)
        finally:
            shutil.rmtree(tempdir)

    def test_keep_graph_on_report_flag_true(self):
        self._check(True)

    def test_keep_graph_on_report_flag_false(self):
        self._check(False)

testing.run_module(__name__, __file__)
<commit_msg>Use dummy functions<commit_after>import os
import shutil
import tempfile
import unittest

import numpy

import chainer
from chainer import configuration
from chainer import function_node
from chainer import links
from chainer import testing
from chainer import training
from chainer.training.extensions import computational_graph as c

class Function1(function_node.FunctionNode):

    def forward(self, inputs):
        return inputs[0],

class Function2(function_node.FunctionNode):

    def forward(self, inputs):
        return inputs[0],

class Dataset(chainer.dataset.DatasetMixin):
    def __init__(self, values):
        self.values = values

    def __len__(self):
        return len(self.values)

    def get_example(self, i):
        return numpy.array([self.values[i]], numpy.float32), numpy.int32(i % 2)

class Model(chainer.Link):
    def __init__(self):
        super(Model, self).__init__()
        self.flag_history = []
        self.l1 = links.Linear(2)
        self.l2 = links.Linear(2)
        self.i = 0

    def __call__(self, x):
        self.flag_history.append(
            configuration.config.keep_graph_on_report)

        h = self.l1(x)
        if self.i == 0:
            h, = Function1().apply((h,))
        else:
            h, = Function2().apply((h,))
        h = self.l2(h)

        self.i += 1
        return h

class TestGraphBuilderKeepGraphOnReport(unittest.TestCase):

    def _run_test(self, tempdir, initial_flag):
        n_data = 4
        n_epochs = 3
        outdir = os.path.join(tempdir, 'testresult')

        # Prepare
        model = Model()
        classifier = links.Classifier(model)
        optimizer = chainer.optimizers.Adam()
        optimizer.setup(classifier)

        dataset = Dataset([i for i in range(n_data)])
        iterator = chainer.iterators.SerialIterator(dataset, 1, shuffle=False)
        updater = training.StandardUpdater(iterator, optimizer)
        trainer = training.Trainer(updater, (n_epochs, 'epoch'), out=outdir)

        extension = c.dump_graph('main/loss', out_name='test.dot')
        trainer.extend(extension)

        # Run
        with chainer.using_config('keep_graph_on_report', initial_flag):
            trainer.run()

        # Check flag history
        self.assertEqual(model.flag_history,
                         [True] + [initial_flag] * (n_data * n_epochs - 1))

        # Check the dumped graph
        graph_path = os.path.join(outdir, 'test.dot')
        with open(graph_path) as f:
            graph_dot = f.read()

        # Check that only the first iteration is dumped
        self.assertIn('Function1', graph_dot)
        self.assertNotIn('Function2', graph_dot)

    def _check(self, initial_flag):
        tempdir = tempfile.mkdtemp()
        try:
            self._run_test(tempdir, initial_flag)
        finally:
            shutil.rmtree(tempdir)

    def test_keep_graph_on_report_flag_true(self):
        self._check(True)

    def test_keep_graph_on_report_flag_false(self):
        self._check(False)

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before>#!/usr/bin/env python2.6
##########################################################################
#
#  Copyright (c) 2011, John Haddon. All rights reserved.
#  Copyright (c) 2011-2012, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#      * Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      * Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided with
#        the distribution.
#
#      * Neither the name of John Haddon nor the names of
#        any other contributors to this software may be used to endorse or
#        promote products derived from this software without specific prior
#        written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import os
import gc
import sys
import ctypes
import signal
import warnings

# Work around cross module rtti errors on linux.
sys.setdlopenflags( sys.getdlopenflags() | ctypes.RTLD_GLOBAL )

# Get rid of the annoying signal handler which turns Ctrl-C into a KeyboardInterrupt exception
signal.signal( signal.SIGINT, signal.SIG_DFL )

# Reenable deprecation warnings - Python2.7 turns them off by default so otherwise we'd never get
# to catch all the naughty deprecated things we do.
warnings.simplefilter( "default", DeprecationWarning )

import IECore

helpText = """Usage :

    gaffer -help                    Print this message
    gaffer -help appName            Print help specific to named application
    gaffer appName [flags]          Run named application with specified flags
    gaffer fileName.gfr [flags]     Run gui application with specified script and flags
    gaffer [flags]                  Run gui application with specified flags
"""

appLoader = IECore.ClassLoader.defaultLoader( "GAFFER_APP_PATHS" )
applicationText = "Installed applications :\n\n" + "\n".join( "    " + x for x in appLoader.classNames() ) + "\n"

def loadApp( appName ) :

	if appName in appLoader.classNames() :
		return appLoader.load( appName )()
	else :
		sys.stderr.write( "ERROR : Application \"%s\" not installed on GAFFER_APP_PATHS\n" % appName )
		sys.stderr.write( "\n" + applicationText )
		sys.exit( 1 )

def checkCleanExit() :

	# Get the Gaffer and GafferUI modules, but only if the app actually
	# imported them. We don't want to force their importation because it's
	# just a waste of time if they weren't used.
	Gaffer = sys.modules.get( "Gaffer" )
	GafferUI = sys.modules.get( "GafferUI" )

	if Gaffer is None and GafferUI is None :
		return

	# Clean up any garbage left behind by Cortex's wrapper mechanism - because
	# the Gaffer.Application itself is derived from IECore.Parameterised, which
	# as far as I can tell is wrapped unnecessarily, we must call this to allow
	# the application to be deleted at all. Note that we're deliberately not also
	# calling gc.collect() - our intention here isn't to clean up on shutdown, but
	# to highlight problems caused by things not cleaning up after themselves during
	# execution. We aim to eliminate all circular references from our code, to avoid
	# garbage collection overhead and to avoid problems caused by referencing Qt widgets
	# which were long since destroyed in C++.
	## \todo Reevaluate the need for this call after Cortex 9 development.
	IECore.RefCounted.collectGarbage()

	# Check for things that shouldn't exist at shutdown, and
	# warn of anything we find.
	scriptNodes = []
	widgets = []
	for o in gc.get_objects() :
		if Gaffer is not None and isinstance( o, Gaffer.ScriptNode ) :
			scriptNodes.append( o )
		elif GafferUI is not None and isinstance( o, GafferUI.Widget ) :
			widgets.append( o )

	if scriptNodes :
		IECore.msg(
			IECore.Msg.Level.Warning,
			"Gaffer shutdown", "%d remaining ScriptNode%s detected. Debugging with gc.get_objects() and gc.get_referrers() is recommended." % (
				len( scriptNodes ),
				"s" if len( scriptNodes ) > 1 else "",
			)
		)

	if widgets :

		count = {}
		for widget in widgets :
			widgetType = widget.__class__.__name__
			count[widgetType] = count.get( widgetType, 0 ) + 1

		summaries = [ "%s (%d)" % ( k, count[k] ) for k in sorted( count.keys() ) ]

		IECore.msg(
			IECore.Msg.Level.Warning,
			"Gaffer shutdown", "%d remaining Widget%s detected : \n\n%s" % (
				len( widgets ),
				"s" if len( widgets ) > 1 else "",
				"\t" + "\n\t".join( summaries )
			)
		)

args = sys.argv[1:]
if args and args[0] in ( "-help", "-h", "--help", "-H" ) :
	if len( args ) > 1 :
		app = loadApp( args[1] )
		formatter = IECore.WrappedTextFormatter( sys.stdout )
		formatter.paragraph( "Name : " + app.path )
		if app.description :
			formatter.paragraph( app.description + "\n" )
		if len( app.parameters().values() ):
			formatter.heading( "Parameters" )
			formatter.indent()
			for p in app.parameters().values() :
				IECore.formatParameterHelp( p, formatter )
			formatter.unindent()
		sys.exit( 0 )
	else :
		sys.stdout.write( helpText )
		sys.stdout.write( "\n" + applicationText )
		sys.exit( 0 )
else :
	appName = "gui"
	appArgs = args
	if len( args ) :
		if not args[0].startswith( "-" ) and not args[0].endswith( ".gfr" ) :
			appName = args[0]
			appArgs = args[1:]

	app = loadApp( appName )
	IECore.ParameterParser().parse( appArgs, app.parameters() )

	result = app.run()

	del app
	checkCleanExit()

	sys.exit( result )

<commit_msg>Added workaround for inexplicable gc.get_objects() behaviour.<commit_after>#!/usr/bin/env python2.6
##########################################################################
#
#  Copyright (c) 2011, John Haddon. All rights reserved.
#  Copyright (c) 2011-2012, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#      * Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      * Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided with
#        the distribution.
#
#      * Neither the name of John Haddon nor the names of
#        any other contributors to this software may be used to endorse or
#        promote products derived from this software without specific prior
#        written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import os
import sys
import ctypes
import signal
import warnings

# Work around cross module rtti errors on linux.
sys.setdlopenflags( sys.getdlopenflags() | ctypes.RTLD_GLOBAL )

# Get rid of the annoying signal handler which turns Ctrl-C into a KeyboardInterrupt exception
signal.signal( signal.SIGINT, signal.SIG_DFL )

# Reenable deprecation warnings - Python2.7 turns them off by default so otherwise we'd never get
# to catch all the naughty deprecated things we do.
warnings.simplefilter( "default", DeprecationWarning )

import IECore

helpText = """Usage :

    gaffer -help                    Print this message
    gaffer -help appName            Print help specific to named application
    gaffer appName [flags]          Run named application with specified flags
    gaffer fileName.gfr [flags]     Run gui application with specified script and flags
    gaffer [flags]                  Run gui application with specified flags
"""

appLoader = IECore.ClassLoader.defaultLoader( "GAFFER_APP_PATHS" )
applicationText = "Installed applications :\n\n" + "\n".join( "    " + x for x in appLoader.classNames() ) + "\n"

def loadApp( appName ) :

	if appName in appLoader.classNames() :
		return appLoader.load( appName )()
	else :
		sys.stderr.write( "ERROR : Application \"%s\" not installed on GAFFER_APP_PATHS\n" % appName )
		sys.stderr.write( "\n" + applicationText )
		sys.exit( 1 )

def checkCleanExit() :

	# Get the Gaffer and GafferUI modules, but only if the app actually
	# imported them. We don't want to force their importation because it's
	# just a waste of time if they weren't used.
	Gaffer = sys.modules.get( "Gaffer" )
	GafferUI = sys.modules.get( "GafferUI" )

	if Gaffer is None and GafferUI is None :
		return

	# Clean up any garbage left behind by Cortex's wrapper mechanism - because
	# the Gaffer.Application itself is derived from IECore.Parameterised, which
	# as far as I can tell is wrapped unnecessarily, we must call this to allow
	# the application to be deleted at all. Note that we're deliberately not also
	# calling gc.collect() - our intention here isn't to clean up on shutdown, but
	# to highlight problems caused by things not cleaning up after themselves during
	# execution. We aim to eliminate all circular references from our code, to avoid
	# garbage collection overhead and to avoid problems caused by referencing Qt widgets
	# which were long since destroyed in C++.
	## \todo Reevaluate the need for this call after Cortex 9 development.
	IECore.RefCounted.collectGarbage()
	# Importing here rather than at the top of the file prevents false
	# positives being reported in gc.get_objects() below. I have no idea why,
	# but if not imported here, get_objects() will report objects which have
	# nothing referring to them and which should be dead, even with an
	# explicit call to gc.collect() beforehand.
	import gc

	# Check for things that shouldn't exist at shutdown, and
	# warn of anything we find.
	scriptNodes = []
	widgets = []
	for o in gc.get_objects() :
		if Gaffer is not None and isinstance( o, Gaffer.ScriptNode ) :
			scriptNodes.append( o )
		elif GafferUI is not None and isinstance( o, GafferUI.Widget ) :
			widgets.append( o )

	if scriptNodes :
		IECore.msg(
			IECore.Msg.Level.Warning,
			"Gaffer shutdown", "%d remaining ScriptNode%s detected. Debugging with gc.get_objects() and gc.get_referrers() is recommended." % (
				len( scriptNodes ),
				"s" if len( scriptNodes ) > 1 else "",
			)
		)

	if widgets :

		count = {}
		for widget in widgets :
			widgetType = widget.__class__.__name__
			count[widgetType] = count.get( widgetType, 0 ) + 1

		summaries = [ "%s (%d)" % ( k, count[k] ) for k in sorted( count.keys() ) ]

		IECore.msg(
			IECore.Msg.Level.Warning,
			"Gaffer shutdown", "%d remaining Widget%s detected : \n\n%s" % (
				len( widgets ),
				"s" if len( widgets ) > 1 else "",
				"\t" + "\n\t".join( summaries )
			)
		)

args = sys.argv[1:]
if args and args[0] in ( "-help", "-h", "--help", "-H" ) :
	if len( args ) > 1 :
		app = loadApp( args[1] )
		formatter = IECore.WrappedTextFormatter( sys.stdout )
		formatter.paragraph( "Name : " + app.path )
		if app.description :
			formatter.paragraph( app.description + "\n" )
		if len( app.parameters().values() ):
			formatter.heading( "Parameters" )
			formatter.indent()
			for p in app.parameters().values() :
				IECore.formatParameterHelp( p, formatter )
			formatter.unindent()
		sys.exit( 0 )
	else :
		sys.stdout.write( helpText )
		sys.stdout.write( "\n" + applicationText )
		sys.exit( 0 )
else :
	appName = "gui"
	appArgs = args
	if len( args ) :
		if not args[0].startswith( "-" ) and not args[0].endswith( ".gfr" ) :
			appName = args[0]
			appArgs = args[1:]

	app = loadApp( appName )
	IECore.ParameterParser().parse( appArgs, app.parameters() )

	result = app.run()

	del app
	checkCleanExit()

	sys.exit( result )

<|endoftext|>


<commit_before>from django.conf import settings
from django.core.management import BaseCommand
from django.utils import timezone

from desecapi.models import Captcha, User

class Command(BaseCommand):

    def handle(self, *args, **kwargs):
        # delete expired captchas
        Captcha.objects.filter(created__lt=timezone.now() - settings.CAPTCHA_VALIDITY_PERIOD).delete()

        # delete inactive users whose activation link expired and who never logged in
        # (this will not delete users who have used their account and were later disabled)
        User.objects.filter(is_active=False, last_login__exact=None,
                            created__lt=timezone.now() - settings.VALIDITY_PERIOD_VERIFICATION_SIGNATURE).delete()
<commit_msg>refactor(): in chores, use method per chore<commit_after>from django.conf import settings
from django.core.management import BaseCommand
from django.utils import timezone

from desecapi import models

class Command(BaseCommand):

    @staticmethod
    def delete_expired_captchas():
        models.Captcha.objects.filter(created__lt=timezone.now() - settings.CAPTCHA_VALIDITY_PERIOD).delete()

    @staticmethod
    def delete_never_activated_users():
        # delete inactive users whose activation link expired and who never logged in
        # (this will not delete users who have used their account and were later disabled)
        models.User.objects.filter(is_active=False, last_login__exact=None,
                            created__lt=timezone.now() - settings.VALIDITY_PERIOD_VERIFICATION_SIGNATURE).delete()

    def handle(self, *args, **kwargs):
        self.delete_expired_captchas()
        self.delete_never_activated_users()
<|endoftext|>


<commit_before># coding: utf-8
import os
import os.path

PATH = "/run/ganeti/kvm-hypervisor" if os.path.exists("/run") else "/var/run/ganeti/kvm-hypervisor"

def discover():
    """
    Returns a list of actually running KVM virtual machines and their pids in
    this manner:
        result[pid] = vm_name
    """
    # TODO: add Xen discovery
    path = os.path.join(PATH, "pid")

    results = {}
    for vm in os.listdir(path):
        vm_path = os.path.join(path, vm)
        results[open(vm_path, "r").readline().strip()] = vm

    return results

def discover_nic(hostname):
    """
    Returns a list of Network Interface Controllers connected with specified
    virtual machine.
    NICs are being read from "/run/ganeti/kvm-hypervisor/nic/HOSTNAME/NUMBER" files
    """
    path = os.path.join(PATH, "nic")

    results = []
    try:
        host_path = os.path.join(path, hostname)
        for F in os.listdir(host_path):
            nic = open(os.path.join(host_path, F), "r").read()
            results.append(nic)
    except OSError:
        pass

    return results

if __name__ == "__main__":
    print discover()
<commit_msg>Pass quietly in discovery when directories are missing<commit_after># coding: utf-8
import os
import os.path

PATH = "/run/ganeti/kvm-hypervisor" if os.path.exists("/run") else "/var/run/ganeti/kvm-hypervisor"

def discover():
    """
    Returns a list of actually running KVM virtual machines and their pids in
    this manner:
        result[pid] = vm_name
    """
    # TODO: add Xen discovery
    path = os.path.join(PATH, "pid")

    results = {}
    try:
        for vm in os.listdir(path):
            vm_path = os.path.join(path, vm)
            results[open(vm_path, "r").readline().strip()] = vm
    except OSError:
        pass

    return results

def discover_nic(hostname):
    """
    Returns a list of Network Interface Controllers connected with specified
    virtual machine.
    NICs are being read from "/run/ganeti/kvm-hypervisor/nic/HOSTNAME/NUMBER" files
    """
    path = os.path.join(PATH, "nic")

    results = []
    try:
        host_path = os.path.join(path, hostname)
        for F in os.listdir(host_path):
            nic = open(os.path.join(host_path, F), "r").read()
            results.append(nic)
    except OSError:
        pass

    return results

if __name__ == "__main__":
    print discover()
<|endoftext|>


<commit_before>import os
# Add parent directory for imports, so activity classes can use elife-api-prototype
parentdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
os.sys.path.insert(0,parentdir)

import boto.swf
import settings as settingsLib
import log
import json
import random
import datetime
import time
import importlib
from optparse import OptionParser

import provider.simpleDB as dblib
import provider.swfmeta as swfmetalib
import starter

"""
Cron job to check for new article S3 XML and start workflows
"""

def start(ENV = "dev"):
  # Specify run environment settings
  settings = settingsLib.get_settings(ENV)

  ping_marker_id = "cron_NewS3XML"

  # Log
  logFile = "starter.log"
  logger = log.logger(logFile, settings.setLevel, ping_marker_id)

  # Data provider
  db = dblib.SimpleDB(settings)
  db.connect()

  # SWF meta data provider
  swfmeta = swfmetalib.SWFMeta(settings)
  swfmeta.connect()

  last_startTimestamp = swfmeta.get_last_completed_workflow_execution_startTimestamp(workflow_id = ping_marker_id)

  # Start a ping workflow as a marker
  start_ping_marker(ping_marker_id, ENV)

  # Check for S3 XML files that were updated since the last run
  date_format = "%Y-%m-%dT%H:%M:%S.000Z"
  time_tuple = time.gmtime(last_startTimestamp)
  last_startDate = time.strftime(date_format, time_tuple)

  xml_item_list = db.elife_get_article_S3_file_items(file_data_type = "xml", latest = True, last_updated_since = last_startDate)
  if(len(xml_item_list) <= 0):
    # No new XML
    pass
  else:
    # Found new XML files

    # Start a Fluidinfo PublishArticle starter
    try:
      starter_name = "starter_PublishArticle"
      import_starter_module(starter_name)
      s = get_starter_module(starter_name)
      s.start(ENV = ENV, last_updated_since = last_startDate)
    except:
      logger.info('Error: %s starting %s' % (ping_marker_id, module_name))
      logger.exception('')

    # Start a LensArticlePublish starter
    try:
      starter_name = "starter_LensArticlePublish"
      import_starter_module(starter_name)
      s = get_starter_module(starter_name)
      s.start(ENV = ENV, all = True)
    except:
      logger.info('Error: %s starting %s' % (ping_marker_id, module_name))
      logger.exception('')

    # Start a LensIndexPublish starter
    try:
      starter_name = "starter.starter_LensIndexPublish"
      import_starter_module(starter_name)
      s = get_starter_module(starter_name)
      s.start(ENV = ENV)
    except:
      logger.info('Error: %s starting %s' % (ping_marker_id, module_name))
      logger.exception('')

def start_ping_marker(workflow_id, ENV = "dev"):
  """
  Start a ping workflow with a unique name to serve as a time marker
  for determining last time this was run
  """

  # Specify run environment settings
  settings = settingsLib.get_settings(ENV)

  workflow_id = workflow_id
  workflow_name = "Ping"
  workflow_version = "1"
  child_policy = None
  execution_start_to_close_timeout = None
  input = None

  conn = boto.swf.layer1.Layer1(settings.aws_access_key_id, settings.aws_secret_access_key)
  try:
    response = conn.start_workflow_execution(settings.domain, workflow_id, workflow_name, workflow_version, settings.default_task_list, child_policy, execution_start_to_close_timeout, input)

  except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
    # There is already a running workflow with that ID, cannot start another
    message = 'SWFWorkflowExecutionAlreadyStartedError: There is already a running workflow with ID %s' % workflow_id
    print message

def get_starter_module(starter_name):
  """
  Given an starter_name, and if the starter module is already
  imported, load the module and return it
  """
  full_path = "starter." + starter_name
  f = eval(full_path)
  return f

def import_starter_module(starter_name):
  """
  Given an starter name as starter_name,
  attempt to lazy load the module when needed
  """
  try:
    module_name = "starter." + starter_name
    importlib.import_module(module_name)
    return True
  except ImportError:
    return False

if __name__ == "__main__":

  # Add options
  parser = OptionParser()
  parser.add_option("-e", "--env", default="dev", action="store", type="string", dest="env", help="set the environment to run, either dev or live")
  (options, args) = parser.parse_args()
  if options.env:
    ENV = options.env

  start(ENV)<commit_msg>Fix starter name.<commit_after>import os
# Add parent directory for imports, so activity classes can use elife-api-prototype
parentdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
os.sys.path.insert(0,parentdir)

import boto.swf
import settings as settingsLib
import log
import json
import random
import datetime
import time
import importlib
from optparse import OptionParser

import provider.simpleDB as dblib
import provider.swfmeta as swfmetalib
import starter

"""
Cron job to check for new article S3 XML and start workflows
"""

def start(ENV = "dev"):
  # Specify run environment settings
  settings = settingsLib.get_settings(ENV)

  ping_marker_id = "cron_NewS3XML"

  # Log
  logFile = "starter.log"
  logger = log.logger(logFile, settings.setLevel, ping_marker_id)

  # Data provider
  db = dblib.SimpleDB(settings)
  db.connect()

  # SWF meta data provider
  swfmeta = swfmetalib.SWFMeta(settings)
  swfmeta.connect()

  last_startTimestamp = swfmeta.get_last_completed_workflow_execution_startTimestamp(workflow_id = ping_marker_id)

  # Start a ping workflow as a marker
  start_ping_marker(ping_marker_id, ENV)

  # Check for S3 XML files that were updated since the last run
  date_format = "%Y-%m-%dT%H:%M:%S.000Z"
  time_tuple = time.gmtime(last_startTimestamp)
  last_startDate = time.strftime(date_format, time_tuple)

  xml_item_list = db.elife_get_article_S3_file_items(file_data_type = "xml", latest = True, last_updated_since = last_startDate)
  if(len(xml_item_list) <= 0):
    # No new XML
    pass
  else:
    # Found new XML files

    # Start a Fluidinfo PublishArticle starter
    try:
      starter_name = "starter_PublishArticle"
      import_starter_module(starter_name)
      s = get_starter_module(starter_name)
      s.start(ENV = ENV, last_updated_since = last_startDate)
    except:
      logger.info('Error: %s starting %s' % (ping_marker_id, module_name))
      logger.exception('')

    # Start a LensArticlePublish starter
    try:
      starter_name = "starter_LensArticlePublish"
      import_starter_module(starter_name)
      s = get_starter_module(starter_name)
      s.start(ENV = ENV, all = True)
    except:
      logger.info('Error: %s starting %s' % (ping_marker_id, module_name))
      logger.exception('')

    # Start a LensIndexPublish starter
    try:
      starter_name = "starter_LensIndexPublish"
      import_starter_module(starter_name)
      s = get_starter_module(starter_name)
      s.start(ENV = ENV)
    except:
      logger.info('Error: %s starting %s' % (ping_marker_id, module_name))
      logger.exception('')

def start_ping_marker(workflow_id, ENV = "dev"):
  """
  Start a ping workflow with a unique name to serve as a time marker
  for determining last time this was run
  """

  # Specify run environment settings
  settings = settingsLib.get_settings(ENV)

  workflow_id = workflow_id
  workflow_name = "Ping"
  workflow_version = "1"
  child_policy = None
  execution_start_to_close_timeout = None
  input = None

  conn = boto.swf.layer1.Layer1(settings.aws_access_key_id, settings.aws_secret_access_key)
  try:
    response = conn.start_workflow_execution(settings.domain, workflow_id, workflow_name, workflow_version, settings.default_task_list, child_policy, execution_start_to_close_timeout, input)

  except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
    # There is already a running workflow with that ID, cannot start another
    message = 'SWFWorkflowExecutionAlreadyStartedError: There is already a running workflow with ID %s' % workflow_id
    print message

def get_starter_module(starter_name):
  """
  Given an starter_name, and if the starter module is already
  imported, load the module and return it
  """
  full_path = "starter." + starter_name
  f = eval(full_path)
  return f

def import_starter_module(starter_name):
  """
  Given an starter name as starter_name,
  attempt to lazy load the module when needed
  """
  try:
    module_name = "starter." + starter_name
    importlib.import_module(module_name)
    return True
  except ImportError:
    return False

if __name__ == "__main__":

  # Add options
  parser = OptionParser()
  parser.add_option("-e", "--env", default="dev", action="store", type="string", dest="env", help="set the environment to run, either dev or live")
  (options, args) = parser.parse_args()
  if options.env:
    ENV = options.env

  start(ENV)<|endoftext|>


<commit_before>##########################################################################
#
#  Copyright (c) 2012, John Haddon. All rights reserved.
#  Copyright (c) 2013-2014, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#      * Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      * Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided with
#        the distribution.
#
#      * Neither the name of John Haddon nor the names of
#        any other contributors to this software may be used to endorse or
#        promote products derived from this software without specific prior
#        written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import IECore

import GafferUI
import GafferSceneUI

## \todo Make this behaviour a part of the preferences.
def __nodeDoubleClick( nodeGraph, node ) :

	GafferUI.NodeEditor.acquire( node )

__nodeDoubleClickConnection = GafferUI.NodeGraph.nodeDoubleClickSignal().connect( __nodeDoubleClick )

def __nodeContextMenu( nodeGraph, node, menuDefinition ) :

	menuDefinition.append( "/Edit...", { "command" : IECore.curry( GafferUI.NodeEditor.acquire, node ) } )

	GafferUI.NodeGraph.appendEnabledPlugMenuDefinitions( nodeGraph, node, menuDefinition )
	GafferUI.NodeGraph.appendConnectionVisibilityMenuDefinitions( nodeGraph, node, menuDefinition )
	GafferUI.DispatcherUI.appendNodeContextMenuDefinitions( nodeGraph, node, menuDefinition )
	GafferUI.BoxUI.appendNodeContextMenuDefinitions( nodeGraph, node, menuDefinition )
	GafferSceneUI.FilteredSceneProcessorUI.appendNodeContextMenuDefinitions( nodeGraph, node, menuDefinition )

__nodeContextMenuConnection = GafferUI.NodeGraph.nodeContextMenuSignal().connect( __nodeContextMenu )
<commit_msg>Added basic colour scheme for nodes via startup/gui/nodeGraph.py.<commit_after>##########################################################################
#
#  Copyright (c) 2012, John Haddon. All rights reserved.
#  Copyright (c) 2013-2014, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#      * Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      * Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided with
#        the distribution.
#
#      * Neither the name of John Haddon nor the names of
#        any other contributors to this software may be used to endorse or
#        promote products derived from this software without specific prior
#        written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import IECore

import Gaffer
import GafferUI
import GafferScene
import GafferSceneUI

##########################################################################
# Colour
##########################################################################

Gaffer.Metadata.registerNodeValue( Gaffer.ExecutableNode, "nodeGadget:color", IECore.Color3f( 0.61, 0.1525, 0.1525 ) )
Gaffer.Metadata.registerPlugValue( Gaffer.ExecutableNode, "requirement", "nodule:color", IECore.Color3f( 0.645, 0.2483, 0.2483 ) )
Gaffer.Metadata.registerPlugValue( Gaffer.ExecutableNode, "requirements.*", "nodule:color", IECore.Color3f( 0.645, 0.2483, 0.2483 ) )
Gaffer.Metadata.registerPlugValue( Gaffer.ExecutableNode, "requirement", "connectionGadget:color", IECore.Color3f( 0.315, 0.0787, 0.0787 ) )
Gaffer.Metadata.registerPlugValue( Gaffer.ExecutableNode, "requirements.*", "connectionGadget:color", IECore.Color3f( 0.315, 0.0787, 0.0787 ) )

Gaffer.Metadata.registerNodeValue( Gaffer.SubGraph, "nodeGadget:color", IECore.Color3f( 0.225 ) )

Gaffer.Metadata.registerPlugValue( GafferScene.SceneNode, "in*", "nodule:color", IECore.Color3f( 0.2346, 0.326, 0.46 ) )
Gaffer.Metadata.registerPlugValue( GafferScene.SceneNode, "out", "nodule:color", IECore.Color3f( 0.2346, 0.326, 0.46 ) )
Gaffer.Metadata.registerPlugValue( GafferScene.ExecutableRender, "in", "nodule:color", IECore.Color3f( 0.2346, 0.326, 0.46 ) )
Gaffer.Metadata.registerPlugValue( GafferScene.InteractiveRender, "in", "nodule:color", IECore.Color3f( 0.2346, 0.326, 0.46 ) )
Gaffer.Metadata.registerPlugValue( GafferScene.Parent, "child", "nodule:color", IECore.Color3f( 0.2346, 0.326, 0.46 ) )

Gaffer.Metadata.registerNodeValue( GafferScene.SceneProcessor, "nodeGadget:color", IECore.Color3f( 0.495, 0.2376, 0.4229 ) )
Gaffer.Metadata.registerNodeValue( GafferScene.SceneElementProcessor, "nodeGadget:color", IECore.Color3f( 0.1886, 0.2772, 0.41 ) )

Gaffer.Metadata.registerPlugValue( GafferScene.FilteredSceneProcessor, "filter", "nodule:color", IECore.Color3f( 0.69, 0.5378, 0.2283 ) )
Gaffer.Metadata.registerPlugValue( GafferScene.Filter, "match", "nodule:color", IECore.Color3f( 0.69, 0.5378, 0.2283 ) )

Gaffer.Metadata.registerNodeValue( GafferScene.Transform, "nodeGadget:color", IECore.Color3f( 0.485, 0.3112, 0.2255 ) )
Gaffer.Metadata.registerNodeValue( GafferScene.Constraint, "nodeGadget:color", IECore.Color3f( 0.485, 0.3112, 0.2255 ) )

Gaffer.Metadata.registerNodeValue( GafferScene.GlobalsProcessor, "nodeGadget:color", IECore.Color3f( 0.255, 0.505, 0.28 ) )

__shaderNoduleColors = {
	Gaffer.FloatPlug.staticTypeId() : IECore.Color3fData( IECore.Color3f( 0.2467, 0.3762, 0.47 ) ),
	Gaffer.Color3fPlug.staticTypeId() : IECore.Color3fData( IECore.Color3f( 0.69, 0.5378, 0.2283 ) ),
	Gaffer.V3fPlug.staticTypeId() : IECore.Color3fData( IECore.Color3f( 0.47, 0.181, 0.181 ) ),
}

def __shaderNoduleColor( plug ) :

	return __shaderNoduleColors.get( plug.typeId(), None )

Gaffer.Metadata.registerPlugValue( GafferScene.Shader, "*", "nodule:color", __shaderNoduleColor )

##########################################################################
# Behaviour
##########################################################################

## \todo Make this behaviour a part of the preferences.
def __nodeDoubleClick( nodeGraph, node ) :

	GafferUI.NodeEditor.acquire( node )

__nodeDoubleClickConnection = GafferUI.NodeGraph.nodeDoubleClickSignal().connect( __nodeDoubleClick )

def __nodeContextMenu( nodeGraph, node, menuDefinition ) :

	menuDefinition.append( "/Edit...", { "command" : IECore.curry( GafferUI.NodeEditor.acquire, node ) } )

	GafferUI.NodeGraph.appendEnabledPlugMenuDefinitions( nodeGraph, node, menuDefinition )
	GafferUI.NodeGraph.appendConnectionVisibilityMenuDefinitions( nodeGraph, node, menuDefinition )
	GafferUI.DispatcherUI.appendNodeContextMenuDefinitions( nodeGraph, node, menuDefinition )
	GafferUI.BoxUI.appendNodeContextMenuDefinitions( nodeGraph, node, menuDefinition )
	GafferSceneUI.FilteredSceneProcessorUI.appendNodeContextMenuDefinitions( nodeGraph, node, menuDefinition )

__nodeContextMenuConnection = GafferUI.NodeGraph.nodeContextMenuSignal().connect( __nodeContextMenu )
<|endoftext|>