<commit_before>from __future__ import absolute_import

import os

from rpython.rlib.objectmodel import specialize
from rpython.rlib.streamio import open_file_as_stream, fdopen_as_stream

from topaz.error import RubyError, print_traceback
from topaz.objects.exceptionobject import W_SystemExit
from topaz.objspace import ObjectSpace

USAGE = "\n".join([
    """Usage: topaz [switches] [--] [programfile] [arguments]""",
#   """  -0[octal]       specify record separator (\0, if no argument)""",
#   """  -a              autosplit mode with -n or -p (splits $_ into $F)""",
#   """  -c              check syntax only""",
#   """  -Cdirectory     cd to directory, before executing your script""",
    """  -d              set debugging flags (set $DEBUG to true)""",
    """  -e 'command'    one line of script. Several -e's allowed. Omit [programfile]""",
#   """  -Eex[:in]       specify the default external and internal character encodings""",
#   """  -Fpattern       split() pattern for autosplit (-a)""",
#   """  -i[extension]   edit ARGV files in place (make backup if extension supplied)""",
    """  -Idirectory     specify $LOAD_PATH directory (may be used more than once)""",
#   """  -l              enable line ending processing""",
    """  -n              assume 'while gets(); ... end' loop around your script""",
#   """  -p              assume loop like -n but print line also like sed""",
    """  -rlibrary       require the library, before executing your script""",
    """  -s              enable some switch parsing for switches after script name""",
    """  -S              look for the script using PATH environment variable""",
#   """  -T[level=1]     turn on tainting checks""",
    """  -v              print version number, then turn on verbose mode""",
    """  -w              turn warnings on for your script""",
    """  -W[level=2]     set warning level; 0=silence, 1=medium, 2=verbose""",
#   """  -x[directory]   strip off text before #!ruby line and perhaps cd to directory""",
    """  --copyright     print the copyright""",
    """  --version       print the version""",
    ""
])
COPYRIGHT = "topaz - Copyright (c) Alex Gaynor and individual contributors\n"

@specialize.memo()
def getspace():
    return ObjectSpace()

def entry_point(argv):
    space = getspace()
    space.setup(argv[0])
    return _entry_point(space, argv)

class CommandLineError(Exception):
    def __init__(self, message):
        self.message = message

class ShortCircuitError(Exception):
    def __init__(self, message):
        self.message = message

def _parse_argv(space, argv):
    flag_globals_w = {
        "$-v": space.w_false,
        "$VERBOSE": space.w_false,
        "$-d": space.w_false,
        "$DEBUG": space.w_false,
        "$-w": space.w_false,
        "$-W": space.newint(1),
        "$-p": space.w_false,
        "$-l": space.w_false,
        "$-a": space.w_false,
    }
    warning_level = None
    do_loop = False
    path = None
    search_path = False
    globalize_switches = False
    globalized_switches = []
    exprs = []
    reqs = []
    load_path_entries = []
    argv_w = []
    idx = 1
    while idx < len(argv):
        arg = argv[idx]
        if arg == "-h" or arg == "--help":
            raise ShortCircuitError(USAGE)
        elif arg == "--copyright":
            raise ShortCircuitError(COPYRIGHT)
        elif arg == "--version":
            raise ShortCircuitError("%s\n" % space.str_w(
                    space.send(
                        space.w_object,
                        space.newsymbol("const_get"),
                        [space.newstr_fromstr("RUBY_DESCRIPTION")]
                    )
                ))
        elif arg == "-v":
            flag_globals_w["$-v"] = space.w_true
            flag_globals_w["$VERBOSE"] = space.w_true
        elif arg == "-d":
            flag_globals_w["$-d"] = space.w_true
            flag_globals_w["$VERBOSE"] = space.w_true
            flag_globals_w["$DEBUG"] = space.w_true
        elif arg == "-w":
            flag_globals_w["$-w"] = space.w_true
            flag_globals_w["$VERBOSE"] = space.w_true
        elif arg == "-e":
            idx += 1
            if idx == len(argv):
                raise CommandLineError("no code specified for -e (RuntimeError)\n")
            exprs.append(argv[idx])
        elif arg.startswith("-e"):
            exprs.append(arg[2:])
        elif arg == "-I":
            idx += 1
            load_path_entries += argv[idx].split(os.pathsep)
        elif arg.startswith("-I"):
            load_path_entries += arg[2:].split(os.pathsep)
        elif arg == "-r":
            idx += 1
            reqs.append(argv[idx])
        elif arg.startswith("-r"):
            reqs.append(arg[2:])
        elif arg.startswith("-W"):
            warning_level = arg[2:]
        elif arg == "-S":
            search_path = True
        elif arg == "-s":
            globalize_switches = True
        elif arg == "-n":
            do_loop = True
        elif arg == "--":
            idx += 1
            break
        else:
            break
        idx += 1
    if idx < len(argv) and not exprs:
        path = argv[idx]
        idx += 1
    while idx < len(argv):
        arg = argv[idx]
        if globalize_switches and arg.startswith("-"):
            globalized_switches.append(arg)
        else:
            argv_w.append(space.newstr_fromstr(arg))
        idx += 1

    if warning_level is not None:
        warning_level_num = 2 if not warning_level.isdigit() else int(warning_level)
        if warning_level_num == 0:
            flag_globals_w["$VERBOSE"] = space.w_nil
        elif warning_level_num == 1:
            flag_globals_w["$VERBOSE"] = space.w_false
        elif warning_level_num >= 2:
            flag_globals_w["$VERBOSE"] = space.w_true

        flag_globals_w["$-W"] = space.newint(warning_level_num)

    return (
        flag_globals_w,
        do_loop,
        path,
        search_path,
        globalized_switches,
        exprs,
        reqs,
        load_path_entries,
        argv_w
    )

def _entry_point(space, argv):
    system, _, _, _, cpu = os.uname()
    platform = "%s-%s" % (cpu, system.lower())
    engine = "topaz"
    version = "1.9.3"
    patchlevel = 125
    description = "%s (ruby-%sp%d) [%s]" % (engine, version, patchlevel, platform)
    space.set_const(space.w_object, "RUBY_ENGINE", space.newstr_fromstr(engine))
    space.set_const(space.w_object, "RUBY_VERSION", space.newstr_fromstr(version))
    space.set_const(space.w_object, "RUBY_PATCHLEVEL", space.newint(patchlevel))
    space.set_const(space.w_object, "RUBY_PLATFORM", space.newstr_fromstr(platform))
    space.set_const(space.w_object, "RUBY_DESCRIPTION", space.newstr_fromstr(description))

    try:
        (
            flag_globals_w,
            do_loop,
            path,
            search_path,
            globalized_switches,
            exprs,
            reqs,
            load_path_entries,
            argv_w
        ) = _parse_argv(space, argv)
    except ShortCircuitError as e:
        os.write(1, e.message)
        return 0
    except CommandLineError as e:
        os.write(2, e.message)
        return 1

    for path_entry in load_path_entries:
        space.send(
            space.w_load_path,
            space.newsymbol("<<"),
            [space.newstr_fromstr(path_entry)]
        )
    for required_lib in reqs:
        space.send(
            space.w_kernel,
            space.newsymbol("require"),
            [space.newstr_fromstr(required_lib)]
        )

    space.set_const(space.w_object, "ARGV", space.newarray(argv_w))
    explicitly_verbose = space.is_true(flag_globals_w["$-v"])
    if explicitly_verbose:
        os.write(1, "%s\n" % description)
    for varname, w_value in flag_globals_w.iteritems():
        space.globals.set(space, varname, w_value)

    if exprs:
        source = "\n".join(exprs)
        path = "-e"
    elif path is not None:
        if search_path:
            for dirname in os.environ["PATH"].split(os.pathsep):
                candidate_path = os.sep.join([dirname, path])
                if os.access(candidate_path, os.R_OK):
                    path = candidate_path
                    break
        try:
            f = open_file_as_stream(path)
        except OSError as e:
            os.write(2, "%s -- %s (LoadError)\n" % (os.strerror(e.errno), path))
            return 1
        try:
            source = f.readall()
        finally:
            f.close()
    elif explicitly_verbose:
        return 0
    else:
        source = fdopen_as_stream(0, "r").readall()
        path = "-"

    for globalized_switch in globalized_switches:
        value = None
        if "=" in globalized_switch:
            globalized_switch, value = globalized_switch.split("=", 1)

        switch_global_var = "$%s" % globalized_switch[1:].replace("-", "_")
        if value is None:
            space.globals.set(space, switch_global_var, space.w_true)
        else:
            space.globals.set(space, switch_global_var, space.newstr_fromstr(value))

    w_program_name = space.newstr_fromstr(path)
    space.globals.set(space, "$0", w_program_name)
    space.globals.set(space, "$PROGRAM_NAME", w_program_name)
    status = 0
    w_exit_error = None
    explicit_status = False
    try:
        if do_loop:
            while True:
                w_line = space.send(space.w_kernel, space.newsymbol("gets"))
                if w_line is space.w_nil:
                    break
                space.execute(source, filepath=path)
        else:
            space.execute(source, filepath=path)
    except RubyError as e:
        explicit_status = True
        w_exc = e.w_value
        if isinstance(w_exc, W_SystemExit):
            status = w_exc.status
        else:
            w_exit_error = w_exc
            status = 1
    exit_handler_status = space.run_exit_handlers()
    if not explicit_status and exit_handler_status != -1:
        status = exit_handler_status
    if w_exit_error is not None:
        print_traceback(space, w_exit_error, path)

    return status
<commit_msg>Trying out a pre-built frame version of Kernel.gets loop<commit_after>from __future__ import absolute_import

import os

from rpython.rlib.objectmodel import specialize
from rpython.rlib.streamio import open_file_as_stream, fdopen_as_stream

from topaz.error import RubyError, print_traceback
from topaz.objects.exceptionobject import W_SystemExit
from topaz.objspace import ObjectSpace

USAGE = "\n".join([
    """Usage: topaz [switches] [--] [programfile] [arguments]""",
#   """  -0[octal]       specify record separator (\0, if no argument)""",
#   """  -a              autosplit mode with -n or -p (splits $_ into $F)""",
#   """  -c              check syntax only""",
#   """  -Cdirectory     cd to directory, before executing your script""",
    """  -d              set debugging flags (set $DEBUG to true)""",
    """  -e 'command'    one line of script. Several -e's allowed. Omit [programfile]""",
#   """  -Eex[:in]       specify the default external and internal character encodings""",
#   """  -Fpattern       split() pattern for autosplit (-a)""",
#   """  -i[extension]   edit ARGV files in place (make backup if extension supplied)""",
    """  -Idirectory     specify $LOAD_PATH directory (may be used more than once)""",
#   """  -l              enable line ending processing""",
    """  -n              assume 'while gets(); ... end' loop around your script""",
#   """  -p              assume loop like -n but print line also like sed""",
    """  -rlibrary       require the library, before executing your script""",
    """  -s              enable some switch parsing for switches after script name""",
    """  -S              look for the script using PATH environment variable""",
#   """  -T[level=1]     turn on tainting checks""",
    """  -v              print version number, then turn on verbose mode""",
    """  -w              turn warnings on for your script""",
    """  -W[level=2]     set warning level; 0=silence, 1=medium, 2=verbose""",
#   """  -x[directory]   strip off text before #!ruby line and perhaps cd to directory""",
    """  --copyright     print the copyright""",
    """  --version       print the version""",
    ""
])
COPYRIGHT = "topaz - Copyright (c) Alex Gaynor and individual contributors\n"

@specialize.memo()
def getspace():
    return ObjectSpace()

def entry_point(argv):
    space = getspace()
    space.setup(argv[0])
    return _entry_point(space, argv)

class CommandLineError(Exception):
    def __init__(self, message):
        self.message = message

class ShortCircuitError(Exception):
    def __init__(self, message):
        self.message = message

def _parse_argv(space, argv):
    flag_globals_w = {
        "$-v": space.w_false,
        "$VERBOSE": space.w_false,
        "$-d": space.w_false,
        "$DEBUG": space.w_false,
        "$-w": space.w_false,
        "$-W": space.newint(1),
        "$-p": space.w_false,
        "$-l": space.w_false,
        "$-a": space.w_false,
    }
    warning_level = None
    do_loop = False
    path = None
    search_path = False
    globalize_switches = False
    globalized_switches = []
    exprs = []
    reqs = []
    load_path_entries = []
    argv_w = []
    idx = 1
    while idx < len(argv):
        arg = argv[idx]
        if arg == "-h" or arg == "--help":
            raise ShortCircuitError(USAGE)
        elif arg == "--copyright":
            raise ShortCircuitError(COPYRIGHT)
        elif arg == "--version":
            raise ShortCircuitError("%s\n" % space.str_w(
                    space.send(
                        space.w_object,
                        space.newsymbol("const_get"),
                        [space.newstr_fromstr("RUBY_DESCRIPTION")]
                    )
                ))
        elif arg == "-v":
            flag_globals_w["$-v"] = space.w_true
            flag_globals_w["$VERBOSE"] = space.w_true
        elif arg == "-d":
            flag_globals_w["$-d"] = space.w_true
            flag_globals_w["$VERBOSE"] = space.w_true
            flag_globals_w["$DEBUG"] = space.w_true
        elif arg == "-w":
            flag_globals_w["$-w"] = space.w_true
            flag_globals_w["$VERBOSE"] = space.w_true
        elif arg == "-e":
            idx += 1
            if idx == len(argv):
                raise CommandLineError("no code specified for -e (RuntimeError)\n")
            exprs.append(argv[idx])
        elif arg.startswith("-e"):
            exprs.append(arg[2:])
        elif arg == "-I":
            idx += 1
            load_path_entries += argv[idx].split(os.pathsep)
        elif arg.startswith("-I"):
            load_path_entries += arg[2:].split(os.pathsep)
        elif arg == "-r":
            idx += 1
            reqs.append(argv[idx])
        elif arg.startswith("-r"):
            reqs.append(arg[2:])
        elif arg.startswith("-W"):
            warning_level = arg[2:]
        elif arg == "-S":
            search_path = True
        elif arg == "-s":
            globalize_switches = True
        elif arg == "-n":
            do_loop = True
        elif arg == "--":
            idx += 1
            break
        else:
            break
        idx += 1
    if idx < len(argv) and not exprs:
        path = argv[idx]
        idx += 1
    while idx < len(argv):
        arg = argv[idx]
        if globalize_switches and arg.startswith("-"):
            globalized_switches.append(arg)
        else:
            argv_w.append(space.newstr_fromstr(arg))
        idx += 1

    if warning_level is not None:
        warning_level_num = 2 if not warning_level.isdigit() else int(warning_level)
        if warning_level_num == 0:
            flag_globals_w["$VERBOSE"] = space.w_nil
        elif warning_level_num == 1:
            flag_globals_w["$VERBOSE"] = space.w_false
        elif warning_level_num >= 2:
            flag_globals_w["$VERBOSE"] = space.w_true

        flag_globals_w["$-W"] = space.newint(warning_level_num)

    return (
        flag_globals_w,
        do_loop,
        path,
        search_path,
        globalized_switches,
        exprs,
        reqs,
        load_path_entries,
        argv_w
    )

def _entry_point(space, argv):
    system, _, _, _, cpu = os.uname()
    platform = "%s-%s" % (cpu, system.lower())
    engine = "topaz"
    version = "1.9.3"
    patchlevel = 125
    description = "%s (ruby-%sp%d) [%s]" % (engine, version, patchlevel, platform)
    space.set_const(space.w_object, "RUBY_ENGINE", space.newstr_fromstr(engine))
    space.set_const(space.w_object, "RUBY_VERSION", space.newstr_fromstr(version))
    space.set_const(space.w_object, "RUBY_PATCHLEVEL", space.newint(patchlevel))
    space.set_const(space.w_object, "RUBY_PLATFORM", space.newstr_fromstr(platform))
    space.set_const(space.w_object, "RUBY_DESCRIPTION", space.newstr_fromstr(description))

    try:
        (
            flag_globals_w,
            do_loop,
            path,
            search_path,
            globalized_switches,
            exprs,
            reqs,
            load_path_entries,
            argv_w
        ) = _parse_argv(space, argv)
    except ShortCircuitError as e:
        os.write(1, e.message)
        return 0
    except CommandLineError as e:
        os.write(2, e.message)
        return 1

    for path_entry in load_path_entries:
        space.send(
            space.w_load_path,
            space.newsymbol("<<"),
            [space.newstr_fromstr(path_entry)]
        )
    for required_lib in reqs:
        space.send(
            space.w_kernel,
            space.newsymbol("require"),
            [space.newstr_fromstr(required_lib)]
        )

    space.set_const(space.w_object, "ARGV", space.newarray(argv_w))
    explicitly_verbose = space.is_true(flag_globals_w["$-v"])
    if explicitly_verbose:
        os.write(1, "%s\n" % description)
    for varname, w_value in flag_globals_w.iteritems():
        space.globals.set(space, varname, w_value)

    if exprs:
        source = "\n".join(exprs)
        path = "-e"
    elif path is not None:
        if search_path:
            for dirname in os.environ["PATH"].split(os.pathsep):
                candidate_path = os.sep.join([dirname, path])
                if os.access(candidate_path, os.R_OK):
                    path = candidate_path
                    break
        try:
            f = open_file_as_stream(path)
        except OSError as e:
            os.write(2, "%s -- %s (LoadError)\n" % (os.strerror(e.errno), path))
            return 1
        try:
            source = f.readall()
        finally:
            f.close()
    elif explicitly_verbose:
        return 0
    else:
        source = fdopen_as_stream(0, "r").readall()
        path = "-"

    for globalized_switch in globalized_switches:
        value = None
        if "=" in globalized_switch:
            globalized_switch, value = globalized_switch.split("=", 1)

        switch_global_var = "$%s" % globalized_switch[1:].replace("-", "_")
        if value is None:
            space.globals.set(space, switch_global_var, space.w_true)
        else:
            space.globals.set(space, switch_global_var, space.newstr_fromstr(value))

    w_program_name = space.newstr_fromstr(path)
    space.globals.set(space, "$0", w_program_name)
    space.globals.set(space, "$PROGRAM_NAME", w_program_name)
    status = 0
    w_exit_error = None
    explicit_status = False
    try:
        if do_loop:
            bc = space.compile(source, path)
            frame = space.create_frame(bc)
            while True:
                w_line = space.send(space.w_kernel, space.newsymbol("gets"))
                if w_line is space.w_nil:
                    break
                with space.getexecutioncontext().visit_frame(frame):
                    space.execute_frame(frame, bc)
        else:
            space.execute(source, filepath=path)
    except RubyError as e:
        explicit_status = True
        w_exc = e.w_value
        if isinstance(w_exc, W_SystemExit):
            status = w_exc.status
        else:
            w_exit_error = w_exc
            status = 1
    exit_handler_status = space.run_exit_handlers()
    if not explicit_status and exit_handler_status != -1:
        status = exit_handler_status
    if w_exit_error is not None:
        print_traceback(space, w_exit_error, path)

    return status
<|endoftext|>


<commit_before>from __future__ import absolute_import

from ckan.lib.plugins import DefaultTranslation
from future import standard_library

from builtins import next

import ckan.plugins as plugins
import ckan.plugins.toolkit as toolkit
from ckanext.scheming.helpers import lang
from ckanext.apicatalog_scheming import cli
import json

try:
    from collections import OrderedDict  # 2.7
except ImportError:
    from sqlalchemy.util import OrderedDict

from . import validators
import logging

standard_library.install_aliases()

log = logging.getLogger(__name__)
_ = toolkit._

_LOCALE_ALIASES = {'en_GB': 'en'}

class Apicatalog_SchemingPlugin(plugins.SingletonPlugin, DefaultTranslation):
    plugins.implements(plugins.IConfigurer)
    plugins.implements(plugins.IValidators)
    plugins.implements(plugins.ITemplateHelpers)
    plugins.implements(plugins.IFacets, inherit=True)
    plugins.implements(plugins.IPackageController, inherit=True)
    plugins.implements(plugins.IClick)
    plugins.implements(plugins.ITranslation)

    # IConfigurer

    def update_config(self, config_):
        toolkit.add_template_directory(config_, 'templates')

    def get_validators(self):
        return {
            'lower_if_exists': validators.lower_if_exists,
            'upper_if_exists': validators.upper_if_exists,
            'valid_resources': validators.valid_resources,
            'only_default_lang_required': validators.only_default_lang_required,
            'keep_old_value_if_missing': validators.keep_old_value_if_missing,
            'default_value': validators.default_value,
            'business_id_validator': validators.business_id_validator,
            'ignore_not_package_maintainer': validators.ignore_not_package_maintainer,
            'create_fluent_tags': validators.create_fluent_tags,
            'convert_to_json_compatible_str_if_str': validators.convert_to_json_compatible_str_if_str,
            'mark_as_modified_in_catalog_if_changed': validators.mark_as_modified_in_catalog_if_changed,
            'override_field_with_default_translation': validators.override_field_with_default_translation,
            'fluent_list': validators.fluent_list,
            'fluent_list_output': validators.fluent_list_output,
            }

    def get_helpers(self):
        return {'scheming_field_only_default_required': scheming_field_only_default_required,
                'scheming_language_text_or_empty': scheming_language_text_or_empty,
                'get_lang_prefix': get_lang_prefix,
                'call_toolkit_function': call_toolkit_function,
                'add_locale_to_source': add_locale_to_source,
                'get_field_from_schema': get_field_from_schema}

    # IFacets

    def dataset_facets(self, facets_dict, package_type):
        lang = get_lang_prefix()
        facets_dict = OrderedDict([
            ('services', _('Services')),
            ('organization', _('Organization')),
            ('vocab_keywords_' + lang, _('Tags')),
            ('res_format', _('Formats'))
            ])
        return facets_dict

    # IPackageController

    def before_index(self, pkg_dict):
        # Map keywords to vocab_keywords_{lang}
        translated_vocabs = ['keywords']
        languages = ['fi', 'sv', 'en']
        for prop_key in translated_vocabs:
            prop_json = pkg_dict.get(prop_key)
            # Add only if not already there
            if not prop_json:
                continue
            prop_value = json.loads(prop_json)
            # Add for each language
            for language in languages:
                if prop_value.get(language):
                    pkg_dict['vocab_%s_%s' % (prop_key, language)] = [tag for tag in prop_value[language]]

        if pkg_dict.get('num_resources', 0) > 0:
            pkg_dict['services'] = "Subsystems with services"
        else:
            pkg_dict['services'] = "Subsystems without services"

        return pkg_dict

    # IClick

    def get_commands(self):
        return cli.get_commands()

def scheming_field_only_default_required(field, lang):
    if (field
            and field.get('only_default_lang_required')
            and lang == toolkit.config.get('ckan.locale_default', 'en')):
        return True
    return False

def add_locale_to_source(kwargs, locale):
    copy = kwargs.copy()
    source = copy.get('data-module-source', None)
    if source:
        copy.update({'data-module-source': source + '_' + locale})
        return copy
    return copy

def scheming_language_text_or_empty(text, prefer_lang=None):
    """
    :param text: {lang: text} dict or text string
    :param prefer_lang: choose this language version if available
    Convert "language-text" to users' language by looking up
    language in dict or using gettext if not a dict
    """
    if not text:
        return u''

    if hasattr(text, 'get'):
        try:
            if prefer_lang is None:
                prefer_lang = lang()
        except TypeError:
            pass  # lang() call will fail when no user language available
        else:
            if prefer_lang in _LOCALE_ALIASES:
                prefer_lang = _LOCALE_ALIASES[prefer_lang]
            try:
                return text[prefer_lang]
            except KeyError:
                return ''

    t = _(text)
    if isinstance(t, str):
        return t.decode('utf-8')
    return t

def get_lang_prefix():
    language = lang()
    if language in _LOCALE_ALIASES:
        language = _LOCALE_ALIASES[language]

    return language

def call_toolkit_function(fn, args, kwargs):
    return getattr(toolkit, fn)(*args, **kwargs)

def create_vocabulary(name, defer=False):
    user = toolkit.get_action('get_site_user')({'ignore_auth': True}, {})
    context = {'user': user['name']}

    try:
        data = {'id': name}
        return toolkit.get_action('vocabulary_show')(context, data)
    except toolkit.ObjectNotFound:
        pass

    log.info("Creating vocab '" + name + "'")
    data = {'name': name}
    try:
        if defer:
            context['defer_commit'] = True
        return toolkit.get_action('vocabulary_create')(context, data)
    except Exception as e:
        log.error('%s' % e)

def create_tag_to_vocabulary(tag, vocab, defer=False):
    user = toolkit.get_action('get_site_user')({'ignore_auth': True}, {})
    context = {'user': user['name']}

    data = {'id': vocab}
    v = toolkit.get_action('vocabulary_show')(context, data)

    data = {
        "name": tag,
        "vocabulary_id": v['id']}

    if defer:
        context['defer_commit'] = True
    try:
        toolkit.get_action('tag_create')(context, data)
    except toolkit.ValidationError:
        pass

def get_field_from_schema(schema, field_name):

    field = next(field for field in schema.get('dataset_fields', []) if field.get('field_name') == field_name)
    return field
<commit_msg>add blank line<commit_after>from __future__ import absolute_import

from ckan.lib.plugins import DefaultTranslation
from future import standard_library

from builtins import next

import ckan.plugins as plugins
import ckan.plugins.toolkit as toolkit
from ckanext.scheming.helpers import lang
from ckanext.apicatalog_scheming import cli
import json

try:
    from collections import OrderedDict  # 2.7
except ImportError:
    from sqlalchemy.util import OrderedDict

from . import validators
import logging

standard_library.install_aliases()

log = logging.getLogger(__name__)
_ = toolkit._

_LOCALE_ALIASES = {'en_GB': 'en'}

class Apicatalog_SchemingPlugin(plugins.SingletonPlugin, DefaultTranslation):
    plugins.implements(plugins.IConfigurer)
    plugins.implements(plugins.IValidators)
    plugins.implements(plugins.ITemplateHelpers)
    plugins.implements(plugins.IFacets, inherit=True)
    plugins.implements(plugins.IPackageController, inherit=True)
    plugins.implements(plugins.IClick)
    plugins.implements(plugins.ITranslation)

    # IConfigurer

    def update_config(self, config_):
        toolkit.add_template_directory(config_, 'templates')

    def get_validators(self):
        return {
            'lower_if_exists': validators.lower_if_exists,
            'upper_if_exists': validators.upper_if_exists,
            'valid_resources': validators.valid_resources,
            'only_default_lang_required': validators.only_default_lang_required,
            'keep_old_value_if_missing': validators.keep_old_value_if_missing,
            'default_value': validators.default_value,
            'business_id_validator': validators.business_id_validator,
            'ignore_not_package_maintainer': validators.ignore_not_package_maintainer,
            'create_fluent_tags': validators.create_fluent_tags,
            'convert_to_json_compatible_str_if_str': validators.convert_to_json_compatible_str_if_str,
            'mark_as_modified_in_catalog_if_changed': validators.mark_as_modified_in_catalog_if_changed,
            'override_field_with_default_translation': validators.override_field_with_default_translation,
            'fluent_list': validators.fluent_list,
            'fluent_list_output': validators.fluent_list_output,
            }

    def get_helpers(self):
        return {'scheming_field_only_default_required': scheming_field_only_default_required,
                'scheming_language_text_or_empty': scheming_language_text_or_empty,
                'get_lang_prefix': get_lang_prefix,
                'call_toolkit_function': call_toolkit_function,
                'add_locale_to_source': add_locale_to_source,
                'get_field_from_schema': get_field_from_schema}

    # IFacets

    def dataset_facets(self, facets_dict, package_type):
        lang = get_lang_prefix()
        facets_dict = OrderedDict([
            ('services', _('Services')),
            ('organization', _('Organization')),
            ('vocab_keywords_' + lang, _('Tags')),
            ('res_format', _('Formats'))
            ])
        return facets_dict

    # IPackageController

    def before_index(self, pkg_dict):
        # Map keywords to vocab_keywords_{lang}
        translated_vocabs = ['keywords']
        languages = ['fi', 'sv', 'en']
        for prop_key in translated_vocabs:
            prop_json = pkg_dict.get(prop_key)
            # Add only if not already there
            if not prop_json:
                continue
            prop_value = json.loads(prop_json)
            # Add for each language
            for language in languages:
                if prop_value.get(language):
                    pkg_dict['vocab_%s_%s' % (prop_key, language)] = [tag for tag in prop_value[language]]

        if pkg_dict.get('num_resources', 0) > 0:
            pkg_dict['services'] = "Subsystems with services"
        else:
            pkg_dict['services'] = "Subsystems without services"

        return pkg_dict

    # IClick

    def get_commands(self):
        return cli.get_commands()

def scheming_field_only_default_required(field, lang):
    if (field
            and field.get('only_default_lang_required')
            and lang == toolkit.config.get('ckan.locale_default', 'en')):
        return True
    return False

def add_locale_to_source(kwargs, locale):
    copy = kwargs.copy()
    source = copy.get('data-module-source', None)
    if source:
        copy.update({'data-module-source': source + '_' + locale})
        return copy
    return copy

def scheming_language_text_or_empty(text, prefer_lang=None):
    """
    :param text: {lang: text} dict or text string
    :param prefer_lang: choose this language version if available
    Convert "language-text" to users' language by looking up
    language in dict or using gettext if not a dict
    """
    if not text:
        return u''

    if hasattr(text, 'get'):
        try:
            if prefer_lang is None:
                prefer_lang = lang()
        except TypeError:
            pass  # lang() call will fail when no user language available
        else:
            if prefer_lang in _LOCALE_ALIASES:
                prefer_lang = _LOCALE_ALIASES[prefer_lang]
            try:
                return text[prefer_lang]
            except KeyError:
                return ''

    t = _(text)
    if isinstance(t, str):
        return t.decode('utf-8')
    return t

def get_lang_prefix():
    language = lang()
    if language in _LOCALE_ALIASES:
        language = _LOCALE_ALIASES[language]

    return language

def call_toolkit_function(fn, args, kwargs):
    return getattr(toolkit, fn)(*args, **kwargs)

def create_vocabulary(name, defer=False):
    user = toolkit.get_action('get_site_user')({'ignore_auth': True}, {})
    context = {'user': user['name']}

    try:
        data = {'id': name}
        return toolkit.get_action('vocabulary_show')(context, data)
    except toolkit.ObjectNotFound:
        pass

    log.info("Creating vocab '" + name + "'")
    data = {'name': name}
    try:
        if defer:
            context['defer_commit'] = True
        return toolkit.get_action('vocabulary_create')(context, data)
    except Exception as e:
        log.error('%s' % e)

def create_tag_to_vocabulary(tag, vocab, defer=False):
    user = toolkit.get_action('get_site_user')({'ignore_auth': True}, {})
    context = {'user': user['name']}

    data = {'id': vocab}
    v = toolkit.get_action('vocabulary_show')(context, data)

    data = {
        "name": tag,
        "vocabulary_id": v['id']}

    if defer:
        context['defer_commit'] = True
    try:
        toolkit.get_action('tag_create')(context, data)
    except toolkit.ValidationError:
        pass

def get_field_from_schema(schema, field_name):

    field = next(field for field in schema.get('dataset_fields', []) if field.get('field_name') == field_name)
    return field
<|endoftext|>


<commit_before>from collections import Counter,defaultdict

def get_stats(X,Y,base='df',stats='',agg='sum',norm='sum'):
	model = {}
	_stats = set(stats.split(' ')+[base,base+'_y'])
	_y = set(Y)

	for s in _stats:
		if s.endswith('_y'):
			model[s] = defaultdict(Counter)
		else:
			model[s] = Counter()

	# ==[ base ]============================================================

	for tokens,y in zip(X,Y):
		if 'tf' in _stats:
			tf = Counter(tokens)
			model['tf_y'][y].update(tf)
			model['tf'].update(tf)

		if 'df' in _stats:
			df = Counter(set(tokens))
			model['df_y'][y].update(df)
			model['df'].update(df)

	pass # TODO drop terms below treshold

	# ==[ stats ]===========================================================

	if 'chi_y' in _stats:
		pass # TODO

	if 'cmfs_y' in _stats:
		cnt_y = {}
		for y in _y:
			cnt_y[y] = Y.count(y) # TODO base=tf
		for t,f in model[base].items():
			for y in _y:
				f_y = model[base+'_y'][y].get(t,0)
				p_tc = 1.0 * f_y / cnt_y[y]
				p_ct = 1.0 * f_y / f
				model['cmfs_y'][y][t] = p_tc * p_ct

	if 'dia_y' in _stats:
		for t,f in model[base].items():
			for y in _y:
				f_y = model[base+'_y'][y].get(t,0)
				model['dia_y'][y][t] = 1.0 * f_y / f

	if 'gini_y' in _stats:
		cnt_y = {}
		for y in _y:
			cnt_y[y] = Y.count(y) # TODO base=tf
		for t,f in model[base].items():
			for y in _y:
				f_y = model[base+'_y'][y].get(t,0)
				p_tc = 1.0 * f_y / cnt_y[y]
				p_ct = 1.0 * f_y / f
				model['gini_y'][y][t] = p_tc*p_tc * p_ct*p_ct

	if 'wcp_y' in _stats:
		pass # TODO

	# TODO - my functions

	# TODO *x_y - ballanced variants

	# ==[ aggregates ]======================================================

	if 'cmfs' in _stats:
		for t in model[base]:
			scores = [model['cmfs_y'][y][t] for y in _y]
			model['cmfs'][t] = max(scores) # TODO agg

	if 'dia' in _stats:
		for t in model[base]:
			scores = [model['dia_y'][y][t] for y in _y]
			model['dia'][t] = max(scores) # agg=sum always==1

	if 'gini' in _stats:
		for t in model[base]:
			scores = [model['gini_y'][y][t] for y in _y]
			model['gini'][t] = sum(scores) # always sum

	return model

if __name__=="__main__":
	X = ['ala ma kota','to jest test work ma','go go power ala ma','go work work kota ma']
	X = [x.split(' ') for x in X]
	Y = [0,1,1,0]
	m = get_stats(X,Y,'tf','gini_y gini')
	#print(m['dia_y'][0])
	#print(m['dia'])
	print(m['gini'])
<commit_msg>text - get_ngrams<commit_after>from collections import Counter,defaultdict
from itertools import islice

def get_stats(X, Y, base='df', stats='', agg='sum', norm='sum'):
	model = {}
	_stats = set(stats.split(' ')+[base,base+'_y'])
	_y = set(Y)

	for s in _stats:
		if s.endswith('_y'):
			model[s] = defaultdict(Counter)
		else:
			model[s] = Counter()

	# ==[ base ]============================================================

	for tokens,y in zip(X,Y):
		if 'tf' in _stats:
			tf = Counter(tokens)
			model['tf_y'][y].update(tf)
			model['tf'].update(tf)

		if 'df' in _stats:
			df = Counter(set(tokens))
			model['df_y'][y].update(df)
			model['df'].update(df)

	pass # TODO drop terms below treshold

	# ==[ stats ]===========================================================

	if 'chi_y' in _stats:
		pass # TODO

	if 'cmfs_y' in _stats:
		cnt_y = {}
		for y in _y:
			cnt_y[y] = Y.count(y) # TODO base=tf
		for t,f in model[base].items():
			for y in _y:
				f_y = model[base+'_y'][y].get(t,0)
				p_tc = 1.0 * f_y / cnt_y[y]
				p_ct = 1.0 * f_y / f
				model['cmfs_y'][y][t] = p_tc * p_ct

	if 'dia_y' in _stats:
		for t,f in model[base].items():
			for y in _y:
				f_y = model[base+'_y'][y].get(t,0)
				p_ct = 1.0 * f_y / f
				model['dia_y'][y][t] = p_ct

	if 'gini_y' in _stats:
		cnt_y = {}
		for y in _y:
			cnt_y[y] = Y.count(y) # TODO base=tf
		for t,f in model[base].items():
			for y in _y:
				f_y = model[base+'_y'][y].get(t,0)
				p_tc = 1.0 * f_y / cnt_y[y]
				p_ct = 1.0 * f_y / f
				model['gini_y'][y][t] = p_tc*p_tc * p_ct*p_ct

	if 'wcp_y' in _stats:
		pass # TODO

	# TODO - my functions

	# TODO *x_y - ballanced variants

	# ==[ aggregates ]======================================================

	if 'cmfs' in _stats:
		for t in model[base]:
			scores = [model['cmfs_y'][y][t] for y in _y]
			model['cmfs'][t] = max(scores) # TODO agg

	if 'dia' in _stats:
		for t in model[base]:
			scores = [model['dia_y'][y][t] for y in _y]
			model['dia'][t] = max(scores) # agg=sum always==1

	if 'gini' in _stats:
		for t in model[base]:
			scores = [model['gini_y'][y][t] for y in _y]
			model['gini'][t] = sum(scores) # always sum

	return model

def get_ngrams(tokens, n, sep=' '):
    iters = [islice(tokens,i,None) for i in range(n)]
    return [sep.join(x) for x in zip(*iters)]

if __name__=="__main__":
	X = ['ala ma kota','to jest test work ma','go go power ala ma','go work work kota ma']
	X = [x.split(' ') for x in X]
	Y = [0,1,1,0]
	m = get_stats(X,Y,'tf','gini_y gini')
	#print(m['dia_y'][0])
	#print(m['dia'])
	print(m['gini'])
	print(get_ngrams('ala ma kota',2,''))
<|endoftext|>


<commit_before>"""
Copyright (c) 2015-2016 Constantine Belev

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

import numpy as np
import scipy as sp

from scipy import sparse, linalg, optimize

from scipy.linalg import rq
from numpy.linalg import svd, qr, norm
from scipy.optimize import minimize_scalar
from scipy.sparse import csr_matrix, csc_matrix, coo_matrix, linalg

from riemannian_optimization.utils.approx_utils import csvd
from riemannian_optimization.lowrank_matrix import ManifoldElement

def conjugate_direction(x_prev, grad_prev, dir_prev, x, grad):
    grad_prev_trans = grad_prev.transport(x)
    dir_prev_trans = dir_prev.transport(x)

    delta = grad - grad_prev_trans
    beta = max(0, delta.scalar_product(grad) / grad_prev_trans.frobenius_norm**2)
    dir_ = -grad + beta * dir_prev_trans<commit_msg>Add first version of conjugated gradients<commit_after>"""
Copyright (c) 2015-2016 Constantine Belev

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

import numpy as np
import scipy as sp

from scipy import sparse

from scipy.sparse import linalg, csr_matrix, csc_matrix
from scipy.optimize import minimize_scalar, line_search
from riemannian_optimization.lowrank_matrix import ManifoldElement
from riemannian_optimization.sparse.utils.retractions import svd_retraction
from riemannian_optimization.sparse.utils.loss_functions import delta_on_sigma_set
from riemannian_optimization.sparse.utils.projections import TangentVector, riemannian_grad_partial

def conjugate_direction(x_prev, grad_prev, dir_prev, x, grad):
    grad_prev_trans = grad_prev.transport(x)
    dir_prev_trans = dir_prev.transport(x)

    delta = grad - grad_prev_trans
    beta = max(0, delta.release().scalar_product(grad.release()) / grad_prev_trans.release().frobenius_norm()**2)
    dir_ = -grad + beta * dir_prev_trans

    angle = grad.release().scalar_product(dir_.release()) / np.sqrt(dir_.release().frobenius_norm()**2 * grad.release().frobenius_norm()**2)
    if angle <= 0.1:
        dir_ = grad
    return dir_

def closed_form_initial_guess(vec, delta, sigma_set):
    n_mat = csc_matrix(vec.release().evaluate(sigma_set)).T
    trace_first = n_mat.multiply(delta.T).sum()
    trace_second = n_mat.multiply(n_mat.T).sum()
    return trace_first / trace_second

def cg(a, sigma_set, r, maxiter=900, eps=1e-9):
    def cost_function(param):
        temp = x + param * conj_dir.release()
        return 0.5 * sp.sparse.linalg.norm(temp.evaluate(sigma_set) - a) ** 2

    def cost_raw(x_val):
        return 0.5 * sp.sparse.linalg.norm(x_val.evaluate(sigma_set) - a) ** 2

    x = ManifoldElement.rand(a.shape, r)
    x_prev = x
    error_history = []
    conj_dir, conj_dir_prev = TangentVector.zero(x), TangentVector.zero(x)
    grad = -TangentVector(x, riemannian_grad_partial(x, a, sigma_set, as_manifold_elements=True))
    for it in range(maxiter):
        delta = delta_on_sigma_set(x, a, sigma_set)
        grad_prev, grad = grad, -TangentVector(x, riemannian_grad_partial(x, a, sigma_set, grad=delta, as_manifold_elements=True))
        error_history.append(grad.release().frobenius_norm())
        if error_history[-1] < eps:
            print('Small grad norm {} is reached at iteration {}'.format(error_history[-1], it))
            return x, it, error_history
        conj_dir_prev, conj_dir = conj_dir,\
                                  conjugate_direction(x_prev, grad_prev, conj_dir_prev, x, grad)
        alpha = minimize_scalar(fun=cost_function, bounds=(0., 10.), method='bounded')['x']
        #alpha_init = closed_form_initial_guess(conj_dir, delta, sigma_set)
        #alpha = line_search(cost_function, xk=alpha_init, )
        m = 0
        for i in range(20):
            if cost_raw(x) - cost_raw(svd_retraction(x + 0.5**m * alpha * conj_dir.release(), r)) >= \
                    -0.0001 * 0.5**m * alpha * grad.release().scalar_product(conj_dir.release()):
                m = i
                break
        print('iter:{}, alpha: {}, m: {} error: {}'.format(it, alpha, m, error_history[-1]))
        x_prev, x = x, svd_retraction(x + 0.5**m * alpha * conj_dir.release(), r)
    print('Error {} is reached at iteration {}. Cannot converge'.format(error_history[-1], maxiter))
    return x, maxiter, error_history<|endoftext|>


<commit_before># $Id$

from simulator import EventProcessor
from params import Parameter
from Numeric import zeros,sqrt
from boundingregion import BoundingBox

def sheet2matrix(x,y,bounds,density):
    left,bottom,right,top = bounds.aarect().lbrt()
    linear_density = sqrt(density)
    col = (x-left) * linear_density
    row = (top-y)  * linear_density

    return int(row),int(col)

def matrix2sheet(row,col,bounds,density):
    left,bottom,right,top = bounds.aarect().lbrt()
    width = right-left
    height = top-bottom
    linear_density = sqrt(density)

    x = (col / linear_density) + left
    y = top - (row / linear_density)

    return x,y

def activation_submatrix(slice_bounds,activation,activation_bounds,density):
    """
    Gets a submatrix of an activation matrix defined by bounding rectangle.
    """
    r1,r2,c1,c2 = input_slice(slice_bounds,activation_bounds,density)

    return activation[r1:r2,c1:c2]

def input_slice(slice_bounds, input_bounds, input_density):
    """
    Gets the parameters for slicing an activation matrix given the slice bounds,
    activation bounds, and density of the activation matrix.

    returns a,b,c,d -- such that an activation matrix M
    can be sliced like this: M[a:b,c:d]

    """
    left,bottom,right,top = slice_bounds.aarect().lbrt()
    toprow,leftcol = sheet2matrix(left,top,input_bounds,input_density)
    bottomrow,rightcol = sheet2matrix(right,bottom,input_bounds,input_density)

    maxrow,maxcol = sheet2matrix(input_bounds.aarect().right(),
                                 input_bounds.aarect().bottom(),
                                 input_bounds,input_density)

    rstart = max(0,toprow)
    rbound = min(maxrow+1,bottomrow+1)
    cstart = max(0,leftcol)
    cbound = min(maxcol+1,rightcol+1)

    return rstart,rbound,cstart,cbound

def bounds2shape(bounds,density):
    left,bottom,right,top = bounds.aarect().lbrt()
    width = right-left
    height = top-bottom
    linear_density = sqrt(density)
    rows = int(height*linear_density)
    cols = int(width*linear_density)
    return rows,cols

class Sheet(EventProcessor):
    """
    The generic base class for neural sheets.

    This class handles the sheet's activity matrix, and it's
    coordinate frame.  It manages two sets of coordinates:

    _Sheet_coordinates_ are specified as (x,y) as on a normal
    cartesian graph; the positive y direction is upward, and the scale
    and origin are arbitrary, and specified by parameters.

    _Matrix_coordinates_ are the (row,col) specification for an
    element in the activation matrix.  The usual matrix coordinate
    specs apply.

    Parameters:

    bounds:   A BoundingBox object indicating the bounds of the sheet.
              [default  (-0.5,-0.5) to (0.5,0.5)]
    density:  The areal density of the sheet [default 100]

    """

    bounds  = Parameter(BoundingBox(points=((-0.5,-0.5),(0.5,0.5))))
    density = Parameter(100)

    def __init__(self,**params):

        super(Sheet,self).__init__(**params)

        linear_density = sqrt(self.density)

        self.debug("linear_density = ",linear_density)
        left,bottom,right,top = self.bounds.aarect().lbrt()
        width,height = right-left,top-bottom
        rows = int(height*linear_density)
        cols = int(width*linear_density)
        self.activation = zeros((rows,cols)) + 0.0
        self.debug('activation.shape =',self.activation.shape)

    def sheet2matrix(self,x,y):
        """
        Coordinate transformation: takes a point (x,y) in sheet
        coordinates and returns the (row,col) of the matrix cell it
        cooresponds to.
        """

        return sheet2matrix(x,y,self.bounds,self.density)

    def matrix2sheet(self,row,col):
        """
        Coordinate transformation: Takes the (row,col) of an element
        in the activity matrix and gives its (x,y) in sheet
        coordinates.
        """
        return matrix2sheet(row,col,self.bounds,self.density)

    def sheet_offset(self):
        """
        Returns the offset of the sheet origin from the lower-left
        corner of the sheet.
        """
        return -self.bounds.aarect().left(),-self.bounds.aarect().bottom()

    def sheet_rows(self):
        rows,cols = self.activation.shape
        coords = [self.matrix2sheet(r,0) for r in range(rows,0,-1)]
        return [y for (x,y) in coords]

    def sheet_cols(self):
        rows,cols = self.activation.shape
        coords = [self.matrix2sheet(0,c) for c in range(cols)]
        return [x for (x,y) in coords]

#####################################
from utils import NxN

class Composer(Sheet):
    """
    A Sheet that combines the activity of 2 or more other sheets into
    a single activity matrix.  When an activity matrix is received on
    an input port, it is added to the the sheet's input buffer
    according to the parameters of that port (see port_configure()
    below).

    """

    def __init__(self,**config):
        super(Composer,self).__init__(**config)
        self.ports = {}
        self.__dirty = False

    def port_configure(self,port,**config):
        """
        Configure a specific input port.
        Port parameters:
           origin = (default (0,0)) The offset in the output matrix
                    where this port's input should be placed.
        """
        if not port in self.ports:
            self.ports[port] = {}

        for k,v in config.items():
            self.ports[port][k] = v

    def pre_sleep(self):
        if self.__dirty:
            self.send_output(data=self.activation)
            self.activation = zeros(self.activation.shape)+0.0
            self.__dirty=False

    def input_event(self,src,src_port,dest_port,data):

        self.verbose("Received %s input from %s." % (NxN(data.shape),src))

        self.__dirty = True

        in_rows, in_cols = data.shape

        # compute the correct position of the input in the buffer
        start_row,start_col = self.sheet2matrix(*self.ports[dest_port]['origin'])
        row_adj,col_adj = src.sheet2matrix(0,0)

        self.debug("origin (row,col) = "+`(start_row,start_col)`)
        self.debug("adjust (row,col) = "+`(row_adj,col_adj)`)

        start_row -= row_adj
        start_col -= col_adj

        # the maximum bounds
        max_row,max_col = self.activation.shape

        self.debug("max_row = %d, max_col = %d" % (max_row,max_col))
        self.debug("in_rows = %d, in_cols = %d" % (in_rows,in_cols))

        end_row = start_row+in_rows
        end_col = start_col+in_cols

        # if the input goes outside the activation, clip it
        left_clip = -min(start_col,0)
        top_clip  = -min(start_row,0)
        right_clip = max(end_col,max_col) - max_col
        bottom_clip = max(end_row,max_row) - max_row

        start_col += left_clip
        start_row += top_clip
        end_col -= right_clip
        end_row -= bottom_clip

        self.debug("start_row = %d,start_col = %d" % (start_row,start_col))
        self.debug("end_row = %d,end_col = %d" % (end_row,end_col))
        self.debug("left_clip = %d" % left_clip)
        self.debug("right_clip = %d" % right_clip)
        self.debug("top_clip = %d" % top_clip)
        self.debug("bottom_clip = %d" % bottom_clip)
        self.debug("activation shape = %s" % NxN(self.activation.shape))

        self.activation[start_row:end_row, start_col:end_col] += data[top_clip:in_rows-bottom_clip,
                                                                      left_clip:in_cols-right_clip]

if __name__ == '__main__':

    # test sheet2matrix
    s = Sheet()

    print 'sheet -> matrix'
    print (0,0), ' -> ', s.sheet2matrix(0,0)
    print (0,0.5), ' -> ', s.sheet2matrix(0,0.5)
    print (0.5,0), ' ->', s.sheet2matrix(0.5,0)
    print (0.5,0.5), ' -> ', s.sheet2matrix(0.5,0.5)

    print 'matrix -> sheet'
    print (0,0), ' -> ', s.matrix2sheet(0,0)
    print (0,100), ' -> ', s.matrix2sheet(0,100)
    print (100,0), ' ->', s.matrix2sheet(100,0)
    print (100,100), ' -> ', s.matrix2sheet(100,100)

<commit_msg>Added sheet_view, add_sheet_view, sheet_view_dict<commit_after># $Id$

from simulator import EventProcessor
from params import Parameter
from Numeric import zeros,sqrt,array
from boundingregion import BoundingBox
from sheetview import SheetView

def sheet2matrix(x,y,bounds,density):
    left,bottom,right,top = bounds.aarect().lbrt()
    linear_density = sqrt(density)
    col = (x-left) * linear_density
    row = (top-y)  * linear_density

    return int(row),int(col)

def matrix2sheet(row,col,bounds,density):
    left,bottom,right,top = bounds.aarect().lbrt()
    width = right-left
    height = top-bottom
    linear_density = sqrt(density)

    x = (col / linear_density) + left
    y = top - (row / linear_density)

    return x,y

def activation_submatrix(slice_bounds,activation,activation_bounds,density):
    """
    Gets a submatrix of an activation matrix defined by bounding rectangle.
    """
    r1,r2,c1,c2 = input_slice(slice_bounds,activation_bounds,density)

    return activation[r1:r2,c1:c2]

def input_slice(slice_bounds, input_bounds, input_density):
    """
    Gets the parameters for slicing an activation matrix given the slice bounds,
    activation bounds, and density of the activation matrix.

    returns a,b,c,d -- such that an activation matrix M
    can be sliced like this: M[a:b,c:d]

    """
    left,bottom,right,top = slice_bounds.aarect().lbrt()
    toprow,leftcol = sheet2matrix(left,top,input_bounds,input_density)
    bottomrow,rightcol = sheet2matrix(right,bottom,input_bounds,input_density)

    maxrow,maxcol = sheet2matrix(input_bounds.aarect().right(),
                                 input_bounds.aarect().bottom(),
                                 input_bounds,input_density)

    rstart = max(0,toprow)
    rbound = min(maxrow+1,bottomrow+1)
    cstart = max(0,leftcol)
    cbound = min(maxcol+1,rightcol+1)

    return rstart,rbound,cstart,cbound

def bounds2shape(bounds,density):
    left,bottom,right,top = bounds.aarect().lbrt()
    width = right-left
    height = top-bottom
    linear_density = sqrt(density)
    rows = int(height*linear_density)
    cols = int(width*linear_density)
    return rows,cols

class Sheet(EventProcessor):
    """
    The generic base class for neural sheets.

    This class handles the sheet's activity matrix, and it's
    coordinate frame.  It manages two sets of coordinates:

    _Sheet_coordinates_ are specified as (x,y) as on a normal
    cartesian graph; the positive y direction is upward, and the scale
    and origin are arbitrary, and specified by parameters.

    _Matrix_coordinates_ are the (row,col) specification for an
    element in the activation matrix.  The usual matrix coordinate
    specs apply.

    Parameters:

    bounds:   A BoundingBox object indicating the bounds of the sheet.
              [default  (-0.5,-0.5) to (0.5,0.5)]
    density:  The areal density of the sheet [default 100]

    sheet_view_dict is a dictionary that stores SheetViews that are
    generated by the plotting environment and registered to Sheets
    to be used later by the bitmap display mechanism.
    """

    bounds  = Parameter(BoundingBox(points=((-0.5,-0.5),(0.5,0.5))))
    density = Parameter(100)

    def __init__(self,**params):

        super(Sheet,self).__init__(**params)

        linear_density = sqrt(self.density)

        self.debug("linear_density = ",linear_density)
        left,bottom,right,top = self.bounds.aarect().lbrt()
        width,height = right-left,top-bottom
        rows = int(height*linear_density)
        cols = int(width*linear_density)
        self.activation = zeros((rows,cols)) + 0.0
        self.debug('activation.shape =',self.activation.shape)
        self.sheet_view_dict = {}

    def sheet_view(self,request='Activation'):
        """
        Create a SheetView object of the current activation of the
        Sheet.  Current implementation gives the raw activation
        matrix.  Uses self.sheet_view_dict if the request is not
        the default.

        The name for an Activation View is the Sheet name plus
        '_Activation', e.g. 'Sheet0002_Activation'.

        Returns None if the View does not exist in this sheet.
        """
        if request == 'Activation':
            activation_copy = array(self.activation)
            new_view = SheetView(activation_copy,self.bounds,
                                 name=(self.name + '_Activation'))
        else:
            if self.sheet_view_dict.has_key(request):
                new_view = self.sheet_view_dict[request]
            else:
                new_view = None
        return new_view

    def add_sheet_view(self,view_name,sheet_view):
        """
        Add a SheetView to the view database in this Sheet object.
        Each view is stored by a dictionary keyword passed into the
        function through view_name.

        Because each SheetView has its own internal name, there is no
        guarantee that the SheetView name will be stored under the key
        name passed into this function.
        """
        self.sheet_view_dict[view_name] = sheet_view

    def sheet2matrix(self,x,y):
        """
        Coordinate transformation: takes a point (x,y) in sheet
        coordinates and returns the (row,col) of the matrix cell it
        cooresponds to.
        """
        return sheet2matrix(x,y,self.bounds,self.density)

    def matrix2sheet(self,row,col):
        """
        Coordinate transformation: Takes the (row,col) of an element
        in the activity matrix and gives its (x,y) in sheet
        coordinates.
        """
        return matrix2sheet(row,col,self.bounds,self.density)

    def sheet_offset(self):
        """
        Returns the offset of the sheet origin from the lower-left
        corner of the sheet.
        """
        return -self.bounds.aarect().left(),-self.bounds.aarect().bottom()

    def sheet_rows(self):
        rows,cols = self.activation.shape
        coords = [self.matrix2sheet(r,0) for r in range(rows,0,-1)]
        return [y for (x,y) in coords]

    def sheet_cols(self):
        rows,cols = self.activation.shape
        coords = [self.matrix2sheet(0,c) for c in range(cols)]
        return [x for (x,y) in coords]

#####################################
from utils import NxN

class Composer(Sheet):
    """
    A Sheet that combines the activity of 2 or more other sheets into
    a single activity matrix.  When an activity matrix is received on
    an input port, it is added to the the sheet's input buffer
    according to the parameters of that port (see port_configure()
    below).

    """

    def __init__(self,**config):
        super(Composer,self).__init__(**config)
        self.ports = {}
        self.__dirty = False

    def port_configure(self,port,**config):
        """
        Configure a specific input port.
        Port parameters:
           origin = (default (0,0)) The offset in the output matrix
                    where this port's input should be placed.
        """
        if not port in self.ports:
            self.ports[port] = {}

        for k,v in config.items():
            self.ports[port][k] = v

    def pre_sleep(self):
        if self.__dirty:
            self.send_output(data=self.activation)
            self.activation = zeros(self.activation.shape)+0.0
            self.__dirty=False

    def input_event(self,src,src_port,dest_port,data):

        self.verbose("Received %s input from %s." % (NxN(data.shape),src))

        self.__dirty = True

        in_rows, in_cols = data.shape

        # compute the correct position of the input in the buffer
        start_row,start_col = self.sheet2matrix(*self.ports[dest_port]['origin'])
        row_adj,col_adj = src.sheet2matrix(0,0)

        self.debug("origin (row,col) = "+`(start_row,start_col)`)
        self.debug("adjust (row,col) = "+`(row_adj,col_adj)`)

        start_row -= row_adj
        start_col -= col_adj

        # the maximum bounds
        max_row,max_col = self.activation.shape

        self.debug("max_row = %d, max_col = %d" % (max_row,max_col))
        self.debug("in_rows = %d, in_cols = %d" % (in_rows,in_cols))

        end_row = start_row+in_rows
        end_col = start_col+in_cols

        # if the input goes outside the activation, clip it
        left_clip = -min(start_col,0)
        top_clip  = -min(start_row,0)
        right_clip = max(end_col,max_col) - max_col
        bottom_clip = max(end_row,max_row) - max_row

        start_col += left_clip
        start_row += top_clip
        end_col -= right_clip
        end_row -= bottom_clip

        self.debug("start_row = %d,start_col = %d" % (start_row,start_col))
        self.debug("end_row = %d,end_col = %d" % (end_row,end_col))
        self.debug("left_clip = %d" % left_clip)
        self.debug("right_clip = %d" % right_clip)
        self.debug("top_clip = %d" % top_clip)
        self.debug("bottom_clip = %d" % bottom_clip)
        self.debug("activation shape = %s" % NxN(self.activation.shape))

        self.activation[start_row:end_row, start_col:end_col] += data[top_clip:in_rows-bottom_clip,
                                                                      left_clip:in_cols-right_clip]

if __name__ == '__main__':

    # test sheet2matrix
    s = Sheet()

    print 'sheet -> matrix'
    print (0,0), ' -> ', s.sheet2matrix(0,0)
    print (0,0.5), ' -> ', s.sheet2matrix(0,0.5)
    print (0.5,0), ' ->', s.sheet2matrix(0.5,0)
    print (0.5,0.5), ' -> ', s.sheet2matrix(0.5,0.5)

    print 'matrix -> sheet'
    print (0,0), ' -> ', s.matrix2sheet(0,0)
    print (0,100), ' -> ', s.matrix2sheet(0,100)
    print (100,0), ' ->', s.matrix2sheet(100,0)
    print (100,100), ' -> ', s.matrix2sheet(100,100)

<|endoftext|>


<commit_before>import pyquil.api as api
import numpy as np
import pyquil.quil as pq
from pyquil.gates import X, I

from grove.bernstein_vazirani.bernstein_vazirani import bernstein_vazirani, \
    oracle_function
import pytest

@pytest.mark.skip(reason="Must add support for Forest connections in testing")
class TestOracleFunction(object):
    def test_one_qubit(self):
        vec_a = np.array([1])
        b = 0
        for x in range(2 ** len(vec_a)):
            _oracle_test_helper(vec_a, b, x)

    def test_two_qubits(self):
        vec_a = np.array([1, 0])
        b = 1
        for x in range(2 ** len(vec_a)):
            _oracle_test_helper(vec_a, b, x)

    def test_three_qubits(self):
        vec_a = np.array([0, 0, 0])
        b = 0
        for x in range(2 ** len(vec_a)):
            _oracle_test_helper(vec_a, b, x)

    def test_four_qubits(self):
        vec_a = np.array([1, 1, 1, 1])
        b = 1
        for x in range(2 ** len(vec_a)):
            _oracle_test_helper(vec_a, b, x)

@pytest.mark.skip(reason="Must add support for Forest connections in testing")
class TestBernsteinVazirani(object):
    def test_one_qubit_all_zeros(self):
        _bv_test_helper(np.array([0]), 0)

    def test_two_qubit_all_ones(self):
        _bv_test_helper(np.array([1, 1]), 1)

    def test_four_qubit_symmetric(self):
        _bv_test_helper(np.array([1, 0, 0, 1]), 1)

    def test_five_qubits_asymmetric(self):
        _bv_test_helper(np.array([0, 0, 1, 0, 1]), 0)

def _bv_test_helper(vec_a, b, trials=1):
    qubits = range(len(vec_a))
    ancilla = len(vec_a)
    oracle = oracle_function(vec_a, b, qubits, ancilla)
    bv_program = bernstein_vazirani(oracle, qubits, ancilla)
    cxn = api.SyncConnection()
    results = cxn.run_and_measure(bv_program, qubits, trials)
    for result in results:
        bv_a = result[::-1]
        assert bv_a == list(vec_a)

def _oracle_test_helper(vec_a, b, x, trials=1):
    qubits = range(len(vec_a))
    ancilla = len(vec_a)

    cxn = api.SyncConnection()
    bitstring = np.binary_repr(x, len(qubits))
    p = pq.Program()

    for i in range(len(bitstring)):
        p.inst(I(i))
        if bitstring[-1 - i] == '1':
            p.inst(X(i))

    p.inst(I(ancilla))
    oracle = oracle_function(vec_a, b, qubits, ancilla)
    p += oracle
    results = cxn.run_and_measure(p, [ancilla], trials)
    a_dot_x = np.binary_repr(int(''.join(map(str, vec_a)), 2) & x).count("1")
    expected = (a_dot_x + b) % 2
    for result in results:
        y = result[0]
        assert y == expected
<commit_msg>Make range in tests into xrange where applicable<commit_after>import pyquil.api as api
import numpy as np
import pyquil.quil as pq
from pyquil.gates import X, I

from grove.bernstein_vazirani.bernstein_vazirani import bernstein_vazirani, \
    oracle_function
import pytest

@pytest.mark.skip(reason="Must add support for Forest connections in testing")
class TestOracleFunction(object):
    def test_one_qubit(self):
        vec_a = np.array([1])
        b = 0
        for x in xrange(2 ** len(vec_a)):
            _oracle_test_helper(vec_a, b, x)

    def test_two_qubits(self):
        vec_a = np.array([1, 0])
        b = 1
        for x in xrange(2 ** len(vec_a)):
            _oracle_test_helper(vec_a, b, x)

    def test_three_qubits(self):
        vec_a = np.array([0, 0, 0])
        b = 0
        for x in xrange(2 ** len(vec_a)):
            _oracle_test_helper(vec_a, b, x)

    def test_four_qubits(self):
        vec_a = np.array([1, 1, 1, 1])
        b = 1
        for x in xrange(2 ** len(vec_a)):
            _oracle_test_helper(vec_a, b, x)

@pytest.mark.skip(reason="Must add support for Forest connections in testing")
class TestBernsteinVazirani(object):
    def test_one_qubit_all_zeros(self):
        _bv_test_helper(np.array([0]), 0)

    def test_two_qubit_all_ones(self):
        _bv_test_helper(np.array([1, 1]), 1)

    def test_four_qubit_symmetric(self):
        _bv_test_helper(np.array([1, 0, 0, 1]), 1)

    def test_five_qubits_asymmetric(self):
        _bv_test_helper(np.array([0, 0, 1, 0, 1]), 0)

def _bv_test_helper(vec_a, b, trials=1):
    qubits = range(len(vec_a))
    ancilla = len(vec_a)
    oracle = oracle_function(vec_a, b, qubits, ancilla)
    bv_program = bernstein_vazirani(oracle, qubits, ancilla)
    cxn = api.SyncConnection()
    results = cxn.run_and_measure(bv_program, qubits, trials)
    for result in results:
        bv_a = result[::-1]
        assert bv_a == list(vec_a)

def _oracle_test_helper(vec_a, b, x, trials=1):
    qubits = range(len(vec_a))
    ancilla = len(vec_a)

    cxn = api.SyncConnection()
    bitstring = np.binary_repr(x, len(qubits))
    p = pq.Program()

    for i in xrange(len(bitstring)):
        p.inst(I(i))
        if bitstring[-1 - i] == '1':
            p.inst(X(i))

    p.inst(I(ancilla))
    oracle = oracle_function(vec_a, b, qubits, ancilla)
    p += oracle
    results = cxn.run_and_measure(p, [ancilla], trials)
    a_dot_x = np.binary_repr(int(''.join(map(str, vec_a)), 2) & x).count("1")
    expected = (a_dot_x + b) % 2
    for result in results:
        y = result[0]
        assert y == expected
<|endoftext|>