<commit_before># pylint: disable=g-bad-file-header
# Copyright 2016 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Configuring the C++ toolchain on macOS."""

load("@bazel_tools//tools/osx:xcode_configure.bzl", "run_xcode_locator")
load(
    "@bazel_tools//tools/cpp:lib_cc_configure.bzl",
    "escape_string",
    "resolve_labels",
    "write_builtin_include_directory_paths",
)
load(
    "@bazel_tools//tools/cpp:unix_cc_configure.bzl",
    "configure_unix_toolchain",
    "get_env",
    "get_escaped_cxx_inc_directories",
)

def _get_escaped_xcode_cxx_inc_directories(repository_ctx, cc, xcode_toolchains):
    """Compute the list of default C++ include paths on Xcode-enabled darwin.

    Args:
      repository_ctx: The repository context.
      cc: The default C++ compiler on the local system.
      xcode_toolchains: A list containing the xcode toolchains available
    Returns:
      include_paths: A list of builtin include paths.
    """

    # TODO(cparsons): Falling back to the default C++ compiler builtin include
    # paths shouldn't be unnecessary once all actions are using xcrun.
    include_dirs = get_escaped_cxx_inc_directories(repository_ctx, cc, "-xc++")
    for toolchain in xcode_toolchains:
        include_dirs.append(escape_string(toolchain.developer_dir))

    # Assume that all paths that point to /Applications/ are built in include paths
    include_dirs.append("/Applications/")
    return include_dirs

def configure_osx_toolchain(repository_ctx, overriden_tools):
    """Configure C++ toolchain on macOS."""
    paths = resolve_labels(repository_ctx, [
        "@bazel_tools//tools/cpp:osx_cc_wrapper.sh.tpl",
        "@bazel_tools//tools/objc:libtool.sh",
        "@bazel_tools//tools/objc:make_hashed_objlist.py",
        "@bazel_tools//tools/objc:xcrunwrapper.sh",
        "@bazel_tools//tools/osx/crosstool:BUILD.tpl",
        "@bazel_tools//tools/osx/crosstool:cc_toolchain_config.bzl",
        "@bazel_tools//tools/osx/crosstool:wrapped_clang.cc",
        "@bazel_tools//tools/osx:xcode_locator.m",
    ])

    env = repository_ctx.os.environ
    should_use_xcode = "BAZEL_USE_XCODE_TOOLCHAIN" in env and env["BAZEL_USE_XCODE_TOOLCHAIN"] == "1"
    xcode_toolchains = []

    # Make the following logic in sync with //tools/cpp:cc_configure.bzl#cc_autoconf_toolchains_impl
    (xcode_toolchains, xcodeloc_err) = run_xcode_locator(
        repository_ctx,
        paths["@bazel_tools//tools/osx:xcode_locator.m"],
    )
    if should_use_xcode and not xcode_toolchains:
        fail("BAZEL_USE_XCODE_TOOLCHAIN is set to 1 but Bazel couldn't find Xcode installed on the " +
             "system. Verify that 'xcode-select -p' is correct.")
    if xcode_toolchains:
        # For Xcode toolchains, there's no reason to use anything other than
        # wrapped_clang, so that we still get the Bazel Xcode placeholder
        # substitution and other behavior for actions that invoke this
        # cc_wrapper.sh script. The wrapped_clang binary is already hardcoded
        # into the Objective-C crosstool actions, anyway, so this ensures that
        # the C++ actions behave consistently.
        cc = repository_ctx.path("wrapped_clang")

        repository_ctx.template(
            "cc_wrapper.sh",
            paths["@bazel_tools//tools/cpp:osx_cc_wrapper.sh.tpl"],
            {
                "%{cc}": escape_string(str(cc)),
                "%{env}": escape_string(get_env(repository_ctx)),
            },
        )
        repository_ctx.symlink(
            paths["@bazel_tools//tools/objc:xcrunwrapper.sh"],
            "xcrunwrapper.sh",
        )
        repository_ctx.symlink(
            paths["@bazel_tools//tools/objc:libtool.sh"],
            "libtool",
        )
        repository_ctx.symlink(
            paths["@bazel_tools//tools/objc:make_hashed_objlist.py"],
            "make_hashed_objlist.py",
        )
        repository_ctx.symlink(
            paths["@bazel_tools//tools/osx/crosstool:cc_toolchain_config.bzl"],
            "cc_toolchain_config.bzl",
        )
        wrapped_clang_src_path = str(repository_ctx.path(
            paths["@bazel_tools//tools/osx/crosstool:wrapped_clang.cc"],
        ))
        xcrun_result = repository_ctx.execute([
            "env",
            "-i",
            "xcrun",
            "--sdk",
            "macosx",
            "clang",
            "-mmacosx-version-min=10.9",
            "-std=c++11",
            "-lc++",
            "-o",
            "wrapped_clang",
            wrapped_clang_src_path,
        ], 30)
        if (xcrun_result.return_code == 0):
            repository_ctx.symlink("wrapped_clang", "wrapped_clang_pp")
        else:
            error_msg = (
                "return code {code}, stderr: {err}, stdout: {out}"
            ).format(
                code = xcrun_result.return_code,
                err = xcrun_result.stderr,
                out = xcrun_result.stdout,
            )
            fail("wrapped_clang failed to generate. Please file an issue at " +
                 "https://github.com/bazelbuild/bazel/issues with the following:\n" +
                 error_msg)

        escaped_include_paths = _get_escaped_xcode_cxx_inc_directories(repository_ctx, cc, xcode_toolchains)
        write_builtin_include_directory_paths(repository_ctx, cc, escaped_include_paths)
        escaped_cxx_include_directories = []
        for path in escaped_include_paths:
            escaped_cxx_include_directories.append(("        \"%s\"," % path))
        if xcodeloc_err:
            escaped_cxx_include_directories.append("# Error: " + xcodeloc_err + "\n")
        repository_ctx.template(
            "BUILD",
            paths["@bazel_tools//tools/osx/crosstool:BUILD.tpl"],
            {"%{cxx_builtin_include_directories}": "\n".join(escaped_cxx_include_directories)},
        )
    else:
        configure_unix_toolchain(repository_ctx, cpu_value = "darwin", overriden_tools = overriden_tools)
<commit_msg>Replace macOS CC path with relative path<commit_after># pylint: disable=g-bad-file-header
# Copyright 2016 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Configuring the C++ toolchain on macOS."""

load("@bazel_tools//tools/osx:xcode_configure.bzl", "run_xcode_locator")
load(
    "@bazel_tools//tools/cpp:lib_cc_configure.bzl",
    "escape_string",
    "resolve_labels",
    "write_builtin_include_directory_paths",
)
load(
    "@bazel_tools//tools/cpp:unix_cc_configure.bzl",
    "configure_unix_toolchain",
    "get_env",
    "get_escaped_cxx_inc_directories",
)

def _get_escaped_xcode_cxx_inc_directories(repository_ctx, cc, xcode_toolchains):
    """Compute the list of default C++ include paths on Xcode-enabled darwin.

    Args:
      repository_ctx: The repository context.
      cc: The default C++ compiler on the local system.
      xcode_toolchains: A list containing the xcode toolchains available
    Returns:
      include_paths: A list of builtin include paths.
    """

    # TODO(cparsons): Falling back to the default C++ compiler builtin include
    # paths shouldn't be unnecessary once all actions are using xcrun.
    include_dirs = get_escaped_cxx_inc_directories(repository_ctx, cc, "-xc++")
    for toolchain in xcode_toolchains:
        include_dirs.append(escape_string(toolchain.developer_dir))

    # Assume that all paths that point to /Applications/ are built in include paths
    include_dirs.append("/Applications/")
    return include_dirs

def configure_osx_toolchain(repository_ctx, overriden_tools):
    """Configure C++ toolchain on macOS."""
    paths = resolve_labels(repository_ctx, [
        "@bazel_tools//tools/cpp:osx_cc_wrapper.sh.tpl",
        "@bazel_tools//tools/objc:libtool.sh",
        "@bazel_tools//tools/objc:make_hashed_objlist.py",
        "@bazel_tools//tools/objc:xcrunwrapper.sh",
        "@bazel_tools//tools/osx/crosstool:BUILD.tpl",
        "@bazel_tools//tools/osx/crosstool:cc_toolchain_config.bzl",
        "@bazel_tools//tools/osx/crosstool:wrapped_clang.cc",
        "@bazel_tools//tools/osx:xcode_locator.m",
    ])

    env = repository_ctx.os.environ
    should_use_xcode = "BAZEL_USE_XCODE_TOOLCHAIN" in env and env["BAZEL_USE_XCODE_TOOLCHAIN"] == "1"
    xcode_toolchains = []

    # Make the following logic in sync with //tools/cpp:cc_configure.bzl#cc_autoconf_toolchains_impl
    (xcode_toolchains, xcodeloc_err) = run_xcode_locator(
        repository_ctx,
        paths["@bazel_tools//tools/osx:xcode_locator.m"],
    )
    if should_use_xcode and not xcode_toolchains:
        fail("BAZEL_USE_XCODE_TOOLCHAIN is set to 1 but Bazel couldn't find Xcode installed on the " +
             "system. Verify that 'xcode-select -p' is correct.")
    if xcode_toolchains:
        # For Xcode toolchains, there's no reason to use anything other than
        # wrapped_clang, so that we still get the Bazel Xcode placeholder
        # substitution and other behavior for actions that invoke this
        # cc_wrapper.sh script. The wrapped_clang binary is already hardcoded
        # into the Objective-C crosstool actions, anyway, so this ensures that
        # the C++ actions behave consistently.
        cc = repository_ctx.path("wrapped_clang")

        cc_path = '"$(dirname "$0")"/wrapped_clang'
        repository_ctx.template(
            "cc_wrapper.sh",
            paths["@bazel_tools//tools/cpp:osx_cc_wrapper.sh.tpl"],
            {
                "%{cc}": escape_string(cc_path),
                "%{env}": escape_string(get_env(repository_ctx)),
            },
        )
        repository_ctx.symlink(
            paths["@bazel_tools//tools/objc:xcrunwrapper.sh"],
            "xcrunwrapper.sh",
        )
        repository_ctx.symlink(
            paths["@bazel_tools//tools/objc:libtool.sh"],
            "libtool",
        )
        repository_ctx.symlink(
            paths["@bazel_tools//tools/objc:make_hashed_objlist.py"],
            "make_hashed_objlist.py",
        )
        repository_ctx.symlink(
            paths["@bazel_tools//tools/osx/crosstool:cc_toolchain_config.bzl"],
            "cc_toolchain_config.bzl",
        )
        wrapped_clang_src_path = str(repository_ctx.path(
            paths["@bazel_tools//tools/osx/crosstool:wrapped_clang.cc"],
        ))
        xcrun_result = repository_ctx.execute([
            "env",
            "-i",
            "xcrun",
            "--sdk",
            "macosx",
            "clang",
            "-mmacosx-version-min=10.9",
            "-std=c++11",
            "-lc++",
            "-o",
            "wrapped_clang",
            wrapped_clang_src_path,
        ], 30)
        if (xcrun_result.return_code == 0):
            repository_ctx.symlink("wrapped_clang", "wrapped_clang_pp")
        else:
            error_msg = (
                "return code {code}, stderr: {err}, stdout: {out}"
            ).format(
                code = xcrun_result.return_code,
                err = xcrun_result.stderr,
                out = xcrun_result.stdout,
            )
            fail("wrapped_clang failed to generate. Please file an issue at " +
                 "https://github.com/bazelbuild/bazel/issues with the following:\n" +
                 error_msg)

        escaped_include_paths = _get_escaped_xcode_cxx_inc_directories(repository_ctx, cc, xcode_toolchains)
        write_builtin_include_directory_paths(repository_ctx, cc, escaped_include_paths)
        escaped_cxx_include_directories = []
        for path in escaped_include_paths:
            escaped_cxx_include_directories.append(("        \"%s\"," % path))
        if xcodeloc_err:
            escaped_cxx_include_directories.append("# Error: " + xcodeloc_err + "\n")
        repository_ctx.template(
            "BUILD",
            paths["@bazel_tools//tools/osx/crosstool:BUILD.tpl"],
            {"%{cxx_builtin_include_directories}": "\n".join(escaped_cxx_include_directories)},
        )
    else:
        configure_unix_toolchain(repository_ctx, cpu_value = "darwin", overriden_tools = overriden_tools)
<|endoftext|>


<commit_before># encoding: utf-8
import pytest
from pymetabiosis.module import import_module
from pymetabiosis.wrapper import MetabiosisWrapper, pypy_convert

def test_getattr_on_module():
    sqlite = import_module("sqlite3")
    assert isinstance(sqlite, MetabiosisWrapper)
    connect = sqlite.connect
    assert isinstance(connect, MetabiosisWrapper)
    assert repr(connect).startswith("<built-in function connect>")

def test_call_function():
    sqlite = import_module("sqlite3")
    connection = sqlite.connect(":memory:")
    assert repr(connection).startswith("<sqlite3.Connection object at ")

def test_pass_wrapper_to_function():
    sqlite = import_module("sqlite3")

    connection = sqlite.connect(":memory:")

    assert repr(connection).startswith("<sqlite3.Connection object at ")

def test_sqlite():
    # Taken from Python 2.7's sqlite doc
    sqlite3 = import_module("sqlite3")

    con = sqlite3.connect(":memory:")
    cur = con.cursor()
    cur.execute("create table people (name_last, age)")

    who = "Yeltsin"
    age = 72

    cur.execute("insert into people values (?, ?)", (who, age))

    cur.execute("select * from people where name_last=:who and age=:age", {"who": who, "age": age})

    ret = cur.fetchone()
    assert ret == (u'Yeltsin', 72)

def test_gettype():
    sqlite3 = import_module("sqlite3")

    assert repr(sqlite3.get_type()) == "<type 'module'>"

def test_convert_return_value():
    builtin = import_module("__builtin__")
    operator = import_module("operator")

    assert builtin.int(32) == 32
    assert builtin.float(3.123) == 3.123

    for s in ['a string']: # TODO 'a string \00yep']:
        assert builtin.str(s) == s

    u = u"some буквы are странные"
    assert builtin.unicode(u) == u

    t = (1, (2.3,))
    assert builtin.tuple(t) == t

    d = {'a': 'b', 1: 2}
    assert builtin.dict(d) == d

    lst = ['a', 1, [2]]
    assert builtin.list(lst) == lst

    assert builtin.bool(True) is True
    assert builtin.bool(False) is False

    assert builtin.bool(None) is False
    assert operator.eq(None, None) is True
    assert operator.eq(None, False) is False

def test_getitem_setitem_delitem():
    builtin = import_module("__builtin__", noconvert=True)

    d = builtin.dict({1: 'foo', (1, 'a'): 'zoo'})
    with pytest.raises(KeyError):
        d[2]
    assert pypy_convert(d[1].obj) == 'foo'
    assert pypy_convert(d[(1, 'a')].obj) == 'zoo'

    key, lst = (1, 2), ['a', 'b']
    d[key] = lst
    assert pypy_convert(d[key].obj) == lst

    with pytest.raises(TypeError):
        d[[1, 2]] = 0

    del d[1]
    with pytest.raises(KeyError):
        d[1]

    with pytest.raises(KeyError):
        del d[2]

def test_str_repr_dir():
    builtin = import_module("__builtin__", noconvert=True)

    assert str(builtin.None) == 'None'
    assert str(builtin.str('a')) == 'a'
    assert repr(builtin.str('a')) == "'a'"

    assert set(['rjust', 'rpartition', 'rstrip', '__le__'])\
            .issubset(dir(builtin.str('a')))

def test_len():
    builtin = import_module("__builtin__", noconvert=True)
    lst = builtin.list([1, 'a'])
    assert len(lst) == 2
    assert len(builtin.list()) == 0
    assert len(builtin.str('abc')) == 3

    with pytest.raises(TypeError):
        len(builtin.iter([1]))

def test_iter():
    builtin = import_module("__builtin__", noconvert=True)
    assert [pypy_convert(x.obj) for x in builtin.list([1, 'a'])] == [1, 'a']
    assert pypy_convert(list(builtin.iter(['a']))[0].obj) == 'a'

def test_exceptions():
    builtin = import_module("__builtin__")

    with pytest.raises(AttributeError):
        builtin.foo

    with pytest.raises(ValueError): # TODO UnicodeDecodeError
        builtin.unicode('\124\323')

def test_no_convert():
    operator = import_module("operator")
    functools = import_module("functools")
    builtin = import_module("__builtin__", noconvert=True)

    lst = builtin.list()

    part = functools.partial(operator.iadd, lst)
    part([1, 2, 3])

    assert pypy_convert(lst.obj) == [1, 2, 3]
<commit_msg>check iter exceptions<commit_after># encoding: utf-8
import pytest
from pymetabiosis.module import import_module
from pymetabiosis.wrapper import MetabiosisWrapper, pypy_convert

def test_getattr_on_module():
    sqlite = import_module("sqlite3")
    assert isinstance(sqlite, MetabiosisWrapper)
    connect = sqlite.connect
    assert isinstance(connect, MetabiosisWrapper)
    assert repr(connect).startswith("<built-in function connect>")

def test_call_function():
    sqlite = import_module("sqlite3")
    connection = sqlite.connect(":memory:")
    assert repr(connection).startswith("<sqlite3.Connection object at ")

def test_pass_wrapper_to_function():
    sqlite = import_module("sqlite3")

    connection = sqlite.connect(":memory:")

    assert repr(connection).startswith("<sqlite3.Connection object at ")

def test_sqlite():
    # Taken from Python 2.7's sqlite doc
    sqlite3 = import_module("sqlite3")

    con = sqlite3.connect(":memory:")
    cur = con.cursor()
    cur.execute("create table people (name_last, age)")

    who = "Yeltsin"
    age = 72

    cur.execute("insert into people values (?, ?)", (who, age))

    cur.execute("select * from people where name_last=:who and age=:age", {"who": who, "age": age})

    ret = cur.fetchone()
    assert ret == (u'Yeltsin', 72)

def test_gettype():
    sqlite3 = import_module("sqlite3")

    assert repr(sqlite3.get_type()) == "<type 'module'>"

def test_convert_return_value():
    builtin = import_module("__builtin__")
    operator = import_module("operator")

    assert builtin.int(32) == 32
    assert builtin.float(3.123) == 3.123

    for s in ['a string']: # TODO 'a string \00yep']:
        assert builtin.str(s) == s

    u = u"some буквы are странные"
    assert builtin.unicode(u) == u

    t = (1, (2.3,))
    assert builtin.tuple(t) == t

    d = {'a': 'b', 1: 2}
    assert builtin.dict(d) == d

    lst = ['a', 1, [2]]
    assert builtin.list(lst) == lst

    assert builtin.bool(True) is True
    assert builtin.bool(False) is False

    assert builtin.bool(None) is False
    assert operator.eq(None, None) is True
    assert operator.eq(None, False) is False

def test_getitem_setitem_delitem():
    builtin = import_module("__builtin__", noconvert=True)

    d = builtin.dict({1: 'foo', (1, 'a'): 'zoo'})
    with pytest.raises(KeyError):
        d[2]
    assert pypy_convert(d[1].obj) == 'foo'
    assert pypy_convert(d[(1, 'a')].obj) == 'zoo'

    key, lst = (1, 2), ['a', 'b']
    d[key] = lst
    assert pypy_convert(d[key].obj) == lst

    with pytest.raises(TypeError):
        d[[1, 2]] = 0

    del d[1]
    with pytest.raises(KeyError):
        d[1]

    with pytest.raises(KeyError):
        del d[2]

def test_str_repr_dir():
    builtin = import_module("__builtin__", noconvert=True)

    assert str(builtin.None) == 'None'
    assert str(builtin.str('a')) == 'a'
    assert repr(builtin.str('a')) == "'a'"

    assert set(['rjust', 'rpartition', 'rstrip', '__le__'])\
            .issubset(dir(builtin.str('a')))

def test_len():
    builtin = import_module("__builtin__", noconvert=True)
    lst = builtin.list([1, 'a'])
    assert len(lst) == 2
    assert len(builtin.list()) == 0
    assert len(builtin.str('abc')) == 3

    with pytest.raises(TypeError):
        len(builtin.iter([1]))

def test_iter():
    builtin = import_module("__builtin__", noconvert=True)
    assert [pypy_convert(x.obj) for x in builtin.list([1, 'a'])] == [1, 'a']
    assert pypy_convert(list(builtin.iter(['a']))[0].obj) == 'a'
    with pytest.raises(TypeError):
        builtin.iter(1)

def test_exceptions():
    builtin = import_module("__builtin__")

    with pytest.raises(AttributeError):
        builtin.foo

    with pytest.raises(ValueError): # TODO UnicodeDecodeError
        builtin.unicode('\124\323')

def test_no_convert():
    operator = import_module("operator")
    functools = import_module("functools")
    builtin = import_module("__builtin__", noconvert=True)

    lst = builtin.list()

    part = functools.partial(operator.iadd, lst)
    part([1, 2, 3])

    assert pypy_convert(lst.obj) == [1, 2, 3]
<|endoftext|>


<commit_before>import json
import urllib2
import boto3
import botocore
from time import sleep, time
import datetime
import argparse
import logging
from operator import itemgetter
from subprocess import check_call
import os
from tempfile import gettempdir

def main():

    def list_all_png_images():
        client = boto3.resource('s3')
        pngs = []
        for obj in client.Bucket(BUCKET).objects.filter(Prefix='images/png/'):
            pngs.append(obj.key)
        return pngs

    def list_all_images_mentioned_in_lists():
        client = boto3.resource('s3')
        lists = list(
            client.Bucket(BUCKET).objects.filter(
                Prefix='images/list/'))
        images = []
        for l in lists:
            data = get_json_file_from_mirror(l.key)
            images += get_images_names_from_list(data)
        return images

    def invalidate_files(files):
        client = boto3.client('cloudfront')
        response = client.create_invalidation(
            DistributionId=DISTRIBUTION_ID,
            InvalidationBatch={
                'Paths': {
                    'Quantity': len(files),
                    'Items': files
                },
                'CallerReference': str(int(time()))
            }
        )
        invalidation_id = response['Invalidation']['Id']
        logging.info(
            'Created invalidation for files {files} with ID {id}'.format(
                files=files, id=invalidation_id))

    def file_exists(file):
        client = boto3.client('s3')
        try:
            client.head_object(
                Bucket=BUCKET,
                Key=file)
        except botocore.exceptions.ClientError as e:
            error_code = int(e.response['Error']['Code'])
            if error_code == 404:
                return False
        return True

    def get_available_dates():
        try:
            data = urllib2.urlopen(
                '{endpoint}/images.php?available_dates'.format(
                    endpoint=API))
            if data.code == 200:
                dates = json.loads(data.read())
                dates.reverse()
                return dates
        except:
            pass
        logging.info('Failed getting available dates')
        return None

    def get_list_by_date(date):
        try:
            data = urllib2.urlopen(
                '{endpoint}/images.php?date={date}'.format(
                    endpoint=API, date=date))
            if data.code == 200:
                return json.loads(data.read())
        except:
            pass
        logging.info('Failed getting list by date for date: {}'.format(date))
        return None

    def get_json_file_from_mirror(list_path):
        client = boto3.client('s3')
        try:
            data = client.get_object(
                Bucket=BUCKET, Key=list_path)['Body'].read()
        except botocore.exceptions.ClientError as e:
            logging.info(
                'Failed getting json file {} from mirror'.format(file))
            return None
        return json.loads(data)

    def get_mirror_list_by_date(date):
        path = 'images/list/images_{date}.json'.format(date=date)
        return get_json_file_from_mirror(path)

    def get_images_names_from_list(image_list):
        images = []
        for l in image_list:
            images.append(l['image'])
        return images

    def get_image_data(image_path):
        for i in range(RETRIES):
            try:
                data = urllib2.urlopen(image_path)
                if data.code == 200:
                    return data
            except:
                sleep(1)
        logging.info(
            'Failed getting image data for image path: {}'.format(image_path))
        return None

    def too_old_to_sync(date):
        now = datetime.datetime.today()
        test_date = datetime.datetime.strptime(date, '%Y-%m-%d')
        delta = now - test_date

        if delta.days > DAYS_TRACK_CHANGES:
            logging.info('Date {} too old'.format(date))
            return True

        return False

    def upload_data(data, key):
        client = boto3.client('s3')

        if isinstance(data, basestring):
            body = data
            content_type = 'text/html; charset=UTF-8'
        else:
            body = data.read()
            content_type = data.info()['Content-type']

        if not args.dryrun:
            client.put_object(
                Body=body,
                Bucket=BUCKET,
                Key=key,
                ContentType=content_type)

    def upload_file(file_name, key):
        client = boto3.client('s3')

        if not args.dryrun:
            client.upload_file(file_name, BUCKET, key)

    def process_images_in_list(image_list):
        images = get_images_names_from_list(image_list)
        failed_images = []
        for image in images:
            logging.info('image: {}'.format(image))
            png = '{endpoint}/png/{image}.png'.format(
                endpoint=ARCHIVE, image=image)
            png_key = 'images/png/{image}.png'.format(image=image)
            logging.info('png key: {}'.format(png_key))
            png_data = get_image_data(png)
            if png_data is None:
                logging.info('Failed image download')
                failed_images.append(image)
                continue
            base_name = os.path.abspath(
                os.path.join(gettempdir(), image))
            local_png_file_name = base_name + '.png'
            local_jpg_file_name = base_name + '.jpg'
            with open(local_png_file_name, 'wb') as f:
                f.write(png_data.read())
            for res in ['2048', '1024', '512', '256', '120']:
                res_string = '{res}x{res}'.format(res=res)
                cmd = 'convert {png} -resize {res} {jpg}'.format(
                    png=local_png_file_name,
                    res=res_string,
                    jpg=local_jpg_file_name)
                check_call(cmd, shell=True)
                # compatibility hack
                if res == '2048':
                    jpg_key = 'images/jpg/{}.jpg'.format(image)
                # NASA thumbnail
                elif res == '120':
                    jpg_key = 'images/thumbs/{}.jpg'.format(image)
                else:
                    jpg_key = 'images/jpg/{}/{}.jpg'.format(res, image)
                logging.info('jpg key: {}'.format(jpg_key))
                upload_file(local_jpg_file_name, jpg_key)
                os.remove(local_jpg_file_name)

            upload_file(local_png_file_name, png_key)
            os.remove(local_png_file_name)

        return failed_images

    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--full',
        help='Sync all dates',
        action='store_true')
    parser.add_argument(
        '--dryrun',
        help='Not writing to mirror',
        action='store_true')
    parser.add_argument(
        '--verbose',
        help='Print debug messages',
        action='store_true')
    parser.add_argument(
        '--dev',
        help='Use dev bucket',
        action='store_true')
    args = parser.parse_args()

    if args.verbose:
        logging.basicConfig(level=logging.INFO)

    if args.dev:
        BUCKET = 'dev.blueturn.earth'
    else:
        BUCKET = 'epic-archive-mirror'

    DISTRIBUTION_ID = 'E2NGB7E5BXXA9J'
    API = 'http://epic.gsfc.nasa.gov/api'
    ARCHIVE = 'http://epic.gsfc.nasa.gov/epic-archive'
    AVAILABLE_DATES_PATH_ON_MIRROR = 'images/available_dates.json'
    LATEST_IMAGES_PATH_ON_MIRROR = 'images/images_latest.json'
    RETRIES = 5
    DAYS_TRACK_CHANGES = 14

    files_to_invalidate = []

    dates = get_available_dates()
    dates_on_mirror = get_json_file_from_mirror(AVAILABLE_DATES_PATH_ON_MIRROR)
    if dates_on_mirror != dates:
        upload_data(
            json.dumps(dates,
                       indent=4),
            AVAILABLE_DATES_PATH_ON_MIRROR)
        files_to_invalidate.append('/' + AVAILABLE_DATES_PATH_ON_MIRROR)

    first_iteration = True
    for date in dates:
        logging.info('date: {}'.format(date))
        if not args.full and too_old_to_sync(date):
            break

        daily_image_list_from_archive = get_mirror_list_by_date(date)
        daily_image_list_from_api = get_list_by_date(date)

        list_of_images_to_download = []
        daily_image_list_to_archive = daily_image_list_from_api
        if args.full or daily_image_list_from_archive is None:
            logging.info('New list')
            list_of_images_to_download = daily_image_list_from_api
        else:
            logging.info('Existing list')
            archive_names = set(
                get_images_names_from_list(daily_image_list_from_archive))
            api_names = set(
                get_images_names_from_list(daily_image_list_from_api))
            new_images = api_names - archive_names
            for new_image in new_images:
                new_image_key = next(
                    (item for item in daily_image_list_from_api if item['image'] == new_image), None)
                new_image_key['new'] = True
                list_of_images_to_download.append(new_image_key)
            daily_image_list_to_archive = daily_image_list_from_archive + \
                list_of_images_to_download

        if len(list_of_images_to_download) > 0:
            failed_images = process_images_in_list(list_of_images_to_download)
            for item in list(daily_image_list_to_archive):
                # remove failed images from list
                if item['image'] in failed_images:
                    daily_image_list_to_archive.remove(item)
                else:
                    # fix coords single to double quotes
                    item['coords'] = item['coords'].replace("'", '"')
            logging.info('New images')
            list_path = 'images/list/images_{date}.json'.format(date=date)
            list_content = sorted(
                daily_image_list_to_archive,
                key=itemgetter('date'))
            upload_data(json.dumps(list_content, indent=4), list_path)
            files_to_invalidate.append('/' + list_path)
            if first_iteration:
                upload_data(
                    json.dumps(list_content, indent=4), LATEST_IMAGES_PATH_ON_MIRROR)
                files_to_invalidate.append('/' + LATEST_IMAGES_PATH_ON_MIRROR)

        first_iteration = False

    if len(files_to_invalidate) > 0:
        invalidate_files(files_to_invalidate)

if __name__ == '__main__':
    main()
<commit_msg>new code<commit_after>import argparse
import urllib2
import logging
import json
import boto3
import os
from tempfile import gettempdir
from subprocess import check_call
import numpy as np
import cv2
import math
import random
import sys
from time import time

class Epic:

    def __init__(self, args, config):
        self.args = args
        self.config = config
        self.s3 = boto3.client('s3')
        self.invalidate_paths = []

    def invalidate(self):
        if not self.args.dryrun and len(self.invalidate_paths) > 0:
            paths = self.invalidate_paths
            client = boto3.client('cloudfront')
            response = client.create_invalidation(
                DistributionId=self.config['distribution_id'],
                InvalidationBatch={
                    'Paths': {
                        'Quantity': len(paths),
                        'Items': paths
                    },
                    'CallerReference': str(int(time()))
                }
            )
            invalidation_id = response['Invalidation']['Id']
            logging.info(
                'Created invalidation for files {} with ID {}'.format(
                    paths, invalidation_id))

    def _read_file_from_mirror(self, bucket, key):
        try:
            return self.s3.get_object(
                Bucket=bucket, Key=key)['Body'].read()
        except botocore.exceptions.ClientError as e:
            logging.info(
                'error reading file from s3://{}/{}'.format(bucket, key))
        return None

    def _read_file_from_url(self, url):
        try:
            data = urllib2.urlopen(url)
            if data.code == 200:
                return data.read()
        except:
            logging.info('error reading file from ' + url)
        return None

    def _read_json(self, data):
        try:
            return json.loads(data)
        except:
            return None

    def missing_dates(self):
        ret = []
        url = self.config['api_url'] + '/images.php?available_dates'
        data = self._read_file_from_url(url)
        dates_from_api = self._read_json(data)
        if self.args.full:
            ret = dates_from_api
        else:
            data = self._read_file_from_mirror(
                self.config['bucket'],
                self.config['available_dates_path'])
            dates_from_mirror = self._read_json(data)
            ret = sorted(set(dates_from_api) - set(dates_from_mirror))
        if len(ret) > 0:
            self._upload_data(
                json.dumps(dates_from_api, indent=4),
                self.config['bucket'],
                self.config['available_dates_path'],
                'application/json')
            self.invalidate_paths.append('/' + self.config['available_dates_path'])
        return ret

    def image_list(self, date):
        url = '{}/images.php?date={}'.format(self.config['api_url'], date)
        data = self._read_file_from_url(url)
        return self._read_json(data)

    def _upload_file(self, path, bucket, key):
        if not self.args.dryrun:
            self.s3.upload_file(path, bucket, key)

    def _upload_data(self, body, bucket, key, content_type):
        if not self.args.dryrun:
            self.s3.put_object(
                Body=body,
                Bucket=bucket,
                Key=key,
                ContentType=content_type)

    def png(self, image_name):
        url = '{}/png/{}.png'.format(self.config['archive_url'], image_name)
        logging.info('Downloading ' + url)
        data = self._read_file_from_url(url)
        filename = os.path.join(gettempdir(), image_name + '.png')
        with open(filename, 'wb') as f:
            f.write(data)
        key = 'images/png/{}.png'.format(image_name)
        logging.info(
            'Uploading to s3://{}/{}'.format(self.config['bucket'], key))
        self._upload_file(filename, self.config['bucket'], key)

    def jpgs(self, image_name):
        for res in self.config['res']:
            res_string = '{res}x{res}'.format(res=res)
            infile = os.path.join(gettempdir(), image_name + '.png')
            outfile = os.path.join(gettempdir(), image_name + '.jpg')
            cmd = 'convert {} -resize {} {}'.format(
                infile, res_string, outfile)
            check_call(cmd, shell=True)
            # compatibility hack
            if res == '2048':
                key = 'images/jpg/{}.jpg'.format(image_name)
            # NASA thumbnail
            elif res == '120':
                key = 'images/thumbs/{}.jpg'.format(image_name)
            else:
                key = 'images/jpg/{}/{}.jpg'.format(res, image_name)
            logging.info(
                'Uploading to s3://{}/{}'.format(self.config['bucket'], key))
            self._upload_file(outfile, self.config['bucket'], key)
            os.remove(outfile)

    def bounding_shapes(self, image_name):
        filename = os.path.join(gettempdir(), image_name + '.png')
        im = cv2.imread(filename, 0)
        height, width = im.shape
        ret, thresh = cv2.threshold(im, 10, 255, cv2.THRESH_BINARY)
        contours, hierarchy = cv2.findContours(
            thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
        area = 0
        idx = 0
        max_area = math.pi * (height / 2)**2
        for index, item in enumerate(contours):
            a = cv2.contourArea(item)
            if a > area and a < max_area:
                area = a
                idx = index
        cnt = contours[idx]
        center, radius = cv2.minEnclosingCircle(cnt)
        cx = center[0] / 2048
        cy = center[1] / 2048
        r = radius / 2048
        logging.info('Circle center, radius: {}, {}'.format((cx, cy), r))
        (ex, ey), (MA, ma), angle = cv2.fitEllipse(cnt)
        points = cv2.ellipse2Poly(
            (int(ex), int(ey)), (int(MA / 2), int(ma / 2)), int(angle), 0, 360, 1)
        npoints = []
        for point in random.sample(points, 5):
            npoints.append((float(point[0]) / 2048, float(point[0]) / 2048))
        cache = {
            'jpg': {
                    'earth_circle': {
                        'center': {'x': cx, 'y': cy},
                        'radius': r
                    },
                    'earth_ellipse': {'points': npoints}
            },
                'png': {
                    'earth_circle': {
                        'center': {'x': cx, 'y': cy},
                        'radius': r
                    },
                    'earth_ellipse': {'points': npoints}
                }
        }
        return cache

    def run(self):
        first = True
        # start from the latest pictures.
        dates = sorted(self.missing_dates(), reverse=True)
        for date in dates:
            images_json = self.image_list(date)
            for image in images_json:
                image_name = image['image']
                self.png(image_name)
                self.jpgs(image_name)
                image['cache'] = self.bounding_shapes(image_name)
                # delete png
                os.remove(os.path.join(gettempdir(), image_name + '.png'))
                # fix json coming from the api
                image['coords'] = image['coords'].replace("'", '"')
            # save latest date json to mirror
            if first:
                self._upload_data(
                    json.dumps(images_json, indent=4),
                    self.config['bucket'],
                    self.config['latest_images_path'],
                    'application/json')
                self.invalidate_paths.append('/' + self.config['latest_images_path'])
            first = False
        self.invalidate()

def main():
    def _parse_arguments():
        parser = argparse.ArgumentParser()
        parser.add_argument(
            '--full',
            help='Full sync',
            action='store_true')
        parser.add_argument(
            '--dryrun',
            help='Not writing to mirror',
            action='store_true')
        parser.add_argument(
            '--verbose',
            help='Print debug messages',
            action='store_true')
        parser.add_argument(
            '--dev',
            help='Use dev bucket',
            action='store_true')
        return parser.parse_args()

    def _config(args):
        if args.verbose:
            logging.basicConfig(level=logging.INFO)

        config = {}

        if args.dev:
            config['bucket'] = 'dev.blueturn.earth'
        else:
            config['bucket'] = 'epic-archive-mirror'

        config['distribution_id'] = 'E2NGB7E5BXXA9J'
        base_url = 'http://epic.gsfc.nasa.gov'
        config['api_url'] = base_url + '/api'
        config['archive_url'] = base_url + '/epic-archive'
        config['available_dates_path'] = 'images/available_dates.json'
        config['latest_images_path'] = 'images/images_latest.json'
        config['retries'] = 5
        config['res'] = ['2048', '1024', '512', '256', '120']
        return config

    args = _parse_arguments()
    config = _config(args)

    epic = Epic(args, config)
    epic.run()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python

import os, sys, random, csv, time
import numpy as np
import datetime as dt
from pyoperant import utils, components, local, hwio
from pyoperant.tricks import two_alt_choice

try:
    import simplejson as json
except ImportError:
    import json

class NumpyAwareJSONEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, np.ndarray):
                return obj.tolist()
        return json.JSONEncoder.default(self, obj)

class EvidenceAccumExperiment(two_alt_choice.TwoAltChoiceExp):
    """docstring for Experiment"""
    def __init__(self, *args, **kwargs):
        super(EvidenceAccumExperiment, self).__init__(*args, **kwargs)
        self.build_transition_matrices()

    def save(self):
        self.snapshot_f = os.path.join(self.parameters['experiment_path'], self.timestamp+'.json')
        with open(self.snapshot_f, 'wb') as config_snap:
            json.dump(self.parameters,
                      config_snap,
                      sort_keys=True,
                      indent=4,
                      cls=NumpyAwareJSONEncoder)

    def build_transition_matrices(self):
        """run through the transitions for each stimulus and generate transition matrixes"""
        n = len(self.parameters['stim_map'])

        for stim_class, class_params in self.parameters['classes'].items():
            arr = np.zeros((n,n),np.float_)
            for a,b in class_params['transitions']:
                arr[a,b] = 1.0

            self.parameters['classes'][stim_class]['evidence_arrays'] = arr
            self.log.debug('added class %s evidence array %s' % (stim_class,arr))

        stim_classes = self.parameters['classes'].keys()
        assert len(stim_classes) == 2

        for this_class in stim_classes:
            other_class = [cl for cl in stim_classes if (cl != this_class)][0]
            trans = self.parameters['odds'] \
                  * self.parameters['classes'][this_class]['evidence_arrays'] \
                  + self.parameters['classes'][other_class]['evidence_arrays']
            trans[0,1:] = 1.0
            trans = np.cumsum(trans,axis=1)
            trans = trans / trans[:,-1][:,None] # I don't get why it suddenly works to append '[:,None]'
            self.parameters['classes'][this_class]['transition_cdf'] = trans
            self.log.debug('added class %s transition cdf %s' % (this_class, trans))

    def get_stimuli(self,trial_class):
        """ take trial class and return a tuple containing the stimulus event to play and a list of additional events

        """

        input_files = []
        motif_ids = []

        # use transition CDF to get iteratively get next motif id
        mid = 0
        for pos in range(self.parameters['strlen_max']):
            mid = (self.parameters['classes'][trial_class]['transition_cdf'][mid] < random.random()).sum()
            motif_ids.append(mid)
        assert len(motif_ids) == self.parameters['strlen_max']

        motifs = [self.parameters['stim_map'][mid] for mid in motif_ids]

        motif_isi = [max(random.gauss(self.parameters['isi_mean'], self.parameters['isi_stdev']),0.0) for mot in motifs]
        motif_isi[-1] = 0.0

        input_files = [(self.parameters['stims'][motif_name], isi) for motif_name, isi in zip(motifs,motif_isi)]
        filename =  os.path.join(self.parameters['stim_path'], ''.join(motifs) + '.wav')
        stim, epochs = utils.concat_wav(input_files,filename)
        stim.name = ''.join(motifs)

        for ep in epochs:
            self.log.debug('old epoch.name: %s' % ep.name)
            for stim_name,f_name in self.parameters['stims'].items():
                if ep.name == f_name:
                    ep.name = stim_name
            self.log.debug('new epoch.name: %s' % ep.name)

        stim.name = ''.join(motifs)

        return stim, epochs

    def trial_pre(self):
        ''' this is where we initialize a trial'''
        # make sure lights are on at the beginning of each trial, prep for trial

        self.new_trial()

        self.this_trial = self.trials[-1]
        motifs = [m for m in self.this_trial.events if (m.label=='motif')]
        min_motif = motifs[self.parameters['strlen_min']-1]
        self.this_trial.annotate(min_wait=min_motif.time+min_motif.duration)
        max_wait = self.this_trial.stimulus_event.duration + self.parameters['response_win']
        self.this_trial.annotate(max_wait=max_wait)
        self.log.debug('created new trial')
        return 'main'

    def analyze_trial(self):
        '''after the trial is complete, perform additional analyses that will be saved'''

        for event in self.this_trial.events:
            event.time += self.this_trial.stimulus_event.time

        # determine the string of motifs the bird heard
        num_mots = 0
        for ev in list(self.this_trial.events):
            if (ev.label=='motif'):
                if (self.this_trial.rt > ev.time) or (self.this_trial.response == 'none'):
                    num_mots += 1
                else:
                    self.this_trial.events.remove(ev) # get rid of motif events the bird didn't hear

        self.this_trial.stimulus = stim_string
        os.remove(self.this_trial.stimulus_event.file_origin)
        self.this_trial.stimulus = self.this_trial.stimulus[:num_mots]

if __name__ == "__main__":

    try: import simplejson as json
    except ImportError: import json

    from pyoperant.local import PANELS

    cmd_line = utils.parse_commandline()
    with open(cmd_line['config_file'], 'rb') as config:
            parameters = json.load(config)

    if parameters['debug']:
        print parameters
        print PANELS

    panel = PANELS[parameters['panel_name']]()

    exp = EvidenceAccumExperiment(panel=panel,**parameters)
    exp.run()

<commit_msg>remove reference to stim_string<commit_after>#!/usr/bin/python

import os, sys, random, csv, time
import numpy as np
import datetime as dt
from pyoperant import utils, components, local, hwio
from pyoperant.tricks import two_alt_choice

try:
    import simplejson as json
except ImportError:
    import json

class NumpyAwareJSONEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, np.ndarray):
                return obj.tolist()
        return json.JSONEncoder.default(self, obj)

class EvidenceAccumExperiment(two_alt_choice.TwoAltChoiceExp):
    """docstring for Experiment"""
    def __init__(self, *args, **kwargs):
        super(EvidenceAccumExperiment, self).__init__(*args, **kwargs)
        self.build_transition_matrices()

    def save(self):
        self.snapshot_f = os.path.join(self.parameters['experiment_path'], self.timestamp+'.json')
        with open(self.snapshot_f, 'wb') as config_snap:
            json.dump(self.parameters,
                      config_snap,
                      sort_keys=True,
                      indent=4,
                      cls=NumpyAwareJSONEncoder)

    def build_transition_matrices(self):
        """run through the transitions for each stimulus and generate transition matrixes"""
        n = len(self.parameters['stim_map'])

        for stim_class, class_params in self.parameters['classes'].items():
            arr = np.zeros((n,n),np.float_)
            for a,b in class_params['transitions']:
                arr[a,b] = 1.0

            self.parameters['classes'][stim_class]['evidence_arrays'] = arr
            self.log.debug('added class %s evidence array %s' % (stim_class,arr))

        stim_classes = self.parameters['classes'].keys()
        assert len(stim_classes) == 2

        for this_class in stim_classes:
            other_class = [cl for cl in stim_classes if (cl != this_class)][0]
            trans = self.parameters['odds'] \
                  * self.parameters['classes'][this_class]['evidence_arrays'] \
                  + self.parameters['classes'][other_class]['evidence_arrays']
            trans[0,1:] = 1.0
            trans = np.cumsum(trans,axis=1)
            trans = trans / trans[:,-1][:,None] # I don't get why it suddenly works to append '[:,None]'
            self.parameters['classes'][this_class]['transition_cdf'] = trans
            self.log.debug('added class %s transition cdf %s' % (this_class, trans))

    def get_stimuli(self,trial_class):
        """ take trial class and return a tuple containing the stimulus event to play and a list of additional events

        """

        input_files = []
        motif_ids = []

        # use transition CDF to get iteratively get next motif id
        mid = 0
        for pos in range(self.parameters['strlen_max']):
            mid = (self.parameters['classes'][trial_class]['transition_cdf'][mid] < random.random()).sum()
            motif_ids.append(mid)
        assert len(motif_ids) == self.parameters['strlen_max']

        motifs = [self.parameters['stim_map'][mid] for mid in motif_ids]

        motif_isi = [max(random.gauss(self.parameters['isi_mean'], self.parameters['isi_stdev']),0.0) for mot in motifs]
        motif_isi[-1] = 0.0

        input_files = [(self.parameters['stims'][motif_name], isi) for motif_name, isi in zip(motifs,motif_isi)]
        filename =  os.path.join(self.parameters['stim_path'], ''.join(motifs) + '.wav')
        stim, epochs = utils.concat_wav(input_files,filename)
        stim.name = ''.join(motifs)

        for ep in epochs:
            self.log.debug('old epoch.name: %s' % ep.name)
            for stim_name,f_name in self.parameters['stims'].items():
                if ep.name == f_name:
                    ep.name = stim_name
            self.log.debug('new epoch.name: %s' % ep.name)

        stim.name = ''.join(motifs)

        return stim, epochs

    def trial_pre(self):
        ''' this is where we initialize a trial'''
        # make sure lights are on at the beginning of each trial, prep for trial

        self.new_trial()

        self.this_trial = self.trials[-1]
        motifs = [m for m in self.this_trial.events if (m.label=='motif')]
        min_motif = motifs[self.parameters['strlen_min']-1]
        self.this_trial.annotate(min_wait=min_motif.time+min_motif.duration)
        max_wait = self.this_trial.stimulus_event.duration + self.parameters['response_win']
        self.this_trial.annotate(max_wait=max_wait)
        self.log.debug('created new trial')
        return 'main'

    def analyze_trial(self):
        '''after the trial is complete, perform additional analyses that will be saved'''

        for event in self.this_trial.events:
            event.time += self.this_trial.stimulus_event.time

        # determine the string of motifs the bird heard
        num_mots = 0
        for ev in list(self.this_trial.events):
            if (ev.label=='motif'):
                if (self.this_trial.rt > ev.time) or (self.this_trial.response == 'none'):
                    num_mots += 1
                else:
                    self.this_trial.events.remove(ev) # get rid of motif events the bird didn't hear

        os.remove(self.this_trial.stimulus_event.file_origin)
        self.this_trial.stimulus = self.this_trial.stimulus[:num_mots]

if __name__ == "__main__":

    try: import simplejson as json
    except ImportError: import json

    from pyoperant.local import PANELS

    cmd_line = utils.parse_commandline()
    with open(cmd_line['config_file'], 'rb') as config:
            parameters = json.load(config)

    if parameters['debug']:
        print parameters
        print PANELS

    panel = PANELS[parameters['panel_name']]()

    exp = EvidenceAccumExperiment(panel=panel,**parameters)
    exp.run()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
""" Contains the AutoCompleteMode """
import logging
from pyqode.qt import QtCore
from pyqode.core.api import TextHelper
from pyqode.core.api.mode import Mode

class AutoCompleteMode(Mode):
    """ Automatically complete quotes and parentheses

    Generic auto complete mode that automatically completes the following
    symbols:

        - " -> "
        - ' -> '
        - ( -> )
        - [ -> ]
        - { -> }
    """
    #: Auto complete mapping, maps input key with completion text.
    MAPPING = {'"': '"', "'": "'", "(": ")", "{": "}", "[": "]"}
    #: The format to use for each symbol in mapping when there is a selection
    SELECTED_QUOTES_FORMATS = {key: '%s%s%s' for key in MAPPING.keys()}
    #: The format to use for each symbol in mapping when there is no selection
    QUOTES_FORMATS = {key: '%s' for key in MAPPING.keys()}

    def __init__(self):
        super(AutoCompleteMode, self).__init__()
        self.logger = logging.getLogger(__name__)
        self._ignore_post = False

    def on_state_changed(self, state):
        if state:
            self.editor.post_key_pressed.connect(self._on_post_key_pressed)
            self.editor.key_pressed.connect(self._on_key_pressed)
        else:
            self.editor.post_key_pressed.disconnect(self._on_post_key_pressed)
            self.editor.key_pressed.disconnect(self._on_key_pressed)

    def _on_post_key_pressed(self, event):
        if not event.isAccepted() and not self._ignore_post:
            txt = event.text()
            next_char = TextHelper(self.editor).get_right_character()
            if txt in self.MAPPING:
                to_insert = self.MAPPING[txt]
                if (not next_char or next_char in self.MAPPING.keys() or
                        next_char in self.MAPPING.values() or
                        next_char.isspace()):
                    TextHelper(self.editor).insert_text(
                        self.QUOTES_FORMATS[txt] % to_insert)
        self._ignore_post = False

    def _on_key_pressed(self, event):
        txt = event.text()
        cursor = self.editor.textCursor()
        from pyqode.qt import QtGui
        assert isinstance(cursor, QtGui.QTextCursor)
        if cursor.hasSelection():
            # quoting of selected text
            if event.text() in self.MAPPING.keys():
                first = event.text()
                last = self.MAPPING[event.text()]
                cursor.insertText(
                    self.SELECTED_QUOTES_FORMATS[event.text()] % (
                        first, cursor.selectedText(), last))
                self.editor.setTextCursor(cursor)
                event.accept()
            else:
                self._ignore_post = True
            return
        next_char = TextHelper(self.editor).get_right_character()
        self.logger.debug('next char: %s', next_char)
        ignore = False
        if event.key() == QtCore.Qt.Key_Backspace:
            # get the character that will get deleted
            tc = self.editor.textCursor()
            tc.movePosition(tc.PreviousCharacter)
            tc.movePosition(tc.NextCharacter, tc.KeepAnchor)
            del_char = tc.selectedText()
            if self.MAPPING[del_char] == next_char:
                tc.beginEditBlock()
                tc.movePosition(tc.NextCharacter, tc.KeepAnchor, 2)
                tc.removeSelectedText()
                tc.endEditBlock()
                ignore = True
        elif txt and next_char == txt and next_char in self.MAPPING:
            ignore = True
        elif event.text() == ')' or event.text() == ']' or event.text() == '}':
            if next_char == ')' or next_char == ']' or next_char == '}':
                ignore = True
        if ignore:
            event.accept()
            TextHelper(self.editor).clear_selection()
            TextHelper(self.editor).move_right()
<commit_msg>Fix a few corner case for the backspace autocompletion<commit_after># -*- coding: utf-8 -*-
""" Contains the AutoCompleteMode """
import logging
from pyqode.qt import QtCore
from pyqode.core.api import TextHelper
from pyqode.core.api.mode import Mode

class AutoCompleteMode(Mode):
    """ Automatically complete quotes and parentheses

    Generic auto complete mode that automatically completes the following
    symbols:

        - " -> "
        - ' -> '
        - ( -> )
        - [ -> ]
        - { -> }
    """
    #: Auto complete mapping, maps input key with completion text.
    MAPPING = {'"': '"', "'": "'", "(": ")", "{": "}", "[": "]"}
    #: The format to use for each symbol in mapping when there is a selection
    SELECTED_QUOTES_FORMATS = {key: '%s%s%s' for key in MAPPING.keys()}
    #: The format to use for each symbol in mapping when there is no selection
    QUOTES_FORMATS = {key: '%s' for key in MAPPING.keys()}

    def __init__(self):
        super(AutoCompleteMode, self).__init__()
        self.logger = logging.getLogger(__name__)
        self._ignore_post = False

    def on_state_changed(self, state):
        if state:
            self.editor.post_key_pressed.connect(self._on_post_key_pressed)
            self.editor.key_pressed.connect(self._on_key_pressed)
        else:
            self.editor.post_key_pressed.disconnect(self._on_post_key_pressed)
            self.editor.key_pressed.disconnect(self._on_key_pressed)

    def _on_post_key_pressed(self, event):
        if not event.isAccepted() and not self._ignore_post:
            txt = event.text()
            next_char = TextHelper(self.editor).get_right_character()
            if txt in self.MAPPING:
                to_insert = self.MAPPING[txt]
                if (not next_char or next_char in self.MAPPING.keys() or
                        next_char in self.MAPPING.values() or
                        next_char.isspace()):
                    TextHelper(self.editor).insert_text(
                        self.QUOTES_FORMATS[txt] % to_insert)
        self._ignore_post = False

    def _on_key_pressed(self, event):
        txt = event.text()
        cursor = self.editor.textCursor()
        from pyqode.qt import QtGui
        assert isinstance(cursor, QtGui.QTextCursor)
        if cursor.hasSelection():
            # quoting of selected text
            if event.text() in self.MAPPING.keys():
                first = event.text()
                last = self.MAPPING[event.text()]
                cursor.insertText(
                    self.SELECTED_QUOTES_FORMATS[event.text()] % (
                        first, cursor.selectedText(), last))
                self.editor.setTextCursor(cursor)
                event.accept()
            else:
                self._ignore_post = True
            return
        next_char = TextHelper(self.editor).get_right_character()
        self.logger.debug('next char: %s', next_char)
        ignore = False
        if event.key() == QtCore.Qt.Key_Backspace:
            # get the character that will get deleted
            tc = self.editor.textCursor()
            pos = tc.position()
            tc.movePosition(tc.Left)
            tc.movePosition(tc.Right, tc.KeepAnchor)
            del_char = tc.selectedText()
            if del_char in self.MAPPING and self.MAPPING[del_char] == next_char:
                tc.beginEditBlock()
                tc.movePosition(tc.Right, tc.KeepAnchor)
                tc.insertText('')
                tc.setPosition(pos - 2)
                tc.endEditBlock()
                self.editor.setTextCursor(tc)
                ignore = True
        elif txt and next_char == txt and next_char in self.MAPPING:
            ignore = True
        elif event.text() == ')' or event.text() == ']' or event.text() == '}':
            if next_char == ')' or next_char == ']' or next_char == '}':
                ignore = True
        if ignore:
            event.accept()
            TextHelper(self.editor).clear_selection()
            TextHelper(self.editor).move_right()
<|endoftext|>


<commit_before>from unittest import TestCase

from ..DAG import  Multigraph
from ..components import GeneratorSource, ShowInputs, BroadcastApplyFunction, ConstantSource, Filter, OneOffProcess
from .. import components
from ..nodes import Component
import asyncio
from ..utils import InputPort, OutputPort,ArrayPort, NewFilePort

async def printer(**kwargs):
    print("Inputs are {}".format(kwargs))
    return

def adder(**kwargs):
    out = sum([item for k, item in kwargs.items() if item])
    return out

async def passer(**kwargs):
    await asyncio.sleep(0.2)
    print(kwargs)
    return {'delayed': kwargs.values()}

source_gen = (i for i in range(100))

class TestMultigraph(TestCase):

    def testNormalPort(self):
        c1 = Component('c1')
        c1.outputs.add(OutputPort('a'))
        c2 = Component('c2')
        c2.inputs.add(InputPort('b'))
        graph = Multigraph()
        graph.connect(c1.outputs['a'], c2.inputs['b'])
        with open('/home/baffelli/normal.dot','w+') as outfile:
            outfile.write(graph.dot())

    def testArrayPort(self):
        c1 = Component('c1')
        c1.outputs.add(OutputPort('a'))
        c2 = Component('c2')
        c2.inputs.add(InputPort('b'))
        c2.inputs.add(InputPort('c'))
        graph = Multigraph()
        graph.connect(c1.outputs['a'], c2.inputs['b'])
        graph.connect(c1.outputs['a'], c2.inputs['c'])
        print(list(graph.iterarcs()))
        with open('/home/baffelli/multiport.dot','w+') as outfile:
            outfile.write(graph.dot())

    def testPortRegister(self):
        c1 = Component('c1')
        c1.outputs.add(OutputPort('a'))
        c1.outputs.add(OutputPort('b'))
        self.assertEquals(c1.outputs.a, c1.outputs['a'])

    def testSendReceive(self):
        c1 = Component('c1')
        c1.outputs.add(OutputPort('a'))
        c2 = Component('c2')
        c2.outputs.add(OutputPort('b'))
        c3 = Component('c3')
        c3.inputs.add(InputPort('in1'))
        c3.inputs.add(InputPort('in2'))
        graph = Multigraph()
        graph.connect(c1.outputs['a'], c3.inputs['in1'])
        graph.connect(c2.outputs['b'], c3.inputs['in2'])
        # graph.set_initial_packet(c3.inputs['in1'], 6)
        async def send(messages):
            for m in messages:
                await asyncio.sleep(0.2)
                [asyncio.ensure_future(c1.outputs['a'].send_to_all(m)), asyncio.ensure_future(c2.outputs['b'].send_to_all(m))]

            asyncio.ensure_future(c1.outputs['a'].close())
            asyncio.ensure_future(c2.outputs['b'].close())
        async def receive():
            while True:
                print(c3.inputs['in1'].queue)
                res, doing = await asyncio.wait([c3.inputs['in1'].receive(),c3.inputs['in2'].receive()], return_when=asyncio.ALL_COMPLETED)
                print('done receiving')
                print(res.pop().result(),res.pop().result())
                # await asyncio.sleep(0)
        futures = [asyncio.ensure_future(send([1,2,3,4,5])),asyncio.ensure_future(receive())]
        loop = asyncio.get_event_loop()
        loop.run_until_complete(futures[1])

    def testLinearPipeline(self):
        source = GeneratorSource('s', source_gen)
        shower = ShowInputs('printer')
        shower.inputs.add(InputPort('in1'))
        graph = Multigraph()
        graph.connect(source.outputs['OUT'], shower.inputs['in1'])
        # graph.set_initial_packet(shower.inputs['in1'],5)

        graph()

    def testSumPipeline(self):
        source1 = GeneratorSource('s1',  (i for i in range(100)))
        source2 = GeneratorSource('s2',  (i for i in range(100)))
        shower = ShowInputs('printer')
        shower.inputs.add(InputPort('in1'))
        summer = BroadcastApplyFunction('summer', adder )
        summer.inputs.add(InputPort('g1'))
        summer.inputs.add(InputPort('g2'))
        summer.outputs.add(OutputPort('sum'))
        graph = Multigraph()
        graph.connect(source1.outputs.OUT, summer.inputs.g1)
        graph.connect(source2.outputs.OUT, summer.inputs.g2)
        graph.connect(summer.outputs.sum, shower.inputs.in1)
        with open('/home/baffelli/sum.dot','w+') as outfile:
            outfile.write(graph.dot())
        graph()

    def testRecursivePipeline(self):
        source1 = GeneratorSource('s1',  (1 for i in range(1000)))
        shower = ShowInputs('printer')
        shower.inputs.add(InputPort('in1'))
        summer = BroadcastApplyFunction('summer', adder )
        summer.inputs.add(InputPort('g1'))
        summer.inputs.add(InputPort('recursion'))
        summer.outputs.add(OutputPort('sum'))
        graph = Multigraph()
        graph.connect(source1.outputs.OUT, summer.inputs.g1)
        graph.connect(summer.outputs.sum, shower.inputs.in1)
        graph.connect(summer.outputs.sum, summer.inputs.recursion)
        graph.set_initial_packet(summer.inputs.recursion, 1)
        with open('/home/baffelli/recursion.dot','w+') as outfile:
            outfile.write(graph.dot())
        graph()

    def testConstantSource(self):
        source1 = ConstantSource('s1', 3)
        shower = ShowInputs('printer')
        shower.inputs.add(InputPort('in1'))
        graph = Multigraph()
        graph.connect(source1.outputs['OUT'], shower.inputs.in1)
        graph()

    def testOneOffProcess(self):
        source1 = ConstantSource('s1', 3)
        source2 = OneOffProcess('of', lambda kwargs: 5)
        source2.outputs.add(OutputPort('s2'))
        summer = BroadcastApplyFunction('summer', adder )
        summer.inputs.add(InputPort('g1'))
        summer.inputs.add(InputPort('g2'))
        summer.outputs.add(OutputPort('sum'))
        shower = ShowInputs('printer', inputs=['in1'])
        graph = Multigraph()
        graph.connect(source1.outputs['OUT'], summer.inputs.g1)
        graph.connect(source2.outputs['s2'], summer.inputs.g2)
        graph.connect(summer.outputs.sum, shower.inputs.in1)
        graph()

    def testInfiniteRecursion(self):
        source1 = ConstantSource('s1', 3)
        shower = ShowInputs('printer', inputs=['in1'])
        summer = BroadcastApplyFunction('summer', adder )
        summer.inputs.add(InputPort('in1'))
        summer.inputs.add(InputPort('recursion'))
        summer.outputs.add(OutputPort('sum'))
        graph = Multigraph()
        graph.connect(source1.outputs.OUT, summer.inputs.in1)
        graph.connect(summer.outputs.sum, summer.inputs.recursion)
        graph.set_initial_packet(summer.inputs.recursion, 0)
        graph.connect(summer.outputs.sum, shower.inputs.in1)
        graph()

    def testSplit(self):
        source1 = ConstantSource('s1', (3,5))
        splitter = components.Split('split in two')
        splitter.outputs.add(OutputPort('a'))
        splitter.outputs.add(OutputPort('b'))
        shower = ShowInputs('printer')
        shower.inputs.add(InputPort('a'))
        shower.inputs.add(InputPort('b'))
        graph = Multigraph()
        graph.connect(source1.outputs.OUT, splitter.inputs.IN)
        graph.connect(splitter.outputs.a, shower.inputs.a)
        graph.connect(splitter.outputs.b, shower.inputs.b)
        graph()

    def testFilter(self):
        def filter_predicate(in1=None):
            if in1 % 2 ==0:
                return True
            else:
                return False
        source1 = GeneratorSource('s1', (i for i in range(100)))
        filt = Filter('filtrator',  filter_predicate,)
        filt.inputs.add(InputPort('in1'))
        filt.outputs.add(OutputPort('out1'))
        shower = ShowInputs('printer', inputs=['in1'])
        graph = Multigraph()
        graph.connect(source1.outputs['OUT'], filt.inputs.in1)
        graph.connect(filt.outputs.out1, shower.inputs.in1)
        graph()

    def testShell(self):
        shell = components.Shell('shell', 'echo a > {outputs.f1}')
        shell.outputs.add(NewFilePort('f1'))
        printer = ShowInputs('show path')
        printer.inputs.add(NewFilePort('f1'))
        graph = Multigraph()
        graph.connect(shell.outputs.)
<commit_msg>Added tests for dynamic output files.<commit_after>from unittest import TestCase

from ..DAG import  Multigraph
from ..components import GeneratorSource, ShowInputs, BroadcastApplyFunction, ConstantSource, Filter, OneOffProcess
from .. import components
from ..nodes import Component
import asyncio
from ..utils import InputPort, OutputPort,ArrayPort, FilePort

import os
import uuid

async def printer(**kwargs):
    print("Inputs are {}".format(kwargs))
    return

def adder(**kwargs):
    out = sum([item for k, item in kwargs.items() if item])
    return out

async def passer(**kwargs):
    await asyncio.sleep(0.2)
    print(kwargs)
    return {'delayed': kwargs.values()}

source_gen = (i for i in range(100))

class TestMultigraph(TestCase):

    def testNormalPort(self):
        c1 = Component('c1')
        c1.outputs.add(OutputPort('a'))
        c2 = Component('c2')
        c2.inputs.add(InputPort('b'))
        graph = Multigraph()
        graph.connect(c1.outputs['a'], c2.inputs['b'])
        with open('/home/baffelli/normal.dot','w+') as outfile:
            outfile.write(graph.dot())

    def testArrayPort(self):
        c1 = Component('c1')
        c1.outputs.add(OutputPort('a'))
        c2 = Component('c2')
        c2.inputs.add(InputPort('b'))
        c2.inputs.add(InputPort('c'))
        graph = Multigraph()
        graph.connect(c1.outputs['a'], c2.inputs['b'])
        graph.connect(c1.outputs['a'], c2.inputs['c'])
        print(list(graph.iterarcs()))
        with open('/home/baffelli/multiport.dot','w+') as outfile:
            outfile.write(graph.dot())

    def testPortUniqueness(self):
        """Test if ports with the same name are unique"""
        p1 = InputPort('b')
        p2 = InputPort('b')
        self.assertFalse(p1 == p2)

    def testPortRegister(self):
        c1 = Component('c1')
        c1.outputs.add(OutputPort('a'))
        c1.outputs.add(OutputPort('b'))
        self.assertEquals(c1.outputs.a, c1.outputs['a'])

    def testSendReceive(self):
        c1 = Component('c1')
        c1.outputs.add(OutputPort('a'))
        c2 = Component('c2')
        c2.outputs.add(OutputPort('b'))
        c3 = Component('c3')
        c3.inputs.add(InputPort('in1'))
        c3.inputs.add(InputPort('in2'))
        graph = Multigraph()
        graph.connect(c1.outputs['a'], c3.inputs['in1'])
        graph.connect(c2.outputs['b'], c3.inputs['in2'])
        # graph.set_initial_packet(c3.inputs['in1'], 6)
        async def send(messages):
            for m in messages:
                await asyncio.sleep(0.2)
                [asyncio.ensure_future(c1.outputs['a'].send_to_all(m)), asyncio.ensure_future(c2.outputs['b'].send_to_all(m))]

            asyncio.ensure_future(c1.outputs['a'].close())
            asyncio.ensure_future(c2.outputs['b'].close())
        async def receive():
            while True:
                print(c3.inputs['in1'].queue)
                res, doing = await asyncio.wait([c3.inputs['in1'].receive(),c3.inputs['in2'].receive()], return_when=asyncio.ALL_COMPLETED)
                print('done receiving')
                print(res.pop().result(),res.pop().result())
                # await asyncio.sleep(0)
        futures = [asyncio.ensure_future(send([1,2,3,4,5])),asyncio.ensure_future(receive())]
        loop = asyncio.get_event_loop()
        loop.run_until_complete(futures[1])

    def testLinearPipeline(self):
        source = GeneratorSource('s', source_gen)
        shower = ShowInputs('printer')
        shower.inputs.add(InputPort('in1'))
        graph = Multigraph(log_path='/home/baffelli/a.txt')
        graph.connect(source.outputs['OUT'], shower.inputs['in1'])
        # graph.set_initial_packet(shower.inputs['in1'],5)

        graph()

    def testSumPipeline(self):
        source1 = GeneratorSource('s1',  (i for i in range(100)))
        source2 = GeneratorSource('s2',  (i for i in range(100)))
        shower = ShowInputs('printer')
        shower.inputs.add(InputPort('in1'))
        summer = BroadcastApplyFunction('summer', adder )
        summer.inputs.add(InputPort('g1'))
        summer.inputs.add(InputPort('g2'))
        summer.outputs.add(OutputPort('sum'))
        graph = Multigraph()
        graph.connect(source1.outputs.OUT, summer.inputs.g1)
        graph.connect(source2.outputs.OUT, summer.inputs.g2)
        graph.connect(summer.outputs.sum, shower.inputs.in1)
        with open('/home/baffelli/sum.dot','w+') as outfile:
            outfile.write(graph.dot())
        graph()

    def testRecursivePipeline(self):
        source1 = GeneratorSource('s1',  (1 for i in range(1000)))
        shower = ShowInputs('printer')
        shower.inputs.add(InputPort('in1'))
        summer = BroadcastApplyFunction('summer', adder )
        summer.inputs.add(InputPort('g1'))
        summer.inputs.add(InputPort('recursion'))
        summer.outputs.add(OutputPort('sum'))
        graph = Multigraph()
        graph.connect(source1.outputs.OUT, summer.inputs.g1)
        graph.connect(summer.outputs.sum, shower.inputs.in1)
        graph.connect(summer.outputs.sum, summer.inputs.recursion)
        graph.set_initial_packet(summer.inputs.recursion, 1)
        with open('/home/baffelli/recursion.dot','w+') as outfile:
            outfile.write(graph.dot())
        graph()

    def testConstantSource(self):
        source1 = ConstantSource('s1', 3)
        shower = ShowInputs('printer')
        shower.inputs.add(InputPort('in1'))
        graph = Multigraph()
        graph.connect(source1.outputs['OUT'], shower.inputs.in1)
        graph()

    def testOneOffProcess(self):
        source1 = ConstantSource('s1', 3)
        source2 = OneOffProcess('of', lambda kwargs: 5)
        source2.outputs.add(OutputPort('s2'))
        summer = BroadcastApplyFunction('summer', adder )
        summer.inputs.add(InputPort('g1'))
        summer.inputs.add(InputPort('g2'))
        summer.outputs.add(OutputPort('sum'))
        shower = ShowInputs('printer', inputs=['in1'])
        graph = Multigraph()
        graph.connect(source1.outputs['OUT'], summer.inputs.g1)
        graph.connect(source2.outputs['s2'], summer.inputs.g2)
        graph.connect(summer.outputs.sum, shower.inputs.in1)
        graph()

    def testInfiniteRecursion(self):
        source1 = ConstantSource('s1', 3)
        shower = ShowInputs('printer', inputs=['in1'])
        summer = BroadcastApplyFunction('summer', adder )
        summer.inputs.add(InputPort('in1'))
        summer.inputs.add(InputPort('recursion'))
        summer.outputs.add(OutputPort('sum'))
        graph = Multigraph()
        graph.connect(source1.outputs.OUT, summer.inputs.in1)
        graph.connect(summer.outputs.sum, summer.inputs.recursion)
        graph.set_initial_packet(summer.inputs.recursion, 0)
        graph.connect(summer.outputs.sum, shower.inputs.in1)
        graph()

    def testSplit(self):
        source1 = ConstantSource('s1', (3,5))
        splitter = components.Split('split in two')
        splitter.outputs.add(OutputPort('a'))
        splitter.outputs.add(OutputPort('b'))
        shower = ShowInputs('printer')
        shower.inputs.add(InputPort('a'))
        shower.inputs.add(InputPort('b'))
        graph = Multigraph()
        graph.connect(source1.outputs.OUT, splitter.inputs.IN)
        graph.connect(splitter.outputs.a, shower.inputs.a)
        graph.connect(splitter.outputs.b, shower.inputs.b)
        graph()

    def testFilter(self):
        def filter_predicate(in1=None):
            if in1 % 2 ==0:
                return True
            else:
                return False
        source1 = GeneratorSource('s1', (i for i in range(100)))
        filt = Filter('filtrator',  filter_predicate,)
        filt.inputs.add(InputPort('in1'))
        filt.outputs.add(OutputPort('out1'))
        shower = ShowInputs('printer', inputs=['in1'])
        graph = Multigraph()
        graph.connect(source1.outputs['OUT'], filt.inputs.in1)
        graph.connect(filt.outputs.out1, shower.inputs.in1)
        graph()

    def testFixedFormatter(self):
        #generate uniquefilename
        unique_file = os.path.dirname(__file__) + '/' + str(uuid.uuid4())
        toucher = components.Shell('shell', "touch {outputs.f1}")
        toucher.outputs.add(FilePort('f1'))
        toucher.FixedFormatter('f1', unique_file)
        printer = ShowInputs('show path')
        printer.inputs.add(FilePort('f1'))
        graph = Multigraph()
        graph.connect(toucher.outputs.f1, printer.inputs.f1)
        graph()
        self.assertTrue(os.path.exists(unique_file))

    def testPatternFormatter(self):
        #Source
        source1 = GeneratorSource('s1', (i for i in range(5)))
        toucher = components.Shell('shell', "echo {inputs.i.value} > {outputs.f1}")
        toucher.outputs.add(FilePort('f1'))
        toucher.inputs.add(InputPort('i'))
        toucher.DynamicFormatter('f1', "{inputs.i.value}.txt")
        printer = ShowInputs('show path')
        printer.inputs.add(FilePort('f1'))
        graph = Multigraph()
        graph.connect(source1.outputs.OUT, toucher.inputs.i)
        graph.connect(toucher.outputs.f1, printer.inputs.f1)
        graph()

    def combinatorialPatternFormatter(self):
        pass

    def testDoNotOverwrite(self):
        #generate uniquefilename
        unique_file = os.path.dirname(__file__) + '/' + 'already_exists'
        toucher = components.Shell('shell', "touch {outputs.f1}")
        toucher.outputs.add(FilePort('f1'))
        toucher.FixedFormatter('f1', unique_file)
        toucher_1 = components.Shell('shell', "touch {outputs.f1}")
        toucher_1.outputs.add(FilePort('f1'))
        toucher_1.FixedFormatter('f1', unique_file)
        printer = ShowInputs('show path')
        printer.inputs.add(FilePort('f1'))
        printer.inputs.add(FilePort('f2'))
        graph = Multigraph()
        graph.connect(toucher.outputs.f1, printer.inputs.f1)
        graph.connect(toucher_1.outputs.f1, printer.inputs.f2)
        graph()
        self.assertTrue(os.path.exists(unique_file))

    def testDynamicFormatter(self):
        #generate uniquefilename
        unique_file = os.path.dirname(__file__) + '/' + str(uuid.uuid4())
        toucher = components.Shell('shell', "touch {outputs.f1}")
        #Write something into the file
        writer = components.Shell('add_something', 'cp {inputs.f1} {outputs.f2}')
        writer.inputs.add(FilePort('f2'))
        writer.outputs.add(FilePort('f3'))
        writer.DynamicFormatter('f2', 'f3', '.a')
        printer = ShowInputs('show path')
        printer.inputs.add(FilePort('f1'))
        graph = Multigraph()
        graph.connect(toucher.outputs.f1,writer.inputs.f2)
        graph.connect(writer.outputs.f3, printer.inputs.f1)
        graph()
<|endoftext|>


<commit_before>""" Update all addons tables in README.md in all OCA repositories

This script must be run in an initially empty directory.
It will clone all OCA repositories from github, update all the addons tables
and push the changes to github.

CAUTION: it is important that this script is run in a directory
dedicated to this purpose, as it will run oca-clone-everything with
the --remove-old-repos option.
"""
import logging
import os
import subprocess
import sys

BRANCHES = ['8.0', '9.0']
EXCLUDE = {
    'maintainer-tools',
    'maintainer-quality-tools',
    'pylint-odoo',
    'OCB',
    'OpenUpgrade',
}

class FatalError(RuntimeError):
    pass

class NonFatalError(RuntimeError):
    pass

def call(cmd, cwd, raise_on_error=True, raise_fatal_error=True, shell=False):
    r = subprocess.call(cmd, cwd=cwd, shell=shell)
    if r != 0 and raise_on_error:
        if not shell:
            cmdstr = ' '.join(cmd)
        else:
            cmdstr = cmd
        msg = "Command '%s' returned %d in %s" % (cmdstr, r, cwd)
        if raise_fatal_error:
            raise FatalError(msg)
        else:
            raise NonFatalError(msg)
    return r

def main():
    # call(['oca-clone-everything', '--remove-old-repos'], cwd='.')
    for d in sorted(os.listdir('.')):
        if not os.path.isdir(os.path.join(d, '.git')):
            continue
        if d in EXCLUDE:
            continue
        sys.stderr.write("============> updating addons table in %s\n" % d)
        for branch in BRANCHES:
            try:
                call(['git', 'checkout', branch], cwd=d,
                     raise_fatal_error=False)
                call(['git', 'reset', '--hard', 'origin/' + branch], cwd=d)
                if not os.path.isfile(os.path.join(d, 'README.md')):
                    continue
                call(['oca-gen-addons-table'], cwd=d,
                     raise_fatal_error=False)
                r = call(['git', 'diff', '--exit-code', 'README.md'], cwd=d,
                         raise_on_error=False)
                if r != 0:
                    call(['git', 'commit',
                          '-m', '[UPD] addons table in README.md', 'README.md'],
                         cwd=d)
                    call(['git', 'push', 'origin'], cwd=d)
            except NonFatalError:
                logging.exception("Error in %s", d, exc_info=True)

if __name__ == '__main__':
    main()
<commit_msg>gen_all_addons_tables: pep8<commit_after>""" Update all addons tables in README.md in all OCA repositories

This script must be run in an initially empty directory.
It will clone all OCA repositories from github, update all the addons tables
and push the changes to github.

CAUTION: it is important that this script is run in a directory
dedicated to this purpose, as it will run oca-clone-everything with
the --remove-old-repos option.
"""
import logging
import os
import subprocess
import sys

BRANCHES = ['8.0', '9.0']
EXCLUDE = {
    'maintainer-tools',
    'maintainer-quality-tools',
    'pylint-odoo',
    'OCB',
    'OpenUpgrade',
}

class FatalError(RuntimeError):
    pass

class NonFatalError(RuntimeError):
    pass

def call(cmd, cwd, raise_on_error=True, raise_fatal_error=True, shell=False):
    r = subprocess.call(cmd, cwd=cwd, shell=shell)
    if r != 0 and raise_on_error:
        if not shell:
            cmdstr = ' '.join(cmd)
        else:
            cmdstr = cmd
        msg = "Command '%s' returned %d in %s" % (cmdstr, r, cwd)
        if raise_fatal_error:
            raise FatalError(msg)
        else:
            raise NonFatalError(msg)
    return r

def main():
    # call(['oca-clone-everything', '--remove-old-repos'], cwd='.')
    for d in sorted(os.listdir('.')):
        if not os.path.isdir(os.path.join(d, '.git')):
            continue
        if d in EXCLUDE:
            continue
        sys.stderr.write("============> updating addons table in %s\n" % d)
        for branch in BRANCHES:
            try:
                call(['git', 'checkout', branch], cwd=d,
                     raise_fatal_error=False)
                call(['git', 'reset', '--hard', 'origin/' + branch], cwd=d)
                if not os.path.isfile(os.path.join(d, 'README.md')):
                    continue
                call(['oca-gen-addons-table'], cwd=d,
                     raise_fatal_error=False)
                r = call(['git', 'diff', '--exit-code', 'README.md'], cwd=d,
                         raise_on_error=False)
                if r != 0:
                    call(['git', 'commit',
                          '-m', '[UPD] addons table in README.md',
                          'README.md'],
                         cwd=d)
                    call(['git', 'push', 'origin'], cwd=d)
            except NonFatalError:
                logging.exception("Error in %s", d, exc_info=True)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>##########################################################################
# Copyright (c) 2012, ETH Zurich.
# All rights reserved.
#
# This file is distributed under the terms in the attached LICENSE file.
# If you do not find this file, copies can be found by writing to:
# ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
##########################################################################

# Quirks:
# * menu.lst is read from hake/menu.lst.arm_gem5, not the one
#    generated by scalebench
# * this is only running in single-core mode, since bootarm=0 is
#    used in above mentioned menu.lst

import os, signal, tempfile, subprocess, shutil, time
import debug, machines
from machines import Machine

GEM5_PATH = '/home/netos/tools/gem5/gem5/build/ARM/'
GEM5_CACHES_ENABLE = '--caches --l2cache'.split()

class Gem5MachineBase(Machine):
	def __init__(self, options):
		super(Gem5MachineBase, self).__init__(options)
		self.child = None
		self.telnet = None
		self.tftp_dir = None
		self.options = options

	def get_coreids(self):
		return range(0, self.get_ncores())

	def get_tickrate(self):
		return None

	def get_boot_timeout(self):
		# we set this to 10 mins since gem5 is very slow
		return 600

	def get_tftp_dir(self):
		if self.tftp_dir is None:
			debug.verbose('creating temporary directory for Gem5 files')
			self.tftp_dir = tempfile.mkdtemp(prefix='harness_gem5_')
			debug.verbose('Gem5 install directory is %s' % self.tftp_dir)
		return self.tftp_dir

	# Use menu.lst in hake/menu.lst.arm_gem5
	def _write_menu_lst(self, data, path):
		pass

	def set_bootmodules(self, modules):
		pass

	def lock(self):
		pass

	def unlock(self):
		pass

	def setup(self):
		pass

	def _get_cmdline(self):
		raise NotImplementedError

	def _kill_child(self):
		# terminate child if running
		if self.child:
			os.kill(self.child.pid, signal.SIGTERM)
			self.child.wait()
			self.child = None

	def reboot(self):
		self._kill_child()
		cmd = self._get_cmdline()
		debug.verbose('starting "%s" in gem5.py:reboot' % ' '.join(cmd))
		devnull = open('/dev/null', 'w')
		self.child = subprocess.Popen(cmd, stderr=devnull)
		# gem5 takes quite a while to come up. If we return right away, telnet will be opened too early and fails to connect
		time.sleep(30)

	def shutdown(self):
		debug.verbose('gem5:shutdown requested');
		debug.verbose('terminating gem5')
		self.child.terminate()
		debug.verbose('terminating telnet')
		self.telnet.terminate()
		# try to cleanup tftp tree if needed
		if self.tftp_dir and os.path.isdir(self.tftp_dir):
			shutil.rmtree(self.tftp_dir, ignore_errors=True)
		self.tftp_dir = None

	def get_output(self):
		# wait a bit to give gem5 time to listen for a telnet connection
		if self.child.poll() != None: # Check if child is down
			print ' '.join(['gem5 is down, return code is ', self.child.returncode])
			return None
		self.telnet = subprocess.Popen(['telnet','localhost','3456'], stdout=subprocess.PIPE)
		return self.telnet.stdout

class Gem5MachineARM(Gem5MachineBase):
	def get_bootarch(self):
		return 'armv7'

	def set_bootmodules(self, modules):
		# store path to kernel for _get_cmdline to use
		tftp_dir = self.get_tftp_dir()
		self.kernel_img = os.path.join(self.options.buildbase, 'release', 'arm_gem5_image')

		#write menu.lst
		path = os.path.join(self.get_tftp_dir(), 'menu.lst')
		self._write_menu_lst(modules.get_menu_data('/'), path)

# SK: did not test this yet, but should work
# @machines.add_machine
# class Gem5MachineARMSingleCore(Gem5MachineARM):
# 	name = 'gem5_arm_1'

# 	def get_ncores(self):
# 		return 1

# 	def _get_cmdline(self):
# 		script_path = os.path.join(self.options.sourcedir, 'tools/arm_gem5', 'gem5script.py')
# 		return (['gem5.fast', script_path, '--kernel=%s'%self.kernel_img, '--n=%s'%self.get_ncores()]
# 				+ GEM5_CACHES_ENABLE)

@machines.add_machine
class Gem5MachineARMMultiCore(Gem5MachineARM):
	name = 'armv7_gem5_2'

	def get_bootarch(self):
		return "armv7"

	def get_ncores(self):
		return 2

	def get_cores_per_socket(self):
		return 1

	def _get_cmdline(self):
		script_path = os.path.join(self.options.sourcedir, 'tools/arm_gem5', 'gem5script.py')
		return ([GEM5_PATH + 'gem5.fast', script_path, \
				 '--kernel=%s'%self.kernel_img, \
				 '--caches', \
				 '--l2cache', \
				 '--n=%s' % self.get_ncores()]
				+ GEM5_CACHES_ENABLE)

# SK: this will not work, since gem5 uses the menu.lst specified in the arm_gem5_image
#    make target. There, only two cores are booted.
# @machines.add_machine
# class Gem5MachineARMMultiCore(Gem5MachineARM):
# 	name = 'gem5_arm_4'

# 	def get_ncores(self):
# 		return 4

# 	def get_cores_per_socket(self):
# 		return 2

# 	def _get_cmdline(self):
# 		script_path = os.path.join(self.options.sourcedir, 'tools/arm_gem5', 'gem5script.py')
# 		return (['gem5.fast', script_path, '--kernel=%s'%self.kernel_img, '--n=%s'%self.get_ncores()]
# 				+ GEM5_CACHES_ENABLE)

<commit_msg>Additional output for telnet to debug armv7_gem5 build error<commit_after>##########################################################################
# Copyright (c) 2012, ETH Zurich.
# All rights reserved.
#
# This file is distributed under the terms in the attached LICENSE file.
# If you do not find this file, copies can be found by writing to:
# ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
##########################################################################

# Quirks:
# * menu.lst is read from hake/menu.lst.arm_gem5, not the one
#    generated by scalebench
# * this is only running in single-core mode, since bootarm=0 is
#    used in above mentioned menu.lst

import os, signal, tempfile, subprocess, shutil, time
import debug, machines
from machines import Machine

GEM5_PATH = '/home/netos/tools/gem5/gem5/build/ARM/'
GEM5_CACHES_ENABLE = '--caches --l2cache'.split()

class Gem5MachineBase(Machine):
	def __init__(self, options):
		super(Gem5MachineBase, self).__init__(options)
		self.child = None
		self.telnet = None
		self.tftp_dir = None
		self.options = options

	def get_coreids(self):
		return range(0, self.get_ncores())

	def get_tickrate(self):
		return None

	def get_boot_timeout(self):
		# we set this to 10 mins since gem5 is very slow
		return 600

	def get_tftp_dir(self):
		if self.tftp_dir is None:
			debug.verbose('creating temporary directory for Gem5 files')
			self.tftp_dir = tempfile.mkdtemp(prefix='harness_gem5_')
			debug.verbose('Gem5 install directory is %s' % self.tftp_dir)
		return self.tftp_dir

	# Use menu.lst in hake/menu.lst.arm_gem5
	def _write_menu_lst(self, data, path):
		pass

	def set_bootmodules(self, modules):
		pass

	def lock(self):
		pass

	def unlock(self):
		pass

	def setup(self):
		pass

	def _get_cmdline(self):
		raise NotImplementedError

	def _kill_child(self):
		# terminate child if running
		if self.child:
			os.kill(self.child.pid, signal.SIGTERM)
			self.child.wait()
			self.child = None

	def reboot(self):
		self._kill_child()
		cmd = self._get_cmdline()
		debug.verbose('starting "%s" in gem5.py:reboot' % ' '.join(cmd))
		devnull = open('/dev/null', 'w')
		self.child = subprocess.Popen(cmd, stderr=devnull)
		# gem5 takes quite a while to come up. If we return right away, telnet will be opened too early and fails to connect
		time.sleep(30)

	def shutdown(self):
		debug.verbose('gem5:shutdown requested');
		debug.verbose('terminating gem5')
		self.child.terminate()
		debug.verbose('terminating telnet')
		self.telnet.terminate()
		# try to cleanup tftp tree if needed
		if self.tftp_dir and os.path.isdir(self.tftp_dir):
			shutil.rmtree(self.tftp_dir, ignore_errors=True)
		self.tftp_dir = None

	def get_output(self):
		# wait a bit to give gem5 time to listen for a telnet connection
		if self.child.poll() != None: # Check if child is down
			print ' '.join(['gem5 is down, return code is ', self.child.returncode])
			return None
		self.telnet = subprocess.Popen(['telnet','-d','localhost','3456'], stdout=subprocess.PIPE)
		return self.telnet.stdout

class Gem5MachineARM(Gem5MachineBase):
	def get_bootarch(self):
		return 'armv7'

	def set_bootmodules(self, modules):
		# store path to kernel for _get_cmdline to use
		tftp_dir = self.get_tftp_dir()
		self.kernel_img = os.path.join(self.options.buildbase, 'release', 'arm_gem5_image')

		#write menu.lst
		path = os.path.join(self.get_tftp_dir(), 'menu.lst')
		self._write_menu_lst(modules.get_menu_data('/'), path)

# SK: did not test this yet, but should work
# @machines.add_machine
# class Gem5MachineARMSingleCore(Gem5MachineARM):
# 	name = 'gem5_arm_1'

# 	def get_ncores(self):
# 		return 1

# 	def _get_cmdline(self):
# 		script_path = os.path.join(self.options.sourcedir, 'tools/arm_gem5', 'gem5script.py')
# 		return (['gem5.fast', script_path, '--kernel=%s'%self.kernel_img, '--n=%s'%self.get_ncores()]
# 				+ GEM5_CACHES_ENABLE)

@machines.add_machine
class Gem5MachineARMMultiCore(Gem5MachineARM):
	name = 'armv7_gem5_2'

	def get_bootarch(self):
		return "armv7"

	def get_ncores(self):
		return 2

	def get_cores_per_socket(self):
		return 1

	def _get_cmdline(self):
		script_path = os.path.join(self.options.sourcedir, 'tools/arm_gem5', 'gem5script.py')
		return ([GEM5_PATH + 'gem5.fast', script_path, \
				 '--kernel=%s'%self.kernel_img, \
				 '--caches', \
				 '--l2cache', \
				 '--n=%s' % self.get_ncores()]
				+ GEM5_CACHES_ENABLE)

# SK: this will not work, since gem5 uses the menu.lst specified in the arm_gem5_image
#    make target. There, only two cores are booted.
# @machines.add_machine
# class Gem5MachineARMMultiCore(Gem5MachineARM):
# 	name = 'gem5_arm_4'

# 	def get_ncores(self):
# 		return 4

# 	def get_cores_per_socket(self):
# 		return 2

# 	def _get_cmdline(self):
# 		script_path = os.path.join(self.options.sourcedir, 'tools/arm_gem5', 'gem5script.py')
# 		return (['gem5.fast', script_path, '--kernel=%s'%self.kernel_img, '--n=%s'%self.get_ncores()]
# 				+ GEM5_CACHES_ENABLE)

<|endoftext|>