<commit_before>
from __future__ import print_function

import psycopg2
import os, sys
import logging
import time

from dbt.compilation import Linker, Compiler
from dbt.templates import BaseCreateTemplate
from dbt.targets import RedshiftTarget
from dbt.source import Source
from dbt.utils import find_model_by_name

from multiprocessing.dummy import Pool as ThreadPool

SCHEMA_PERMISSION_DENIED_MESSAGE = """The user '{user}' does not have sufficient permissions to create the schema '{schema}'.
Either create the schema  manually, or adjust the permissions of the '{user}' user."""

RELATION_PERMISSION_DENIED_MESSAGE = """The user '{user}' does not have sufficient permissions to create the model '{model}'  in the schema '{schema}'.
Please adjust the permissions of the '{user}' user on the '{schema}' schema.
With a superuser account, execute the following commands, then re-run dbt.

grant usage, create on schema "{schema}" to "{user}";
grant select on all tables in schema "{schema}" to "{user}";"""

RELATION_NOT_OWNER_MESSAGE = """The user '{user}' does not have sufficient permissions to drop the model '{model}' in the schema '{schema}'.
This is likely because the relation was created by a different user. Either delete the model "{schema}"."{model}" manually,
or adjust the permissions of the '{user}' user in the '{schema}' schema."""

class RunModelResult(object):
    def __init__(self, model, error=None, skip=False, status=None):
        self.model = model
        self.error = error
        self.skip  = skip
        self.status = status

    @property
    def errored(self):
        return self.error is not None

    @property
    def skipped(self):
        return self.skip

class Runner:
    def __init__(self, project, target_path, run_mode):
        self.logger = logging.getLogger(__name__)
        self.project = project
        self.target_path = target_path
        self.run_mode = run_mode

    def get_compiled_models(self):
        return Source(self.project).get_compiled(self.target_path, self.run_mode)

    def get_target(self):
        target_cfg = self.project.run_environment()
        return RedshiftTarget(target_cfg)

    def deserialize_graph(self):
        linker = Linker()
        base_target_path = self.project['target-path']
        filename = 'graph-{}.yml'.format(self.run_mode)
        graph_file = os.path.join(base_target_path, filename)
        linker.read_graph(graph_file)

        return linker

    def create_schema(self, schema_name):
        target = self.get_target()
        with target.get_handle() as handle:
            with handle.cursor() as cursor:
                cursor.execute('create schema if not exists "{}"'.format(schema_name))

    def get_schemas(self):
        target = self.get_target()
        existing = []
        with target.get_handle() as handle:
            with handle.cursor() as cursor:
                cursor.execute('select nspname from pg_catalog.pg_namespace')

                existing = [name for (name,) in cursor.fetchall()]
        return existing

    def create_schema_or_exit(self, schema_name):

        target_cfg = self.project.run_environment()
        user = target_cfg['user']

        try:
            self.create_schema(schema_name)
        except psycopg2.ProgrammingError as e:
            if "permission denied for" in e.diag.message_primary:
                raise RuntimeError(SCHEMA_PERMISSION_DENIED_MESSAGE.format(schema=schema_name, user=user))
            else:
                raise e

    def query_for_existing(self, target, schema):
        sql = """
            select tablename as name, 'table' as type from pg_tables where schemaname = '{schema}'
                union all
            select viewname as name, 'view' as type from pg_views where schemaname = '{schema}' """.format(schema=schema)

        with target.get_handle() as handle:
            with handle.cursor() as cursor:
                cursor.execute(sql)
                existing = [(name, relation_type) for (name, relation_type) in cursor.fetchall()]

        return dict(existing)

    def get_drop_statement(self, schema, relation, relation_type):
        return 'drop {relation_type} if exists "{schema}"."{relation}" cascade'.format(schema=schema, relation_type=relation_type, relation=relation)

    def drop(self, target, model, relation, relation_type):
        sql = self.get_drop_statement(target.schema, relation, relation_type)
        self.logger.info("dropping %s %s.%s", relation_type, target.schema, relation)
        self.execute_and_handle_permissions(target, sql, model, relation)
        self.logger.info("dropped %s %s.%s", relation_type, target.schema, relation)

    def __do_execute(self, target, sql, model):
        with target.get_handle() as handle:
            with handle.cursor() as cursor:
                try:
                    self.logger.debug("SQL: %s", sql)
                    pre = time.time()
                    cursor.execute(sql)
                    post = time.time()
                    self.logger.debug("SQL status: %s in %d seconds", cursor.statusmessage, post-pre)
                    return cursor.statusmessage
                except Exception as e:
                    e.model = model
                    self.logger.exception("Error running SQL: %s", sql)
                    raise e

    def drop_models(self, models):
        target = self.get_target()

        existing = self.query_for_existing(target, target.schema);
        for model in models:
            model_name = model.fqn[-1]
            self.drop(target, model, model.name, existing[model_name])

    def get_model_by_fqn(self, models, fqn):
        for model in models:
            if tuple(model.fqn) == tuple(fqn):
                return model
        raise RuntimeError("Couldn't find a compiled model with fqn: '{}'".format(fqn))

    def execute_wrapped_model(self, data):
        target = data['target']
        model = data['model']
        tmp_drop_type = data['tmp_drop_type']
        final_drop_type = data['final_drop_type']

        error = None
        try:
            status = self.execute_model(target, model, tmp_drop_type, final_drop_type)
        except (RuntimeError, psycopg2.ProgrammingError) as e:
            error = "Error executing {filepath}\n{error}".format(filepath=model.filepath, error=str(e).strip())
            status = "ERROR"

        return RunModelResult(model, error=error, status=status)

    def execute_and_handle_permissions(self, target, query, model, model_name):
        try:
            return self.__do_execute(target, query, model)
        except psycopg2.ProgrammingError as e:
            error_data = {"model": model_name, "schema": target.schema, "user": target.user}
            if 'must be owner of relation' in e.diag.message_primary:
                raise RuntimeError(RELATION_NOT_OWNER_MESSAGE.format(**error_data))
            elif "permission denied for" in e.diag.message_primary:
                raise RuntimeError(RELATION_PERMISSION_DENIED_MESSAGE.format(**error_data))
            else:
                raise e

    def rename(self, target, model):
        rename_query = model.rename_query(target.schema)
        self.logger.info("renaming model %s.%s --> %s.%s", target.schema, model.tmp_name(), target.schema, model.name)
        self.execute_and_handle_permissions(target, rename_query, model, model.name)
        self.logger.info("renamed model %s.%s --> %s.%s", target.schema, model.tmp_name(), target.schema, model.name)

    def execute_model(self, target, model, tmp_drop_type, final_drop_type):
        self.logger.info("executing model %s", model)

        if tmp_drop_type is not None:
            self.drop(target, model, model.tmp_name(), tmp_drop_type)

        status = self.execute_and_handle_permissions(target, model.contents, model, model.tmp_name())

        if final_drop_type is not None:
            self.drop(target, model, model.name, final_drop_type)

        if not model.is_incremental:
            self.rename(target, model)

        return status

    def execute_models(self, linker, models, limit_to=None):
        target = self.get_target()

        dependency_list = linker.as_dependency_list(limit_to)
        num_models = sum([len(node_list) for node_list in dependency_list])

        if num_models == 0:
            print("WARNING: No models to run in '{}'. Try checking your model configs and running `dbt compile`".format(self.target_path))
            return []

        existing = self.query_for_existing(target, target.schema);

        def wrap_fqn(target, models, existing, fqn):
            model = self.get_model_by_fqn(models, fqn)

            if model.is_incremental:
                tmp_drop_type = None
                final_drop_type = None
            else:
                tmp_drop_type = existing.get(model.tmp_name(), None)
                final_drop_type = existing.get(model.name, None)
            return {"model" : model, "target": target, "tmp_drop_type": tmp_drop_type, 'final_drop_type': final_drop_type}

        # we can only pass one arg to the self.execute_model method below. Pass a dict w/ all the data we need
        model_dependency_list = []
        for node_list in dependency_list:
            level = []
            for fqn in node_list:
                node = linker.get_node(fqn)
                if node.get('materialized') == 'ephemeral':
                    continue
                else:
                    wrapped = wrap_fqn(target, models, existing, fqn)
                    level.append(wrapped)
            model_dependency_list.append(level)

        num_threads = target.threads
        print("Concurrency: {} threads (target='{}')".format(num_threads, self.project['run-target']))
        print("Running!")

        pool = ThreadPool(num_threads)

        failed_models = set()

        model_results = []
        for model_list in model_dependency_list:
            failed_nodes = [tuple(model.fqn) for model in failed_models]

            models_to_execute = [data for data in model_list if not linker.is_child_of(failed_nodes, tuple(data['model'].fqn))]
            models_to_skip = [data for data in model_list if linker.is_child_of(failed_nodes, tuple(data['model'].fqn))]

            for i, data in enumerate(models_to_skip):
                model = data['model']
                model_result = RunModelResult(model, skip=True)
                model_results.append(model_result)
                print("{} of {} -- SKIP relation {}.{} because parent failed".format(len(model_results), num_models, target.schema, model_result.model.name))

            for i, model in enumerate(models_to_execute):
                print_vars = {
                    "progress": 1 + i + len(model_results),
                    "total" : num_models,
                    "schema": target.schema,
                    "model_name": model['model'].name,
                    "model_type": model['model'].materialization,
                    "info": "START"
                }

                output = "{progress} of {total} -- {info} {model_type} model {schema}.{model_name} ".format(**print_vars)
                print("{} [Running]".format(output.ljust(80, ".")))

            run_model_results = pool.map(self.execute_wrapped_model, models_to_execute)

            for run_model_result in run_model_results:
                model_results.append(run_model_result)

                print_vars = {
                    "progress": len(model_results),
                    "total" : num_models,
                    "schema": target.schema,
                    "model_name": run_model_result.model.name,
                    "model_type": run_model_result.model.materialization,
                    "info": "ERROR creating" if run_model_result.errored else "OK created"
                }

                output = "{progress} of {total} -- {info} {model_type} model {schema}.{model_name} ".format(**print_vars)
                print("{} [{}]".format(output.ljust(80, "."), run_model_result.status))

                if run_model_result.errored:
                    failed_models.add(run_model_result.model)
                    print(run_model_result.error)

        pool.close()
        pool.join()

        return model_results

    def run(self, specified_models=None):
        linker = self.deserialize_graph()
        compiled_models = self.get_compiled_models()

        limit_to = None
        if specified_models is not None:
            limit_to = []
            for model_name in specified_models:
                try:
                    model = find_model_by_name(compiled_models, model_name)
                    limit_to.append(tuple(model.fqn))
                except RuntimeError as e:
                    print("ERROR: {}".format(str(e)))
                    print("Exiting")
                    return[]

        target_cfg = self.project.run_environment()
        schema_name = target_cfg['schema']

        try:
            schemas = self.get_schemas()

            if schema_name not in schemas:
                self.create_schema_or_exit(schema_name)

            return self.execute_models(linker, compiled_models, limit_to)
        except psycopg2.OperationalError as e:
            print("ERROR: Could not connect to the target database. Try `dbt debug` for more information")
            print(str(e))
            sys.exit(1)
<commit_msg>count models after ephemeral models have been culled<commit_after>
from __future__ import print_function

import psycopg2
import os, sys
import logging
import time

from dbt.compilation import Linker, Compiler
from dbt.templates import BaseCreateTemplate
from dbt.targets import RedshiftTarget
from dbt.source import Source
from dbt.utils import find_model_by_name

from multiprocessing.dummy import Pool as ThreadPool

SCHEMA_PERMISSION_DENIED_MESSAGE = """The user '{user}' does not have sufficient permissions to create the schema '{schema}'.
Either create the schema  manually, or adjust the permissions of the '{user}' user."""

RELATION_PERMISSION_DENIED_MESSAGE = """The user '{user}' does not have sufficient permissions to create the model '{model}'  in the schema '{schema}'.
Please adjust the permissions of the '{user}' user on the '{schema}' schema.
With a superuser account, execute the following commands, then re-run dbt.

grant usage, create on schema "{schema}" to "{user}";
grant select on all tables in schema "{schema}" to "{user}";"""

RELATION_NOT_OWNER_MESSAGE = """The user '{user}' does not have sufficient permissions to drop the model '{model}' in the schema '{schema}'.
This is likely because the relation was created by a different user. Either delete the model "{schema}"."{model}" manually,
or adjust the permissions of the '{user}' user in the '{schema}' schema."""

class RunModelResult(object):
    def __init__(self, model, error=None, skip=False, status=None):
        self.model = model
        self.error = error
        self.skip  = skip
        self.status = status

    @property
    def errored(self):
        return self.error is not None

    @property
    def skipped(self):
        return self.skip

class Runner:
    def __init__(self, project, target_path, run_mode):
        self.logger = logging.getLogger(__name__)
        self.project = project
        self.target_path = target_path
        self.run_mode = run_mode

    def get_compiled_models(self):
        return Source(self.project).get_compiled(self.target_path, self.run_mode)

    def get_target(self):
        target_cfg = self.project.run_environment()
        return RedshiftTarget(target_cfg)

    def deserialize_graph(self):
        linker = Linker()
        base_target_path = self.project['target-path']
        filename = 'graph-{}.yml'.format(self.run_mode)
        graph_file = os.path.join(base_target_path, filename)
        linker.read_graph(graph_file)

        return linker

    def create_schema(self, schema_name):
        target = self.get_target()
        with target.get_handle() as handle:
            with handle.cursor() as cursor:
                cursor.execute('create schema if not exists "{}"'.format(schema_name))

    def get_schemas(self):
        target = self.get_target()
        existing = []
        with target.get_handle() as handle:
            with handle.cursor() as cursor:
                cursor.execute('select nspname from pg_catalog.pg_namespace')

                existing = [name for (name,) in cursor.fetchall()]
        return existing

    def create_schema_or_exit(self, schema_name):

        target_cfg = self.project.run_environment()
        user = target_cfg['user']

        try:
            self.create_schema(schema_name)
        except psycopg2.ProgrammingError as e:
            if "permission denied for" in e.diag.message_primary:
                raise RuntimeError(SCHEMA_PERMISSION_DENIED_MESSAGE.format(schema=schema_name, user=user))
            else:
                raise e

    def query_for_existing(self, target, schema):
        sql = """
            select tablename as name, 'table' as type from pg_tables where schemaname = '{schema}'
                union all
            select viewname as name, 'view' as type from pg_views where schemaname = '{schema}' """.format(schema=schema)

        with target.get_handle() as handle:
            with handle.cursor() as cursor:
                cursor.execute(sql)
                existing = [(name, relation_type) for (name, relation_type) in cursor.fetchall()]

        return dict(existing)

    def get_drop_statement(self, schema, relation, relation_type):
        return 'drop {relation_type} if exists "{schema}"."{relation}" cascade'.format(schema=schema, relation_type=relation_type, relation=relation)

    def drop(self, target, model, relation, relation_type):
        sql = self.get_drop_statement(target.schema, relation, relation_type)
        self.logger.info("dropping %s %s.%s", relation_type, target.schema, relation)
        self.execute_and_handle_permissions(target, sql, model, relation)
        self.logger.info("dropped %s %s.%s", relation_type, target.schema, relation)

    def __do_execute(self, target, sql, model):
        with target.get_handle() as handle:
            with handle.cursor() as cursor:
                try:
                    self.logger.debug("SQL: %s", sql)
                    pre = time.time()
                    cursor.execute(sql)
                    post = time.time()
                    self.logger.debug("SQL status: %s in %d seconds", cursor.statusmessage, post-pre)
                    return cursor.statusmessage
                except Exception as e:
                    e.model = model
                    self.logger.exception("Error running SQL: %s", sql)
                    raise e

    def drop_models(self, models):
        target = self.get_target()

        existing = self.query_for_existing(target, target.schema);
        for model in models:
            model_name = model.fqn[-1]
            self.drop(target, model, model.name, existing[model_name])

    def get_model_by_fqn(self, models, fqn):
        for model in models:
            if tuple(model.fqn) == tuple(fqn):
                return model
        raise RuntimeError("Couldn't find a compiled model with fqn: '{}'".format(fqn))

    def execute_wrapped_model(self, data):
        target = data['target']
        model = data['model']
        tmp_drop_type = data['tmp_drop_type']
        final_drop_type = data['final_drop_type']

        error = None
        try:
            status = self.execute_model(target, model, tmp_drop_type, final_drop_type)
        except (RuntimeError, psycopg2.ProgrammingError) as e:
            error = "Error executing {filepath}\n{error}".format(filepath=model.filepath, error=str(e).strip())
            status = "ERROR"

        return RunModelResult(model, error=error, status=status)

    def execute_and_handle_permissions(self, target, query, model, model_name):
        try:
            return self.__do_execute(target, query, model)
        except psycopg2.ProgrammingError as e:
            error_data = {"model": model_name, "schema": target.schema, "user": target.user}
            if 'must be owner of relation' in e.diag.message_primary:
                raise RuntimeError(RELATION_NOT_OWNER_MESSAGE.format(**error_data))
            elif "permission denied for" in e.diag.message_primary:
                raise RuntimeError(RELATION_PERMISSION_DENIED_MESSAGE.format(**error_data))
            else:
                raise e

    def rename(self, target, model):
        rename_query = model.rename_query(target.schema)
        self.logger.info("renaming model %s.%s --> %s.%s", target.schema, model.tmp_name(), target.schema, model.name)
        self.execute_and_handle_permissions(target, rename_query, model, model.name)
        self.logger.info("renamed model %s.%s --> %s.%s", target.schema, model.tmp_name(), target.schema, model.name)

    def execute_model(self, target, model, tmp_drop_type, final_drop_type):
        self.logger.info("executing model %s", model)

        if tmp_drop_type is not None:
            self.drop(target, model, model.tmp_name(), tmp_drop_type)

        status = self.execute_and_handle_permissions(target, model.contents, model, model.tmp_name())

        if final_drop_type is not None:
            self.drop(target, model, model.name, final_drop_type)

        if not model.is_incremental:
            self.rename(target, model)

        return status

    def execute_models(self, linker, models, limit_to=None):
        target = self.get_target()

        dependency_list = linker.as_dependency_list(limit_to)
        existing = self.query_for_existing(target, target.schema);

        def wrap_fqn(target, models, existing, fqn):
            model = self.get_model_by_fqn(models, fqn)

            if model.is_incremental:
                tmp_drop_type = None
                final_drop_type = None
            else:
                tmp_drop_type = existing.get(model.tmp_name(), None)
                final_drop_type = existing.get(model.name, None)
            return {"model" : model, "target": target, "tmp_drop_type": tmp_drop_type, 'final_drop_type': final_drop_type}

        # we can only pass one arg to the self.execute_model method below. Pass a dict w/ all the data we need
        model_dependency_list = []
        for node_list in dependency_list:
            level = []
            for fqn in node_list:
                node = linker.get_node(fqn)
                if node.get('materialized') == 'ephemeral':
                    continue
                else:
                    wrapped = wrap_fqn(target, models, existing, fqn)
                    level.append(wrapped)
            model_dependency_list.append(level)

        num_models = sum([len(node_list) for node_list in model_dependency_list])

        if num_models == 0:
            print("WARNING: No models to run in '{}'. Try checking your model configs and running `dbt compile`".format(self.target_path))
            return []

        num_threads = target.threads
        print("Concurrency: {} threads (target='{}')".format(num_threads, self.project['run-target']))
        print("Running!")

        pool = ThreadPool(num_threads)

        failed_models = set()

        model_results = []
        for model_list in model_dependency_list:
            failed_nodes = [tuple(model.fqn) for model in failed_models]

            models_to_execute = [data for data in model_list if not linker.is_child_of(failed_nodes, tuple(data['model'].fqn))]
            models_to_skip = [data for data in model_list if linker.is_child_of(failed_nodes, tuple(data['model'].fqn))]

            for i, data in enumerate(models_to_skip):
                model = data['model']
                model_result = RunModelResult(model, skip=True)
                model_results.append(model_result)
                print("{} of {} -- SKIP relation {}.{} because parent failed".format(len(model_results), num_models, target.schema, model_result.model.name))

            for i, model in enumerate(models_to_execute):
                print_vars = {
                    "progress": 1 + i + len(model_results),
                    "total" : num_models,
                    "schema": target.schema,
                    "model_name": model['model'].name,
                    "model_type": model['model'].materialization,
                    "info": "START"
                }

                output = "{progress} of {total} -- {info} {model_type} model {schema}.{model_name} ".format(**print_vars)
                print("{} [Running]".format(output.ljust(80, ".")))

            run_model_results = pool.map(self.execute_wrapped_model, models_to_execute)

            for run_model_result in run_model_results:
                model_results.append(run_model_result)

                print_vars = {
                    "progress": len(model_results),
                    "total" : num_models,
                    "schema": target.schema,
                    "model_name": run_model_result.model.name,
                    "model_type": run_model_result.model.materialization,
                    "info": "ERROR creating" if run_model_result.errored else "OK created"
                }

                output = "{progress} of {total} -- {info} {model_type} model {schema}.{model_name} ".format(**print_vars)
                print("{} [{}]".format(output.ljust(80, "."), run_model_result.status))

                if run_model_result.errored:
                    failed_models.add(run_model_result.model)
                    print(run_model_result.error)

        pool.close()
        pool.join()

        return model_results

    def run(self, specified_models=None):
        linker = self.deserialize_graph()
        compiled_models = self.get_compiled_models()

        limit_to = None
        if specified_models is not None:
            limit_to = []
            for model_name in specified_models:
                try:
                    model = find_model_by_name(compiled_models, model_name)
                    limit_to.append(tuple(model.fqn))
                except RuntimeError as e:
                    print("ERROR: {}".format(str(e)))
                    print("Exiting")
                    return[]

        target_cfg = self.project.run_environment()
        schema_name = target_cfg['schema']

        try:
            schemas = self.get_schemas()

            if schema_name not in schemas:
                self.create_schema_or_exit(schema_name)

            return self.execute_models(linker, compiled_models, limit_to)
        except psycopg2.OperationalError as e:
            print("ERROR: Could not connect to the target database. Try `dbt debug` for more information")
            print(str(e))
            sys.exit(1)
<|endoftext|>


<commit_before>from pliers.extractors import (MetricExtractor, BertLMExtractor,
                               merge_results)
from pliers.stimuli import SeriesStim, ComplexTextStim
from pliers.tests.utils import get_test_data_path
import numpy as np
import scipy
from pathlib import Path
import pytest
from os import environ

def test_metric_extractor():

    def dummy(array):
        return array[0]

    def dummy_list(array):
        return array[0], array[1]

    f = Path(get_test_data_path(), 'text', 'test_lexical_dictionary.txt')
    stim = SeriesStim(data=np.linspace(1., 4., 20), onset=2., duration=.5)
    stim_file = SeriesStim(filename=f, column='frequency', sep='\t',
                           index_col='text')

    ext_single = MetricExtractor(functions='numpy.mean')
    ext_idx = MetricExtractor(functions='numpy.mean',
                              subset_idx=['for', 'testing', 'text'])
    ext_multiple = MetricExtractor(functions=['numpy.mean', 'numpy.min',
                                              scipy.stats.entropy, dummy,
                                              dummy_list])
    ext_names = MetricExtractor(functions=['numpy.mean', 'numpy.min',
                                           scipy.stats.entropy, dummy,
                                           dummy_list, 'tensorflow.reduce_mean'],
                                var_names=['mean', 'min', 'entropy',
                                           'custom1', 'custom2', 'tf_mean'])
    ext_lambda = MetricExtractor(functions='lambda x: -np.max(x)',
                                 var_names='custom_function')

    r = ext_single.transform(stim)
    r_file = ext_single.transform(stim_file)
    r_file_idx = ext_idx.transform(stim_file)
    r_multiple = ext_multiple.transform(stim)
    r_names = ext_names.transform(stim)
    r_lambda = ext_lambda.transform(stim)

    r_df = r.to_df()
    r_file_df = r_file.to_df()
    r_file_idx_df = r_file_idx.to_df()
    r_multiple_df = r_multiple.to_df()
    r_long = r_multiple.to_df(format='long')
    r_names_df = r_names.to_df()
    r_lambda_df = r_lambda.to_df()

    for res in [r_df, r_file_df, r_multiple_df]:
        assert res.shape[0] == 1
    assert r_long.shape[0] == len(ext_multiple.functions)
    assert r_df['onset'][0] == 2
    assert r_df['duration'][0] == .5
    assert r_df['mean'][0] == 2.5
    assert np.isclose(r_file_df['mean'][0], 11.388, rtol=0.001)
    assert np.isclose(r_file_idx_df['mean'][0], 12.582, rtol=0.001)
    assert all([m in r_multiple_df.columns for m in ['mean', 'entropy']])
    assert r_multiple_df['amin'][0] == 1.
    assert r_multiple_df['dummy'][0] == 1.
    assert r_multiple_df['dummy_list'][0][0] == np.linspace(1., 4., 20)[0]
    assert r_multiple_df['dummy_list'][0][1] == np.linspace(1., 4., 20)[1]
    assert type(r_multiple_df['dummy_list'][0]) == np.ndarray
    assert r_names_df.columns[-3] == 'custom1'
    assert r_names_df.columns[-2] == 'custom2'
    assert r_names_df.columns[-1] == 'tf_mean'
    assert np.isclose(r_names_df['mean'][0], r_names_df['tf_mean'][0])
    assert r_lambda_df['custom_function'][0] == -4

# Change test to use simpler extractor
@pytest.mark.skipif(
    environ.get('skip_high_memory', False) == 'true', reason='high memory')
def test_metric_er_as_stim():
    stim = ComplexTextStim(text='This is [MASK] test')
    ext_bert = BertLMExtractor(return_softmax=True)
    ext_metric = MetricExtractor(functions='numpy.sum')
    r = ext_metric.transform(ext_bert.transform(stim))
    df = merge_results(r, extractor_names=False)
    assert np.isclose(df['sum'][0], 1)
<commit_msg>replace bert extractor with vader in metric extractor tests<commit_after>from pliers.extractors import (MetricExtractor,
                               VADERSentimentExtractor,
                               merge_results)
from pliers.stimuli import SeriesStim, TextStim
from pliers.tests.utils import get_test_data_path
import numpy as np
import scipy
from pathlib import Path
import pytest
from os import environ

def test_metric_extractor():

    def dummy(array):
        return array[0]

    def dummy_list(array):
        return array[0], array[1]

    f = Path(get_test_data_path(), 'text', 'test_lexical_dictionary.txt')
    stim = SeriesStim(data=np.linspace(1., 4., 20), onset=2., duration=.5)
    stim_file = SeriesStim(filename=f, column='frequency', sep='\t',
                           index_col='text')

    ext_single = MetricExtractor(functions='numpy.mean')
    ext_idx = MetricExtractor(functions='numpy.mean',
                              subset_idx=['for', 'testing', 'text'])
    ext_multiple = MetricExtractor(functions=['numpy.mean', 'numpy.min',
                                              scipy.stats.entropy, dummy,
                                              dummy_list])
    ext_names = MetricExtractor(functions=['numpy.mean', 'numpy.min',
                                           scipy.stats.entropy, dummy,
                                           dummy_list, 'tensorflow.reduce_mean'],
                                var_names=['mean', 'min', 'entropy',
                                           'custom1', 'custom2', 'tf_mean'])
    ext_lambda = MetricExtractor(functions='lambda x: -np.max(x)',
                                 var_names='custom_function')

    r = ext_single.transform(stim)
    r_file = ext_single.transform(stim_file)
    r_file_idx = ext_idx.transform(stim_file)
    r_multiple = ext_multiple.transform(stim)
    r_names = ext_names.transform(stim)
    r_lambda = ext_lambda.transform(stim)

    r_df = r.to_df()
    r_file_df = r_file.to_df()
    r_file_idx_df = r_file_idx.to_df()
    r_multiple_df = r_multiple.to_df()
    r_long = r_multiple.to_df(format='long')
    r_names_df = r_names.to_df()
    r_lambda_df = r_lambda.to_df()

    for res in [r_df, r_file_df, r_multiple_df]:
        assert res.shape[0] == 1
    assert r_long.shape[0] == len(ext_multiple.functions)
    assert r_df['onset'][0] == 2
    assert r_df['duration'][0] == .5
    assert r_df['mean'][0] == 2.5
    assert np.isclose(r_file_df['mean'][0], 11.388, rtol=0.001)
    assert np.isclose(r_file_idx_df['mean'][0], 12.582, rtol=0.001)
    assert all([m in r_multiple_df.columns for m in ['mean', 'entropy']])
    assert r_multiple_df['amin'][0] == 1.
    assert r_multiple_df['dummy'][0] == 1.
    assert r_multiple_df['dummy_list'][0][0] == np.linspace(1., 4., 20)[0]
    assert r_multiple_df['dummy_list'][0][1] == np.linspace(1., 4., 20)[1]
    assert type(r_multiple_df['dummy_list'][0]) == np.ndarray
    assert r_names_df.columns[-3] == 'custom1'
    assert r_names_df.columns[-2] == 'custom2'
    assert r_names_df.columns[-1] == 'tf_mean'
    assert np.isclose(r_names_df['mean'][0], r_names_df['tf_mean'][0])
    assert r_lambda_df['custom_function'][0] == -4

def test_metric_er_as_stim():
    stim = TextStim(text='This is a test')
    ext_sent = VADERSentimentExtractor()
    ext_metric = MetricExtractor(functions='numpy.sum',
                                 subset_idx=[f'sentiment_{d}'
                                             for d in ['neg','pos','neu']])
    r = ext_metric.transform(ext_sent.transform(stim))
    df = merge_results(r, extractor_names=False)
    assert np.isclose(df['sum'][0], 1)
<|endoftext|>


<commit_before>"""
 SlipStream Client
 =====
 Copyright (C) 2015 SixSq Sarl (sixsq.com)
 =====
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

import time

from libcloud.compute.types import Provider
from libcloud.compute.types import NodeState
from libcloud.compute.providers import get_driver
import libcloud.security

import slipstream.util as util
import slipstream.exceptions.Exceptions as Exceptions

from slipstream.util import override
from slipstream.NodeDecorator import NodeDecorator
from slipstream.cloudconnectors.BaseCloudConnector import BaseCloudConnector

def getConnector(configHolder):
    return getConnectorClass()(configHolder)

def getConnectorClass():
    return OpenStackClientCloud

def searchInObjectList(list_, propertyName, propertyValue):
    for element in list_:
        if isinstance(element, dict):
            if element.get(propertyName) == propertyValue:
                return element
        else:
            if getattr(element, propertyName) == propertyValue:
                return element
    return None

class OpenStackClientCloud(BaseCloudConnector):
    cloudName = 'openstack'

    def __init__(self, configHolder):
        libcloud.security.VERIFY_SSL_CERT = False

        super(OpenStackClientCloud, self).__init__(configHolder)

        self._set_capabilities(contextualization=True,
                               orchestrator_can_kill_itself_or_its_vapp=True)

        self.flavors = []
        self.images = []
        self.networks = []
        self.security_groups = []
        self.tempPrivateKey = None

    @override
    def _initialization(self, user_info):
        util.printStep('Initialize the OpenStack connector.')
        self._thread_local.driver = self._get_driver(user_info)
        self.flavors = self._thread_local.driver.list_sizes()

        if self.is_deployment():
            self._get_iaas_attributes()
            self._import_keypair(user_info)
        elif self.is_build_image():
            self._get_iaas_attributes()
            self._create_keypair_and_set_on_user_info(user_info)

    @override
    def _finalization(self, user_info):
        try:
            kp_name = user_info.get_keypair_name()
            self._delete_keypair(kp_name)
        # pylint: disable=W0703
        except Exception:
            pass

    def _get_iaas_attributes(self):
        self.images = self._thread_local.driver.list_images()
        self.networks = self._thread_local.driver.ex_list_networks()
        self.security_groups = self._thread_local.driver.ex_list_security_groups()

    def _build_image(self, user_info, node_instance):
        return self._build_image_on_openstack(user_info, node_instance)

    def _build_image_on_openstack(self, user_info, node_instance):
        self._thread_local.driver = self._get_driver(user_info)
        listener = self._get_listener()

        if not user_info.get_private_key() and self.tempPrivateKey:
            user_info.set_private_key(self.tempPrivateKey)
        machine_name = node_instance.get_name()

        vm = self._get_vm(machine_name)

        util.printAndFlush("\n  node_instance: %s \n" % str(node_instance))
        util.printAndFlush("\n  VM: %s \n" % str(vm))

        ip_address = self._vm_get_ip(vm)
        vm_id = self._vm_get_id(vm)
        instance = vm['instance']

        self._wait_instance_in_running_state(vm_id)

        self._build_image_increment(user_info, node_instance, ip_address)

        util.printStep('Creation of the new Image.')
        listener.write_for(machine_name, 'Saving the image')
        newImg = self._thread_local.driver.ex_save_image(instance,
                                                         node_instance.get_image_short_name(),
                                                         metadata=None)

        self._wait_image_creation_completed(newImg.id)
        listener.write_for(machine_name, 'Image saved !')

        return newImg.id

    @override
    def _start_image(self, user_info, node_instance, vm_name):
        self._thread_local.driver = self._get_driver(user_info)
        return self._start_image_on_openstack(user_info, node_instance, vm_name)

    def _start_image_on_openstack(self, user_info, node_instance, vm_name):
        image_id = node_instance.get_image_id()
        instance_type = node_instance.get_instance_type()
        keypair = user_info.get_keypair_name()
        security_groups = self._get_security_groups_for_node_instance(node_instance)
        flavor = searchInObjectList(self.flavors, 'name', instance_type)
        image = searchInObjectList(self.images, 'id', image_id)
        contextualization_script = self.is_build_image() and '' or self._get_bootstrap_script(node_instance)

        if flavor is None:
            raise Exceptions.ParameterNotFoundException("Couldn't find the specified flavor: %s" % instance_type)
        if image is None:
            raise Exceptions.ParameterNotFoundException("Couldn't find the specified image: %s" % image_id)

        # extract mappings for Public and Private networks from the connector instance
        network = None
        network_type = node_instance.get_network_type()
        if network_type == 'Public':
            network_name = user_info.get_public_network_name()
            network = searchInObjectList(self.networks, 'name', network_name)
        elif network_type == 'Private':
            network_name = user_info.get_private_network_name()
            network = searchInObjectList(self.networks, 'name', network_name)

        kwargs = {"name": vm_name,
                  "size": flavor,
                  "image": image,
                  "ex_keyname": keypair,
                  "ex_userdata": contextualization_script,
                  "ex_security_groups": security_groups}

        if network is not None:
            kwargs["networks"] = [network]

        instance = self._thread_local.driver.create_node(**kwargs)

        vm = dict(networkType=node_instance.get_network_type(),
                  instance=instance,
                  ip='',
                  id=instance.id)
        return vm

    @override
    def list_instances(self):
        return self._thread_local.driver.list_nodes()

    def _get_security_groups_for_node_instance(self, node_instance):
        security_groups = []
        security_groups_not_found = []

        for sg_name in node_instance.get_security_groups():
            if not sg_name:
                continue
            sgs = [sg for sg in self.security_groups if sg.name == sg_name.strip()]

            if len(sgs) < 1:
                security_groups_not_found.append(sg_name)
            else:
                security_groups.append(sgs[0])

        if security_groups_not_found:
            raise Exceptions.ParameterNotFoundException("Couldn't find the following security groups: %s"
                                                        % ', '.join(security_groups_not_found))

        return security_groups

    @override
    def _create_allow_all_security_group(self):
        sg_name = NodeDecorator.SECURITY_GROUP_ALLOW_ALL_NAME
        sg_desc = NodeDecorator.SECURITY_GROUP_ALLOW_ALL_DESCRIPTION
        driver = self._thread_local.driver

        if any([sg.name == sg_name for sg in self.security_groups]):
            return

        sg = driver.ex_create_security_group(sg_name, sg_desc)
        driver.ex_create_security_group_rule(sg, 'tcp', 1, 65535, cidr='0.0.0.0/0')
        driver.ex_create_security_group_rule(sg, 'udp', 1, 65535, cidr='0.0.0.0/0')
        driver.ex_create_security_group_rule(sg, 'icmp', -1, -1, cidr='0.0.0.0/0')

        # Update the cached list of available security groups
        self.security_groups = driver.ex_list_security_groups()

    @override
    def _stop_deployment(self):
        for _, vm in self.get_vms().items():
            vm['instance'].destroy()

    @override
    def _stop_vms_by_ids(self, ids):
        for node in self._thread_local.driver.list_nodes():
            if node.id in ids:
                node.destroy()

    def _get_driver(self, user_info):
        driverOpenStack = get_driver(Provider.OPENSTACK)
        isHttps = user_info.get_cloud_endpoint().lower().startswith('https://')

        return driverOpenStack(user_info.get_cloud_username(),
                               user_info.get_cloud_password(),
                               secure=isHttps,
                               ex_tenant_name=user_info.get_cloud('tenant.name'),
                               ex_force_auth_url=user_info.get_cloud_endpoint(),
                               ex_force_auth_version='2.0_password',
                               ex_force_service_type=user_info.get_cloud('service.type'),
                               ex_force_service_name=user_info.get_cloud('service.name'),
                               ex_force_service_region=user_info.get_cloud('service.region'))

    @override
    def _vm_get_ip(self, vm):
        return vm['ip']

    @override
    def _vm_get_id(self, vm):
        return vm['id']

    def _get_vm_size(self, vm_instance):
        try:
            size = [i for i in self.flavors if i.id == vm_instance.extra.get('flavorId')][0]
        except IndexError:
            return None
        else:
            return size

    @override
    def _vm_get_ip_from_list_instances(self, vm_instance):
        return self._get_instance_ip_address(vm_instance)

    @override
    def _vm_get_cpu(self, vm_instance):
        size = self._get_vm_size(vm_instance)
        if size:
            return size.vcpus

    @override
    def _vm_get_ram(self, vm_instance):
        size = self._get_vm_size(vm_instance)
        if size:
            return size.ram

    @override
    def _vm_get_root_disk(self, vm_instance):
        size = self._get_vm_size(vm_instance)
        if size:
            return size.disk

    @override
    def _vm_get_instance_type(self, vm_instance):
        size = self._get_vm_size(vm_instance)
        if size:
            return size.name

    def _get_instance_ip_address(self, instance, ip_type='', strict=True):
        if ip_type.lower() == 'private':
            return (len(instance.private_ips) != 0) and instance.private_ips[0] or (len(instance.public_ips) != 0 and not strict) and instance.public_ips[0] or ''
        elif ip_type.lower() == 'public':
            return (len(instance.public_ips) != 0) and instance.public_ips[0] or (len(instance.private_ips) != 0 and not strict) and instance.private_ips[0] or ''
        else:
            return (len(instance.public_ips) != 0) and instance.public_ips[0] or (len(instance.private_ips) != 0) and instance.private_ips[0] or ''

    @override
    def _wait_and_get_instance_ip_address(self, vm):
        timeWait = 300
        timeStop = time.time() + timeWait

        while time.time() < timeStop:
            time.sleep(1)

            ipType = vm['networkType']
            vmId = vm['id']

            instances = self._thread_local.driver.list_nodes()
            instance = searchInObjectList(instances, 'id', vmId)
            ip = self._get_instance_ip_address(instance, ipType or '')
            if ip:
                vm['ip'] = ip
                return vm

        try:
            ip = self._get_instance_ip_address(instance, ipType or '', False)
        # pylint: disable=W0703
        except Exception:
            pass

        if ip:
            vm['ip'] = ip
            return vm

        raise Exceptions.ExecutionException(
            'Timed out while waiting for IPs to be assigned to instances: %s' % vmId)

    def _wait_instance_in_running_state(self, instanceId):
        timeWait = 300
        timeStop = time.time() + timeWait

        state = ''
        while state != NodeState.RUNNING:
            if time.time() > timeStop:
                raise Exceptions.ExecutionException(
                    'Timed out while waiting for instance "%s" enter in running state'
                    % instanceId)
            time.sleep(1)
            node = self._thread_local.driver.list_nodes()
            state = searchInObjectList(node, 'id', instanceId).state

    def _wait_image_creation_completed(self, imageId):
        timeWait = 600
        timeStop = time.time() + timeWait

        imgState = None
        while imgState == None:
            if time.time() > timeStop:
                raise Exceptions.ExecutionException(
                    'Timed out while waiting for image "%s" to be created' % imageId)
            time.sleep(1)
            images = self._thread_local.driver.list_images()
            imgState = searchInObjectList(images, 'id', imageId)

    def _import_keypair(self, user_info):
        kp_name = 'ss-key-%i' % int(time.time())
        public_key = user_info.get_public_keys()
        try:
            kp = self._thread_local.driver.ex_import_keypair_from_string(kp_name, public_key)
        except Exception as ex:
            raise Exceptions.ExecutionException('Cannot import the public key. Reason: %s' % ex)
        kp_name = kp.name
        user_info.set_keypair_name(kp_name)
        return kp_name

    def _create_keypair_and_set_on_user_info(self, user_info):
        kp_name = 'ss-build-image-%i' % int(time.time())
        kp = self._thread_local.driver.ex_create_keypair(kp_name)
        user_info.set_private_key(kp.private_key)
        user_info.set_keypair_name(kp.name)
        self.tempPrivateKey = kp.private_key
        return kp.name

    def _delete_keypair(self, kp_name):
        kp = searchInObjectList(self._thread_local.driver.list_key_pairs(), 'name', kp_name)
        self._thread_local.driver.delete_key_pair(kp)

<commit_msg>OpenStack: Improved error messages.<commit_after>"""
 SlipStream Client
 =====
 Copyright (C) 2015 SixSq Sarl (sixsq.com)
 =====
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

import time

from libcloud.common.types import InvalidCredsError, LibcloudError
from libcloud.compute.types import Provider, NodeState
from libcloud.compute.providers import get_driver
import libcloud.security

import slipstream.util as util
import slipstream.exceptions.Exceptions as Exceptions

from slipstream.util import override
from slipstream.NodeDecorator import NodeDecorator
from slipstream.cloudconnectors.BaseCloudConnector import BaseCloudConnector

def getConnector(configHolder):
    return getConnectorClass()(configHolder)

def getConnectorClass():
    return OpenStackClientCloud

def searchInObjectList(list_, propertyName, propertyValue):
    for element in list_:
        if isinstance(element, dict):
            if element.get(propertyName) == propertyValue:
                return element
        else:
            if getattr(element, propertyName) == propertyValue:
                return element
    return None

class OpenStackClientCloud(BaseCloudConnector):
    cloudName = 'openstack'

    def __init__(self, configHolder):
        libcloud.security.VERIFY_SSL_CERT = False

        super(OpenStackClientCloud, self).__init__(configHolder)

        self._set_capabilities(contextualization=True,
                               orchestrator_can_kill_itself_or_its_vapp=True)

        self.flavors = []
        self.images = []
        self.networks = []
        self.security_groups = []
        self.tempPrivateKey = None

    @override
    def _initialization(self, user_info):
        util.printStep('Initialize the OpenStack connector.')
        self._thread_local.driver = self._get_driver(user_info)

        try:
            self.flavors = self._thread_local.driver.list_sizes()
        except InvalidCredsError as e:
            raise Exceptions.ValidationException("Invalid Cloud credentials. "
                                                 "Please check your Cloud username, password, project name and domain "
                                                 "(if applicable) in your SlipStream user profile.")
        except LibcloudError as e:
            if e.message == 'Could not find specified endpoint':
                raise Exceptions.ValidationException("Invalid Cloud configuration. "
                                                     "Please ask your SlipStream administrator to check the region, "
                                                     "service-type and service-name.")
            else:
                raise

        if self.is_deployment():
            self._get_iaas_attributes()
            self._import_keypair(user_info)
        elif self.is_build_image():
            self._get_iaas_attributes()
            self._create_keypair_and_set_on_user_info(user_info)

    @override
    def _finalization(self, user_info):
        try:
            kp_name = user_info.get_keypair_name()
            self._delete_keypair(kp_name)
        # pylint: disable=W0703
        except Exception:
            pass

    def _get_iaas_attributes(self):
        self.images = self._thread_local.driver.list_images()
        self.networks = self._thread_local.driver.ex_list_networks()
        self.security_groups = self._thread_local.driver.ex_list_security_groups()

    def _build_image(self, user_info, node_instance):
        return self._build_image_on_openstack(user_info, node_instance)

    def _build_image_on_openstack(self, user_info, node_instance):
        self._thread_local.driver = self._get_driver(user_info)
        listener = self._get_listener()

        if not user_info.get_private_key() and self.tempPrivateKey:
            user_info.set_private_key(self.tempPrivateKey)
        machine_name = node_instance.get_name()

        vm = self._get_vm(machine_name)

        util.printAndFlush("\n  node_instance: %s \n" % str(node_instance))
        util.printAndFlush("\n  VM: %s \n" % str(vm))

        ip_address = self._vm_get_ip(vm)
        vm_id = self._vm_get_id(vm)
        instance = vm['instance']

        self._wait_instance_in_running_state(vm_id)

        self._build_image_increment(user_info, node_instance, ip_address)

        util.printStep('Creation of the new Image.')
        listener.write_for(machine_name, 'Saving the image')
        newImg = self._thread_local.driver.ex_save_image(instance,
                                                         node_instance.get_image_short_name(),
                                                         metadata=None)

        self._wait_image_creation_completed(newImg.id)
        listener.write_for(machine_name, 'Image saved !')

        return newImg.id

    @override
    def _start_image(self, user_info, node_instance, vm_name):
        self._thread_local.driver = self._get_driver(user_info)
        return self._start_image_on_openstack(user_info, node_instance, vm_name)

    def _start_image_on_openstack(self, user_info, node_instance, vm_name):
        image_id = node_instance.get_image_id()
        instance_type = node_instance.get_instance_type()
        keypair = user_info.get_keypair_name()
        security_groups = self._get_security_groups_for_node_instance(node_instance)
        flavor = searchInObjectList(self.flavors, 'name', instance_type)
        image = searchInObjectList(self.images, 'id', image_id)
        contextualization_script = self.is_build_image() and '' or self._get_bootstrap_script(node_instance)

        if flavor is None:
            raise Exceptions.ParameterNotFoundException("Couldn't find the specified flavor: %s" % instance_type)
        if image is None:
            raise Exceptions.ParameterNotFoundException("Couldn't find the specified image: %s" % image_id)

        # extract mappings for Public and Private networks from the connector instance
        network = None
        network_type = node_instance.get_network_type()
        if network_type == 'Public':
            network_name = user_info.get_public_network_name()
            network = searchInObjectList(self.networks, 'name', network_name)
        elif network_type == 'Private':
            network_name = user_info.get_private_network_name()
            network = searchInObjectList(self.networks, 'name', network_name)

        kwargs = {"name": vm_name,
                  "size": flavor,
                  "image": image,
                  "ex_keyname": keypair,
                  "ex_userdata": contextualization_script,
                  "ex_security_groups": security_groups}

        if network is not None:
            kwargs["networks"] = [network]

        instance = self._thread_local.driver.create_node(**kwargs)

        vm = dict(networkType=node_instance.get_network_type(),
                  instance=instance,
                  ip='',
                  id=instance.id)
        return vm

    @override
    def list_instances(self):
        return self._thread_local.driver.list_nodes()

    def _get_security_groups_for_node_instance(self, node_instance):
        security_groups = []
        security_groups_not_found = []

        for sg_name in node_instance.get_security_groups():
            if not sg_name:
                continue
            sgs = [sg for sg in self.security_groups if sg.name == sg_name.strip()]

            if len(sgs) < 1:
                security_groups_not_found.append(sg_name)
            else:
                security_groups.append(sgs[0])

        if security_groups_not_found:
            raise Exceptions.ParameterNotFoundException("Couldn't find the following security groups: %s"
                                                        % ', '.join(security_groups_not_found))

        return security_groups

    @override
    def _create_allow_all_security_group(self):
        sg_name = NodeDecorator.SECURITY_GROUP_ALLOW_ALL_NAME
        sg_desc = NodeDecorator.SECURITY_GROUP_ALLOW_ALL_DESCRIPTION
        driver = self._thread_local.driver

        if any([sg.name == sg_name for sg in self.security_groups]):
            return

        sg = driver.ex_create_security_group(sg_name, sg_desc)
        driver.ex_create_security_group_rule(sg, 'tcp', 1, 65535, cidr='0.0.0.0/0')
        driver.ex_create_security_group_rule(sg, 'udp', 1, 65535, cidr='0.0.0.0/0')
        driver.ex_create_security_group_rule(sg, 'icmp', -1, -1, cidr='0.0.0.0/0')

        # Update the cached list of available security groups
        self.security_groups = driver.ex_list_security_groups()

    @override
    def _stop_deployment(self):
        for _, vm in self.get_vms().items():
            vm['instance'].destroy()

    @override
    def _stop_vms_by_ids(self, ids):
        for node in self._thread_local.driver.list_nodes():
            if node.id in ids:
                node.destroy()

    def _get_driver(self, user_info):
        driverOpenStack = get_driver(Provider.OPENSTACK)
        isHttps = user_info.get_cloud_endpoint().lower().startswith('https://')

        return driverOpenStack(user_info.get_cloud_username(),
                               user_info.get_cloud_password(),
                               secure=isHttps,
                               ex_tenant_name=user_info.get_cloud('tenant.name'),
                               ex_force_auth_url=user_info.get_cloud_endpoint(),
                               ex_force_auth_version='2.0_password',
                               ex_force_service_type=user_info.get_cloud('service.type'),
                               ex_force_service_name=user_info.get_cloud('service.name'),
                               ex_force_service_region=user_info.get_cloud('service.region'))

    @override
    def _vm_get_ip(self, vm):
        return vm['ip']

    @override
    def _vm_get_id(self, vm):
        return vm['id']

    def _get_vm_size(self, vm_instance):
        try:
            size = [i for i in self.flavors if i.id == vm_instance.extra.get('flavorId')][0]
        except IndexError:
            return None
        else:
            return size

    @override
    def _vm_get_ip_from_list_instances(self, vm_instance):
        return self._get_instance_ip_address(vm_instance)

    @override
    def _vm_get_cpu(self, vm_instance):
        size = self._get_vm_size(vm_instance)
        if size:
            return size.vcpus

    @override
    def _vm_get_ram(self, vm_instance):
        size = self._get_vm_size(vm_instance)
        if size:
            return size.ram

    @override
    def _vm_get_root_disk(self, vm_instance):
        size = self._get_vm_size(vm_instance)
        if size:
            return size.disk

    @override
    def _vm_get_instance_type(self, vm_instance):
        size = self._get_vm_size(vm_instance)
        if size:
            return size.name

    def _get_instance_ip_address(self, instance, ip_type='', strict=True):
        if ip_type.lower() == 'private':
            return (len(instance.private_ips) != 0) and instance.private_ips[0] or (len(instance.public_ips) != 0 and not strict) and instance.public_ips[0] or ''
        elif ip_type.lower() == 'public':
            return (len(instance.public_ips) != 0) and instance.public_ips[0] or (len(instance.private_ips) != 0 and not strict) and instance.private_ips[0] or ''
        else:
            return (len(instance.public_ips) != 0) and instance.public_ips[0] or (len(instance.private_ips) != 0) and instance.private_ips[0] or ''

    @override
    def _wait_and_get_instance_ip_address(self, vm):
        timeWait = 300
        timeStop = time.time() + timeWait

        while time.time() < timeStop:
            time.sleep(1)

            ipType = vm['networkType']
            vmId = vm['id']

            instances = self._thread_local.driver.list_nodes()
            instance = searchInObjectList(instances, 'id', vmId)
            ip = self._get_instance_ip_address(instance, ipType or '')
            if ip:
                vm['ip'] = ip
                return vm

        try:
            ip = self._get_instance_ip_address(instance, ipType or '', False)
        # pylint: disable=W0703
        except Exception:
            pass

        if ip:
            vm['ip'] = ip
            return vm

        raise Exceptions.ExecutionException(
            'Timed out while waiting for IPs to be assigned to instances: %s' % vmId)

    def _wait_instance_in_running_state(self, instanceId):
        timeWait = 300
        timeStop = time.time() + timeWait

        state = ''
        while state != NodeState.RUNNING:
            if time.time() > timeStop:
                raise Exceptions.ExecutionException(
                    'Timed out while waiting for instance "%s" enter in running state'
                    % instanceId)
            time.sleep(1)
            node = self._thread_local.driver.list_nodes()
            state = searchInObjectList(node, 'id', instanceId).state

    def _wait_image_creation_completed(self, imageId):
        timeWait = 600
        timeStop = time.time() + timeWait

        imgState = None
        while imgState == None:
            if time.time() > timeStop:
                raise Exceptions.ExecutionException(
                    'Timed out while waiting for image "%s" to be created' % imageId)
            time.sleep(1)
            images = self._thread_local.driver.list_images()
            imgState = searchInObjectList(images, 'id', imageId)

    def _import_keypair(self, user_info):
        kp_name = 'ss-key-%i' % int(time.time())
        public_key = user_info.get_public_keys()
        try:
            kp = self._thread_local.driver.ex_import_keypair_from_string(kp_name, public_key)
        except Exception as ex:
            raise Exceptions.ExecutionException('Cannot import the public key. Reason: %s' % ex)
        kp_name = kp.name
        user_info.set_keypair_name(kp_name)
        return kp_name

    def _create_keypair_and_set_on_user_info(self, user_info):
        kp_name = 'ss-build-image-%i' % int(time.time())
        kp = self._thread_local.driver.ex_create_keypair(kp_name)
        user_info.set_private_key(kp.private_key)
        user_info.set_keypair_name(kp.name)
        self.tempPrivateKey = kp.private_key
        return kp.name

    def _delete_keypair(self, kp_name):
        kp = searchInObjectList(self._thread_local.driver.list_key_pairs(), 'name', kp_name)
        self._thread_local.driver.delete_key_pair(kp)

<|endoftext|>


<commit_before>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2016, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

helpStr = """
Script to analyze the stored patterns in a knn classifier
"""

import argparse
import csv
import numpy
from textwrap import TextWrapper

from htmresearch.support.register_regions import registerAllResearchRegions
from htmresearch.support.csv_helper import readDataAndReshuffle
from htmresearch.frameworks.nlp.classification_model import ClassificationModel

wrapper = TextWrapper(width=100)

def analyzeModel(args, model, documentTextMap):
  """
  Test the given model on testData and print out accuracy.

  Accuracy is calculated as follows. Each token in a document votes for a single
  category. The document is classified with the category that received the most
  votes. Note that it is possible for a token and/or document to receive no
  votes, in which case it is counted as a misclassification.
  """
  knn = model.getClassifier()

  # It would be better if there were proper accessors for this information
  numPatterns = knn._numPatterns
  inputWidth = float(knn._Memory.nCols())
  sparsities = numpy.zeros(numPatterns)
  nonZeroBits = numpy.zeros(numPatterns)

  print "Number of patterns stored in KNN=",numPatterns
  print "Dimensionality of input patterns=",inputWidth
  for i in range(numPatterns):
    nonZeroBits[i] = len(knn.getPattern(i, sparseBinaryForm=True))
    sparsities[i] = (100.0*nonZeroBits[i]/inputWidth)

    # For debugging. You can print out specific documents like this:
    # words = len(documentTextMap[knn.getPartitionId(i)].split())
    # if sparsities[i] < 15 and words > 160:
    #   print wrapper.fill(documentTextMap[knn.getPartitionId(i)])
    #   print

  # Compute min/max/mean sparsities
  print "Min sparsity:     %5.3f"%(sparsities.min())
  print "Max sparsity:     %5.3f"%(sparsities.max())
  print "Average sparsity: %5.3f"%(sparsities.mean())
  print "Sparsity stdev:   %5.3f"%(sparsities.std())

  with open(args.outFile, "wb") as outFile:
    csvWriter = csv.writer(outFile)
    csvWriter.writerow(["words", "sparsity", "bytes"])
    sortedIndices = sparsities.argsort()
    for i in sortedIndices:
      bytes = len(documentTextMap[knn.getPartitionId(i)])
      words = len(documentTextMap[knn.getPartitionId(i)].split())
      csvWriter.writerow([words, sparsities[i], bytes])

def runExperiment(args):
  """
  Create model according to args, train on training data, save model,
  restore model, test on test data.
  """

  (trainingData, labelRefs, documentCategoryMap,
   documentTextMap) = readDataAndReshuffle(args,
                         [8,9,10,5,6,11,13,0,1,2,3,4,7,12,14])

  model = ClassificationModel.load(args.modelDir)

  analyzeModel(args, model, documentTextMap)

  return model

if __name__ == "__main__":

  parser = argparse.ArgumentParser(
    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    description=helpStr
  )

  parser.add_argument("--numLabels",
                      default=13,
                      type=int,
                      help="Number of unique labels to train on.")
  parser.add_argument("-m", "--modelDir",
                      default="docfp.checkpoint",
                      help="Checkpoint directory where model is saved.")
  parser.add_argument("--dataPath",
                      default=None,
                      help="CSV file containing original labeled dataset.")
  parser.add_argument("--outFile",
                      default="temp.csv",
                      help="Some statistics will be output to this CSV file.")
  parser.add_argument("-v", "--verbosity",
                      default=1,
                      type=int,
                      help="verbosity level of printouts")
  args = parser.parse_args()

  model = runExperiment(args)
<commit_msg>cleanup<commit_after>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2016, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

helpStr = """
Script to analyze the stored patterns in a knn classifier
"""

import argparse
import csv
import numpy
from textwrap import TextWrapper

from htmresearch.support.csv_helper import readDataAndReshuffle
from htmresearch.frameworks.nlp.classification_model import ClassificationModel

wrapper = TextWrapper(width=100)

def analyzeModel(args, model, documentTextMap):
  """
  Test the given model on testData and print out accuracy.

  Accuracy is calculated as follows. Each token in a document votes for a single
  category. The document is classified with the category that received the most
  votes. Note that it is possible for a token and/or document to receive no
  votes, in which case it is counted as a misclassification.
  """
  knn = model.getClassifier()

  # It would be better if there were proper accessors for this information
  numPatterns = knn._numPatterns
  inputWidth = float(knn._Memory.nCols())
  sparsities = numpy.zeros(numPatterns)
  nonZeroBits = numpy.zeros(numPatterns)

  print "Number of patterns stored in KNN=",numPatterns
  print "Dimensionality of input patterns=",inputWidth
  for i in range(numPatterns):
    nonZeroBits[i] = len(knn.getPattern(i, sparseBinaryForm=True))
    sparsities[i] = (100.0*nonZeroBits[i]/inputWidth)

    # For debugging. You can print out specific documents like this:
    # words = len(documentTextMap[knn.getPartitionId(i)].split())
    # if sparsities[i] < 15 and words > 160:
    #   print wrapper.fill(documentTextMap[knn.getPartitionId(i)])
    #   print

  # Compute min/max/mean sparsities
  print "Min sparsity:     %5.3f"%(sparsities.min())
  print "Max sparsity:     %5.3f"%(sparsities.max())
  print "Average sparsity: %5.3f"%(sparsities.mean())
  print "Sparsity stdev:   %5.3f"%(sparsities.std())

  with open(args.outFile, "wb") as outFile:
    csvWriter = csv.writer(outFile)
    csvWriter.writerow(["words", "sparsity", "bytes"])
    sortedIndices = sparsities.argsort()
    for i in sortedIndices:
      bytes = len(documentTextMap[knn.getPartitionId(i)])
      words = len(documentTextMap[knn.getPartitionId(i)].split())
      csvWriter.writerow([words, sparsities[i], bytes])

def runExperiment(args):
  """
  Create model according to args, train on training data, save model,
  restore model, test on test data.
  """

  (trainingData, labelRefs, documentCategoryMap,
   documentTextMap) = readDataAndReshuffle(args,
                         [8,9,10,5,6,11,13,0,1,2,3,4,7,12,14])

  model = ClassificationModel.load(args.modelDir)

  analyzeModel(args, model, documentTextMap)

  return model

if __name__ == "__main__":

  parser = argparse.ArgumentParser(
    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    description=helpStr
  )

  parser.add_argument("--numLabels",
                      default=13,
                      type=int,
                      help="Number of unique labels to train on.")
  parser.add_argument("-m", "--modelDir",
                      default="docfp.checkpoint",
                      help="Checkpoint directory where model is saved.")
  parser.add_argument("--dataPath",
                      default=None,
                      help="CSV file containing original labeled dataset.")
  parser.add_argument("--outFile",
                      default="temp.csv",
                      help="Some statistics will be output to this CSV file.")
  parser.add_argument("-v", "--verbosity",
                      default=1,
                      type=int,
                      help="verbosity level of printouts")
  args = parser.parse_args()

  model = runExperiment(args)
<|endoftext|>


<commit_before>import os, fcntl, termios, struct
import time

def my_forkpty():

    (master_fd, slave_fd) = os.openpty()

    if (master_fd < 0  or  slave_fd < 0):
        raise ExceptionPexpect("Forkpty failed")

    # slave_name = ptsname(master_fd);

    pid = os.fork();
    if pid == -1:
        raise ExceptionPexpect("Forkpty failed")
    elif pid == 0: # Child
        if 'TIOCNOTTY' in dir(termios):
        #        /* Some platforms require an explicit detach of the
        #        current controlling tty before closing stdin, stdout, stderr.
        #        OpenBSD says that this is obsolete, but doesn't hurt. */
            fd = os.open("/dev/tty", os.O_RDWR | os.O_NOCTTY)
            if fd >= 0:
                s = struct.pack("H", 0)
                fcntl.ioctl(fd, termios.TIOCNOTTY, s);
                os.close(fd)

        # The setsid() system call will place the process into its own session
        # which has the effect of disassociating it from the controlling terminal.
        # This is known to be true for OpenBSD.
        os.setsid()
        # except:            return posix_error();

        # Verify that we are disconnected from the controlling tty.
        try:
            fd = os.open("/dev/tty", os.O_RDWR | os.O_NOCTTY)
            os.close(fd)
            raise ExceptionPexpect("Forkpty failed")
        except:
            pass
        if 'TIOCSCTTY' in dir(termios):
            # Make the pseudo terminal the controlling terminal for this process
            # (the process must not currently have a controlling terminal).
            s = struct.pack("H", 0)
            if fcntl.ioctl(slave_fd, termios.TIOCSCTTY, s) < 0:
                raise ExceptionPexpect("Forkpty failed")

#        # Verify that we can open to the slave pty file. */
#        fd = os.open(slave_name, os.O_RDWR);
#        if fd < 0:
#            raise ExceptionPexpect("Forkpty failed")
#        else:
#            os.close(fd);

        # Verify that we now have a controlling tty.
        fd = os.open("/dev/tty", os.O_WRONLY)
        if fd < 0:
            raise ExceptionPexpect("This process could not get a controlling tty.")
        else:
            os.close(fd)

        os.close(master_fd)
        os.dup2(slave_fd, 0)
        os.dup2(slave_fd, 1)
        os.dup2(slave_fd, 2)
        if slave_fd > 2:
            os.close(slave_fd)
        pid = 0

    else:
        # PARENT
        os.close(slave_fd);

    if pid == -1:
        raise ExceptionPexpect("This process could not get a controlling tty.")
#    if (pid == 0)
#        PyOS_AfterFork();

    return (pid, master_fd)

pid, fd = my_forkpty ()
if pid == 0: # child
    print 'I am not a robot!'
else:
    print '(pid, fd) = (%d, %d)' % (pid, fd)
    time.sleep(1) # Give the child a chance to print.
    print 'Robots always say:', os.read(fd,100)
    os.close(fd)

<commit_msg>Nothing fucking works on Solaris...<commit_after>import os, fcntl, termios
import time

def my_forkpty():

    (master_fd, slave_fd) = os.openpty()

    if (master_fd < 0  or  slave_fd < 0):
        raise ExceptionPexpect("Forkpty failed")

    # slave_name = ptsname(master_fd);

    pid = os.fork();
    if pid == -1:
        raise ExceptionPexpect("Forkpty failed")
    elif pid == 0: # Child
        if hasattr(termios, 'TIOCNOTTY'):
        #        Some platforms require an explicit detach of the
        #        current controlling tty before closing stdin, stdout, stderr.
        #        OpenBSD says that this is obsolete, but doesn't hurt.
            try:
                fd = os.open("/dev/tty", os.O_RDWR | os.O_NOCTTY)
            except:
                pass
            else: #if fd >= 0:
                fcntl.ioctl(fd, termios.TIOCNOTTY, 0)
                os.close(fd)

        # The setsid() system call will place the process into its own session
        # which has the effect of disassociating it from the controlling terminal.
        # This is known to be true for OpenBSD.
        os.setsid()
        # except:            return posix_error();

        # Verify that we are disconnected from the controlling tty.
        try:
            fd = os.open("/dev/tty", os.O_RDWR | os.O_NOCTTY)
            os.close(fd)
            raise ExceptionPexpect("Forkpty failed")
        except:
            pass
        if 'TIOCSCTTY' in dir(termios):
            # Make the pseudo terminal the controlling terminal for this process
            # (the process must not currently have a controlling terminal).
            if fcntl.ioctl(slave_fd, termios.TIOCSCTTY, '') < 0:
                raise ExceptionPexpect("Forkpty failed")

#        # Verify that we can open to the slave pty file. */
#        fd = os.open(slave_name, os.O_RDWR);
#        if fd < 0:
#            raise ExceptionPexpect("Forkpty failed")
#        else:
#            os.close(fd);

        # Verify that we now have a controlling tty.
        fd = os.open("/dev/tty", os.O_WRONLY)
        if fd < 0:
            raise ExceptionPexpect("This process could not get a controlling tty.")
        else:
            os.close(fd)

        os.close(master_fd)
        os.dup2(slave_fd, 0)
        os.dup2(slave_fd, 1)
        os.dup2(slave_fd, 2)
        if slave_fd > 2:
            os.close(slave_fd)
        pid = 0

    else:
        # PARENT
        os.close(slave_fd);

    if pid == -1:
        raise ExceptionPexpect("This process could not get a controlling tty.")
#    if (pid == 0)
#        PyOS_AfterFork();

    return (pid, master_fd)

pid, fd = my_forkpty ()
if pid == 0: # child
    print 'I am not a robot!'
else:
    print '(pid, fd) = (%d, %d)' % (pid, fd)
    time.sleep(1) # Give the child a chance to print.
    print 'Robots always say:', os.read(fd,100)
    os.close(fd)

<|endoftext|>


<commit_before>import os
from vhdl_toolkit.formater import formatVhdl
from vhdl_toolkit.synthetisator.interfaceLevel.unitFromHdl import UnitFromHdl
import shutil
from vhdl_toolkit.synthetisator.vhdlSerializer import VhdlSerializer
from vhdl_toolkit.hdlObjects.entity import Entity
from vhdl_toolkit.hdlObjects.architecture import Architecture
from vhdl_toolkit.synthetisator.vhdlCodeWrap import VhdlCodeWrap
from vhdl_toolkit.synthetisator.interfaceLevel.unit import Unit
from python_toolkit.fileHelpers import find_files
from vhdl_toolkit.parser import Parser

def synthetizeCls(cls, name=None, multithread=True):
    u = cls(multithread=multithread)
    u._loadDeclarations()
    u._loadImplementations()
    return formatVhdl(
                     "\n".join([ VhdlSerializer.asHdl(x) for x in u._synthesise(name)])
                     )

def synthetizeAndSave(unit, folderName='.', name=None):
    header = None
    os.makedirs(folderName, exist_ok=True)
    files = set()

    for o in [ x for x in unit._synthesise(name)]:
        if isinstance(o, VhdlCodeWrap):
            header = o
            fName = None
        elif isinstance(o, Entity):
            fName = o.name + "_ent.vhd"
        elif isinstance(o, Architecture):
            fName = o.entityName + "_" + o.name + "_arch.vhd"
        elif isinstance(o, UnitFromHdl):
            fName = None
            for fn in o._hdlSources:
                shutil.copy2(fn, folderName)
                files.add(fn)
        else:
            raise Exception("Do not know how to serialize %s" % (repr(o)))

        if fName is not None:
            fp = os.path.join(folderName, fName)
            files.add(fp)

            with open(fp, 'w') as f:
                f.write(formatVhdl(
                     "\n".join([ VhdlSerializer.asHdl(x) for x in [header, o]])
                        ))
    return files

def fileSyntaxCheck(fileName, lang, timeoutInterval=20):
    """
    Perform syntax check on whole file (only in java parser)
    """
    p = Parser.spotLoadingProc(fileName, lang, hierarchyOnly=True, debug=False)
    p.communicate(timeout=timeoutInterval)

def syntaxCheck(unitOrFileName):
    if isinstance(unitOrFileName, Unit):
        d = "__pycache__/" + unitOrFileName._name
        synthetizeAndSave(unitOrFileName, d)
        for f in find_files(d, '*.vhd', recursive=True):
            fileSyntaxCheck(f, Parser.VHDL)
        for f in find_files(d, '*.v', recursive=True):
            fileSyntaxCheck(f, Parser.VERILOG)

    elif isinstance(unitOrFileName, str):
        fileSyntaxCheck(f, Parser.langFromExtension(unitOrFileName))
    else:
        raise  NotImplementedError("Not implemented for '%'" % (repr(unitOrFileName)))

def synthetizeAsIpcore(unit, folderName=".", name=None):
    # too simple to implement -> useless
    raise NotImplementedError()

<commit_msg>lib copy fix<commit_after>import os
from vhdl_toolkit.formater import formatVhdl
from vhdl_toolkit.synthetisator.interfaceLevel.unitFromHdl import UnitFromHdl
import shutil
from vhdl_toolkit.synthetisator.vhdlSerializer import VhdlSerializer
from vhdl_toolkit.hdlObjects.entity import Entity
from vhdl_toolkit.hdlObjects.architecture import Architecture
from vhdl_toolkit.synthetisator.vhdlCodeWrap import VhdlCodeWrap
from vhdl_toolkit.synthetisator.interfaceLevel.unit import Unit
from python_toolkit.fileHelpers import find_files
from vhdl_toolkit.parser import Parser

def synthetizeCls(cls, name=None, multithread=True):
    u = cls(multithread=multithread)
    u._loadAll()
    return formatVhdl(
                     "\n".join([ VhdlSerializer.asHdl(x) for x in u._synthesise(name)])
                     )

def synthetizeAndSave(unit, folderName='.', name=None):
    unit._loadAll()
    header = None
    os.makedirs(folderName, exist_ok=True)
    files = set()

    for o in [ x for x in unit._synthesise(name)]:
        if isinstance(o, VhdlCodeWrap):
            header = o
            fName = None
        elif isinstance(o, Entity):
            fName = o.name + "_ent.vhd"
        elif isinstance(o, Architecture):
            fName = o.entityName + "_" + o.name + "_arch.vhd"
        elif isinstance(o, UnitFromHdl):
            fName = None
            for fn in o._hdlSources:
                if isinstance(fn, str):
                    shutil.copy2(fn, folderName)
                    files.add(fn)
        else:
            raise Exception("Do not know how to serialize %s" % (repr(o)))

        if fName is not None:
            fp = os.path.join(folderName, fName)
            files.add(fp)

            with open(fp, 'w') as f:
                f.write(formatVhdl(
                     "\n".join([ VhdlSerializer.asHdl(x) for x in [header, o]])
                        ))
    return files

def fileSyntaxCheck(fileName, lang, timeoutInterval=20):
    """
    Perform syntax check on whole file (only in java parser)
    """
    p = Parser.spotLoadingProc(fileName, lang, hierarchyOnly=True, debug=False)
    p.communicate(timeout=timeoutInterval)

def syntaxCheck(unitOrFileName):
    if isinstance(unitOrFileName, Unit):
        d = "__pycache__/" + unitOrFileName._name
        synthetizeAndSave(unitOrFileName, d)
        for f in find_files(d, '*.vhd', recursive=True):
            fileSyntaxCheck(f, Parser.VHDL)
        for f in find_files(d, '*.v', recursive=True):
            fileSyntaxCheck(f, Parser.VERILOG)

    elif isinstance(unitOrFileName, str):
        fileSyntaxCheck(f, Parser.langFromExtension(unitOrFileName))
    else:
        raise  NotImplementedError("Not implemented for '%'" % (repr(unitOrFileName)))

def synthetizeAsIpcore(unit, folderName=".", name=None):
    # too simple to implement -> useless
    raise NotImplementedError()

<|endoftext|>