<commit_before>import csv
import os
import subprocess
import threading

# Gather the packages to test.

PREFIX = './packages/node_modules/'
CISCOSPARK = os.path.join(PREFIX, '@ciscospark')
WEBEX = os.path.join(PREFIX, '@webex')

PROD_ENV_VARS = {
  # 'ACL_SERVICE_URL': 'https://acl-a.wbx2.com/acl/api/v1', ?
  'ATLAS_SERVICE_URL': 'https://atlas-a.wbx2.com/admin/api/v1',
  'CONVERSATION_SERVICE': 'https://conv-a.wbx2.com/conversation/api/v1',
  'ENCRYPTION_SERVICE_URL': 'https://encryption-a.wbx2.com',
  'IDBROKER_BASE_URL': 'https://idbroker.webex.com',
  'IDENTITY_BASE_URL': 'https://identity.webex.com',
  'WDM_SERVICE_URL': 'https://wdm-a.wbx2.com/wdm/api/v1',
  # Logging
  'ENABLE_VERBOSE_NETWORK_LOGGING': 'true'
}

INT_ENV_VARS = {
  # Environments
  'ACL_SERVICE_URL': 'https://acl-intb.ciscospark.com/acl/api/v1',
  'ATLAS_SERVICE_URL': 'https://atlas-intb.ciscospark.com/admin/api/v1',
  'CONVERSATION_SERVICE': 'https://conversation-intb.ciscospark.com/conversation/api/v1',
  'ENCRYPTION_SERVICE_URL': 'https://encryption-intb.ciscospark.com/encryption/api/v1',
  'HYDRA_SERVICE_URL': 'https://hydra-intb.ciscospark.com/v1',
  'IDBROKER_BASE_URL': 'https://idbrokerbts.webex.com',
  'IDENTITY_BASE_URL': 'https://identitybts.webex.com',
  'WDM_SERVICE_URL': 'https://wdm-intb.ciscospark.com/wdm/api/v1',
  'WHISTLER_API_SERVICE_URL': 'https://whistler.onint.ciscospark.com/api/v1',
  # Logging
  'ENABLE_VERBOSE_NETWORK_LOGGING': 'true'
}

OUTPUT_DIR = 'output'
OUTPUT_FILE_PATH = os.path.join(OUTPUT_DIR, 'test-comparison.csv')

TEST_COMMAND = 'npm test -- --package %s'

SKIP_PACKAGES = [
  '@webex/bin-sauce-connect', # needs Sauce started
  # '@webex/plugin-meetings', # no tests
  # '@webex/test-helper-server' # no tests
  # '@ciscospark/internal-plugin-calendar', # no tests
  # '@ciscospark/plugin-webhooks' # no tests
]

def should_include_package(path_name, name):
  scoped_name = os.path.join(os.path.basename(path_name), name)
  return os.path.isdir(os.path.join(path_name, name)) and scoped_name not in SKIP_PACKAGES

def get_package_names(path_name):
  namespace = path_name.replace(PREFIX, '')
  return [os.path.join(namespace, name) for name in os.listdir(path_name) if should_include_package(path_name, name)]

def run_subprocess(bash_command, env_vars):
  env = os.environ.copy()
  env.update(env_vars)
  process = subprocess.Popen(bash_command.split(), stdout=subprocess.PIPE, env=env)

  output, error = process.communicate()
  return process.returncode # , output, error

class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'

def print_result(return_code, prefix='Tests are a...'):
  if return_code == 0:
    print(bcolors.OKGREEN + prefix + 'success.' + bcolors.ENDC)
  else:
    print(bcolors.FAIL + prefix + 'failure.' + bcolors.ENDC)

def run_test(package, environment):
  env_vars = INT_ENV_VARS if environment is 'integration' else PROD_ENV_VARS
  print(bcolors.OKBLUE + 'Testing `%s` on %s...' % (package, environment) + bcolors.ENDC)
  bash_command = TEST_COMMAND % package
  return_code = run_subprocess(bash_command, env_vars)
  print_result(return_code, prefix='Testing `%s` on %s...' % (package, environment))
  return return_code

def run_env_tests(package, writer, csv_file):
  prod_return_code = run_test(package, 'production')
  int_return_code = run_test(package, 'integration')
  writer.writerow([package, prod_return_code, int_return_code])
  csv_file.flush()

def main():
  ciscospark_packages = get_package_names(CISCOSPARK)
  webex_packages = get_package_names(WEBEX)
  packages = ciscospark_packages + webex_packages
  print ('Skipping %d packages: %s' % (len(SKIP_PACKAGES), ', '.join(SKIP_PACKAGES)))
  print('Testing %d packages...' % len(packages))

  try:
    os.mkdir(OUTPUT_DIR)
  except OSError:
    pass

  threads = []

  with open(OUTPUT_FILE_PATH, 'wb') as csv_file:
    writer = csv.writer(csv_file, quoting=csv.QUOTE_MINIMAL)
    writer.writerow(['Package', 'Production exit code', 'Integration exit code'])

    for package in packages:
      run_env_tests(package, writer, csv_file)

    # threads = [threading.Thread(target=run_env_tests, args=(package, writer, csv_file)) for package in packages]
    # for thread in threads:
    #   thread.start()
    # for thread in threads:
    #   thread.join()

  print('Wrote output to: %s' % OUTPUT_FILE_PATH)
  print('Done.')

if __name__ == "__main__":
  main()
<commit_msg>fix(test.py): use the public integration URL for Hydra<commit_after>import csv
import os
import subprocess
import threading

# Gather the packages to test.

PREFIX = './packages/node_modules/'
CISCOSPARK = os.path.join(PREFIX, '@ciscospark')
WEBEX = os.path.join(PREFIX, '@webex')

PROD_ENV_VARS = {
  # 'ACL_SERVICE_URL': 'https://acl-a.wbx2.com/acl/api/v1', ?
  'ATLAS_SERVICE_URL': 'https://atlas-a.wbx2.com/admin/api/v1',
  'CONVERSATION_SERVICE': 'https://conv-a.wbx2.com/conversation/api/v1',
  'ENCRYPTION_SERVICE_URL': 'https://encryption-a.wbx2.com',
  'IDBROKER_BASE_URL': 'https://idbroker.webex.com',
  'IDENTITY_BASE_URL': 'https://identity.webex.com',
  'WDM_SERVICE_URL': 'https://wdm-a.wbx2.com/wdm/api/v1',
  # Logging
  'ENABLE_VERBOSE_NETWORK_LOGGING': 'true'
}

INT_ENV_VARS = {
  # Environments
  'ACL_SERVICE_URL': 'https://acl-intb.ciscospark.com/acl/api/v1',
  'ATLAS_SERVICE_URL': 'https://atlas-intb.ciscospark.com/admin/api/v1',
  'CONVERSATION_SERVICE': 'https://conversation-intb.ciscospark.com/conversation/api/v1',
  'ENCRYPTION_SERVICE_URL': 'https://encryption-intb.ciscospark.com/encryption/api/v1',
  # Do not use 'https://hydra-intb.ciscospark.com/v1' for Hydra. CI expects 'apialpha'.
  'HYDRA_SERVICE_URL': 'https://apialpha.ciscospark.com/v1/',
  'IDBROKER_BASE_URL': 'https://idbrokerbts.webex.com',
  'IDENTITY_BASE_URL': 'https://identitybts.webex.com',
  'WDM_SERVICE_URL': 'https://wdm-intb.ciscospark.com/wdm/api/v1',
  'WHISTLER_API_SERVICE_URL': 'https://whistler.onint.ciscospark.com/api/v1',
  # Logging
  'ENABLE_VERBOSE_NETWORK_LOGGING': 'true'
}

OUTPUT_DIR = 'output'
OUTPUT_FILE_PATH = os.path.join(OUTPUT_DIR, 'test-comparison.csv')

TEST_COMMAND = 'npm test -- --package %s'

SKIP_PACKAGES = [
  '@webex/bin-sauce-connect', # needs Sauce started
  # '@webex/plugin-meetings', # no tests
  # '@webex/test-helper-server' # no tests
  # '@ciscospark/internal-plugin-calendar', # no tests
  # '@ciscospark/plugin-webhooks' # no tests
]

def should_include_package(path_name, name):
  scoped_name = os.path.join(os.path.basename(path_name), name)
  return os.path.isdir(os.path.join(path_name, name)) and scoped_name not in SKIP_PACKAGES

def get_package_names(path_name):
  namespace = path_name.replace(PREFIX, '')
  return [os.path.join(namespace, name) for name in os.listdir(path_name) if should_include_package(path_name, name)]

def run_subprocess(bash_command, env_vars):
  env = os.environ.copy()
  env.update(env_vars)
  process = subprocess.Popen(bash_command.split(), stdout=subprocess.PIPE, env=env)

  output, error = process.communicate()
  return process.returncode # , output, error

class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'

def print_result(return_code, prefix='Tests are a...'):
  if return_code == 0:
    print(bcolors.OKGREEN + prefix + 'success.' + bcolors.ENDC)
  else:
    print(bcolors.FAIL + prefix + 'failure.' + bcolors.ENDC)

def run_test(package, environment):
  env_vars = INT_ENV_VARS if environment is 'integration' else PROD_ENV_VARS
  print(bcolors.OKBLUE + 'Testing `%s` on %s...' % (package, environment) + bcolors.ENDC)
  bash_command = TEST_COMMAND % package
  return_code = run_subprocess(bash_command, env_vars)
  print_result(return_code, prefix='Testing `%s` on %s...' % (package, environment))
  return return_code

def run_env_tests(package, writer, csv_file):
  prod_return_code = run_test(package, 'production')
  int_return_code = run_test(package, 'integration')
  writer.writerow([package, prod_return_code, int_return_code])
  csv_file.flush()

def main():
  ciscospark_packages = get_package_names(CISCOSPARK)
  webex_packages = get_package_names(WEBEX)
  packages = ciscospark_packages + webex_packages
  print ('Skipping %d packages: %s' % (len(SKIP_PACKAGES), ', '.join(SKIP_PACKAGES)))
  print('Testing %d packages...' % len(packages))

  try:
    os.mkdir(OUTPUT_DIR)
  except OSError:
    pass

  threads = []

  with open(OUTPUT_FILE_PATH, 'wb') as csv_file:
    writer = csv.writer(csv_file, quoting=csv.QUOTE_MINIMAL)
    writer.writerow(['Package', 'Production exit code', 'Integration exit code'])

    for package in packages:
      run_env_tests(package, writer, csv_file)

    # threads = [threading.Thread(target=run_env_tests, args=(package, writer, csv_file)) for package in packages]
    # for thread in threads:
    #   thread.start()
    # for thread in threads:
    #   thread.join()

  print('Wrote output to: %s' % OUTPUT_FILE_PATH)
  print('Done.')

if __name__ == "__main__":
  main()
<|endoftext|>


<commit_before>from lib.config import Config
from slackclient import SlackClient
from flask import abort
from youtube import Youtube
from random import randint
from datetime import datetime

class Tubey():

    buttons = [{
                    "name": "send",
                    "text": "Send",
                    "type": "button",
                    "style": "primary",
                    "value": "Send that pineapple"
                },
                {
                    "name": "shuffle",
                    "text": "Shuffle",
                    "type": "button",
                    "value": "Shuffle it up!"
                },
                {
                    "name": "cancel",
                    "text": "Cancel",
                    "type": "button",
                    "style": "danger",
                    "value": "Cancel dis plz"
                 }]

    def __init__(self):
        # Cache the client in memory
        self._bot_client = None
        self._user_client = None
        self.clients = {'bot': self._bot_client, 'user': self._user_client}

    def send_message(self, params, type="Message"):
        # Sends message to the user/channel
        # Type should either be Message or Ephemeral
        if 'as_user' in params.keys() and params['as_user']:
            client = self.get_client(type='user')
        else:
            client = self.get_client(type='bot')

        client.api_call("chat.post" + type, **params)

    def get_client(self, type='bot'):
        # Fetch a cached slack client or create one and return it
        # type is either 'bot' or 'user'
        if self.clients[type] is not None:
            return self.clients[type]

        print("NEW CLIENT ALERT FOR: ", type)

        token = Config.get_variable('tubey', type + '_oauth_token')
        sc = SlackClient(token)
        self.clients[type] = sc
        return sc

    def suggest_video(self, query, channel, user, is_shuffle):
        # Sends a video suggestion in an ephemeral message
        videos = self.search(query)
        num_vids = len(videos)
        suggested_video = videos[randint(0, num_vids) % num_vids]

        published_date = datetime.strptime(suggested_video['snippet']['publishedAt'][0:10], "%Y-%m-%d").date().strftime(
            '%B %d, %Y')
        channel_name = suggested_video['snippet']['channelTitle']
        video_title = suggested_video['snippet']['title']
        description = suggested_video['snippet']['description']
        thumbnail = suggested_video['snippet']['thumbnails']['high']['url']
        id = suggested_video['id']['videoId']

        self.buttons[0]['value'] = id
        self.buttons[1]['value'] = query

        params = {
            'unfurl_links': False,
            'channel': channel,
            'user': user,
            "attachments": [{
                "title": "Video title: {}".format(video_title),
                "text": "Channel name: {}\nPublished date: {}\nVideo description: {}".format(channel_name, published_date, description),
                "title_link": "https://www.youtube.com/watch?v=".format(id),
                "image_url": thumbnail,
                "fallback": "You need to upgrade your Slack client to use this command!",
                "color": "#CD201F",
                "attachment_type": "default",
                "actions": self.buttons,
                "callback_id": "primary_menu"
            }]
        }

        if is_shuffle:
            params['replace_original'] = True
            return params
        else:
            response = self.send_message(params, type="Ephemeral")
            print(response)

    def send_video(self, video_id, channel):
        # Sends a chosen video to the channel
        params =  {
            "channel": channel,
            "text": "https://www.youtube.com/watch?v={}".format(video_id),
            "as_user": True
        }

        response = self.send_message(params)

        return {"delete_original": True}

    def send_channel(self, channel_id, channel_name, thumbnail_url):
        # Sends the youtube channel to the active user/slack channel
        channel_url = "https://www.youtube.com/channel/{}".format(channel_id)
        params = {'channel': 'tubeydev',
                  'text': "Here's the channel:",
                  'attachments': [
                      {'fallback': channel_name + ' thumbnail',
                       'title': channel_name,
                       "title_link": channel_url,
                       "image_url": thumbnail_url
                       }]
                  }
        self.send_message(params)

    def search(self, search_query):
        youtube = Youtube()
        results = youtube.query(search_query)

        return results['videos']

    def verify_token(self, token):
        verif_token = Config.get_variable('tubey', 'verif_token')

        # Validate the request parameters
        if token != verif_token:
            abort(401)  # Unauthorized request. If you're not Slack, go away

if __name__ == "__main__":
    # Should probably put a legitimate sample run here...I'll start it off
    pass
<commit_msg>Add _ prefix to private variables in slack.py<commit_after>from lib.config import Config
from slackclient import SlackClient
from flask import abort
from youtube import Youtube
from random import randint
from datetime import datetime

class Tubey():

    buttons = [{
                    "name": "send",
                    "text": "Send",
                    "type": "button",
                    "style": "primary",
                    "value": "Send that pineapple"
                },
                {
                    "name": "shuffle",
                    "text": "Shuffle",
                    "type": "button",
                    "value": "Shuffle it up!"
                },
                {
                    "name": "cancel",
                    "text": "Cancel",
                    "type": "button",
                    "style": "danger",
                    "value": "Cancel dis plz"
                 }]

    def __init__(self):
        # Cache the client in memory
        self._bot_client = None
        self._user_client = None
        self._clients = {'bot': self._bot_client, 'user': self._user_client}

    def send_message(self, params, type="Message"):
        # Sends message to the user/channel
        # Type should either be Message or Ephemeral
        if 'as_user' in params.keys() and params['as_user']:
            _client = self.get_client(type='user')
        else:
            _client = self.get_client(type='bot')

        _client.api_call("chat.post" + type, **params)

    def get_client(self, type='bot'):
        # Fetch a cached slack client or create one and return it
        # type is either 'bot' or 'user'
        if self._clients[type] is not None:
            return self._clients[type]

        print("NEW CLIENT ALERT FOR: ", type)

        _token = Config.get_variable('tubey', type + '_oauth_token')
        _sc = SlackClient(_token)
        self._clients[type] = _sc
        return _sc

    def suggest_video(self, query, channel, user, is_shuffle):
        # Sends a video suggestion in an ephemeral message
        videos = self.search(query)
        num_vids = len(videos)
        suggested_video = videos[randint(0, num_vids) % num_vids]

        published_date = datetime.strptime(suggested_video['snippet']['publishedAt'][0:10], "%Y-%m-%d").date().strftime(
            '%B %d, %Y')
        channel_name = suggested_video['snippet']['channelTitle']
        video_title = suggested_video['snippet']['title']
        description = suggested_video['snippet']['description']
        thumbnail = suggested_video['snippet']['thumbnails']['high']['url']
        id = suggested_video['id']['videoId']

        self.buttons[0]['value'] = id
        self.buttons[1]['value'] = query

        params = {
            'unfurl_links': False,
            'channel': channel,
            'user': user,
            "attachments": [{
                "title": "Video title: {}".format(video_title),
                "text": "Channel name: {}\nPublished date: {}\nVideo description: {}".format(channel_name, published_date, description),
                "title_link": "https://www.youtube.com/watch?v=".format(id),
                "image_url": thumbnail,
                "fallback": "You need to upgrade your Slack client to use this command!",
                "color": "#CD201F",
                "attachment_type": "default",
                "actions": self.buttons,
                "callback_id": "primary_menu"
            }]
        }

        if is_shuffle:
            params['replace_original'] = True
            return params
        else:
            response = self.send_message(params, type="Ephemeral")
            print(response)

    def send_video(self, video_id, channel):
        # Sends a chosen video to the channel
        params =  {
            "channel": channel,
            "text": "https://www.youtube.com/watch?v={}".format(video_id),
            "as_user": True
        }

        response = self.send_message(params)

        return {"delete_original": True}

    def send_channel(self, channel_id, channel_name, thumbnail_url):
        # Sends the youtube channel to the active user/slack channel
        channel_url = "https://www.youtube.com/channel/{}".format(channel_id)
        params = {'channel': 'tubeydev',
                  'text': "Here's the channel:",
                  'attachments': [
                      {'fallback': channel_name + ' thumbnail',
                       'title': channel_name,
                       "title_link": channel_url,
                       "image_url": thumbnail_url
                       }]
                  }
        self.send_message(params)

    def search(self, search_query):
        youtube = Youtube()
        results = youtube.query(search_query)

        return results['videos']

    def verify_token(self, token):
        verif_token = Config.get_variable('tubey', 'verif_token')

        # Validate the request parameters
        if token != verif_token:
            abort(401)  # Unauthorized request. If you're not Slack, go away

if __name__ == "__main__":
    # Should probably put a legitimate sample run here...I'll start it off
    pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# (c) 2014 Bjoern Ricks <bjoern.ricks@gmail.com>
#
# See LICENSE comming with the source of 'trex' for details.
#

from django.conf.urls import patterns, include, url
from django.contrib import admin

from trex.views.project import (
    ProjectListCreateAPIView, ProjectDetailAPIView)

urlpatterns = patterns(
    '',
    url(r"^admin/", include(admin.site.urls)),
    url(r"^projects/$", ProjectListCreateAPIView.as_view(),
        name="project-list"),
    url(r"^projects/(?P<pk>[0-9]+)/$", ProjectDetailAPIView.as_view(),
        name="project-details"),
)
<commit_msg>Use standard view name for project details<commit_after># -*- coding: utf-8 -*-
#
# (c) 2014 Bjoern Ricks <bjoern.ricks@gmail.com>
#
# See LICENSE comming with the source of 'trex' for details.
#

from django.conf.urls import patterns, include, url
from django.contrib import admin

from trex.views.project import (
    ProjectListCreateAPIView, ProjectDetailAPIView)

urlpatterns = patterns(
    '',
    url(r"^admin/", include(admin.site.urls)),
    url(r"^projects/$", ProjectListCreateAPIView.as_view(),
        name="project-list"),
    url(r"^projects/(?P<pk>[0-9]+)/$", ProjectDetailAPIView.as_view(),
        name="project-detail"),
)
<|endoftext|>


<commit_before>import itertools
import typing
from typing import TypeVar, Callable, Generic, Iterable

from tek.tools import find  # type: ignore

from tryp.maybe import Maybe

A = TypeVar('A', covariant=True)
B = TypeVar('B')

def flatten(l: Iterable[Iterable[A]]) -> Iterable[A]:
    return list(itertools.chain.from_iterable(l))  # type: ignore

class List(typing.List[A], Generic[A]):

    def __init__(self, *elements):
        typing.List.__init__(self, elements)

    @staticmethod
    def wrap(l: Iterable[B]) -> 'List[B]':
        return List(*l)

    def lift(self, index: int) -> Maybe[A]:
        return Maybe.from_call(self.__getitem__, index, exc=IndexError)

    def map(self, f: Callable[[A], B]) -> 'List[B]':
        return List.wrap(list(map(f, self)))

    def smap(self, f: Callable[[A], B]) -> 'List[B]':
        return List.wrap(list(itertools.starmap(f, self)))

    def flat_map(self, f: Callable[[A], 'Iterable[B]']) -> 'List[B]':
        return List.wrap(flatten(map(f, self)))

    def foreach(self, f: Callable[[A], B]) -> None:
        for el in self:
            f(el)

    def find(self, f: Callable[[A], bool]):
        return Maybe(find(f, self))

    def filter(self, f: Callable[[A], bool]):
        return List.wrap(filter(f, self))

    @property
    def is_empty(self):
        return self.length == 0

    @property
    def length(self):
        return len(self)

    @property
    def head(self):
        return self.lift(0)

    @property
    def last(self):
        return self.lift(-1)

    @property
    def distinct(self):
        seen = set()
        return List.wrap(x for x in self if x not in seen and not seen.add(x))

    def __add__(self, other: typing.List[A]) -> 'List[A]':
        return List.wrap(typing.List.__add__(self, other))

__all__ = ['List']
<commit_msg>List.flatten<commit_after>import itertools
import typing
from typing import TypeVar, Callable, Generic, Iterable

from tek.tools import find  # type: ignore

from tryp.maybe import Maybe

A = TypeVar('A', covariant=True)
B = TypeVar('B')

def flatten(l: Iterable[Iterable[A]]) -> Iterable[A]:
    return list(itertools.chain.from_iterable(l))  # type: ignore

class List(typing.List[A], Generic[A]):

    def __init__(self, *elements):
        typing.List.__init__(self, elements)

    @staticmethod
    def wrap(l: Iterable[B]) -> 'List[B]':
        return List(*l)

    def lift(self, index: int) -> Maybe[A]:
        return Maybe.from_call(self.__getitem__, index, exc=IndexError)

    def map(self, f: Callable[[A], B]) -> 'List[B]':
        return List.wrap(list(map(f, self)))

    def smap(self, f: Callable[[A], B]) -> 'List[B]':
        return List.wrap(list(itertools.starmap(f, self)))

    def flat_map(self, f: Callable[[A], 'Iterable[B]']) -> 'List[B]':
        return List.wrap(flatten(map(f, self)))

    @property
    def flatten(self):
        return self.flat_map(lambda a: a)

    def foreach(self, f: Callable[[A], B]) -> None:
        for el in self:
            f(el)

    def find(self, f: Callable[[A], bool]):
        return Maybe(find(f, self))

    def filter(self, f: Callable[[A], bool]):
        return List.wrap(filter(f, self))

    @property
    def is_empty(self):
        return self.length == 0

    @property
    def length(self):
        return len(self)

    @property
    def head(self):
        return self.lift(0)

    @property
    def last(self):
        return self.lift(-1)

    @property
    def distinct(self):
        seen = set()
        return List.wrap(x for x in self if x not in seen and not seen.add(x))

    def __add__(self, other: typing.List[A]) -> 'List[A]':
        return List.wrap(typing.List.__add__(self, other))

__all__ = ['List']
<|endoftext|>


<commit_before>"""
ttt.gtest
~~~~~~~~~~~~
This module implements the gtest execution wrapper. It will run a gtest binary
in a subprocess and capture its output to track the outcome of test execution.
:copyright: (c) yerejm
"""
import collections
import re
import termstyle
import os
import sys

from ttt.executor import PASSED, FAILED, CRASHED
from ttt.terminal import Terminal

TESTCASE_START_RE = re.compile('^\[----------\] \d+ tests? from (.*?)$')
TESTCASE_END_RE = re.compile(
    '^\[----------\] \d+ tests? from (.*?) \(\d+ ms total\)$'
)
TEST_START_RE = re.compile('^\[ RUN      \] (.*?)$')
TEST_END_RE = re.compile('^\[  (FAILED |     OK) \] (.*?)$')
TESTCASE_TIME_RE = re.compile(
    '^\[==========\] \d tests? from \d test cases? ran. \((\d+) ms total\)$'
)

# The patterns above are to match against the relevant output of a gtest run.
# TESTCASE refers to a group of TESTs. There can be more than one TESTCASE per
# gtest run.
#
# The output of a gtest binary on success:
#
# Running main() from gtest_main.cc
# [==========] Running 4 tests from 1 test case.
# [----------] Global test environment set-up.
# [----------] 4 tests from core
# [ RUN      ] core.ok
# [       OK ] core.ok (0 ms)
# [ RUN      ] core.okshadow
# [       OK ] core.okshadow (0 ms)
# [ RUN      ] core.notok
# [       OK ] core.notok (0 ms)
# [ RUN      ] core.blah
# [       OK ] core.blah (0 ms)
# [----------] 4 tests from core (1 ms total)
#
# [----------] Global test environment tear-down
# [==========] 4 tests from 1 test case ran. (1 ms total)
# [  PASSED  ] 4 tests.
#
# The output of a gtest binary on failure:
#
# Running main() from gtest_main.cc
# [==========] Running 4 tests from 1 test case.
# [----------] Global test environment set-up.
# [----------] 4 tests from core
# [ RUN      ] core.ok
# /path/to/project/test/test_core.cc:12: Failure
# Value of: 4
# Expected: ok()
# Which is: 42
# [  FAILED  ] core.ok (0 ms)
# [ RUN      ] core.okshadow
# [       OK ] core.okshadow (0 ms)
# [ RUN      ] core.notok
# [       OK ] core.notok (0 ms)
# [ RUN      ] core.blah
# [       OK ] core.blah (0 ms)
# [----------] 4 tests from core (0 ms total)
#
# [----------] Global test environment tear-down
# [==========] 4 tests from 1 test case ran. (0 ms total)
# [  PASSED  ] 3 tests.
# [  FAILED  ] 1 test, listed below:
# [  FAILED  ] core.ok
#
#  1 FAILED TEST
#

def testcase_starts_at(line):
    """Indicates if the line is the start of a testcase."""
    return TESTCASE_START_RE.match(line)

def testcase_ends_at(line):
    """Indicates if the line is the end of a testcase."""
    return TESTCASE_END_RE.match(line)

def test_starts_at(line):
    """Indicates if the line is the start of a test."""
    return TEST_START_RE.match(line)

def test_ends_at(line):
    """Indicates if the line is the end of a test."""
    return TEST_END_RE.match(line)

def test_elapsed_at(line):
    """Indicates if the line contains the time statistic for the testcase."""
    return TESTCASE_TIME_RE.match(line)

class GTest(object):
    """Representation of the execution, output capture and output parsing of a
    gtest-based binary.

    By default, the output of a :class:`GTest` object when executing its test
    is a summary, noting the test source file name, the test case, and a . or F
    to indicate the pass or fail state respectively for each test.
    e.g. sanity/test/test_core.cc :: core ....
    """
    WAITING_TESTCASE, WAITING_TEST, IN_TEST = range(3)

    def __init__(self, source, executable, term=None):
        """Creates a representation of a GTest binary.

        :param source: Path of the test source file
        :param executable: Path of the test executable
        :param term: (optional) Terminal object to send output of test
        execution. Default Terminal() will send no output.
        """
        if not source:
            raise Exception('Invalid source')
        if not executable:
            raise Exception('Invalid executable')

        self._source = source
        self._executable = executable
        self._term = term if term else Terminal()
        self.reset()

    def out(self, *args, **kwargs):
        """Print at the specific verbosity level."""
        t = self._term
        verbose = kwargs.pop("verbose", 0)
        if (verbose == t.verbosity):
            t.writeln(*args, **kwargs)

    def reset(self):
        self._output = []
        self._error = []
        self._tests = collections.OrderedDict()
        self._state = GTest.WAITING_TESTCASE
        self._testcase = None
        self._test = None
        self._elapsed = 0
        self._pass_count = 0
        self._fail_count = 0

    def passes(self):
        """The number of passing tests detected in the latest test run."""
        return self._pass_count

    def fails(self):
        """The number of failing tests detected in the latest test run."""
        return self._fail_count

    def source(self):
        """The relative path to the test source file."""
        return self._source

    def executable(self):
        """The absolute path to the gtest binary executable."""
        return self._executable

    def run_time(self):
        """The elapsed time in milliseconds to run the tests."""
        return self._elapsed

    def execute(self, test_filters):
        """Executes the test executable, with this instance as a line listener.

        :param test_filters: a list of tests identified by name to be executed.
            This is a passed through as a colon separated string to the
            --gtest_filter command line option.
        :return a list of failing tests identified by name
        """
        from ttt.subproc import streamed_call
        command = [self.executable()]
        if test_filters:
            command.append("--gtest_filter={}".format(':'.join(test_filters)))
        self.reset()
        self.out("Executing {}".format(" ".join(command)), verbose=2)
        rc, stdout, stderr = streamed_call(command, listener=self)
        self.out(command, verbose=2)
        if stdout:
            self.out(os.linesep.join(stdout), verbose=2)
        if stderr:
            self.out(os.linesep.join(stderr), verbose=2)

        if rc != 0:
            # TODO Handle non-test crash?

            # probably crashed: note the test that did it and accept that
            # remaining tests do not run
            if self._test is not None:
                self._tests[self._test] = (
                    CRASHED,
                    [signalstring(rc)] + self._output,
                    self._error,
                )
                self._fail_count += 1
                self.out(' {}'.format(signalstring(rc)),
                         decorator=[termstyle.bold, termstyle.red],
                         verbose=0)
        return self.failures()

    def failures(self):
        """Gets the list of tests that failed by name."""
        # results[0] is the first item in the results tuple. This is the
        # boolean indicator of test failure.
        return [
            test for test, results in self._tests.items()
            if results[0] != PASSED
        ]

    def __call__(self, channel, line):
        """Listener interface for lines output during test execution.

        :class:`GTest` presents this callable interface intended for use by the
        :class:`SystemContext`.streamed_call as a line listener. Each line
        output during the execution of the tests is fed into __call__ (see
        execute()).
        """

        self.line(line)
        if channel == sys.stdout:
            self._output.append(line)
        else:
            self._error.append(line)

        # Track what the test execution is currently doing as a state.
        if self._state == GTest.WAITING_TESTCASE and testcase_starts_at(line):
            self.begin_testcase(line)
            self._state = GTest.WAITING_TEST
        elif self._state == GTest.WAITING_TEST and testcase_ends_at(line):
            self.end_testcase(line)
            self._state = GTest.WAITING_TESTCASE
        elif self._state == GTest.WAITING_TEST and test_starts_at(line):
            self.begin_test(line)
            self._state = GTest.IN_TEST
        elif self._state == GTest.IN_TEST and test_ends_at(line):
            self.end_test(line)
            self._state = GTest.WAITING_TEST
        elif self._state == GTest.WAITING_TESTCASE:
            match = test_elapsed_at(line)
            if match:
                self._elapsed = int(match.group(1))
        return None

    def line(self, line):
        """Line handler that will recolourise lines in the same manner as gtest
        when verbose mode is enabled.

        A gtest detects where it is being run and colorises output accordingly.
        This means that when run in a subprocess, it will not colorise output.
        """
        leader = line[:13]
        trailer = line[13:]

        decorator = [
            termstyle.bold,
            termstyle.red if '[  FAILED  ]' in line else termstyle.green
        ] if '[' in leader else []
        self.out(leader, decorator=decorator, end='', verbose=1)
        self.out(trailer, verbose=1)

    def begin_testcase(self, line):
        """Tracks when a test case starts.

        Output is suppressed in verbose mode because line() will have output
        the gtest actual output.
        """
        testcase = line[line.rfind(' ') + 1:]
        self._testcase = testcase

        self.out('{} :: {} '.format(str(self._source), testcase),
                 end='', verbose=0)

    def end_testcase(self, line):
        """Tracks when a test case ends.

        Output is suppressed in verbose mode because line() will have output
        the gtest actual output.
        """
        self._testcase = None

        self.out(verbose=0)

    def begin_test(self, line):
        """Tracks when a test starts."""
        test = line[line.rfind(' ') + 1:]
        self._test = test
        self._output = []
        self._error = []

    def end_test(self, line):
        """Tracks when a test ends and whether it passed or failed.

        When not in verbose mode, a summary result is output (. for success, F
        for failure).

        Output is suppressed in verbose mode because line() will have output
        the gtest actual output.

        If the test failed, the output is captured.
        """
        if self._testcase is None:
            raise Exception('Invalid current testcase')
        if self._test is None:
            raise Exception('Invalid current test')
        failed = '[  FAILED  ]' in line

        # windows crash is a failure
        seh = False
        for l in self._output:
            if 'error: SEH exception' in l:
                seh = True
                break
        outcome = PASSED
        if seh:
            outcome = CRASHED
            self._output = ['SEH Exception'] + self._output
        elif failed:
            outcome = FAILED

        self._tests[self._test] = (
            outcome,
            self._output[:-1],  # cut the [ OK/FAILED ] line
            self._error[:],
        )

        if failed:
            self._fail_count += 1
            self.out('X' if seh else 'F', end='', verbose=0)
        else:
            self._pass_count += 1
            self.out('.', end='', verbose=0)
        self._test = None
        self._output = []
        self._error = []

    def results(self):
        """Gets the test results of the last test execution.

        This may not contain all possible tests due to the presence of a test
        filter (see execute()).

        This is a Dict() of test name key to a tuple of:
          - pass/fail/crash indicator
          - whatever appeared on the stdout stream during test execution
          - whatever appeared on the stderr stream during test execution
        """
        return self._tests

    def test_results(self, testname):
        """Gets the test results for a particular test."""
        return self._tests[testname]

def signalstring(value):
    coresignals = {
        # windows
        1: 'ERROR',
        3: 'ABORT',
        # posix
        -3: 'SIGQUIT',
        -4: 'SIGILL',
        -5: 'SIGTRAP',
        -6: 'SIGABRT',
        -7: 'SIGEMT',
        -8: 'SIGFPE',
        -10: 'SIGBUS',
        -11: 'SIGSEGV',
        -12: 'SIGSYS',
    }
    if value in coresignals:
        return coresignals[value]
    else:
        return "CRASH"
<commit_msg>fix deprecations<commit_after>"""
ttt.gtest
~~~~~~~~~~~~
This module implements the gtest execution wrapper. It will run a gtest binary
in a subprocess and capture its output to track the outcome of test execution.
:copyright: (c) yerejm
"""
import collections
import re
import termstyle
import os
import sys

from ttt.executor import PASSED, FAILED, CRASHED
from ttt.terminal import Terminal

TESTCASE_START_RE = re.compile('^\\[----------\\] \\d+ tests? from (.*?)$')
TESTCASE_END_RE = re.compile(
    '^\\[----------\\] \\d+ tests? from (.*?) \\(\\d+ ms total\\)$'
)
TEST_START_RE = re.compile('^\\[ RUN      \\] (.*?)$')
TEST_END_RE = re.compile('^\\[  (FAILED |     OK) \\] (.*?)$')
TESTCASE_TIME_RE = re.compile(
    '^\\[==========\\] \\d tests? from \\d test cases? ran. '
    '\\((\\d+) ms total\\)$'
)

# The patterns above are to match against the relevant output of a gtest run.
# TESTCASE refers to a group of TESTs. There can be more than one TESTCASE per
# gtest run.
#
# The output of a gtest binary on success:
#
# Running main() from gtest_main.cc
# [==========] Running 4 tests from 1 test case.
# [----------] Global test environment set-up.
# [----------] 4 tests from core
# [ RUN      ] core.ok
# [       OK ] core.ok (0 ms)
# [ RUN      ] core.okshadow
# [       OK ] core.okshadow (0 ms)
# [ RUN      ] core.notok
# [       OK ] core.notok (0 ms)
# [ RUN      ] core.blah
# [       OK ] core.blah (0 ms)
# [----------] 4 tests from core (1 ms total)
#
# [----------] Global test environment tear-down
# [==========] 4 tests from 1 test case ran. (1 ms total)
# [  PASSED  ] 4 tests.
#
# The output of a gtest binary on failure:
#
# Running main() from gtest_main.cc
# [==========] Running 4 tests from 1 test case.
# [----------] Global test environment set-up.
# [----------] 4 tests from core
# [ RUN      ] core.ok
# /path/to/project/test/test_core.cc:12: Failure
# Value of: 4
# Expected: ok()
# Which is: 42
# [  FAILED  ] core.ok (0 ms)
# [ RUN      ] core.okshadow
# [       OK ] core.okshadow (0 ms)
# [ RUN      ] core.notok
# [       OK ] core.notok (0 ms)
# [ RUN      ] core.blah
# [       OK ] core.blah (0 ms)
# [----------] 4 tests from core (0 ms total)
#
# [----------] Global test environment tear-down
# [==========] 4 tests from 1 test case ran. (0 ms total)
# [  PASSED  ] 3 tests.
# [  FAILED  ] 1 test, listed below:
# [  FAILED  ] core.ok
#
#  1 FAILED TEST
#

def testcase_starts_at(line):
    """Indicates if the line is the start of a testcase."""
    return TESTCASE_START_RE.match(line)

def testcase_ends_at(line):
    """Indicates if the line is the end of a testcase."""
    return TESTCASE_END_RE.match(line)

def test_starts_at(line):
    """Indicates if the line is the start of a test."""
    return TEST_START_RE.match(line)

def test_ends_at(line):
    """Indicates if the line is the end of a test."""
    return TEST_END_RE.match(line)

def test_elapsed_at(line):
    """Indicates if the line contains the time statistic for the testcase."""
    return TESTCASE_TIME_RE.match(line)

class GTest(object):
    """Representation of the execution, output capture and output parsing of a
    gtest-based binary.

    By default, the output of a :class:`GTest` object when executing its test
    is a summary, noting the test source file name, the test case, and a . or F
    to indicate the pass or fail state respectively for each test.
    e.g. sanity/test/test_core.cc :: core ....
    """
    WAITING_TESTCASE, WAITING_TEST, IN_TEST = range(3)

    def __init__(self, source, executable, term=None):
        """Creates a representation of a GTest binary.

        :param source: Path of the test source file
        :param executable: Path of the test executable
        :param term: (optional) Terminal object to send output of test
        execution. Default Terminal() will send no output.
        """
        if not source:
            raise Exception('Invalid source')
        if not executable:
            raise Exception('Invalid executable')

        self._source = source
        self._executable = executable
        self._term = term if term else Terminal()
        self.reset()

    def out(self, *args, **kwargs):
        """Print at the specific verbosity level."""
        t = self._term
        verbose = kwargs.pop("verbose", 0)
        if (verbose == t.verbosity):
            t.writeln(*args, **kwargs)

    def reset(self):
        self._output = []
        self._error = []
        self._tests = collections.OrderedDict()
        self._state = GTest.WAITING_TESTCASE
        self._testcase = None
        self._test = None
        self._elapsed = 0
        self._pass_count = 0
        self._fail_count = 0

    def passes(self):
        """The number of passing tests detected in the latest test run."""
        return self._pass_count

    def fails(self):
        """The number of failing tests detected in the latest test run."""
        return self._fail_count

    def source(self):
        """The relative path to the test source file."""
        return self._source

    def executable(self):
        """The absolute path to the gtest binary executable."""
        return self._executable

    def run_time(self):
        """The elapsed time in milliseconds to run the tests."""
        return self._elapsed

    def execute(self, test_filters):
        """Executes the test executable, with this instance as a line listener.

        :param test_filters: a list of tests identified by name to be executed.
            This is a passed through as a colon separated string to the
            --gtest_filter command line option.
        :return a list of failing tests identified by name
        """
        from ttt.subproc import streamed_call
        command = [self.executable()]
        if test_filters:
            command.append("--gtest_filter={}".format(':'.join(test_filters)))
        self.reset()
        self.out("Executing {}".format(" ".join(command)), verbose=2)
        rc, stdout, stderr = streamed_call(command, listener=self)
        self.out(command, verbose=2)
        if stdout:
            self.out(os.linesep.join(stdout), verbose=2)
        if stderr:
            self.out(os.linesep.join(stderr), verbose=2)

        if rc != 0:
            # TODO Handle non-test crash?

            # probably crashed: note the test that did it and accept that
            # remaining tests do not run
            if self._test is not None:
                self._tests[self._test] = (
                    CRASHED,
                    [signalstring(rc)] + self._output,
                    self._error,
                )
                self._fail_count += 1
                self.out(' {}'.format(signalstring(rc)),
                         decorator=[termstyle.bold, termstyle.red],
                         verbose=0)
        return self.failures()

    def failures(self):
        """Gets the list of tests that failed by name."""
        # results[0] is the first item in the results tuple. This is the
        # boolean indicator of test failure.
        return [
            test for test, results in self._tests.items()
            if results[0] != PASSED
        ]

    def __call__(self, channel, line):
        """Listener interface for lines output during test execution.

        :class:`GTest` presents this callable interface intended for use by the
        :class:`SystemContext`.streamed_call as a line listener. Each line
        output during the execution of the tests is fed into __call__ (see
        execute()).
        """

        self.line(line)
        if channel == sys.stdout:
            self._output.append(line)
        else:
            self._error.append(line)

        # Track what the test execution is currently doing as a state.
        if self._state == GTest.WAITING_TESTCASE and testcase_starts_at(line):
            self.begin_testcase(line)
            self._state = GTest.WAITING_TEST
        elif self._state == GTest.WAITING_TEST and testcase_ends_at(line):
            self.end_testcase(line)
            self._state = GTest.WAITING_TESTCASE
        elif self._state == GTest.WAITING_TEST and test_starts_at(line):
            self.begin_test(line)
            self._state = GTest.IN_TEST
        elif self._state == GTest.IN_TEST and test_ends_at(line):
            self.end_test(line)
            self._state = GTest.WAITING_TEST
        elif self._state == GTest.WAITING_TESTCASE:
            match = test_elapsed_at(line)
            if match:
                self._elapsed = int(match.group(1))
        return None

    def line(self, line):
        """Line handler that will recolourise lines in the same manner as gtest
        when verbose mode is enabled.

        A gtest detects where it is being run and colorises output accordingly.
        This means that when run in a subprocess, it will not colorise output.
        """
        leader = line[:13]
        trailer = line[13:]

        decorator = [
            termstyle.bold,
            termstyle.red if '[  FAILED  ]' in line else termstyle.green
        ] if '[' in leader else []
        self.out(leader, decorator=decorator, end='', verbose=1)
        self.out(trailer, verbose=1)

    def begin_testcase(self, line):
        """Tracks when a test case starts.

        Output is suppressed in verbose mode because line() will have output
        the gtest actual output.
        """
        testcase = line[line.rfind(' ') + 1:]
        self._testcase = testcase

        self.out('{} :: {} '.format(str(self._source), testcase),
                 end='', verbose=0)

    def end_testcase(self, line):
        """Tracks when a test case ends.

        Output is suppressed in verbose mode because line() will have output
        the gtest actual output.
        """
        self._testcase = None

        self.out(verbose=0)

    def begin_test(self, line):
        """Tracks when a test starts."""
        test = line[line.rfind(' ') + 1:]
        self._test = test
        self._output = []
        self._error = []

    def end_test(self, line):
        """Tracks when a test ends and whether it passed or failed.

        When not in verbose mode, a summary result is output (. for success, F
        for failure).

        Output is suppressed in verbose mode because line() will have output
        the gtest actual output.

        If the test failed, the output is captured.
        """
        if self._testcase is None:
            raise Exception('Invalid current testcase')
        if self._test is None:
            raise Exception('Invalid current test')
        failed = '[  FAILED  ]' in line

        # windows crash is a failure
        seh = False
        for l in self._output:
            if 'error: SEH exception' in l:
                seh = True
                break
        outcome = PASSED
        if seh:
            outcome = CRASHED
            self._output = ['SEH Exception'] + self._output
        elif failed:
            outcome = FAILED

        self._tests[self._test] = (
            outcome,
            self._output[:-1],  # cut the [ OK/FAILED ] line
            self._error[:],
        )

        if failed:
            self._fail_count += 1
            self.out('X' if seh else 'F', end='', verbose=0)
        else:
            self._pass_count += 1
            self.out('.', end='', verbose=0)
        self._test = None
        self._output = []
        self._error = []

    def results(self):
        """Gets the test results of the last test execution.

        This may not contain all possible tests due to the presence of a test
        filter (see execute()).

        This is a Dict() of test name key to a tuple of:
          - pass/fail/crash indicator
          - whatever appeared on the stdout stream during test execution
          - whatever appeared on the stderr stream during test execution
        """
        return self._tests

    def test_results(self, testname):
        """Gets the test results for a particular test."""
        return self._tests[testname]

def signalstring(value):
    coresignals = {
        # windows
        1: 'ERROR',
        3: 'ABORT',
        # posix
        -3: 'SIGQUIT',
        -4: 'SIGILL',
        -5: 'SIGTRAP',
        -6: 'SIGABRT',
        -7: 'SIGEMT',
        -8: 'SIGFPE',
        -10: 'SIGBUS',
        -11: 'SIGSEGV',
        -12: 'SIGSYS',
    }
    if value in coresignals:
        return coresignals[value]
    else:
        return "CRASH"
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
/***************************************************************************
 Svir
                                 A QGIS plugin
 OpenQuake Social Vulnerability and Integrated Risk
                              -------------------
        begin                : 2013-10-24
        copyright            : (C) 2013 by GEM Foundation
        email                : devops@openquake.org
 ***************************************************************************/

# Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.
"""
import math
from numpy import mean, std, argwhere, amax, amin, log10, log
from types import NoneType
import qgis
from utils import Register

TRANSFORMATION_ALGS = Register()
RANK_VARIANTS = ('AVERAGE', 'MIN', 'MAX', 'DENSE', 'ORDINAL')
QUADRATIC_VARIANTS = ('INCREASING', 'DECREASING')
LOG10_VARIANTS = ('INCREMENT BY ONE IF ZEROS ARE FOUND',
                  'IGNORE ZEROS')

def transform(features_dict, algorithm, variant_name="", inverse=False):
    """
    Use the chosen algorithm (and optional variant) on the input dict, and
    return a dict containing the transformed values with the original ids
    """
    # make a copy of the dictionary containing features (ids and values) and
    # remove elements containing missing values, so the transformation is
    # performed only on the subset of valid values
    f_dict_copy = features_dict.copy()
    # elements with null value will be saved in another dict, so they can be
    # re-added afterwards
    dict_of_null_values = {}
    for key, value in f_dict_copy.iteritems():
        if type(value) in (qgis.QPyNullVariant, NoneType):
            dict_of_null_values[key] = value
    for key in dict_of_null_values.keys():
        del f_dict_copy[key]
    transformed_list, invalid_input_values = algorithm(
        f_dict_copy.values(), variant_name, inverse)
    transformed_dict = dict(zip(f_dict_copy.keys(), transformed_list))
    # add to the transformed_dict the null elements that were removed
    transformed_dict.update(dict_of_null_values)
    return transformed_dict, invalid_input_values

@TRANSFORMATION_ALGS.add('RANK')
def rank(input_list, variant_name="AVERAGE", inverse=False):
    """Assign ranks to data, dealing with ties appropriately.

    Args:
        input_list: the list of numbers to rank
        variant_name: available variants are
            [AVERAGE, MIN, MAX, DENSE, ORDINAL] and they correspond to
            different strategies on how to cope with ties (default: AVERAGE)
        inverse: instead of giving the highest rank to the biggest input value,
            give the highest rank to the smallest input value
    Returns:
        list of ranks corresponding to the input data
    Raises:
        NotImplementedError: if variant_name is not implemented
    """
    input_copy = input_list[:]
    len_input_list = len(input_list)
    rank_list = [0] * len_input_list
    previous_ties = 0
    if not inverse:  # high values get high ranks
        # obtain a value above the maximum value contained in input_list,
        # so it will never be picked as the minimum element of the list
        above_max_input = max(input_list) + 1
        curr_idx = 1
        while curr_idx <= len_input_list:
            # get the list of indices of the min elements of input_copy.
            # note that we might have ties.
            bottom_indices = argwhere(
                input_copy == amin(input_copy)).flatten().tolist()
            bottom_amount = len(bottom_indices)
            for bottom_idx in bottom_indices:
                if variant_name == "AVERAGE":
                    # e.g., if 4 inputs are equal, and they would receive
                    # ranks from 3 to 6, all of them will obtain rank 4.5
                    # Afterwards the ranks increase from 7 on.
                    rank_list[bottom_idx] = \
                        (2 * curr_idx + bottom_amount - 1) / 2.0
                elif variant_name == "MIN":
                    # e.g., if 4 inputs are equal, and they would receive
                    # ranks from 3 to 6, all of them will obtain rank 3.
                    # Afterwards the ranks increase from 7 on.
                    rank_list[bottom_idx] = curr_idx
                elif variant_name == "MAX":
                    # e.g., if 4 inputs are equal, and they would receive
                    # ranks from 3 to 6, all of them will obtain rank 6
                    # Afterwards the ranks increase from 7 on.
                    rank_list[bottom_idx] = curr_idx + bottom_amount - 1
                elif variant_name == "DENSE":
                    # the same as for "MIN", but instead of ranking the
                    # next elements counting from 7 on, the ranks will
                    # increase from 4 on (no "jumps").
                    rank_list[bottom_idx] = curr_idx - previous_ties
                elif variant_name == "ORDINAL":
                    # the ties are ranked in a "ordinal" way, assigning
                    # the smallest rank to the leftmost tie found, and so on.
                    rank_list[bottom_idx] = curr_idx
                    curr_idx += 1
                else:
                    raise NotImplementedError(
                        "%s variant not implemented" % variant_name)
                # "discard" the tie, so it will not be found as the min in
                # the next iteration (instead of removing it, which would
                # modify the list indices, I make it bigger than the
                # biggest input)
                input_copy[bottom_idx] = above_max_input
            if variant_name != "ORDINAL":
                # necessary adjustment in case of "ORDINAL"
                curr_idx += bottom_amount
            # necessary for the "DENSE" variant, to keep a count of ties
            previous_ties += bottom_amount - 1

    else:  # inverse, i.e., small inputs get high ranks
        # same as the direct methods, but proceeding top->down
        below_min_input = min(input_list) - 1
        curr_idx = len_input_list
        while curr_idx > 0:
            top_indices = argwhere(
                input_copy == amax(input_copy)).flatten().tolist()
            top_amount = len(top_indices)
            for top_idx in top_indices:
                if variant_name == "AVERAGE":
                    rank_list[top_idx] = \
                        len_input_list - (2 * curr_idx - top_amount - 1) / 2.0
                elif variant_name == "MIN":
                    rank_list[top_idx] = len_input_list - curr_idx + 1
                elif variant_name == "MAX":
                    rank_list[top_idx] = len_input_list - curr_idx + top_amount
                elif variant_name == "DENSE":
                    rank_list[top_idx] = \
                        len_input_list - curr_idx - previous_ties + 1
                elif variant_name == "ORDINAL":
                    rank_list[top_idx] = len_input_list - curr_idx + 1
                    curr_idx -= 1
                else:
                    raise NotImplementedError(
                        "%s variant not implemented" % variant_name)
                input_copy[top_idx] = below_min_input
            if variant_name != "ORDINAL":
                curr_idx -= top_amount
            previous_ties += top_amount - 1
    return rank_list, None

@TRANSFORMATION_ALGS.add('Z_SCORE')
def z_score(input_list, variant_name=None, inverse=False):
    """
    Direct:
        Transformed(e_i) = (e_i - mean(e)) / stddev(e)
    Inverse:
        Multiply each input by -1, before doing exactly the same
    """
    if variant_name:
        raise NotImplementedError("%s variant not implemented" % variant_name)
    mean_val = mean(input_list)
    stddev_val = std(input_list)
    input_copy = input_list[:]
    if inverse:
        # multiply each input_list element by -1
        input_copy[:] = [-x for x in input_list]
    output_list = [
        1.0 * (num - mean_val) / stddev_val for num in input_copy]
    return output_list, None

@TRANSFORMATION_ALGS.add('MIN_MAX')
def min_max(input_list, variant_name=None, inverse=False):
    """
    Direct:
        Transformed(e_i) = (e_i - min(e)) / (max(e) - min(e))
    Inverse:
        Transformed(e_i) = 1 - [(e_i - min(e)) / (max(e) - min(e))]

    """
    if variant_name:
        raise NotImplementedError("%s variant not implemented" % variant_name)
    list_min = min(input_list)
    list_max = max(input_list)
    # Get the range of the list
    list_range = float(list_max - list_min)
    # Transform
    if inverse:
        output_list = map(
            lambda x: 1.0 - ((x - list_min) / list_range), input_list)
    else:
        output_list = map(
            lambda x: (x - list_min) / list_range, input_list)
    return output_list, None

@TRANSFORMATION_ALGS.add('LOG10')
def log10_(input_list,
           variant_name='IGNORE ZEROS',
           inverse=False):
    """
    Accept only input_list containing positive (or zero) values
    In case of zeros:
        * the variant IGNORE ZEROS produces NULL as output when any input is
          zero
        * the variant INCREMENT BY ONE IF ZEROS ARE FOUND increments all input
          data by 1
    Then use numpy.log10 function to perform the log10 transformation on the
    list of values
    """
    if inverse:
        raise NotImplementedError(
            "Inverse transformation for log10 is not implemented")
    if variant_name not in LOG10_VARIANTS:
        raise NotImplementedError(
            "%s variant not implemented" % variant_name)
    if any(n < 0 for n in input_list):
        raise ValueError("log10 transformation can not be performed if "
                         "the field contains negative values")
    if any(n == 0 for n in input_list):
        if variant_name == 'INCREMENT BY ONE IF ZEROS ARE FOUND':
            corrected_input = [input_value + 1 for input_value in input_list]
            output_list = list(log10(corrected_input))
            return output_list, None
        elif variant_name == 'IGNORE ZEROS':
            output_list = []
            for input_value in input_list:
                if input_value == 0:
                    output_value = qgis.QPyNullVariant(float)
                    output_list.append(output_value)
                else:
                    output_list.append(log10(input_value))
            return output_list, None
    output_list = list(log10(input_list))
    return output_list, None

@TRANSFORMATION_ALGS.add('QUADRATIC')
def simple_quadratic(input_list, variant_name="INCREASING", inverse=False):
    """
    Simple quadratic transformation (bottom = 0)
        quadratic(e_i) = (e_i - bottom)^2 / (max(e) - bottom)^2
    ==>
        simple_quadratic(e_i) = e_i^2 / max(e)^2

    Inverse:
        For each output x, the final output will be 1 - x
    """
    bottom = 0.0
    max_input = max(input_list)
    if max_input - bottom == 0:
        raise ZeroDivisionError("It is impossible to perform the "
                                "transformation if the maximum "
                                "input value is 0")
    squared_range = (max_input - bottom) ** 2
    if variant_name == "INCREASING":
        output_list = map(
            lambda x: (x - bottom) ** 2 / squared_range, input_list)
    elif variant_name == "DECREASING":
        output_list = map(
            lambda x: (max_input - (x - bottom)) ** 2 / squared_range,
            input_list)

    else:
        raise NotImplementedError("%s variant not implemented" % variant_name)
    if inverse:
        output_list[:] = [1.0 - x for x in output_list]
    return output_list, None

@TRANSFORMATION_ALGS.add('SIGMOID')
def sigmoid(input_list, variant_name="", inverse=False):
    """
    Logistic sigmoid function:
        f(x) = 1 / [1 + e^(-x)]

    Inverse function:
        f(y) =  ln[y / (1 - y)]
    """
    if variant_name:
        raise NotImplementedError("%s variant not implemented" % variant_name)
    output_list = []
    invalid_input_values = []
    if inverse:
        for y in input_list:
            try:
                output = log(y / (1 - y))
            except:
                output = qgis.QPyNullVariant(float)
                invalid_input_values.append(y)
            output_list.append(output)
    else:  # direct
        for x in input_list:
            try:
                output = 1 / (1 + math.exp(-x))
            except:
                output = qgis.QPyNullVariant(float)
                invalid_input_values.append(x)
            output_list.append(output)
    return output_list, invalid_input_values
<commit_msg>Fix corner-case (range zero) in min_max transformation<commit_after># -*- coding: utf-8 -*-
"""
/***************************************************************************
 Svir
                                 A QGIS plugin
 OpenQuake Social Vulnerability and Integrated Risk
                              -------------------
        begin                : 2013-10-24
        copyright            : (C) 2013 by GEM Foundation
        email                : devops@openquake.org
 ***************************************************************************/

# Copyright (c) 2010-2013, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.
"""
import math
from numpy import mean, std, argwhere, amax, amin, log10, log
from types import NoneType
import qgis
from utils import Register

TRANSFORMATION_ALGS = Register()
RANK_VARIANTS = ('AVERAGE', 'MIN', 'MAX', 'DENSE', 'ORDINAL')
QUADRATIC_VARIANTS = ('INCREASING', 'DECREASING')
LOG10_VARIANTS = ('INCREMENT BY ONE IF ZEROS ARE FOUND',
                  'IGNORE ZEROS')

def transform(features_dict, algorithm, variant_name="", inverse=False):
    """
    Use the chosen algorithm (and optional variant) on the input dict, and
    return a dict containing the transformed values with the original ids
    """
    # make a copy of the dictionary containing features (ids and values) and
    # remove elements containing missing values, so the transformation is
    # performed only on the subset of valid values
    f_dict_copy = features_dict.copy()
    # elements with null value will be saved in another dict, so they can be
    # re-added afterwards
    dict_of_null_values = {}
    for key, value in f_dict_copy.iteritems():
        if type(value) in (qgis.QPyNullVariant, NoneType):
            dict_of_null_values[key] = value
    for key in dict_of_null_values.keys():
        del f_dict_copy[key]
    transformed_list, invalid_input_values = algorithm(
        f_dict_copy.values(), variant_name, inverse)
    transformed_dict = dict(zip(f_dict_copy.keys(), transformed_list))
    # add to the transformed_dict the null elements that were removed
    transformed_dict.update(dict_of_null_values)
    return transformed_dict, invalid_input_values

@TRANSFORMATION_ALGS.add('RANK')
def rank(input_list, variant_name="AVERAGE", inverse=False):
    """Assign ranks to data, dealing with ties appropriately.

    Args:
        input_list: the list of numbers to rank
        variant_name: available variants are
            [AVERAGE, MIN, MAX, DENSE, ORDINAL] and they correspond to
            different strategies on how to cope with ties (default: AVERAGE)
        inverse: instead of giving the highest rank to the biggest input value,
            give the highest rank to the smallest input value
    Returns:
        list of ranks corresponding to the input data
    Raises:
        NotImplementedError: if variant_name is not implemented
    """
    input_copy = input_list[:]
    len_input_list = len(input_list)
    rank_list = [0] * len_input_list
    previous_ties = 0
    if not inverse:  # high values get high ranks
        # obtain a value above the maximum value contained in input_list,
        # so it will never be picked as the minimum element of the list
        above_max_input = max(input_list) + 1
        curr_idx = 1
        while curr_idx <= len_input_list:
            # get the list of indices of the min elements of input_copy.
            # note that we might have ties.
            bottom_indices = argwhere(
                input_copy == amin(input_copy)).flatten().tolist()
            bottom_amount = len(bottom_indices)
            for bottom_idx in bottom_indices:
                if variant_name == "AVERAGE":
                    # e.g., if 4 inputs are equal, and they would receive
                    # ranks from 3 to 6, all of them will obtain rank 4.5
                    # Afterwards the ranks increase from 7 on.
                    rank_list[bottom_idx] = \
                        (2 * curr_idx + bottom_amount - 1) / 2.0
                elif variant_name == "MIN":
                    # e.g., if 4 inputs are equal, and they would receive
                    # ranks from 3 to 6, all of them will obtain rank 3.
                    # Afterwards the ranks increase from 7 on.
                    rank_list[bottom_idx] = curr_idx
                elif variant_name == "MAX":
                    # e.g., if 4 inputs are equal, and they would receive
                    # ranks from 3 to 6, all of them will obtain rank 6
                    # Afterwards the ranks increase from 7 on.
                    rank_list[bottom_idx] = curr_idx + bottom_amount - 1
                elif variant_name == "DENSE":
                    # the same as for "MIN", but instead of ranking the
                    # next elements counting from 7 on, the ranks will
                    # increase from 4 on (no "jumps").
                    rank_list[bottom_idx] = curr_idx - previous_ties
                elif variant_name == "ORDINAL":
                    # the ties are ranked in a "ordinal" way, assigning
                    # the smallest rank to the leftmost tie found, and so on.
                    rank_list[bottom_idx] = curr_idx
                    curr_idx += 1
                else:
                    raise NotImplementedError(
                        "%s variant not implemented" % variant_name)
                # "discard" the tie, so it will not be found as the min in
                # the next iteration (instead of removing it, which would
                # modify the list indices, I make it bigger than the
                # biggest input)
                input_copy[bottom_idx] = above_max_input
            if variant_name != "ORDINAL":
                # necessary adjustment in case of "ORDINAL"
                curr_idx += bottom_amount
            # necessary for the "DENSE" variant, to keep a count of ties
            previous_ties += bottom_amount - 1

    else:  # inverse, i.e., small inputs get high ranks
        # same as the direct methods, but proceeding top->down
        below_min_input = min(input_list) - 1
        curr_idx = len_input_list
        while curr_idx > 0:
            top_indices = argwhere(
                input_copy == amax(input_copy)).flatten().tolist()
            top_amount = len(top_indices)
            for top_idx in top_indices:
                if variant_name == "AVERAGE":
                    rank_list[top_idx] = \
                        len_input_list - (2 * curr_idx - top_amount - 1) / 2.0
                elif variant_name == "MIN":
                    rank_list[top_idx] = len_input_list - curr_idx + 1
                elif variant_name == "MAX":
                    rank_list[top_idx] = len_input_list - curr_idx + top_amount
                elif variant_name == "DENSE":
                    rank_list[top_idx] = \
                        len_input_list - curr_idx - previous_ties + 1
                elif variant_name == "ORDINAL":
                    rank_list[top_idx] = len_input_list - curr_idx + 1
                    curr_idx -= 1
                else:
                    raise NotImplementedError(
                        "%s variant not implemented" % variant_name)
                input_copy[top_idx] = below_min_input
            if variant_name != "ORDINAL":
                curr_idx -= top_amount
            previous_ties += top_amount - 1
    return rank_list, None

@TRANSFORMATION_ALGS.add('Z_SCORE')
def z_score(input_list, variant_name=None, inverse=False):
    """
    Direct:
        Transformed(e_i) = (e_i - mean(e)) / stddev(e)
    Inverse:
        Multiply each input by -1, before doing exactly the same
    """
    if variant_name:
        raise NotImplementedError("%s variant not implemented" % variant_name)
    mean_val = mean(input_list)
    stddev_val = std(input_list)
    input_copy = input_list[:]
    if inverse:
        # multiply each input_list element by -1
        input_copy[:] = [-x for x in input_list]
    output_list = [
        1.0 * (num - mean_val) / stddev_val for num in input_copy]
    return output_list, None

@TRANSFORMATION_ALGS.add('MIN_MAX')
def min_max(input_list, variant_name=None, inverse=False):
    """
    Direct:
        Transformed(e_i) = (e_i - min(e)) / (max(e) - min(e))
    Inverse:
        Transformed(e_i) = 1 - [(e_i - min(e)) / (max(e) - min(e))]

    """
    if variant_name:
        raise NotImplementedError("%s variant not implemented" % variant_name)
    list_min = min(input_list)
    list_max = max(input_list)
    # Get the range of the list
    list_range = float(list_max - list_min)
    if list_range == 0:
        raise ValueError("The min_max transformation can not be performed"
                         " if the range of valid values (max-min) is zero.")
    # Transform
    if inverse:
        output_list = map(
            lambda x: 1.0 - ((x - list_min) / list_range), input_list)
    else:
        output_list = map(
            lambda x: (x - list_min) / list_range, input_list)
    return output_list, None

@TRANSFORMATION_ALGS.add('LOG10')
def log10_(input_list,
           variant_name='IGNORE ZEROS',
           inverse=False):
    """
    Accept only input_list containing positive (or zero) values
    In case of zeros:
        * the variant IGNORE ZEROS produces NULL as output when any input is
          zero
        * the variant INCREMENT BY ONE IF ZEROS ARE FOUND increments all input
          data by 1
    Then use numpy.log10 function to perform the log10 transformation on the
    list of values
    """
    if inverse:
        raise NotImplementedError(
            "Inverse transformation for log10 is not implemented")
    if variant_name not in LOG10_VARIANTS:
        raise NotImplementedError(
            "%s variant not implemented" % variant_name)
    if any(n < 0 for n in input_list):
        raise ValueError("log10 transformation can not be performed if "
                         "the field contains negative values")
    if any(n == 0 for n in input_list):
        if variant_name == 'INCREMENT BY ONE IF ZEROS ARE FOUND':
            corrected_input = [input_value + 1 for input_value in input_list]
            output_list = list(log10(corrected_input))
            return output_list, None
        elif variant_name == 'IGNORE ZEROS':
            output_list = []
            for input_value in input_list:
                if input_value == 0:
                    output_value = qgis.QPyNullVariant(float)
                    output_list.append(output_value)
                else:
                    output_list.append(log10(input_value))
            return output_list, None
    output_list = list(log10(input_list))
    return output_list, None

@TRANSFORMATION_ALGS.add('QUADRATIC')
def simple_quadratic(input_list, variant_name="INCREASING", inverse=False):
    """
    Simple quadratic transformation (bottom = 0)
        quadratic(e_i) = (e_i - bottom)^2 / (max(e) - bottom)^2
    ==>
        simple_quadratic(e_i) = e_i^2 / max(e)^2

    Inverse:
        For each output x, the final output will be 1 - x
    """
    bottom = 0.0
    max_input = max(input_list)
    if max_input - bottom == 0:
        raise ZeroDivisionError("It is impossible to perform the "
                                "transformation if the maximum "
                                "input value is 0")
    squared_range = (max_input - bottom) ** 2
    if variant_name == "INCREASING":
        output_list = map(
            lambda x: (x - bottom) ** 2 / squared_range, input_list)
    elif variant_name == "DECREASING":
        output_list = map(
            lambda x: (max_input - (x - bottom)) ** 2 / squared_range,
            input_list)

    else:
        raise NotImplementedError("%s variant not implemented" % variant_name)
    if inverse:
        output_list[:] = [1.0 - x for x in output_list]
    return output_list, None

@TRANSFORMATION_ALGS.add('SIGMOID')
def sigmoid(input_list, variant_name="", inverse=False):
    """
    Logistic sigmoid function:
        f(x) = 1 / [1 + e^(-x)]

    Inverse function:
        f(y) =  ln[y / (1 - y)]
    """
    if variant_name:
        raise NotImplementedError("%s variant not implemented" % variant_name)
    output_list = []
    invalid_input_values = []
    if inverse:
        for y in input_list:
            try:
                output = log(y / (1 - y))
            except:
                output = qgis.QPyNullVariant(float)
                invalid_input_values.append(y)
            output_list.append(output)
    else:  # direct
        for x in input_list:
            try:
                output = 1 / (1 + math.exp(-x))
            except:
                output = qgis.QPyNullVariant(float)
                invalid_input_values.append(x)
            output_list.append(output)
    return output_list, invalid_input_values
<|endoftext|>


<commit_before>import __orange__
__pool = {}

class Text():

    id = None
    x = None
    y = None
    z = None
    tcolor = None
    ocolor = None
    size = None

    _ehandlers = {}

    def __init__(self, id, text, x, y, z, tcolor = 0xFFFFFFFF, ocolor = 0xFFFFFFFF, size = 20):
        self.id = id
        self.x = x
        self.y = y
        self.z = z
        self.tcolor = tcolor
        self.ocolor = ocolor
        self.size = size

    def delete(self):
        __orange__.Delete3DText(self.id)
        del __pool[self.id]

    def getID(self):
        return self.id

    def getPosition(self):
        return (self.x, self.y, self.z)

    def getColors(self):
        return (self.tcolor, self.ocolor)

    def getSize(self):
        return self.size

    def getText(self):
        return self.text

def create(text, x, y, z, tcolor = 0xFFFFFFFF, ocolor = 0xFFFFFFFF, size = 20):
    global __pool

    text = Text(__orange__.Create3DTextForAll(text, x, y, z, tcolor, ocolor, size), text, x, y, z, tcolor, ocolor, size)
    __pool[text.id] = text
    return text

def getByID(id):
    global __pool

    if isinstance(id, int):
        if id in __pool.keys():
            return __pool[id]
        return False
    else:
        raise TypeError('3DText ID must be an integer')

def getAll():
    return __pool<commit_msg>text lib: fixed deletion & added deleteByID function<commit_after>import __orange__
__pool = {}

class Text():

    id = None
    x = None
    y = None
    z = None
    tcolor = None
    ocolor = None
    size = None

    _ehandlers = {}

    def __init__(self, id, text, x, y, z, tcolor = 0xFFFFFFFF, ocolor = 0xFFFFFFFF, size = 20):
        self.id = id
        self.x = x
        self.y = y
        self.z = z
        self.tcolor = tcolor
        self.ocolor = ocolor
        self.size = size

    def delete(self):
        deleteByID(self.id)

    def getID(self):
        return self.id

    def getPosition(self):
        return (self.x, self.y, self.z)

    def getColors(self):
        return (self.tcolor, self.ocolor)

    def getSize(self):
        return self.size

    def getText(self):
        return self.text

def create(text, x, y, z, tcolor = 0xFFFFFFFF, ocolor = 0xFFFFFFFF, size = 20):
    global __pool

    text = Text(__orange__.Create3DTextForAll(text, x, y, z, tcolor, ocolor, size), text, x, y, z, tcolor, ocolor, size)
    __pool[text.id] = text
    return text

def deleteByID(id):
    global __pool

    if isinstance(id, int):
        if id in __pool.keys():
            del __pool[id]
            return __orange__.Delete3DText(id)
        else:
            return False
    else:
        raise TypeError('3DText ID must be an integer')

def getByID(id):
    global __pool

    if isinstance(id, int):
        if id in __pool.keys():
            return __pool[id]
        return False
    else:
        raise TypeError('3DText ID must be an integer')

def getAll():
    return __pool<|endoftext|>