<commit_before>import random
import string
import os
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from bs4 import BeautifulSoup
import urlparse
import urllib2

folder_name = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(8))
cur_dir = os.curdir

def open_webpage(url):
    driver = webdriver.PhantomJS()
    driver.get(url)
    WebDriverWait(driver=driver, timeout=10)
    source = driver.page_source
    soup = BeautifulSoup(source)
    make_links_absolute(soup, url)
    folder_path = cur_dir + '/' + folder_name
    global folder_path
    os.mkdir(folder_path)
    download_images(soup)

def make_links_absolute(soup, url):
    for tag in soup.findAll('a', href=True):
        if urlparse.urlparse(tag['href']).scheme == '':
            tag['href'] = urlparse.urljoin(url, tag['href'])
    for img in soup.findAll('img', src=True):
        if urlparse.urlparse(img['src']).scheme == '':
            img['src'] = urlparse.urljoin(url, img['src'])

def download_images(soup):
    for img in soup.findAll('img', src=True):
        img_source = urllib2.urlopen(img['src']).read()
        parse_url = urlparse.urlparse(img['src'])
        folders_to_make = '/'.join(parse_url.path.split('/')[0:-1])
        print folders_to_make
        os.makedirs(folder_path + folders_to_make)
        f = open(folder_path + parse_url.path, 'r+')
        f.write(img_source)
        f.close()

open_webpage('http://www.google.com')
<commit_msg>Download iamges<commit_after>import random
import string
import os
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from bs4 import BeautifulSoup
import urlparse
import urllib2

folder_name = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(8))
cur_dir = os.curdir

def open_webpage(url):
    driver = webdriver.PhantomJS()
    driver.get(url)
    WebDriverWait(driver=driver, timeout=10)
    source = driver.page_source
    soup = BeautifulSoup(source)
    make_links_absolute(soup, url)
    folder_path = cur_dir + '/' + folder_name
    global folder_path
    os.mkdir(folder_path)
    download_images(soup)

def make_links_absolute(soup, url):
    for tag in soup.findAll('a', href=True):
        if urlparse.urlparse(tag['href']).scheme == '':
            tag['href'] = urlparse.urljoin(url, tag['href'])
    for img in soup.findAll('img', src=True):
        if urlparse.urlparse(img['src']).scheme == '':
            img['src'] = urlparse.urljoin(url, img['src'])

def download_images(soup):
    urls = dict()
    for img in soup.findAll('img', src=True):
        img_source = urllib2.urlopen(img['src']).read()
        parse_url = urlparse.urlparse(img['src'])
        urls[img['src']] = parse_url.path.split('/')[-1]
        folders_to_make = '/'.join(parse_url.path.split('/')[0:-1])
        print folders_to_make
        os.makedirs(folder_path + folders_to_make)
        f = open(folder_path + parse_url.path, 'wb')
        f.write(img_source)
        f.close()

open_webpage('http://www.google.com')
<|endoftext|>


<commit_before>#!/usr/bin/python

#from deluge.log import LOG as log
from deluge.ui.client import client
import deluge.component as component
from twisted.internet import reactor, defer
import time
import datetime
import logging
from urlparse import urlparse
import argparse

# Logging
log = logging.getLogger("core")
logger = logging.getLogger()
handler = logging.StreamHandler()
FORMAT = "%(asctime)-15s %(levelname)-8s %(name)-11s %(message)s"
formatter = logging.Formatter(FORMAT)
handler.setFormatter(formatter)
logger.addHandler(handler)

status_keys = ["state",
               "save_path",
               "tracker",
               "tracker_status",
               "next_announce",
               "name",
               "total_size",
               "progress",
               "num_seeds",
               "total_seeds",
               "num_peers",
               "total_peers",
               "eta",
               "download_payload_rate",
               "upload_payload_rate",
               "ratio",
               "distributed_copies",
               "num_pieces",
               "piece_length",
               "total_done",
               "files",
               "file_priorities",
               "file_progress",
               "peers",
               "is_seed",
               "is_finished",
               "active_time",
               "seeding_time",
               "time_added"
               ]

non_fatal_errors = ["timed out", "Connection timed out"]

count = 0
torrent_ids = []

# CONFIGURATION
maxlimits = {'bitmetv.org': 50,
             'tvtorrents.com': 50}

max_ratio = 10
orphan_limit = 5
free_space_limit = 5

def endSession(esresult):
    if esresult:
        print esresult
        reactor.stop()
    else:
        client.disconnect()
        log.debug("Client disconnected.")
        reactor.stop()

def printReport(rresult):
    log.debug("TOTAL TORRENTS: %i" % (count))
    endSession(None)

def print_info(status):
    print("--State: %s" % (status["state"]))
    print("--Added: %s" % (time.ctime(status["time_added"])))
    added = datetime.datetime.fromtimestamp(status["time_added"])
    td = datetime.datetime.now() - added
    hours, minutes, seconds = td.seconds // 3600, td.seconds // 60 % 60, td.seconds % 60
    print("--Age: %dd %dh %dm %ds" % (td.days, hours, minutes, seconds))
    print("--Ratio: %s" % (status["ratio"]))
    print("--Tracker: %s" % (status["tracker"]))

def log_removal(status, reason=None):
    print("Added for removal: %s" % (status["name"]))
    if reason:
        print("--Reason: %s" % reason)
    else:
        print("--Reason: %s" % (status["tracker_status"]))
    print_info(status)

def on_torrents_status(all_torrents):
    global filtertime
    tlist=[]

    torrents_by_tracker = {}

    # order torrents by tracker
    for torrent_id, status in all_torrents.items():
        if status['tracker']:
            tracker = urlparse(status['tracker']).hostname
        else:
            tracker = "No tracker"

        torrents_by_tracker.setdefault(tracker, []).append((torrent_id, status))

    if log.getEffectiveLevel() == logging.getLevelName("DEBUG"):
        log.debug("Tracker torrent counts:")
        for tracker, torrents in torrents_by_tracker.items():
            log.debug("%-25s count: %d", tracker, len(torrents))

    total_delete_count = 0

    # Delete oldest torrents from sites with max count reached
    if args['delete_maxcount']:
        # go through sites with set maximum linits
        for tracker, limit in maxlimits.items():
            limit_count = 0
            # count number of torrents on tracker
            for torrent_id, status in all_torrents.items():
                if tracker in status['tracker']:
                    limit_count += 1

            log.debug("%d torrents found for tracker %s (limit %d)" % (limit_count, tracker, limit))

            # over or at limit, start deleting
            if limit_count >= limit:
                # delete up to limit + 1 to make room
                delete_count = limit_count - limit
                print("Tracker %s is %d over limit" % (tracker, delete_count))
                delete_count += 1
                print("Deleting %d oldest torrents to make room" % delete_count)

                # start deleting from the oldest onwards
                for torrent_id, status in sorted(all_torrents.items(), key=lambda item: item[1]["time_added"]):
                    if tracker in status['tracker']:
                        tlist.append(client.core.remove_torrent(torrent_id, True))
                        delete_count = delete_count - 1
                        total_delete_count += 1
                        log_removal(status, "tracker %s over limit" % tracker)
                        if delete_count <= 0:
                            break

    # Remove torrents with no tracker
    if args['delete_orphans']:
        counter = 0
        for torrent_id, status in torrents_by_tracker.get("No tracker", []):
            # Don't count timed out trackers as no tracker torrents
            # this happens f.ex. when deluge is started with a big torrent load,
            # not all torrents can connect in time
            if not any(reason in status["tracker_status"] for reason in non_fatal_errors):
                added = datetime.datetime.fromtimestamp(status["time_added"])
                td = datetime.datetime.now() - added
                hours, minutes, seconds = td.seconds // 3600, td.seconds // 60 % 60, td.seconds % 60
                # don't delete torrents under 1d old as orphans, it might be a connection issue
                if td.days < 1:
                    print("Skipping %s" % status["name"])
                    print("Reason: under 1d old")
                    print_info(status)
                    continue
                tlist.append(client.core.remove_torrent(torrent_id, True))
                total_delete_count +=1
                log_removal(status)
                counter += 1
            # Max 5 at a time
            if counter > orphan_limit: break

    # delete torrents over maximum ratio
    if args['maximum_ratio']:
        for torrent_id, status in sorted(all_torrents.items(), key=lambda item: item[1]["ratio"]):
            if status['ratio'] > max_ratio:
                total_delete_count +=1
                tlist.append(client.core.remove_torrent(torrent_id, True))
                log_removal(status, "Torrent over maximum ratio (%d)" % max_ratio)

    # Only delete oldest to free space if nothing else has been deleted during this run
    if total_delete_count == 0 and args['free_space']:
        counter = 0
        for torrent_id, status in sorted(all_torrents.items(), key=lambda item: item[1]["time_added"]):
            tlist.append(client.core.remove_torrent(torrent_id, True))
            log_removal(status, "Free disk space needed")
            counter += 1
            if counter >= free_space_limit: break

    # for torrent_id, status in all_torrents.items():
    #     # Filter out torrents with no issues
    #     if status["tracker_status"].endswith("Announce OKXX"):
    #         log.debug("Current torrent id is: %s" % (torrent_id))
    #         log.debug("--Torrent name is: %s" % (status["name"]))
    #         log.debug("--Torrent state is: %s" % (status["state"]))
    #         log.debug("--Torrent ratio is: %s" % (status["ratio"]))
    #         log.debug("--Torrent DL rate is: %s" % (status["download_payload_rate"]))
    #         log.debug("--Torrent UL rate is: %s" % (status["upload_payload_rate"]))
    #         log.debug("--Torrent tracker is: %s" % (status["tracker_status"]))

    global count
    count = len(all_torrents)

    defer.DeferredList(tlist).addCallback(printReport)

def on_session_state(result):
    client.core.get_torrents_status({"id": result}, status_keys).addCallback(on_torrents_status)

def on_connect_success(result):
    log.debug("Connection was successful!")
    curtime = time.time()
    log.debug("Current unix time is %i" % (curtime))
    # connected, start querying session state
    client.core.get_session_state().addCallback(on_session_state)

if __name__ == "__main__":
    global args

    parser = argparse.ArgumentParser(description="Automated Deluge Manager")
    parser.add_argument("--delete-orphans", action="store_true")
    parser.add_argument("--delete-maxcount", action="store_true")
    parser.add_argument("--free-space", action="store_true")
    parser.add_argument("--maximum-ratio", action="store_true")
    parser.add_argument("--cron", action="store_false")

    parsed_args = parser.parse_args()

    args = vars(parsed_args)

    cliconnect = client.connect()
    is_interactive = args['cron']

    # turn on logging if running interactively
    if is_interactive:
        log.setLevel(logging.DEBUG)

    cliconnect.addCallbacks(on_connect_success, endSession, errbackArgs=("Connection failed: check settings and try again."))

    reactor.run()
<commit_msg>Move a few noisy outputs to interactive mode only<commit_after>#!/usr/bin/python

#from deluge.log import LOG as log
from deluge.ui.client import client
import deluge.component as component
from twisted.internet import reactor, defer
import time
import datetime
import logging
from urlparse import urlparse
import argparse

# Logging
log = logging.getLogger("core")
logger = logging.getLogger()
handler = logging.StreamHandler()
FORMAT = "%(asctime)-15s %(levelname)-8s %(name)-11s %(message)s"
formatter = logging.Formatter(FORMAT)
handler.setFormatter(formatter)
logger.addHandler(handler)

status_keys = ["state",
               "save_path",
               "tracker",
               "tracker_status",
               "next_announce",
               "name",
               "total_size",
               "progress",
               "num_seeds",
               "total_seeds",
               "num_peers",
               "total_peers",
               "eta",
               "download_payload_rate",
               "upload_payload_rate",
               "ratio",
               "distributed_copies",
               "num_pieces",
               "piece_length",
               "total_done",
               "files",
               "file_priorities",
               "file_progress",
               "peers",
               "is_seed",
               "is_finished",
               "active_time",
               "seeding_time",
               "time_added"
               ]

non_fatal_errors = ["timed out", "Connection timed out"]

count = 0
torrent_ids = []

# CONFIGURATION
maxlimits = {'bitmetv.org': 50,
             'tvtorrents.com': 50}

max_ratio = 10
orphan_limit = 5
free_space_limit = 5

def endSession(esresult):
    if esresult:
        print esresult
        reactor.stop()
    else:
        client.disconnect()
        log.debug("Client disconnected.")
        reactor.stop()

def printReport(rresult):
    log.debug("TOTAL TORRENTS: %i" % (count))
    endSession(None)

def print_info(status):
    print("--State: %s" % (status["state"]))
    print("--Added: %s" % (time.ctime(status["time_added"])))
    added = datetime.datetime.fromtimestamp(status["time_added"])
    td = datetime.datetime.now() - added
    hours, minutes, seconds = td.seconds // 3600, td.seconds // 60 % 60, td.seconds % 60
    print("--Age: %dd %dh %dm %ds" % (td.days, hours, minutes, seconds))
    print("--Ratio: %s" % (status["ratio"]))
    print("--Tracker: %s" % (status["tracker"]))

def log_removal(status, reason=None):
    print("Added for removal: %s" % (status["name"]))
    if reason:
        print("--Reason: %s" % reason)
    else:
        print("--Reason: %s" % (status["tracker_status"]))
    print_info(status)

def on_torrents_status(all_torrents):
    global filtertime
    tlist=[]

    torrents_by_tracker = {}

    # order torrents by tracker
    for torrent_id, status in all_torrents.items():
        if status['tracker']:
            tracker = urlparse(status['tracker']).hostname
        else:
            tracker = "No tracker"

        torrents_by_tracker.setdefault(tracker, []).append((torrent_id, status))

    if log.getEffectiveLevel() == logging.getLevelName("DEBUG"):
        log.debug("Tracker torrent counts:")
        for tracker, torrents in torrents_by_tracker.items():
            log.debug("%-25s count: %d", tracker, len(torrents))

    total_delete_count = 0

    # Delete oldest torrents from sites with max count reached
    if args['delete_maxcount']:
        # go through sites with set maximum linits
        for tracker, limit in maxlimits.items():
            limit_count = 0
            # count number of torrents on tracker
            for torrent_id, status in all_torrents.items():
                if tracker in status['tracker']:
                    limit_count += 1

            log.debug("%d torrents found for tracker %s (limit %d)" % (limit_count, tracker, limit))

            # over or at limit, start deleting
            if limit_count >= limit:
                # delete up to limit + 1 to make room
                delete_count = limit_count - limit
                delete_count += 1

                if is_interactive:
                    print("Tracker %s is %d over limit" % (tracker, delete_count))
                    print("Deleting %d oldest torrents to make room" % delete_count)

                # start deleting from the oldest onwards
                for torrent_id, status in sorted(all_torrents.items(), key=lambda item: item[1]["time_added"]):
                    if tracker in status['tracker']:
                        tlist.append(client.core.remove_torrent(torrent_id, True))

                        delete_count = delete_count - 1
                        total_delete_count += 1

                        if is_interactive:
                            log_removal(status, "tracker %s over limit" % tracker)
                        if delete_count <= 0:
                            break

    # Remove torrents with no tracker
    if args['delete_orphans']:
        counter = 0
        for torrent_id, status in torrents_by_tracker.get("No tracker", []):
            # Don't count timed out trackers as no tracker torrents
            # this happens f.ex. when deluge is started with a big torrent load,
            # not all torrents can connect in time
            if not any(reason in status["tracker_status"] for reason in non_fatal_errors):
                added = datetime.datetime.fromtimestamp(status["time_added"])
                td = datetime.datetime.now() - added
                hours, minutes, seconds = td.seconds // 3600, td.seconds // 60 % 60, td.seconds % 60
                # don't delete torrents under 1d old as orphans, it might be a connection issue
                if td.days < 1:
                    if is_interactive:
                        print("Skipping %s" % status["name"])
                        print("Reason: under 1d old")
                        print_info(status)
                    continue
                tlist.append(client.core.remove_torrent(torrent_id, True))
                total_delete_count +=1
                log_removal(status, "Orphan torrent")
                counter += 1
            # Max 5 at a time
            if counter > orphan_limit: break

    # delete torrents over maximum ratio
    if args['maximum_ratio']:
        for torrent_id, status in sorted(all_torrents.items(), key=lambda item: item[1]["ratio"]):
            if status['ratio'] > max_ratio:
                total_delete_count +=1
                tlist.append(client.core.remove_torrent(torrent_id, True))
                log_removal(status, "Torrent over maximum ratio (%d)" % max_ratio)

    # Only delete oldest to free space if nothing else has been deleted during this run
    if total_delete_count == 0 and args['free_space']:
        counter = 0
        for torrent_id, status in sorted(all_torrents.items(), key=lambda item: item[1]["time_added"]):
            tlist.append(client.core.remove_torrent(torrent_id, True))
            if is_interactive:
                log_removal(status, "Free disk space needed")
            counter += 1
            if counter >= free_space_limit: break

    # for torrent_id, status in all_torrents.items():
    #     # Filter out torrents with no issues
    #     if status["tracker_status"].endswith("Announce OKXX"):
    #         log.debug("Current torrent id is: %s" % (torrent_id))
    #         log.debug("--Torrent name is: %s" % (status["name"]))
    #         log.debug("--Torrent state is: %s" % (status["state"]))
    #         log.debug("--Torrent ratio is: %s" % (status["ratio"]))
    #         log.debug("--Torrent DL rate is: %s" % (status["download_payload_rate"]))
    #         log.debug("--Torrent UL rate is: %s" % (status["upload_payload_rate"]))
    #         log.debug("--Torrent tracker is: %s" % (status["tracker_status"]))

    global count
    count = len(all_torrents)

    defer.DeferredList(tlist).addCallback(printReport)

def on_session_state(result):
    client.core.get_torrents_status({"id": result}, status_keys).addCallback(on_torrents_status)

def on_connect_success(result):
    log.debug("Connection was successful!")
    curtime = time.time()
    log.debug("Current unix time is %i" % (curtime))
    # connected, start querying session state
    client.core.get_session_state().addCallback(on_session_state)

if __name__ == "__main__":
    global args

    parser = argparse.ArgumentParser(description="Automated Deluge Manager")
    parser.add_argument("--delete-orphans", action="store_true")
    parser.add_argument("--delete-maxcount", action="store_true")
    parser.add_argument("--free-space", action="store_true")
    parser.add_argument("--maximum-ratio", action="store_true")
    parser.add_argument("--cron", action="store_false")

    parsed_args = parser.parse_args()

    args = vars(parsed_args)

    cliconnect = client.connect()
    is_interactive = args['cron']

    # turn on logging if running interactively
    if is_interactive:
        log.setLevel(logging.DEBUG)

    cliconnect.addCallbacks(on_connect_success, endSession, errbackArgs=("Connection failed: check settings and try again."))

    reactor.run()
<|endoftext|>


<commit_before>from sklearn.metrics import roc_auc_score, average_precision_score
import torch.nn as nn
import pandas as pd
import os
import numpy as np

### Evaluator for graph classification
class Evaluator:
    def __init__(self, name):
        self.name = name

        meta_info = pd.read_csv(os.path.join(os.path.dirname(__file__), "master.csv"), index_col = 0)
        if not self.name in meta_info:
            print(self.name)
            error_mssg = "Invalid dataset name {}.\n".format(self.name)
            error_mssg += "Available datasets are as follows:\n"
            error_mssg += "\n".join(meta_info.keys())
            raise ValueError(error_mssg)

        self.num_tasks = int(meta_info[self.name]["num tasks"])
        self.task_type = meta_info[self.name]["task type"]

    def _parse_and_check_input(self, input_dict):
        if self.task_type == "binary classification" or self.task_type == "regression":
            if not "y_true" in input_dict:
                RuntimeError("Missing key of y_true")
            if not "y_pred" in input_dict:
                RuntimeError("Missing key of y_pred")

            y_true, y_pred = input_dict["y_true"], input_dict["y_pred"]

            """
                y_true: numpy ndarray of shape (num_node, num_tasks)
                y_pred: numpy ndarray of shape (num_node, num_tasks)
            """
            ## check type
            if not (isinstance(y_true, np.ndarray) and isinstance(y_true, np.ndarray)):
                raise RuntimeError("Arguments to Evaluator need to be numpy ndarray")

            if not y_true.shape == y_pred.shape:
                raise RuntimeError("Shape of y_true and y_pred must be the same")

            if not y_true.ndim == 2:
                raise RuntimeError("y_true and y_pred mush to 2-dim arrray, {}-dim array given".format(y_true.ndim))

            if not y_true.shape[1] == self.num_tasks:
                raise RuntimeError("Number of tasks for {} should be {} but {} given".format(self.name, self.num_tasks, y_true.shape[1]))

            return y_true, y_pred

        else:
            raise ValueError("Undefined task type %s" (self.task_type))

    def eval(self, input_dict):
        """
            y_true: numpy ndarray of shape (num_data, num_tasks)
            y_pred: numpy ndarray of shape (num_data, num_tasks)

        """

        if self.task_type == "binary classification":
            y_true, y_pred = self._parse_and_check_input(input_dict)
            return self._eval_bincls(y_true, y_pred)
        elif self.task_type == "regression":
            y_true, y_pred = self._parse_and_check_input(input_dict)
            return self._eval_regression(y_true, y_pred)
        else:
            raise ValueError("Undefined task type %s" (self.task_type))

    @property
    def expected_input_format(self):
        desc = "==== Expected input format of Evaluator for {}\n".format(self.name)
        if self.task_type == "binary classification":
            desc += "{\"y_true\": y_true, \"y_pred\": y_pred}\n"
            desc += "- y_true: numpy.ndarray of shape (num_graph, num_task)\n"
            desc += "- y_pred: numpy ndarray of shape (num_graph, num_task)\n"
            desc += "where y_pred stores score values (for computing ROC-AUC and Average Precision),\n"
            desc += "num_task is {}, and ".format(self.num_tasks)
            desc += "each row corresponds to one graph.\n"
        elif self.task_type == "regression":
            desc += "{\"y_true\": y_true, \"y_pred\": y_pred}\n"
            desc += "- y_true: numpy.ndarray of shape (num_graph, num_task)\n"
            desc += "- y_pred: numpy ndarray of shape (num_graph, num_task)\n"
            desc += "where num_task is {}, and ".format(self.num_tasks)
            desc += "each row corresponds to one graph.\n"
        else:
            raise ValueError("Undefined task type %s" (self.task_type))

        return desc

    @property
    def expected_output_format(self):
        desc = "==== Expected output format of Evaluator for {}\n".format(self.name)
        if self.task_type == "binary classification":
            desc += "{\"rocauc\": rocauc, \"ap\": ap}\n"
            desc += "- rocauc (float): ROC-AUC score averaged across {} task(s)\n".format(self.num_tasks)
            desc += "- ap (float): Average Precision score averaged across {} task(s)\n".format(self.num_tasks)
        elif self.task_type == "regression":
            desc += "{\"mae\": mae, \"rmse\": rmse}\n"
            desc += "- mae (float): mean absolute error averaged across {} task(s)\n".format(self.num_tasks)
            desc += "- rmse (float): root mean squared error averaged across {} task(s)\n".format(self.num_tasks)
        else:
            raise ValueError("Undefined task type %s" (self.task_type))

        return desc

    def _eval_bincls(self, y_true, y_pred):
        """
            compute ROC-AUC and AP score averaged across tasks
        """

        rocauc_list = []
        ap_list = []

        for i in range(y_true.shape[1]):
            #AUC is only defined when there is at least one positive data.
            if np.sum(y_true[:,i] == 1) > 0 and np.sum(y_true[:,i] == 0) > 0:
                is_valid = y_true[:,i] == y_true[:,i]
                rocauc_list.append(roc_auc_score(y_true[is_valid,i], y_pred[is_valid,i]))
                ap_list.append(average_precision_score(y_true[is_valid,i], y_pred[is_valid,i]))

        if len(rocauc_list) == 0:
            raise RuntimeError("No positively labeled data available. Cannot compute ROC-AUC.")

        return {"rocauc": sum(rocauc_list)/len(rocauc_list), "ap": sum(ap_list)/len(ap_list)}

    def _eval_regression(self, y_true, y_pred):
        """
            compute MAE and RMSE score averaged across tasks
        """
        mae_list = []
        rmse_list = []

        for i in range(y_true.shape[1]):
            is_valid = y_true[:,i] == y_true[:,i]
            mae_list.append(np.abs(y_true[is_valid] - y_pred[is_valid]).mean())
            rmse_list.append(np.sqrt(((y_true[is_valid] - y_pred[is_valid])**2).mean()))

        return {"mae": sum(mae_list)/len(mae_list), "rmse": sum(rmse_list)/len(rmse_list)}

if __name__ == "__main__":
    ### binary classification case
    evaluator = Evaluator("ogbg-mol-tox21")
    print(evaluator.expected_input_format)
    print(evaluator.expected_output_format)
    y_true = np.random.randint(2, size = (100,12))
    y_pred = np.random.randn(100,12)
    input_dict = {"y_true": y_true, "y_pred": y_pred}
    result = evaluator.eval(input_dict)
    print(result)

    ### regression case
    evaluator = Evaluator("ogbg-mol-lipo")
    print(evaluator.expected_input_format)
    print(evaluator.expected_output_format)
    y_true = np.random.randn(100,1)
    y_pred = np.random.randn(100,1)
    input_dict = {"y_true": y_true, "y_pred": y_pred}
    result = evaluator.eval(input_dict)
    print(result)

<commit_msg>remove import torch;<commit_after>from sklearn.metrics import roc_auc_score, average_precision_score
import pandas as pd
import os
import numpy as np

### Evaluator for graph classification
class Evaluator:
    def __init__(self, name):
        self.name = name

        meta_info = pd.read_csv(os.path.join(os.path.dirname(__file__), "master.csv"), index_col = 0)
        if not self.name in meta_info:
            print(self.name)
            error_mssg = "Invalid dataset name {}.\n".format(self.name)
            error_mssg += "Available datasets are as follows:\n"
            error_mssg += "\n".join(meta_info.keys())
            raise ValueError(error_mssg)

        self.num_tasks = int(meta_info[self.name]["num tasks"])
        self.task_type = meta_info[self.name]["task type"]

    def _parse_and_check_input(self, input_dict):
        if self.task_type == "binary classification" or self.task_type == "regression":
            if not "y_true" in input_dict:
                RuntimeError("Missing key of y_true")
            if not "y_pred" in input_dict:
                RuntimeError("Missing key of y_pred")

            y_true, y_pred = input_dict["y_true"], input_dict["y_pred"]

            """
                y_true: numpy ndarray of shape (num_node, num_tasks)
                y_pred: numpy ndarray of shape (num_node, num_tasks)
            """
            ## check type
            if not (isinstance(y_true, np.ndarray) and isinstance(y_true, np.ndarray)):
                raise RuntimeError("Arguments to Evaluator need to be numpy ndarray")

            if not y_true.shape == y_pred.shape:
                raise RuntimeError("Shape of y_true and y_pred must be the same")

            if not y_true.ndim == 2:
                raise RuntimeError("y_true and y_pred mush to 2-dim arrray, {}-dim array given".format(y_true.ndim))

            if not y_true.shape[1] == self.num_tasks:
                raise RuntimeError("Number of tasks for {} should be {} but {} given".format(self.name, self.num_tasks, y_true.shape[1]))

            return y_true, y_pred

        else:
            raise ValueError("Undefined task type %s" (self.task_type))

    def eval(self, input_dict):
        """
            y_true: numpy ndarray of shape (num_data, num_tasks)
            y_pred: numpy ndarray of shape (num_data, num_tasks)

        """

        if self.task_type == "binary classification":
            y_true, y_pred = self._parse_and_check_input(input_dict)
            return self._eval_bincls(y_true, y_pred)
        elif self.task_type == "regression":
            y_true, y_pred = self._parse_and_check_input(input_dict)
            return self._eval_regression(y_true, y_pred)
        else:
            raise ValueError("Undefined task type %s" (self.task_type))

    @property
    def expected_input_format(self):
        desc = "==== Expected input format of Evaluator for {}\n".format(self.name)
        if self.task_type == "binary classification":
            desc += "{\"y_true\": y_true, \"y_pred\": y_pred}\n"
            desc += "- y_true: numpy.ndarray of shape (num_graph, num_task)\n"
            desc += "- y_pred: numpy ndarray of shape (num_graph, num_task)\n"
            desc += "where y_pred stores score values (for computing ROC-AUC and Average Precision),\n"
            desc += "num_task is {}, and ".format(self.num_tasks)
            desc += "each row corresponds to one graph.\n"
        elif self.task_type == "regression":
            desc += "{\"y_true\": y_true, \"y_pred\": y_pred}\n"
            desc += "- y_true: numpy.ndarray of shape (num_graph, num_task)\n"
            desc += "- y_pred: numpy ndarray of shape (num_graph, num_task)\n"
            desc += "where num_task is {}, and ".format(self.num_tasks)
            desc += "each row corresponds to one graph.\n"
        else:
            raise ValueError("Undefined task type %s" (self.task_type))

        return desc

    @property
    def expected_output_format(self):
        desc = "==== Expected output format of Evaluator for {}\n".format(self.name)
        if self.task_type == "binary classification":
            desc += "{\"rocauc\": rocauc, \"ap\": ap}\n"
            desc += "- rocauc (float): ROC-AUC score averaged across {} task(s)\n".format(self.num_tasks)
            desc += "- ap (float): Average Precision score averaged across {} task(s)\n".format(self.num_tasks)
        elif self.task_type == "regression":
            desc += "{\"mae\": mae, \"rmse\": rmse}\n"
            desc += "- mae (float): mean absolute error averaged across {} task(s)\n".format(self.num_tasks)
            desc += "- rmse (float): root mean squared error averaged across {} task(s)\n".format(self.num_tasks)
        else:
            raise ValueError("Undefined task type %s" (self.task_type))

        return desc

    def _eval_bincls(self, y_true, y_pred):
        """
            compute ROC-AUC and AP score averaged across tasks
        """

        rocauc_list = []
        ap_list = []

        for i in range(y_true.shape[1]):
            #AUC is only defined when there is at least one positive data.
            if np.sum(y_true[:,i] == 1) > 0 and np.sum(y_true[:,i] == 0) > 0:
                is_valid = y_true[:,i] == y_true[:,i]
                rocauc_list.append(roc_auc_score(y_true[is_valid,i], y_pred[is_valid,i]))
                ap_list.append(average_precision_score(y_true[is_valid,i], y_pred[is_valid,i]))

        if len(rocauc_list) == 0:
            raise RuntimeError("No positively labeled data available. Cannot compute ROC-AUC.")

        return {"rocauc": sum(rocauc_list)/len(rocauc_list), "ap": sum(ap_list)/len(ap_list)}

    def _eval_regression(self, y_true, y_pred):
        """
            compute MAE and RMSE score averaged across tasks
        """
        mae_list = []
        rmse_list = []

        for i in range(y_true.shape[1]):
            is_valid = y_true[:,i] == y_true[:,i]
            mae_list.append(np.abs(y_true[is_valid] - y_pred[is_valid]).mean())
            rmse_list.append(np.sqrt(((y_true[is_valid] - y_pred[is_valid])**2).mean()))

        return {"mae": sum(mae_list)/len(mae_list), "rmse": sum(rmse_list)/len(rmse_list)}

if __name__ == "__main__":
    ### binary classification case
    evaluator = Evaluator("ogbg-mol-tox21")
    print(evaluator.expected_input_format)
    print(evaluator.expected_output_format)
    y_true = np.random.randint(2, size = (100,12))
    y_pred = np.random.randn(100,12)
    input_dict = {"y_true": y_true, "y_pred": y_pred}
    result = evaluator.eval(input_dict)
    print(result)

    ### regression case
    evaluator = Evaluator("ogbg-mol-lipo")
    print(evaluator.expected_input_format)
    print(evaluator.expected_output_format)
    y_true = np.random.randn(100,1)
    y_pred = np.random.randn(100,1)
    input_dict = {"y_true": y_true, "y_pred": y_pred}
    result = evaluator.eval(input_dict)
    print(result)

<|endoftext|>


<commit_before>"""
Class for reading spikegadgets files.
Only continuous signals are supported at the moment.

https://spikegadgets.com/spike-products/

Documentation of the format:
https://bitbucket.org/mkarlsso/trodes/wiki/Configuration

Note :
  * this file format have multiple version. news version include the gain for scaling.
     The actual implementation do not contain this feature because we don't have
     files to test this. So now the gain is "hardcoded" to 1. and so units
     is not handled correctly.

The ".rec" file format contains:
  * a first  text part with information in an XML structure
  * a second part for the binary buffer

Author: Samuel Garcia
"""
from .baserawio import (BaseRawIO, _signal_channel_dtype, _signal_stream_dtype,
    _spike_channel_dtype, _event_channel_dtype)

import numpy as np

from xml.etree import ElementTree

class SpikeGadgetsRawIO(BaseRawIO):
    extensions = ['rec']
    rawmode = 'one-file'

    def __init__(self, filename='', selected_streams=None):
        """
        Class for reading spikegadgets files.
        Only continuous signals are supported at the moment.

        https://spikegadgets.com/spike-products/

        Args:
            filename: str
                The filename
            selected_streams: None, list, str
                sublist of streams to load/expose to API
                useful for spikeextractor when one stream only is needed.
                For instance streams = ['ECU', 'trodes']
                'trodes' is name for ephy channel (ntrodes)
        """
        BaseRawIO.__init__(self)
        self.filename = filename
        self.selected_streams = selected_streams

    def _source_name(self):
        return self.filename

    def _parse_header(self):
        # parse file until "</Configuration>"
        header_size = None
        with open(self.filename, mode='rb') as f:
            while True:
                line = f.readline()
                if b"</Configuration>" in line:
                    header_size = f.tell()
                    break

            if header_size is None:
                ValueError("SpikeGadgets: the xml header does not contain '</Configuration>'")

            f.seek(0)
            header_txt = f.read(header_size).decode('utf8')

        # explore xml header
        root = ElementTree.fromstring(header_txt)
        gconf = sr = root.find('GlobalConfiguration')
        hconf = root.find('HardwareConfiguration')
        sconf = root.find('SpikeConfiguration')

        self._sampling_rate = float(hconf.attrib['samplingRate'])
        num_ephy_channels = int(hconf .attrib['numChannels'])

        # explore sub stream and count packet size
        # first bytes is 0x55
        packet_size = 1
        stream_bytes = {}
        for device in hconf:
            stream_id = device.attrib['name']
            num_bytes = int(device.attrib['numBytes'])
            stream_bytes[stream_id] = packet_size
            packet_size += num_bytes

        # timesteamps 4 uint32
        self._timestamp_byte = packet_size
        packet_size += 4

        packet_size += 2 * num_ephy_channels

        # read the binary part lazily
        raw_memmap = np.memmap(self.filename, mode='r', offset=header_size, dtype='<u1')

        num_packet = raw_memmap.size // packet_size
        raw_memmap = raw_memmap[:num_packet * packet_size]
        self._raw_memmap = raw_memmap.reshape(-1, packet_size)

        # create signal channels
        stream_ids = []
        signal_streams = []
        signal_channels = []

        # walk in xml device and keep only "analog" one
        self._mask_channels_bytes = {}
        for device in hconf:
            stream_id = device.attrib['name']
            for channel in device:

                if 'interleavedDataIDByte' in channel.attrib:
                    # TODO LATER: deal with "headstageSensor" which have interleaved
                    continue

                if channel.attrib['dataType'] == 'analog':

                    if stream_id not in stream_ids:
                        stream_ids.append(stream_id)
                        stream_name = stream_id
                        signal_streams.append((stream_name, stream_id))
                        self._mask_channels_bytes[stream_id] = []

                    name = channel.attrib['id']
                    chan_id = channel.attrib['id']
                    dtype = 'int16'
                    # TODO LATER : handle gain correctly according the file version
                    units = ''
                    gain = 1.
                    offset = 0.
                    signal_channels.append((name, chan_id, self._sampling_rate, 'int16',
                                         units, gain, offset, stream_id))

                    num_bytes = stream_bytes[stream_id] + int(channel.attrib['startByte'])
                    chan_mask = np.zeros(packet_size, dtype='bool')
                    chan_mask[num_bytes] = True
                    chan_mask[num_bytes + 1] = True
                    self._mask_channels_bytes[stream_id].append(chan_mask)

        if num_ephy_channels > 0:
            stream_id = 'trodes'
            stream_name = stream_id
            signal_streams.append((stream_name, stream_id))
            self._mask_channels_bytes[stream_id] = []

            chan_ind = 0
            for trode in sconf:
                for schan in trode:
                    name = 'trode' + trode.attrib['id'] + 'chan' + schan.attrib['hwChan']
                    chan_id = schan.attrib['hwChan']
                    # TODO LATER : handle gain correctly according the file version
                    units = ''
                    gain = 1.
                    offset = 0.
                    signal_channels.append((name, chan_id, self._sampling_rate, 'int16',
                                         units, gain, offset, stream_id))

                    chan_mask = np.zeros(packet_size, dtype='bool')
                    num_bytes = packet_size - 2 * num_ephy_channels + 2 * chan_ind
                    chan_mask[num_bytes] = True
                    chan_mask[num_bytes + 1] = True
                    self._mask_channels_bytes[stream_id].append(chan_mask)

                    chan_ind += 1

        # make mask as array (used in _get_analogsignal_chunk(...))
        self._mask_streams = {}
        for stream_id, l in self._mask_channels_bytes.items():
            mask = np.array(l)
            self._mask_channels_bytes[stream_id] = mask
            self._mask_streams[stream_id] = np.any(mask, axis=0)

        signal_streams = np.array(signal_streams, dtype=_signal_stream_dtype)
        signal_channels = np.array(signal_channels, dtype=_signal_channel_dtype)

        # remove some stream if no wanted
        if self.selected_streams is not None:
            if isinstance(self.selected_streams, str):
                self.selected_streams = [self.selected_streams]
            assert isinstance(self.selected_streams, list)

            keep = np.in1d(signal_streams['id'], self.selected_streams)
            signal_streams = signal_streams[keep]

            keep = np.in1d(signal_channels['stream_id'], self.selected_streams)
            signal_channels = signal_channels[keep]

        # No events channels
        event_channels = []
        event_channels = np.array(event_channels, dtype=_event_channel_dtype)

        # No spikes  channels
        spike_channels = []
        spike_channels = np.array(spike_channels, dtype=_spike_channel_dtype)

        # fille into header dict
        self.header = {}
        self.header['nb_block'] = 1
        self.header['nb_segment'] = [1]
        self.header['signal_streams'] = signal_streams
        self.header['signal_channels'] = signal_channels
        self.header['spike_channels'] = spike_channels
        self.header['event_channels'] = event_channels

        self._generate_minimal_annotations()
        # info from GlobalConfiguration in xml are copied to block and seg annotations
        bl_ann = self.raw_annotations['blocks'][0]
        seg_ann = self.raw_annotations['blocks'][0]['segments'][0]
        for ann in (bl_ann, seg_ann):
            ann.update(gconf.attrib)

    def _segment_t_start(self, block_index, seg_index):
        return 0.

    def _segment_t_stop(self, block_index, seg_index):
        size = self._raw_memmap.shape[0]
        t_stop = size / self._sampling_rate
        return t_stop

    def _get_signal_size(self, block_index, seg_index, stream_index):
        size = self._raw_memmap.shape[0]
        return size

    def _get_signal_t_start(self, block_index, seg_index, stream_index):
        return 0.

    def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, stream_index,
                                channel_indexes):
        stream_id = self.header['signal_streams'][stream_index]['id']

        raw_unit8 = self._raw_memmap[i_start:i_stop]

        num_chan = len(self._mask_channels_bytes[stream_id])
        re_order = None
        if channel_indexes is None:
            # no loop : entire stream mask
            stream_mask = self._mask_streams[stream_id]
        else:
            # accumulate mask
            if isinstance(channel_indexes, slice):
                chan_inds = np.arange(num_chan)[channel_indexes]
            else:
                chan_inds = channel_indexes

                if np.any(np.diff(channel_indexes) < 0):
                    # handle channel are not ordered
                    sorted_channel_indexes = np.sort(channel_indexes)
                    re_order = np.array([list(sorted_channel_indexes).index(ch)
                                         for ch in channel_indexes])

            stream_mask = np.zeros(raw_unit8.shape[1], dtype='bool')
            for chan_ind in chan_inds:
                chan_mask = self._mask_channels_bytes[stream_id][chan_ind]
                stream_mask |= chan_mask

        # this copies the data from the memmap into memory
        raw_unit8_mask = raw_unit8[:, stream_mask]
        shape = raw_unit8_mask.shape
        shape = (shape[0], shape[1] // 2)
        # reshape the and retype by view
        raw_unit16 = raw_unit8_mask.flatten().view('int16').reshape(shape)

        if re_order is not None:
            raw_unit16 = raw_unit16[:, re_order]

        return raw_unit16
<commit_msg>Julia suggestion<commit_after>"""
Class for reading spikegadgets files.
Only continuous signals are supported at the moment.

https://spikegadgets.com/spike-products/

Documentation of the format:
https://bitbucket.org/mkarlsso/trodes/wiki/Configuration

Note :
  * this file format have multiple version. news version include the gain for scaling.
     The actual implementation do not contain this feature because we don't have
     files to test this. So now the gain is "hardcoded" to 1. and so units
     is not handled correctly.

The ".rec" file format contains:
  * a first  text part with information in an XML structure
  * a second part for the binary buffer

Author: Samuel Garcia
"""
from .baserawio import (BaseRawIO, _signal_channel_dtype, _signal_stream_dtype,
    _spike_channel_dtype, _event_channel_dtype)

import numpy as np

from xml.etree import ElementTree

class SpikeGadgetsRawIO(BaseRawIO):
    extensions = ['rec']
    rawmode = 'one-file'

    def __init__(self, filename='', selected_streams=None):
        """
        Class for reading spikegadgets files.
        Only continuous signals are supported at the moment.

        Initialize a SpikeGadgetsRawIO for a single ".rec" file.

        Args:
            filename: str
                The filename
            selected_streams: None, list, str
                sublist of streams to load/expose to API
                useful for spikeextractor when one stream only is needed.
                For instance streams = ['ECU', 'trodes']
                'trodes' is name for ephy channel (ntrodes)
        """
        BaseRawIO.__init__(self)
        self.filename = filename
        self.selected_streams = selected_streams

    def _source_name(self):
        return self.filename

    def _parse_header(self):
        # parse file until "</Configuration>"
        header_size = None
        with open(self.filename, mode='rb') as f:
            while True:
                line = f.readline()
                if b"</Configuration>" in line:
                    header_size = f.tell()
                    break

            if header_size is None:
                ValueError("SpikeGadgets: the xml header does not contain '</Configuration>'")

            f.seek(0)
            header_txt = f.read(header_size).decode('utf8')

        # explore xml header
        root = ElementTree.fromstring(header_txt)
        gconf = sr = root.find('GlobalConfiguration')
        hconf = root.find('HardwareConfiguration')
        sconf = root.find('SpikeConfiguration')

        self._sampling_rate = float(hconf.attrib['samplingRate'])
        num_ephy_channels = int(hconf .attrib['numChannels'])

        # explore sub stream and count packet size
        # first bytes is 0x55
        packet_size = 1
        stream_bytes = {}
        for device in hconf:
            stream_id = device.attrib['name']
            num_bytes = int(device.attrib['numBytes'])
            stream_bytes[stream_id] = packet_size
            packet_size += num_bytes

        # timesteamps 4 uint32
        self._timestamp_byte = packet_size
        packet_size += 4

        packet_size += 2 * num_ephy_channels

        # read the binary part lazily
        raw_memmap = np.memmap(self.filename, mode='r', offset=header_size, dtype='<u1')

        num_packet = raw_memmap.size // packet_size
        raw_memmap = raw_memmap[:num_packet * packet_size]
        self._raw_memmap = raw_memmap.reshape(-1, packet_size)

        # create signal channels
        stream_ids = []
        signal_streams = []
        signal_channels = []

        # walk in xml device and keep only "analog" one
        self._mask_channels_bytes = {}
        for device in hconf:
            stream_id = device.attrib['name']
            for channel in device:

                if 'interleavedDataIDByte' in channel.attrib:
                    # TODO LATER: deal with "headstageSensor" which have interleaved
                    continue

                if channel.attrib['dataType'] == 'analog':

                    if stream_id not in stream_ids:
                        stream_ids.append(stream_id)
                        stream_name = stream_id
                        signal_streams.append((stream_name, stream_id))
                        self._mask_channels_bytes[stream_id] = []

                    name = channel.attrib['id']
                    chan_id = channel.attrib['id']
                    dtype = 'int16'
                    # TODO LATER : handle gain correctly according the file version
                    units = ''
                    gain = 1.
                    offset = 0.
                    signal_channels.append((name, chan_id, self._sampling_rate, 'int16',
                                         units, gain, offset, stream_id))

                    num_bytes = stream_bytes[stream_id] + int(channel.attrib['startByte'])
                    chan_mask = np.zeros(packet_size, dtype='bool')
                    chan_mask[num_bytes] = True
                    chan_mask[num_bytes + 1] = True
                    self._mask_channels_bytes[stream_id].append(chan_mask)

        if num_ephy_channels > 0:
            stream_id = 'trodes'
            stream_name = stream_id
            signal_streams.append((stream_name, stream_id))
            self._mask_channels_bytes[stream_id] = []

            chan_ind = 0
            for trode in sconf:
                for schan in trode:
                    name = 'trode' + trode.attrib['id'] + 'chan' + schan.attrib['hwChan']
                    chan_id = schan.attrib['hwChan']
                    # TODO LATER : handle gain correctly according the file version
                    units = ''
                    gain = 1.
                    offset = 0.
                    signal_channels.append((name, chan_id, self._sampling_rate, 'int16',
                                         units, gain, offset, stream_id))

                    chan_mask = np.zeros(packet_size, dtype='bool')
                    num_bytes = packet_size - 2 * num_ephy_channels + 2 * chan_ind
                    chan_mask[num_bytes] = True
                    chan_mask[num_bytes + 1] = True
                    self._mask_channels_bytes[stream_id].append(chan_mask)

                    chan_ind += 1

        # make mask as array (used in _get_analogsignal_chunk(...))
        self._mask_streams = {}
        for stream_id, l in self._mask_channels_bytes.items():
            mask = np.array(l)
            self._mask_channels_bytes[stream_id] = mask
            self._mask_streams[stream_id] = np.any(mask, axis=0)

        signal_streams = np.array(signal_streams, dtype=_signal_stream_dtype)
        signal_channels = np.array(signal_channels, dtype=_signal_channel_dtype)

        # remove some stream if no wanted
        if self.selected_streams is not None:
            if isinstance(self.selected_streams, str):
                self.selected_streams = [self.selected_streams]
            assert isinstance(self.selected_streams, list)

            keep = np.in1d(signal_streams['id'], self.selected_streams)
            signal_streams = signal_streams[keep]

            keep = np.in1d(signal_channels['stream_id'], self.selected_streams)
            signal_channels = signal_channels[keep]

        # No events channels
        event_channels = []
        event_channels = np.array(event_channels, dtype=_event_channel_dtype)

        # No spikes  channels
        spike_channels = []
        spike_channels = np.array(spike_channels, dtype=_spike_channel_dtype)

        # fille into header dict
        self.header = {}
        self.header['nb_block'] = 1
        self.header['nb_segment'] = [1]
        self.header['signal_streams'] = signal_streams
        self.header['signal_channels'] = signal_channels
        self.header['spike_channels'] = spike_channels
        self.header['event_channels'] = event_channels

        self._generate_minimal_annotations()
        # info from GlobalConfiguration in xml are copied to block and seg annotations
        bl_ann = self.raw_annotations['blocks'][0]
        seg_ann = self.raw_annotations['blocks'][0]['segments'][0]
        for ann in (bl_ann, seg_ann):
            ann.update(gconf.attrib)

    def _segment_t_start(self, block_index, seg_index):
        return 0.

    def _segment_t_stop(self, block_index, seg_index):
        size = self._raw_memmap.shape[0]
        t_stop = size / self._sampling_rate
        return t_stop

    def _get_signal_size(self, block_index, seg_index, stream_index):
        size = self._raw_memmap.shape[0]
        return size

    def _get_signal_t_start(self, block_index, seg_index, stream_index):
        return 0.

    def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, stream_index,
                                channel_indexes):
        stream_id = self.header['signal_streams'][stream_index]['id']

        raw_unit8 = self._raw_memmap[i_start:i_stop]

        num_chan = len(self._mask_channels_bytes[stream_id])
        re_order = None
        if channel_indexes is None:
            # no loop : entire stream mask
            stream_mask = self._mask_streams[stream_id]
        else:
            # accumulate mask
            if isinstance(channel_indexes, slice):
                chan_inds = np.arange(num_chan)[channel_indexes]
            else:
                chan_inds = channel_indexes

                if np.any(np.diff(channel_indexes) < 0):
                    # handle channel are not ordered
                    sorted_channel_indexes = np.sort(channel_indexes)
                    re_order = np.array([list(sorted_channel_indexes).index(ch)
                                         for ch in channel_indexes])

            stream_mask = np.zeros(raw_unit8.shape[1], dtype='bool')
            for chan_ind in chan_inds:
                chan_mask = self._mask_channels_bytes[stream_id][chan_ind]
                stream_mask |= chan_mask

        # this copies the data from the memmap into memory
        raw_unit8_mask = raw_unit8[:, stream_mask]
        shape = raw_unit8_mask.shape
        shape = (shape[0], shape[1] // 2)
        # reshape the and retype by view
        raw_unit16 = raw_unit8_mask.flatten().view('int16').reshape(shape)

        if re_order is not None:
            raw_unit16 = raw_unit16[:, re_order]

        return raw_unit16
<|endoftext|>


<commit_before>"""OneSignal App Client class."""
from .base_client import OneSignalBaseClient

class OneSignalAppClient(OneSignalBaseClient):
    """OneSignal Client."""
    ENDPOINTS = {
        'notifications': 'notifications',
        'csv_export': 'players/csv_export?app_id=%s'
    }

    def __init__(self, app_id, app_api_key):
        """
        Initializes the OneSignal Client.

        :param app_id: OneSignal App ID.
        Found under OneSignal Dashboard > App Settings > Keys & IDs
        :type app_id: string
        :param app_api_key: Application REST API key.
        Found under OneSignal Dashboard > App Settings > Keys & IDs
        :type app_api_key: string
        """
        self.app_id = app_id
        self.app_api_key = app_api_key
        self.mode = self.MODE_APP

    def get_headers(self):
        """
        Build default headers for requests.
        :return: Returns dict which contains the headers
        """
        return self._get_headers()

    def create_notification(self, notification):
        """
        Creates a new notification.
        """
        payload = notification.get_payload_for_request()
        return self.post(self._url(self.ENDPOINTS['notifications']),
                         payload=payload)

    def csv_export(self):
        """
        Request a CSV export from OneSignal.
        :return: Returns the request result.
        """
        endpoint = self.ENDPOINTS['csv_export'] % (self.app_id)
        return self.post(self._url(endpoint))
<commit_msg>Improve doc<commit_after>"""OneSignal App Client class."""
from .base_client import OneSignalBaseClient

class OneSignalAppClient(OneSignalBaseClient):
    """OneSignal Client."""
    ENDPOINTS = {
        'notifications': 'notifications',
        'csv_export': 'players/csv_export?app_id=%s'
    }

    def __init__(self, app_id, app_api_key):
        """
        Initializes the OneSignal Client.

        :param app_id: OneSignal App ID.
        Found under OneSignal Dashboard > App Settings > Keys & IDs
        :type app_id: string
        :param app_api_key: Application REST API key.
        Found under OneSignal Dashboard > App Settings > Keys & IDs
        :type app_api_key: string
        """
        self.app_id = app_id
        self.app_api_key = app_api_key
        self.mode = self.MODE_APP

    def get_headers(self):
        """
        Build default headers for requests.
        :return: Returns dict which contains the headers
        """
        return self._get_headers()

    def create_notification(self, notification):
        """
        Creates a new notification.
        :param notification: onesignalclient.notification.Notification object
        """
        payload = notification.get_payload_for_request()
        return self.post(self._url(self.ENDPOINTS['notifications']),
                         payload=payload)

    def csv_export(self):
        """
        Request a CSV export from OneSignal.
        :return: Returns the request result.
        """
        endpoint = self.ENDPOINTS['csv_export'] % (self.app_id)
        return self.post(self._url(endpoint))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import division, with_statement
import re
from collections import OrderedDict
from acq4.devices.Camera import Camera, CameraTask
from PyQt4 import QtCore
import time, sys, traceback
from numpy import *
from acq4.util.metaarray import *
import acq4.util.ptime as ptime
from acq4.util.Mutex import Mutex
from acq4.util.debug import *

class MicroManagerCamera(Camera):
    """Camera device that uses MicroManager to provide imaging.

    Configuration keys:

    * mmAdapterName
    * mmDeviceName
    """
    def __init__(self, manager, config, name):
        try:
            import MMCorePy
        except ImportError:
            if sys.platform != 'win32':
                raise
            # MM does not install itself to standard path. User should take care of this,
            # but we can make a guess..
            MM_PATH = os.path.join('C:', os.path.sep, 'Program Files','Micro-Manager-1.4')
            sys.path.append(MM_PATH)
            os.environ['PATH'] = MM_PATH + ';' + os.environ['PATH']
            #sys.path.append('C:\\Program Files\\Micro-Manager-1.4')
            try:
                import MMCorePy
            finally:
                sys.path.pop()

        self.camName = str(name)  # we will use this name as the handle to the MM camera
        self.mmc = MMCorePy.CMMCore()

        self.camLock = Mutex(Mutex.Recursive)  ## Lock to protect access to camera
        self._config = config
        Camera.__init__(self, manager, config, name)  ## superclass will call setupCamera when it is ready.
        self.acqBuffer = None
        self.frameId = 0
        self.lastFrameTime = None

    def setupCamera(self):
        # sanity check for MM adapter and device name
        adapterName = self._config['mmAdapterName']
        allAdapters = self.mmc.getDeviceAdapterNames()
        if adapterName not in allAdapters:
            raise ValueError("Adapter name '%s' is not valid. Options are: %s" % (adapterName, allAdapters))
        deviceName = self._config['mmDeviceName']
        allDevices = self.mmc.getAvailableDevices(adapterName)
        if deviceName not in allDevices:
            raise ValueError("Device name '%s' is not valid for adapter '%s'. Options are: %s" % (deviceName, adapterName, allDevices))

        self.mmc.loadDevice(self.camName, adapterName, deviceName)
        self.mmc.initializeDevice(self.camName)

        self._readAllParams()
        # import pprint
        # pprint.pprint(dict(self.listParams()))

    def startCamera(self):
        with self.camLock:
            self.mmc.startContinuousSequenceAcquisition(0)

    def stopCamera(self):
        with self.camLock:
            self.mmc.stopSequenceAcquisition()
            self.acqBuffer = None

    def _acquireFrames(self, n=1):
        self.mmc.setCameraDevice(self.camName)
        self.mmc.startSequenceAcquisition(n, 0, True)
        frames = []
        for i in range(n):
            start = time.time()
            while self.mmc.getRemainingImageCount() == 0:
                time.sleep(0.005)
                if time.time() - start > 10.0:
                    raise Exception("Timed out waiting for camera frame.")
            frames.append(self.mmc.popNextImage().T[np.newaxis, ...])
        self.mmc.stopSequenceAcquisition()
        return np.concatenate(frames, axis=0)

    def newFrames(self):
        """Return a list of all frames acquired since the last call to newFrames."""

        with self.camLock:
            nFrames = self.mmc.getRemainingImageCount()
            if nFrames == 0:
                return []

        now = ptime.time()
        if self.lastFrameTime is None:
            self.lastFrameTime = now

        dt = (now - self.lastFrameTime) / nFrames
        frames = []
        with self.camLock:
            for i in range(nFrames):
                frame = {}
                frame['time'] = self.lastFrameTime + (dt * (i+1))
                frame['id'] = self.frameId
                frame['data'] = self.mmc.popNextImage().T
                frames.append(frame)
                self.frameId += 1

        self.lastFrame = frame
        self.lastFrameTime = now
        return frames

    def quit(self):
        self.mmc.stopSequenceAcquisition()
        self.mmc.unloadDevice(self.camName)

    def _readAllParams(self):
        # these are parameters expected for all cameras
        defaultParams = ['exposure', 'binningX', 'binningY', 'regionX', 'regionY', 'regionW', 'regionH', 'triggerMode', 'bitDepth']

        with self.camLock:
            params = OrderedDict([(n, None) for n in defaultParams])

            properties = self.mmc.getDevicePropertyNames(self.camName)
            for prop in properties:
                vals = self.mmc.getAllowedPropertyValues(self.camName, prop)
                if vals == ():
                    if self.mmc.hasPropertyLimits(self.camName, prop):
                        vals = (
                            self.mmc.getPropertyLowerLimit(self.camName, prop),
                            self.mmc.getPropertyUpperLimit(self.camName, prop),
                        )
                else:
                    vals = list(vals)
                readonly = self.mmc.isPropertyReadOnly(self.camName, prop)

                # translate standard properties to the names / formats that we expect
                if prop == 'Exposure':
                    prop = 'exposure'
                    # convert ms to s
                    vals = tuple([v * 1e-3 for v in vals])
                elif prop == 'Binning':
                    vals = [v.split('x') for v in vals]
                    params['binningX'] = ([int(v[0]) for v in vals], not readonly, True, [])
                    params['binningY'] = ([int(v[1]) for v in vals], not readonly, True, [])
                    continue
                elif prop == 'Trigger':
                    prop = 'triggerMode'
                    vals = [{'NORMAL': 'Normal', 'START': 'TriggerStart'}[v] for v in vals]
                elif prop == 'PixelType':
                    prop = 'bitDepth'
                    vals = [int(bd.rstrip('bit')) for bd in vals]

                params[prop] = (vals, not readonly, True, [])

            # Reset ROI to full frame so we know the native resolution
            self.mmc.setCameraDevice(self.camName)
            self.mmc.setProperty(self.camName, 'Binning', '1x1')
            self.mmc.clearROI()
            rgn = self.mmc.getROI(self.camName)
            self._sensorSize = rgn[2:]

            params.update({
                'regionX': [(0, rgn[2]-1, 1), True, True, []],
                'regionY': [(0, rgn[3]-1, 1), True, True, []],
                'regionW': [(1, rgn[2], 1), True, True, []],
                'regionH': [(1, rgn[3], 1), True, True, []],
            })

        self._allParams = params

    def listParams(self, params=None):
        """List properties of specified parameters, or of all parameters if None"""
        if params is None:
            return self._allParams.copy()
        if isinstance(params, basestring):
            return self._allParams[params]
        return dict([(p, self._allParams[p]) for p in params])

    def setParams(self, params, autoRestart=True, autoCorrect=True):
        # umanager will refuse to set params while camera is running,
        # so autoRestart doesn't make sense in this context
        if self.isRunning():
            restart = True
            self.stop()
        else:
            restart = False

        newVals = {}
        for k,v in params.items():
            self._setParam(k, v, autoCorrect=autoCorrect)
            if k == 'binning':
                newVals['binningX'], newVals['binningY'] = self.getParam(k)
            elif k == 'region':
                newVals['regionX'], newVals['regionY'], newVals['regionW'], newVals['regionH'] = self.getParam(k)
            else:
                newVals[k] = self.getParam(k)
        self.sigParamsChanged.emit(newVals)

        if restart:
            self.start()

        needRestart = False
        return (newVals, needRestart)

    def setParam(self, param, value, autoCorrect=True, autoRestart=True):
        return self.setParams({param: value}, autoCorrect=autoCorrect, autoRestart=autoRestart)

    def _setParam(self, param, value, autoCorrect=True):
        if param == 'region':
            value = (value[0], value[1], value[2], value[3])
            self.mmc.setCameraDevice(self.camName)
            self.mmc.setROI(*value)
            return

        if param.startswith('region'):
            rgn = list(self.mmc.getROI(self.camName))
            if param[-1] == 'X':
                rgn[0] = value
            elif param[-1] == 'Y':
                rgn[1] = value
            elif param[-1] == 'W':
                rgn[2] = value
            elif param[-1] == 'H':
                rgn[3] = value
            self.mmc.setCameraDevice(self.camName)
            self.mmc.setROI(*rgn)
            return

        if param.startswith('binning'):
            if param == 'binningX':
                y = self.getParam('binningY')
                value = (value, y)
                param = 'binning'
            elif param == 'binningY':
                x = self.getParam('binningX')
                value = (x, value)
                param = 'binning'
            value = '%dx%d' % value
            param = 'Binning'

        elif param == 'exposure':
            # s to ms
            value = value * 1e3
            param = 'Exposure'

        elif param == 'triggerMode':
            value = {'Normal': 'NORMAL', 'TriggerStart': 'START'}[value]
            param = 'Trigger'

        with self.camLock:
            self.mmc.setProperty(self.camName, str(param), str(value))

    def getParams(self, params=None):
        if params is None:
            params = self.listParams().keys()
        return dict([(p, self.getParam(p)) for p in params])

    def getParam(self, param):
        if param == 'sensorSize':
            return self._sensorSize
        elif param.startswith('region'):
            rgn = self.mmc.getROI(self.camName)
            if param == 'region':
                return rgn
            i = ['regionX', 'regionY', 'regionW', 'regionH'].index(param)
            return rgn[i]

        paramTrans = {
            'exposure': 'Exposure',
            'binning': 'Binning',
            'binningX': 'Binning',
            'binningY': 'Binning',
            'triggerMode': 'Trigger',
            'bitDepth': 'PixelType',
        }.get(param, param)
        with self.camLock:
            val = self.mmc.getProperty(self.camName, str(paramTrans))

        # coerce to int or float if possible
        try:
            val = int(val)
        except ValueError:
            try:
                val = float(val)
            except ValueError:
                pass

        if param == 'binningY':
            return int(val.split('x')[1])
        elif param == 'binningX':
            return int(val.split('x')[0])
        elif param == 'binning':
            return tuple([int(b) for b in val.split('x')])
        elif param == 'exposure':
            # ms to s
            val = val * 1e-3
        elif param == 'bitDepth':
            return int(val.rstrip('bit'))

        return val

<commit_msg>adjusted binning settings<commit_after># -*- coding: utf-8 -*-
from __future__ import division, with_statement
import re
from collections import OrderedDict
from acq4.devices.Camera import Camera, CameraTask
from PyQt4 import QtCore
import time, sys, traceback
from numpy import *
from acq4.util.metaarray import *
import acq4.util.ptime as ptime
from acq4.util.Mutex import Mutex
from acq4.util.debug import *

class MicroManagerCamera(Camera):
    """Camera device that uses MicroManager to provide imaging.

    Configuration keys:

    * mmAdapterName
    * mmDeviceName
    """
    def __init__(self, manager, config, name):
        try:
            import MMCorePy
        except ImportError:
            if sys.platform != 'win32':
                raise
            # MM does not install itself to standard path. User should take care of this,
            # but we can make a guess..
            MM_PATH = os.path.join('C:', os.path.sep, 'Program Files','Micro-Manager-1.4')
            sys.path.append(MM_PATH)
            os.environ['PATH'] = MM_PATH + ';' + os.environ['PATH']
            #sys.path.append('C:\\Program Files\\Micro-Manager-1.4')
            try:
                import MMCorePy
            finally:
                sys.path.pop()

        self.camName = str(name)  # we will use this name as the handle to the MM camera
        self.mmc = MMCorePy.CMMCore()

        self.camLock = Mutex(Mutex.Recursive)  ## Lock to protect access to camera
        self._config = config
        Camera.__init__(self, manager, config, name)  ## superclass will call setupCamera when it is ready.
        self.acqBuffer = None
        self.frameId = 0
        self.lastFrameTime = None

    def setupCamera(self):
        # sanity check for MM adapter and device name
        adapterName = self._config['mmAdapterName']
        allAdapters = self.mmc.getDeviceAdapterNames()
        if adapterName not in allAdapters:
            raise ValueError("Adapter name '%s' is not valid. Options are: %s" % (adapterName, allAdapters))
        deviceName = self._config['mmDeviceName']
        allDevices = self.mmc.getAvailableDevices(adapterName)
        if deviceName not in allDevices:
            raise ValueError("Device name '%s' is not valid for adapter '%s'. Options are: %s" % (deviceName, adapterName, allDevices))

        self.mmc.loadDevice(self.camName, adapterName, deviceName)
        self.mmc.initializeDevice(self.camName)

        self._readAllParams()
        # import pprint
        # pprint.pprint(dict(self.listParams()))

    def startCamera(self):
        with self.camLock:
            self.mmc.startContinuousSequenceAcquisition(0)

    def stopCamera(self):
        with self.camLock:
            self.mmc.stopSequenceAcquisition()
            self.acqBuffer = None

    def _acquireFrames(self, n=1):
        self.mmc.setCameraDevice(self.camName)
        self.mmc.startSequenceAcquisition(n, 0, True)
        frames = []
        for i in range(n):
            start = time.time()
            while self.mmc.getRemainingImageCount() == 0:
                time.sleep(0.005)
                if time.time() - start > 10.0:
                    raise Exception("Timed out waiting for camera frame.")
            frames.append(self.mmc.popNextImage().T[np.newaxis, ...])
        self.mmc.stopSequenceAcquisition()
        return np.concatenate(frames, axis=0)

    def newFrames(self):
        """Return a list of all frames acquired since the last call to newFrames."""

        with self.camLock:
            nFrames = self.mmc.getRemainingImageCount()
            if nFrames == 0:
                return []

        now = ptime.time()
        if self.lastFrameTime is None:
            self.lastFrameTime = now

        dt = (now - self.lastFrameTime) / nFrames
        frames = []
        with self.camLock:
            for i in range(nFrames):
                frame = {}
                frame['time'] = self.lastFrameTime + (dt * (i+1))
                frame['id'] = self.frameId
                frame['data'] = self.mmc.popNextImage().T
                frames.append(frame)
                self.frameId += 1

        self.lastFrame = frame
        self.lastFrameTime = now
        return frames

    def quit(self):
        self.mmc.stopSequenceAcquisition()
        self.mmc.unloadDevice(self.camName)

    def _readAllParams(self):
        # these are parameters expected for all cameras
        defaultParams = ['exposure', 'binningX', 'binningY', 'regionX', 'regionY', 'regionW', 'regionH', 'triggerMode', 'bitDepth']

        with self.camLock:
            params = OrderedDict([(n, None) for n in defaultParams])

            properties = self.mmc.getDevicePropertyNames(self.camName)
            for prop in properties:
                vals = self.mmc.getAllowedPropertyValues(self.camName, prop)
                if vals == ():
                    if self.mmc.hasPropertyLimits(self.camName, prop):
                        vals = (
                            self.mmc.getPropertyLowerLimit(self.camName, prop),
                            self.mmc.getPropertyUpperLimit(self.camName, prop),
                        )
                else:
                    vals = list(vals)
                readonly = self.mmc.isPropertyReadOnly(self.camName, prop)

                # translate standard properties to the names / formats that we expect
                if prop == 'Exposure':
                    prop = 'exposure'
                    # convert ms to s
                    vals = tuple([v * 1e-3 for v in vals])
                elif prop == 'Binning':
                    vals = [v.split('x') for v in vals]
                    params['binningX'] = ([int(v[0 % len(v)]) for v in vals], not readonly, True, [])
                    params['binningY'] = ([int(v[1 % len(v)]) for v in vals], not readonly, True, [])
                    #params['binningX'] = ([int(v[0]) for v in vals], not readonly, True, [])
                    #params['binningY'] = ([int(v[1]) for v in vals], not readonly, True, [])
                    continue
                elif prop == 'Trigger':
                    prop = 'triggerMode'
                    vals = [{'NORMAL': 'Normal', 'START': 'TriggerStart'}[v] for v in vals]
                elif prop == 'PixelType':
                    prop = 'bitDepth'
                    vals = [int(bd.rstrip('bit')) for bd in vals]

                params[prop] = (vals, not readonly, True, [])

            # Reset ROI to full frame so we know the native resolution
            self.mmc.setCameraDevice(self.camName)
            self.mmc.setProperty(self.camName, 'Binning', '1')
            #self.mmc.setProperty(self.camName, 'Binning', '1x1')
            self.mmc.clearROI()
            rgn = self.mmc.getROI(self.camName)
            self._sensorSize = rgn[2:]

            params.update({
                'regionX': [(0, rgn[2]-1, 1), True, True, []],
                'regionY': [(0, rgn[3]-1, 1), True, True, []],
                'regionW': [(1, rgn[2], 1), True, True, []],
                'regionH': [(1, rgn[3], 1), True, True, []],
            })

        self._allParams = params

    def listParams(self, params=None):
        """List properties of specified parameters, or of all parameters if None"""
        if params is None:
            return self._allParams.copy()
        if isinstance(params, basestring):
            return self._allParams[params]
        return dict([(p, self._allParams[p]) for p in params])

    def setParams(self, params, autoRestart=True, autoCorrect=True):
        # umanager will refuse to set params while camera is running,
        # so autoRestart doesn't make sense in this context
        if self.isRunning():
            restart = True
            self.stop()
        else:
            restart = False

        newVals = {}
        for k,v in params.items():
            self._setParam(k, v, autoCorrect=autoCorrect)
            if k == 'binning':
                newVals['binningX'], newVals['binningY'] = self.getParam(k)
            elif k == 'region':
                newVals['regionX'], newVals['regionY'], newVals['regionW'], newVals['regionH'] = self.getParam(k)
            else:
                newVals[k] = self.getParam(k)
        self.sigParamsChanged.emit(newVals)

        if restart:
            self.start()

        needRestart = False
        return (newVals, needRestart)

    def setParam(self, param, value, autoCorrect=True, autoRestart=True):
        return self.setParams({param: value}, autoCorrect=autoCorrect, autoRestart=autoRestart)

    def _setParam(self, param, value, autoCorrect=True):
        if param == 'region':
            value = (value[0], value[1], value[2], value[3])
            self.mmc.setCameraDevice(self.camName)
            self.mmc.setROI(*value)
            return

        if param.startswith('region'):
            rgn = list(self.mmc.getROI(self.camName))
            if param[-1] == 'X':
                rgn[0] = value
            elif param[-1] == 'Y':
                rgn[1] = value
            elif param[-1] == 'W':
                rgn[2] = value
            elif param[-1] == 'H':
                rgn[3] = value
            self.mmc.setCameraDevice(self.camName)
            self.mmc.setROI(*rgn)
            return

        if param.startswith('binning'):
            if param == 'binningX':
                y = self.getParam('binningY')
                value = (value, y)
                param = 'binning'
            elif param == 'binningY':
                x = self.getParam('binningX')
                value = (x, value)
                param = 'binning'
            value = '%dx%d' % value
            param = 'Binning'

        elif param == 'exposure':
            # s to ms
            value = value * 1e3
            param = 'Exposure'

        elif param == 'triggerMode':
            value = {'Normal': 'NORMAL', 'TriggerStart': 'START'}[value]
            param = 'Trigger'

        with self.camLock:
            self.mmc.setProperty(self.camName, str(param), str(value))

    def getParams(self, params=None):
        if params is None:
            params = self.listParams().keys()
        return dict([(p, self.getParam(p)) for p in params])

    def getParam(self, param):
        if param == 'sensorSize':
            return self._sensorSize
        elif param.startswith('region'):
            rgn = self.mmc.getROI(self.camName)
            if param == 'region':
                return rgn
            i = ['regionX', 'regionY', 'regionW', 'regionH'].index(param)
            return rgn[i]

        paramTrans = {
            'exposure': 'Exposure',
            'binning': 'Binning',
            'binningX': 'Binning',
            'binningY': 'Binning',
            'triggerMode': 'Trigger',
            'bitDepth': 'PixelType',
        }.get(param, param)
        with self.camLock:
            val = self.mmc.getProperty(self.camName, str(paramTrans))

        # coerce to int or float if possible
        try:
            val = int(val)
        except ValueError:
            try:
                val = float(val)
            except ValueError:
                pass

        if param == 'binningY':
            return int(val.split('x')[1])
        elif param == 'binningX':
            return int(val.split('x')[0])
        elif param == 'binning':
            return tuple([int(b) for b in val.split('x')])
        elif param == 'exposure':
            # ms to s
            val = val * 1e-3
        elif param == 'bitDepth':
            return int(val.rstrip('bit'))

        return val

<|endoftext|>


<commit_before>import logging
import functools
import opbeat
import sys

import pyramid.tweens

from opbeat.instrumentation import control

from pyramid import events
from pyramid import httpexceptions

from opbeat_pyramid import tweens

NO_DEFAULT_PROVIDED = {}
DEFAULT_UNKNOWN_ROUTE_TEXT = 'Unknown Route'
TRUTHY_VALUES = {True, 'true', 'yes', 'on'}

DEFAULT_UNSAFE_SETTINGS_TERMS = 'token,password,passphrase,secret,private,key'
OPBEAT_SETTING_PREFIX = 'opbeat.'

control.instrument()

logger = logging.getLogger(__name__)

def get_opbeat_setting(request, name, default=NO_DEFAULT_PROVIDED):
    setting_name = OPBEAT_SETTING_PREFIX + name
    result = request.registry.settings.get(setting_name, default)

    if result is NO_DEFAULT_PROVIDED:
        raise ValueError('Setting ' + setting_name + ' is required.')

    return result

def get_opbeat_client_cache(request):
    if not hasattr(request.registry, '_opbeat_clients'):
        request.registry._opbeat_clients = {}

    return request.registry._opbeat_clients

def create_opbeat_client(request, app_id):
    secret_token = get_opbeat_setting(request, 'secret_token')
    organization_id = get_opbeat_setting(request, 'organization_id')

    return opbeat.Client(
        secret_token=secret_token,
        organization_id=organization_id,
        app_id=app_id,
    )

def opbeat_client_factory(request):
    clients = get_opbeat_client_cache(request)

    app_id = get_opbeat_setting(request, 'app_id')
    client = clients.get(app_id)

    if client:
        return client

    clients[app_id] = create_opbeat_client(request, app_id)

    return clients[app_id]

def setting_is_enabled(request, setting_name):
    setting = request.registry.settings.get(setting_name, False)

    if setting is True:
        return True

    return setting.lower() in TRUTHY_VALUES

def is_opbeat_enabled(request):
    'opbeat.enabled'

def get_request_module_name(request):
    return get_opbeat_setting(request, 'module_name', default='UNKNOWN_MODULE')

def get_unsafe_settings_terms(request):
    private_terms = get_opbeat_setting(
        request,
        'unsafe_setting_terms',
        default=None,
    )

    if private_terms is None:
        return DEFAULT_UNSAFE_SETTINGS_TERMS

    return set(private_terms.split(','))

def get_safe_settings(request):
    unsafe_terms = get_unsafe_settings_terms(request)
    result = {}

    for key in request.registry.settings.keys():
        for term in unsafe_terms:
            if term in key:
                continue

        result[key] = request.registry.settings[key]

    return result

def should_ignore_exception(request, exc):
    if is_http_exception(exc):
        return False

    return get_opbeat_setting(request, 'ignore_http_exceptions', default=False)

def capture_exception(request, exc_info, extra):
    client = opbeat_client_factory(request)

    data = {
        'http': {
            'url': get_full_request_url(request),
            'method': request.method,
            'query_string': request.query_string,
        }
    }

    try:
        return client.capture_exception(exc_info, data=data, extra=extra)

    except Exception as e:
        # NOTE: This should not be allowed until we know which exception we are
        # looking for here.
        pass

def get_full_request_url(request):
    """ Get the full URL for a given request. """

    scheme = request.scheme + '://'
    host = request.host + ':' + request.host_port
    path = request.path
    return scheme + host + path

def handle_exception(request, exc_info):
    if should_ignore_exception(request, exc_info):
        return

    details = get_safe_settings(request)

    details.update({
        'client_ip_address': request.client_addr,
        'logging_successful': 'true',
        'url': request.url,
        'user_agent': request.user_agent,
    })

    logger.error('An error occured. Sending to opbeat.', exc_info=exc_info)
    return capture_exception(request, exc_info, details)

def get_exception_for_request(request):
    exc_info = getattr(request, 'exc_info', None)

    if exc_info is not None:
        return exc_info

    return sys.exc_info()

def opbeat_tween(handler, registry, request):
    try:
        response = handler(request)
    except Exception as exc:
        handle_exception(request, sys.exc_info())
        raise

    exc_info = get_exception_for_request(request)

    if exc_info is not None:
        handle_exception(request, exc_info)

    return response

@tweens.tween_config(over=[
    pyramid.tweens.EXCVIEW,
    'pyramid_tm.tm_tween_factory',
])
def opbeat_tween_factory(handler, registry):
    return functools.partial(opbeat_tween, handler, registry)

def is_http_exception(exc_info):
    if exc_info and exc_info[1]:
        exc_info = exc_info[1]

    return isinstance(exc_info, httpexceptions.HTTPException)

def get_status_code(request):
    # Handles an edge-case where `request.response` isn't the actual response.
    # This can occur whenever a view needs to create a new response instead of
    # using the one attached to the request.
    if is_http_exception(request.exc_info):
        return request.exc_info[1].code

    return request.response.status_code

def get_route_name(request):
    if request.view_name:
        return request.view_name

    elif request.matched_route and request.matched_route.name:
        module_name = get_request_module_name(request)
        return module_name + '.' + request.matched_route.name

    return get_opbeat_setting(
        request,
        'unknown_route_name',
        DEFAULT_UNKNOWN_ROUTE_TEXT,
    )

def on_request_finished(request):
    if not is_opbeat_enabled(request):
        return

    client = opbeat_client_factory(request)
    client.end_transaction(get_route_name(request), get_status_code(request))

@events.subscriber(events.NewRequest)
def on_request_begin(event):
    request = event.request

    if not is_opbeat_enabled(request):
        return

    client = opbeat_client_factory(request)
    client.begin_transaction(get_request_module_name(request))
    request.add_finished_callback(on_request_finished)
<commit_msg>Allow storing settings in environment variables.<commit_after>import logging
import functools
import opbeat
import os
import sys

import pyramid.tweens

from opbeat.instrumentation import control

from pyramid import events
from pyramid import httpexceptions

from opbeat_pyramid import tweens

NO_DEFAULT_PROVIDED = {}
DEFAULT_UNKNOWN_ROUTE_TEXT = 'Unknown Route'
TRUTHY_VALUES = {True, 'true', 'yes', 'on'}

DEFAULT_UNSAFE_SETTINGS_TERMS = 'token,password,passphrase,secret,private,key'
OPBEAT_SETTING_PREFIX = 'opbeat.'

control.instrument()

logger = logging.getLogger(__name__)

def get_opbeat_setting(request, name, default=NO_DEFAULT_PROVIDED):
    setting_name = OPBEAT_SETTING_PREFIX + name

    environment_override = os.environ.get(setting_name.replace('.', '_').upper())
    if environment_override:
        return environment_override

    result = request.registry.settings.get(setting_name, default)

    if result is NO_DEFAULT_PROVIDED:
        raise ValueError('Setting ' + setting_name + ' is required.')

    return result

def get_opbeat_client_cache(request):
    if not hasattr(request.registry, '_opbeat_clients'):
        request.registry._opbeat_clients = {}

    return request.registry._opbeat_clients

def create_opbeat_client(request, app_id):
    secret_token = get_opbeat_setting(request, 'secret_token')
    organization_id = get_opbeat_setting(request, 'organization_id')

    return opbeat.Client(
        secret_token=secret_token,
        organization_id=organization_id,
        app_id=app_id,
    )

def opbeat_client_factory(request):
    clients = get_opbeat_client_cache(request)

    app_id = get_opbeat_setting(request, 'app_id')
    client = clients.get(app_id)

    if client:
        return client

    clients[app_id] = create_opbeat_client(request, app_id)

    return clients[app_id]

def setting_is_enabled(request, setting_name):
    setting = request.registry.settings.get(setting_name, False)

    if setting is True:
        return True

    return setting.lower() in TRUTHY_VALUES

def is_opbeat_enabled(request):
    'opbeat.enabled'

def get_request_module_name(request):
    return get_opbeat_setting(request, 'module_name', default='UNKNOWN_MODULE')

def get_unsafe_settings_terms(request):
    private_terms = get_opbeat_setting(
        request,
        'unsafe_setting_terms',
        default=None,
    )

    if private_terms is None:
        return DEFAULT_UNSAFE_SETTINGS_TERMS

    return set(private_terms.split(','))

def get_safe_settings(request):
    unsafe_terms = get_unsafe_settings_terms(request)
    result = {}

    for key in request.registry.settings.keys():
        for term in unsafe_terms:
            if term in key:
                continue

        result[key] = request.registry.settings[key]

    return result

def should_ignore_exception(request, exc):
    if is_http_exception(exc):
        return False

    return get_opbeat_setting(request, 'ignore_http_exceptions', default=False)

def capture_exception(request, exc_info, extra):
    client = opbeat_client_factory(request)

    data = {
        'http': {
            'url': get_full_request_url(request),
            'method': request.method,
            'query_string': request.query_string,
        }
    }

    try:
        return client.capture_exception(exc_info, data=data, extra=extra)

    except Exception as e:
        # NOTE: This should not be allowed until we know which exception we are
        # looking for here.
        pass

def get_full_request_url(request):
    """ Get the full URL for a given request. """

    scheme = request.scheme + '://'
    host = request.host + ':' + request.host_port
    path = request.path
    return scheme + host + path

def handle_exception(request, exc_info):
    if should_ignore_exception(request, exc_info):
        return

    details = get_safe_settings(request)

    details.update({
        'client_ip_address': request.client_addr,
        'logging_successful': 'true',
        'url': request.url,
        'user_agent': request.user_agent,
    })

    logger.error('An error occured. Sending to opbeat.', exc_info=exc_info)
    return capture_exception(request, exc_info, details)

def get_exception_for_request(request):
    exc_info = getattr(request, 'exc_info', None)

    if exc_info is not None:
        return exc_info

    return sys.exc_info()

def opbeat_tween(handler, registry, request):
    try:
        response = handler(request)
    except Exception as exc:
        handle_exception(request, sys.exc_info())
        raise

    exc_info = get_exception_for_request(request)

    if exc_info is not None:
        handle_exception(request, exc_info)

    return response

@tweens.tween_config(over=[
    pyramid.tweens.EXCVIEW,
    'pyramid_tm.tm_tween_factory',
])
def opbeat_tween_factory(handler, registry):
    return functools.partial(opbeat_tween, handler, registry)

def is_http_exception(exc_info):
    if exc_info and exc_info[1]:
        exc_info = exc_info[1]

    return isinstance(exc_info, httpexceptions.HTTPException)

def get_status_code(request):
    # Handles an edge-case where `request.response` isn't the actual response.
    # This can occur whenever a view needs to create a new response instead of
    # using the one attached to the request.
    if is_http_exception(request.exc_info):
        return request.exc_info[1].code

    return request.response.status_code

def get_route_name(request):
    if request.view_name:
        return request.view_name

    elif request.matched_route and request.matched_route.name:
        module_name = get_request_module_name(request)
        return module_name + '.' + request.matched_route.name

    return get_opbeat_setting(
        request,
        'unknown_route_name',
        DEFAULT_UNKNOWN_ROUTE_TEXT,
    )

def on_request_finished(request):
    if not is_opbeat_enabled(request):
        return

    client = opbeat_client_factory(request)
    client.end_transaction(get_route_name(request), get_status_code(request))

@events.subscriber(events.NewRequest)
def on_request_begin(event):
    request = event.request

    if not is_opbeat_enabled(request):
        return

    client = opbeat_client_factory(request)
    client.begin_transaction(get_request_module_name(request))
    request.add_finished_callback(on_request_finished)
<|endoftext|>


<commit_before>CLICK_CONTEXT_SETTINGS = {
    'max_content_width': 100,
    'help_option_names': ['-h', '--help'],
}
<commit_msg>Add printable_dumpling shared helper function<commit_after>import json

import pygments
import pygments.lexers
import pygments.formatters

CLICK_CONTEXT_SETTINGS = {
    'max_content_width': 100,
    'help_option_names': ['-h', '--help'],
}

def printable_dumpling(contents, colorize=True):
    """
    Converts dumpling contents to a pretty-printed and optionally colorized
    output string.

    :param contents: Dumpling contents to generate printable string from.
    :param colorize: Whether to colorize the dumpling string.
    :return: Pretty-printed string representing the dumpling contents.
    """
    contents_printable = json.dumps(
        contents,
        sort_keys=True,
        indent=4,
        separators=(',', ': ')
    )

    if colorize:
        contents_printable = pygments.highlight(
            contents_printable,
            pygments.lexers.JsonLexer(),
            pygments.formatters.TerminalFormatter(),
        ).rstrip()

    return contents_printable
<|endoftext|>