<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import ujson
import requests

import pyslicer.exceptions as exceptions

from pyslicer.core.handler_response import SDHandlerResponse
from pyslicer.core.requester import Requester

class SlicingDiceAPI(object):
    """A python interface to make requests in Slicing Dice API"""

    BASE_URL = os.environ.get(
        'SD_API_ADDRESS',
        'https://api.slicingdice.com/v1')

    def __init__(
        self, master_key=None, write_key=None, read_key=None,
            custom_key=None, use_ssl=True, timeout=60):
        """Instantiate a new SlicerDicer object.

        Keyword arguments:
        key(string or SlicerKey) -- Key(s) to access API
        use_ssl(bool) -- Define if the request uses verification SSL for
            HTTPS requests. Defaults False.(Optional)
        timeout(int) -- Define timeout to request,
            defaults 30 secs(Optional).
        """
        self.keys = self._organize_keys(
            master_key, custom_key, read_key, write_key)
        self._api_key = self._get_key()[0]
        self._requester = Requester(use_ssl, timeout)
        self.__status_code = None
        self.__headers = None

    @property
    def status_code(self):
        return self.__status_code

    @property
    def headers(self):
        return self.__headers

    @staticmethod
    def _organize_keys(master_key, custom_key, read_key, write_key):
        return {
            "master_key": master_key,
            "custom_key": custom_key,
            "read_key": read_key,
            "write_key": write_key
        }

    def _get_key(self):
        if self.keys["master_key"] is not None:
            return [self.keys["master_key"], 2]
        elif self.keys["custom_key"] is not None:
            return [self.keys["custom_key"], 2]
        elif self.keys["write_key"] is not None:
            return [self.keys["write_key"], 1]
        elif self.keys["read_key"] is not None:
            return [self.keys["read_key"], 0]
        raise exceptions.InvalidSlicingDiceKeysException("You need put a key.")

    def _check_key(self, key_level):
        """Select automatically a key to make the request in Slicing Dice

        Keyword arguments:
        key_level(int) -- Define the key level needed
        """
        current_key_level = self._get_key()
        if current_key_level[1] == 2:
            return current_key_level[0]
        if current_key_level[1] != key_level:
            raise exceptions.InvalidSlicingDiceKeysException(
                "This key is not allowed to perform this operation.")
        return current_key_level[0]

    def _make_request(self, url, req_type, key_level, json_data=None):
        """Returns a object request result

        Keyword arguments:
        url(string) -- the url to make a request
        req_type(string) -- the request type (POST, PUT, DELETE or GET)
        key_level(int) -- Define the key level needed
        json_data(json) -- The json to use on request (default None)
        """
        self._check_key(key_level)
        headers = {'Content-Type': 'application/json',
                   'Authorization': self._api_key}

        req = None

        if req_type == "post":
            req = self._requester.post(
                url,
                data=json_data,
                headers=headers)
        elif req_type == "get":
            req = self._requester.get(
                url,
                headers=headers)

        elif req_type == "delete":
            req = self._requester.get(
                url,
                headers=headers)

        elif req_type == "put":
            req = self._requester.put(
                url,
                data=json_data,
                headers=headers)

        return self._handler_request(req)

    def _handler_request(self, req):
        """Handler request response

        Keyword arguments:
        req -- the request object
        """
        if req is None:
            raise exceptions.SlicingDiceException("Bad request.")

        try:
            result = ujson.loads(req.text)
        except ValueError:
            raise exceptions.InternalException

        sd_response = SDHandlerResponse(
            result=result,
            status_code=req.status_code,
            headers=req.headers)

        if sd_response.request_successful():
            if self._check_request(req):
                self._set_properties_values(sd_response)
                return sd_response.result

    @staticmethod
    def _check_request(request):
        """Check if the request was successful

        Keyword arguments:
        request -- A object request result
        """
        if request.status_code is not requests.codes.ok:
            raise exceptions.SlicingDiceHTTPError(
                "HTTP status code: {}".format(request.status_code))
        return True

    def _set_properties_values(self, sd_response):
        """Set current status code and header request response in objects

        Keyword arguments:
        sd_response -- A request object
        """
        self.__status_code = int(sd_response.status_code)
        self.__headers = dict(sd_response.headers)
<commit_msg>Fix InternalException on api<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import ujson
import requests

import pyslicer.exceptions as exceptions

from pyslicer.core.handler_response import SDHandlerResponse
from pyslicer.core.requester import Requester

class SlicingDiceAPI(object):
    """A python interface to make requests in Slicing Dice API"""

    BASE_URL = os.environ.get(
        'SD_API_ADDRESS',
        'https://api.slicingdice.com/v1')

    def __init__(
        self, master_key=None, write_key=None, read_key=None,
            custom_key=None, use_ssl=True, timeout=60):
        """Instantiate a new SlicerDicer object.

        Keyword arguments:
        key(string or SlicerKey) -- Key(s) to access API
        use_ssl(bool) -- Define if the request uses verification SSL for
            HTTPS requests. Defaults False.(Optional)
        timeout(int) -- Define timeout to request,
            defaults 30 secs(Optional).
        """
        self.keys = self._organize_keys(
            master_key, custom_key, read_key, write_key)
        self._api_key = self._get_key()[0]
        self._requester = Requester(use_ssl, timeout)
        self.__status_code = None
        self.__headers = None

    @property
    def status_code(self):
        return self.__status_code

    @property
    def headers(self):
        return self.__headers

    @staticmethod
    def _organize_keys(master_key, custom_key, read_key, write_key):
        return {
            "master_key": master_key,
            "custom_key": custom_key,
            "read_key": read_key,
            "write_key": write_key
        }

    def _get_key(self):
        if self.keys["master_key"] is not None:
            return [self.keys["master_key"], 2]
        elif self.keys["custom_key"] is not None:
            return [self.keys["custom_key"], 2]
        elif self.keys["write_key"] is not None:
            return [self.keys["write_key"], 1]
        elif self.keys["read_key"] is not None:
            return [self.keys["read_key"], 0]
        raise exceptions.InvalidSlicingDiceKeysException("You need put a key.")

    def _check_key(self, key_level):
        """Select automatically a key to make the request in Slicing Dice

        Keyword arguments:
        key_level(int) -- Define the key level needed
        """
        current_key_level = self._get_key()
        if current_key_level[1] == 2:
            return current_key_level[0]
        if current_key_level[1] != key_level:
            raise exceptions.InvalidSlicingDiceKeysException(
                "This key is not allowed to perform this operation.")
        return current_key_level[0]

    def _make_request(self, url, req_type, key_level, json_data=None):
        """Returns a object request result

        Keyword arguments:
        url(string) -- the url to make a request
        req_type(string) -- the request type (POST, PUT, DELETE or GET)
        key_level(int) -- Define the key level needed
        json_data(json) -- The json to use on request (default None)
        """
        self._check_key(key_level)
        headers = {'Content-Type': 'application/json',
                   'Authorization': self._api_key}

        req = None

        if req_type == "post":
            req = self._requester.post(
                url,
                data=json_data,
                headers=headers)
        elif req_type == "get":
            req = self._requester.get(
                url,
                headers=headers)

        elif req_type == "delete":
            req = self._requester.get(
                url,
                headers=headers)

        elif req_type == "put":
            req = self._requester.put(
                url,
                data=json_data,
                headers=headers)

        return self._handler_request(req)

    def _handler_request(self, req):
        """Handler request response

        Keyword arguments:
        req -- the request object
        """
        if req is None:
            raise exceptions.SlicingDiceException("Bad request.")

        try:
            result = ujson.loads(req.text)
        except ValueError as e:
            raise exceptions.InternalException("Error while trying to load Json: %s" % e.message)

        sd_response = SDHandlerResponse(
            result=result,
            status_code=req.status_code,
            headers=req.headers)

        if sd_response.request_successful():
            if self._check_request(req):
                self._set_properties_values(sd_response)
                return sd_response.result

    @staticmethod
    def _check_request(request):
        """Check if the request was successful

        Keyword arguments:
        request -- A object request result
        """
        if request.status_code is not requests.codes.ok:
            raise exceptions.SlicingDiceHTTPError(
                "HTTP status code: {}".format(request.status_code))
        return True

    def _set_properties_values(self, sd_response):
        """Set current status code and header request response in objects

        Keyword arguments:
        sd_response -- A request object
        """
        self.__status_code = int(sd_response.status_code)
        self.__headers = dict(sd_response.headers)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
pytest_instafail
~~~~~~~~~~~~~~~~

py.test plugin to show failures instantly.

:copyright: see LICENSE for details
:license: BSD, see LICENSE for more details.
"""
import pytest
import pytest, py
import sys
import random
import os
import json
import time
import re
from _pytest.terminal import TerminalReporter

class TerminalColors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    GRAY = '\033[1;30m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'

    def disable(self):
        self.HEADER = ''
        self.OKBLUE = ''
        self.OKGREEN = ''
        self.WARNING = ''
        self.FAIL = ''
        self.ENDC = ''
        self.GRAY = ''

bcolors = TerminalColors()

class EtaLogger:
    def __init__(self):
        self.settings_path = os.path.expanduser("~/.pytest-sugar.json")
        try:
            self.collected = json.loads(
                open(self.settings_path).read()
            )
        except IOError:
            print "FUCK"
            self.collected = {}
        #print "-----------------"
        #print self.collected

    def collect(self, time, filename=None, hash=None):
        self.collected[filename] = time
        self.save()

    def estimate(self, paths):
        estimated_time = 0
        for path in paths:
            if path in self.collected:
                estimated_time += self.collected[path]
        return estimated_time

    def save_session(
        self,
        tests_duration,
        session_duration
    ):
        tests_sum = 0
        for path, time in tests_duration.items():
            self.collect(
                filename=path,
                time=time
            )
            tests_sum += time

        extra_time = session_duration - tests_sum
        self.collect(
            filename=os.getcwd(),
            time=extra_time
        )
        self.save()

    def save(self):
        #print "WRITING TO ",self.settings_path
        fp = open(self.settings_path, 'w')
        fp.write(json.dumps(self.collected))
        fp.close()

def flatten(l):
    for x in l:
        if isinstance(x, (list, tuple)):
            for y in flatten(x):
                yield y
        else:
            yield x

def pytest_collection_modifyitems(session, config, items):
    for i in items:
        config.pluginmanager.getplugin('terminalreporter').tests_count += 1

def pytest_addoption(parser):
    group = parser.getgroup("terminal reporting", "reporting", after="general")
    group._addoption(
        '--nosugar', action="store_false", dest="sugar", default=True,
        help=(
            "disable pytest-sugar"
        )
    )

def real_string_length(string):
    for color in [
        bcolors.HEADER,
        bcolors.OKBLUE,
        bcolors.OKGREEN,
        bcolors.GRAY,
        bcolors.WARNING,
        bcolors.FAIL,
        bcolors.ENDC
    ]:
        string = string.replace(color, '')
    return len(string)

@pytest.mark.trylast
def pytest_configure(config):
    if config.option.sugar:
        # Get the standard terminal reporter plugin and replace it with our
        standard_reporter = config.pluginmanager.getplugin('terminalreporter')
        sugar_reporter = InstafailingTerminalReporter(standard_reporter)
        config.pluginmanager.unregister(standard_reporter)
        config.pluginmanager.register(sugar_reporter, 'terminalreporter')

def pytest_report_teststatus(report):
    if report.passed:
        letter = bcolors.OKGREEN+u'✓'+bcolors.ENDC
    elif report.skipped:
        letter = bcolors.OKBLUE+u's'+bcolors.ENDC
    elif report.failed:
        letter = bcolors.FAIL+u'⨯'+bcolors.ENDC
        if report.when != "call":
            letter = bcolors.FAIL+u'ₓ'+bcolors.ENDC
    return report.outcome, letter, report.outcome.upper()

class InstafailingTerminalReporter(TerminalReporter):
    def __init__(self, reporter):
        #pytest_collectreport = self.pytest_collectreport
        TerminalReporter.__init__(self, reporter.config)
        self.writer = self._tw
        self.eta_logger = EtaLogger()
        self.paths_left = []
        self.tests_count = 0
        self.tests_taken = 0
        self.current_line = u''
        self.currentfspath2 = ''
        self.time_taken = {}
        self.reports = []
        self.unreported_errors = []

    def get_estimate(self):
        session_duration = py.std.time.time() - self._sessionstarttime

        estimate = 0
        estimate += self.eta_logger.estimate(self.paths_left)
        estimate += self.eta_logger.estimate([os.getcwd()])
        estimate -= session_duration
        return estimate

    def report_collect(self, final=False):
        pass

    def pytest_collectreport(self, report):
        TerminalReporter.pytest_collectreport(self, report)
        if report.location[0]:
            self.paths_left.append(
                os.path.join(os.getcwd(), report.location[0])
            )
        if report.failed:
            self.rewrite("")
            self.print_failure(report)

    def pytest_sessionstart(self, session):
        self._sessionstarttime = py.std.time.time()
        verinfo = ".".join(map(str, sys.version_info[:3]))
        self.write_line("Test session starts (%s, %s)" % (sys.platform, verinfo), bold=True)
        lines = self.config.hook.pytest_report_header(config=self.config)
        lines.reverse()
        for line in flatten(lines):
            self.write_line(line)

    def write_fspath_result(self, fspath, res):
        return

    def insert_progress(self):
        def get_estimate():
            #return str(round(self.get_estimate(),2))
            seconds = int(round(self.get_estimate()))
            minutes = seconds/60
            seconds = seconds%60
            return "%im %is" % (minutes, seconds)

        def get_progress_bar():
            blocks = [
                u'█',
                u'▉',
                u'▉',
                u'▊',
                u'▊',
                u'▋',
                u'▋',
                u'▌',
                u'▌',
                u'▍',
                u'▍',
                u'▎',
                u'▎',
                u'▏',
                u'▏'
            ]
            length = 10
            p = float(self.tests_taken) / self.tests_count
            floored = int(round(p * length))
            progressbar = u''
            progressbar += "%i%% " % round(p*100)
            progressbar += bcolors.OKGREEN
            progressbar += blocks[0] * floored
            progressbar += bcolors.ENDC
            progressbar += bcolors.GRAY + blocks[0] * (length - floored) + bcolors.ENDC
            return progressbar

        append_string = get_progress_bar()
        return self.append_string(append_string)

    def append_string(self, append_string=''):
        console_width = self._tw.fullwidth
        full_line = self.current_line + " " * (console_width  - real_string_length(self.current_line))
        full_line = full_line[0:-(len(append_string)-15)] + append_string
        return full_line

    def overwrite(self, line):
        sys.stdout.write("\r" + self.append_string(line))
        sys.stdout.flush()

    def pytest_runtest_logreport(self, report):
        self.reports.append(report)
        if report.outcome == 'failed':
            print ""
            self.print_failure(report)
            # Ignore other reports or it will cause duplicated letters
        if report.when == 'setup':
            self.setup_timer = time.time()
        if report.when == 'teardown':
            self.tests_taken += 1
            path = os.path.join(os.getcwd(), report.location[0])
            time_taken = time.time() - self.setup_timer
            if not path in self.time_taken:
                self.time_taken[path] = 0
            self.time_taken[path] += time_taken
        if report.when == 'call':
            if report.fspath != self.currentfspath2:
                self.currentfspath2 = report.fspath
                basename = os.path.basename(report.fspath)
                self.current_line = (
                    u"   " +
                    bcolors.GRAY +
                    report.fspath[0:-len(basename)] +
                    bcolors.ENDC +
                    report.fspath[-len(basename):] +
                    u" "
                )
                print ""

            rep = report
            res = pytest_report_teststatus(report=report)
            cat, letter, word = res
            self.current_line = self.current_line + letter

            self.overwrite(self.insert_progress())

            self.stats.setdefault(cat, []).append(rep)
            if not letter and not word:
                return
            if self.verbosity > 0:
                if isinstance(word, tuple):
                    word, markup = word
                else:
                    if rep.passed:
                        markup = {'green':True}
                    elif rep.failed:
                        markup = {'red':True}
                    elif rep.skipped:
                        markup = {'yellow':True}
                line = self._locationline(str(rep.fspath), *rep.location)
                if not hasattr(rep, 'node'):
                    self.write_ensure_prefix(line, word, **markup)
                else:
                    self.ensure_newline()
                    if hasattr(rep, 'node'):
                        self._tw.write("[%s] " % rep.node.gateway.id)
                    self._tw.write(word, **markup)
                    self._tw.write(" " + line)
                    self.currentfspath = -2

    def count(self, key):
        if self.stats.get(key):
            return len(self.stats.get(key))
        else:
            return 0

    def summary_stats(self):
        session_duration = py.std.time.time() - self._sessionstarttime

        self.eta_logger.save_session(self.time_taken, session_duration)

        print "\nResults (%.2fs):" % round(session_duration,2)
        if self.count('passed') > 0:
            self.write_line(
                "   %d passed" % self.count('passed') +
                bcolors.ENDC
            )

        if self.count('failed') > 0:
            self.write_line(
                bcolors.FAIL+
                "   %d failed" % self.count('failed') +
                bcolors.ENDC
            )
            for report in self.reports:
                if report.outcome == 'failed':
                    print "      - %s" % (
                        self._getcrashline(report)
                    )

        if self.count('skipped') > 0:
            self.write_line(
                bcolors.GRAY+
                "   %d skipped" % self.count('skipped') +
                bcolors.ENDC
            )

        if self.count('deselected') > 0:
            self.write_line(
                bcolors.GRAY+
                "   %d deselected" % self.count('deselected') +
                bcolors.ENDC
            )

    def summary_failures(self):
        # Prevent failure summary from being shown since we already
        # show the failure instantly after failure has occured.
        pass

    def summary_errors(self):
        # Prevent error summary from being shown since we already
        # show the error instantly after error has occured.
        pass

    def print_failure(self, report):
        if self.config.option.tbstyle != "no":
            if self.config.option.tbstyle == "line":
                line = self._getcrashline(report)
                self.write_line(line)
            else:
                msg = self._getfailureheadline(report)
                if not hasattr(report, 'when'):
                    msg = "ERROR collecting " + msg
                elif report.when == "setup":
                    msg = "ERROR at setup of " + msg
                elif report.when == "teardown":
                    msg = "ERROR at teardown of " + msg
                self.write_line('')
                self.write_sep(u"―", msg)
                self._outrep_summary(report)
<commit_msg>left in a print statement.<commit_after># -*- coding: utf-8 -*-
"""
pytest_instafail
~~~~~~~~~~~~~~~~

py.test plugin to show failures instantly.

:copyright: see LICENSE for details
:license: BSD, see LICENSE for more details.
"""
import pytest
import pytest, py
import sys
import random
import os
import json
import time
import re
from _pytest.terminal import TerminalReporter

class TerminalColors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    GRAY = '\033[1;30m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'

    def disable(self):
        self.HEADER = ''
        self.OKBLUE = ''
        self.OKGREEN = ''
        self.WARNING = ''
        self.FAIL = ''
        self.ENDC = ''
        self.GRAY = ''

bcolors = TerminalColors()

class EtaLogger:
    def __init__(self):
        self.settings_path = os.path.expanduser("~/.pytest-sugar.json")
        try:
            self.collected = json.loads(
                open(self.settings_path).read()
            )
        except IOError:
            self.collected = {}
        #print "-----------------"
        #print self.collected

    def collect(self, time, filename=None, hash=None):
        self.collected[filename] = time
        self.save()

    def estimate(self, paths):
        estimated_time = 0
        for path in paths:
            if path in self.collected:
                estimated_time += self.collected[path]
        return estimated_time

    def save_session(
        self,
        tests_duration,
        session_duration
    ):
        tests_sum = 0
        for path, time in tests_duration.items():
            self.collect(
                filename=path,
                time=time
            )
            tests_sum += time

        extra_time = session_duration - tests_sum
        self.collect(
            filename=os.getcwd(),
            time=extra_time
        )
        self.save()

    def save(self):
        #print "WRITING TO ",self.settings_path
        fp = open(self.settings_path, 'w')
        fp.write(json.dumps(self.collected))
        fp.close()

def flatten(l):
    for x in l:
        if isinstance(x, (list, tuple)):
            for y in flatten(x):
                yield y
        else:
            yield x

def pytest_collection_modifyitems(session, config, items):
    for i in items:
        config.pluginmanager.getplugin('terminalreporter').tests_count += 1

def pytest_addoption(parser):
    group = parser.getgroup("terminal reporting", "reporting", after="general")
    group._addoption(
        '--nosugar', action="store_false", dest="sugar", default=True,
        help=(
            "disable pytest-sugar"
        )
    )

def real_string_length(string):
    for color in [
        bcolors.HEADER,
        bcolors.OKBLUE,
        bcolors.OKGREEN,
        bcolors.GRAY,
        bcolors.WARNING,
        bcolors.FAIL,
        bcolors.ENDC
    ]:
        string = string.replace(color, '')
    return len(string)

@pytest.mark.trylast
def pytest_configure(config):
    if config.option.sugar:
        # Get the standard terminal reporter plugin and replace it with our
        standard_reporter = config.pluginmanager.getplugin('terminalreporter')
        sugar_reporter = InstafailingTerminalReporter(standard_reporter)
        config.pluginmanager.unregister(standard_reporter)
        config.pluginmanager.register(sugar_reporter, 'terminalreporter')

def pytest_report_teststatus(report):
    if report.passed:
        letter = bcolors.OKGREEN+u'✓'+bcolors.ENDC
    elif report.skipped:
        letter = bcolors.OKBLUE+u's'+bcolors.ENDC
    elif report.failed:
        letter = bcolors.FAIL+u'⨯'+bcolors.ENDC
        if report.when != "call":
            letter = bcolors.FAIL+u'ₓ'+bcolors.ENDC
    return report.outcome, letter, report.outcome.upper()

class InstafailingTerminalReporter(TerminalReporter):
    def __init__(self, reporter):
        #pytest_collectreport = self.pytest_collectreport
        TerminalReporter.__init__(self, reporter.config)
        self.writer = self._tw
        self.eta_logger = EtaLogger()
        self.paths_left = []
        self.tests_count = 0
        self.tests_taken = 0
        self.current_line = u''
        self.currentfspath2 = ''
        self.time_taken = {}
        self.reports = []
        self.unreported_errors = []

    def get_estimate(self):
        session_duration = py.std.time.time() - self._sessionstarttime

        estimate = 0
        estimate += self.eta_logger.estimate(self.paths_left)
        estimate += self.eta_logger.estimate([os.getcwd()])
        estimate -= session_duration
        return estimate

    def report_collect(self, final=False):
        pass

    def pytest_collectreport(self, report):
        TerminalReporter.pytest_collectreport(self, report)
        if report.location[0]:
            self.paths_left.append(
                os.path.join(os.getcwd(), report.location[0])
            )
        if report.failed:
            self.rewrite("")
            self.print_failure(report)

    def pytest_sessionstart(self, session):
        self._sessionstarttime = py.std.time.time()
        verinfo = ".".join(map(str, sys.version_info[:3]))
        self.write_line("Test session starts (%s, %s)" % (sys.platform, verinfo), bold=True)
        lines = self.config.hook.pytest_report_header(config=self.config)
        lines.reverse()
        for line in flatten(lines):
            self.write_line(line)

    def write_fspath_result(self, fspath, res):
        return

    def insert_progress(self):
        def get_estimate():
            #return str(round(self.get_estimate(),2))
            seconds = int(round(self.get_estimate()))
            minutes = seconds/60
            seconds = seconds%60
            return "%im %is" % (minutes, seconds)

        def get_progress_bar():
            blocks = [
                u'█',
                u'▉',
                u'▉',
                u'▊',
                u'▊',
                u'▋',
                u'▋',
                u'▌',
                u'▌',
                u'▍',
                u'▍',
                u'▎',
                u'▎',
                u'▏',
                u'▏'
            ]
            length = 10
            p = float(self.tests_taken) / self.tests_count
            floored = int(round(p * length))
            progressbar = u''
            progressbar += "%i%% " % round(p*100)
            progressbar += bcolors.OKGREEN
            progressbar += blocks[0] * floored
            progressbar += bcolors.ENDC
            progressbar += bcolors.GRAY + blocks[0] * (length - floored) + bcolors.ENDC
            return progressbar

        append_string = get_progress_bar()
        return self.append_string(append_string)

    def append_string(self, append_string=''):
        console_width = self._tw.fullwidth
        full_line = self.current_line + " " * (console_width  - real_string_length(self.current_line))
        full_line = full_line[0:-(len(append_string)-15)] + append_string
        return full_line

    def overwrite(self, line):
        sys.stdout.write("\r" + self.append_string(line))
        sys.stdout.flush()

    def pytest_runtest_logreport(self, report):
        self.reports.append(report)
        if report.outcome == 'failed':
            print ""
            self.print_failure(report)
            # Ignore other reports or it will cause duplicated letters
        if report.when == 'setup':
            self.setup_timer = time.time()
        if report.when == 'teardown':
            self.tests_taken += 1
            path = os.path.join(os.getcwd(), report.location[0])
            time_taken = time.time() - self.setup_timer
            if not path in self.time_taken:
                self.time_taken[path] = 0
            self.time_taken[path] += time_taken
        if report.when == 'call':
            if report.fspath != self.currentfspath2:
                self.currentfspath2 = report.fspath
                basename = os.path.basename(report.fspath)
                self.current_line = (
                    u"   " +
                    bcolors.GRAY +
                    report.fspath[0:-len(basename)] +
                    bcolors.ENDC +
                    report.fspath[-len(basename):] +
                    u" "
                )
                print ""

            rep = report
            res = pytest_report_teststatus(report=report)
            cat, letter, word = res
            self.current_line = self.current_line + letter

            self.overwrite(self.insert_progress())

            self.stats.setdefault(cat, []).append(rep)
            if not letter and not word:
                return
            if self.verbosity > 0:
                if isinstance(word, tuple):
                    word, markup = word
                else:
                    if rep.passed:
                        markup = {'green':True}
                    elif rep.failed:
                        markup = {'red':True}
                    elif rep.skipped:
                        markup = {'yellow':True}
                line = self._locationline(str(rep.fspath), *rep.location)
                if not hasattr(rep, 'node'):
                    self.write_ensure_prefix(line, word, **markup)
                else:
                    self.ensure_newline()
                    if hasattr(rep, 'node'):
                        self._tw.write("[%s] " % rep.node.gateway.id)
                    self._tw.write(word, **markup)
                    self._tw.write(" " + line)
                    self.currentfspath = -2

    def count(self, key):
        if self.stats.get(key):
            return len(self.stats.get(key))
        else:
            return 0

    def summary_stats(self):
        session_duration = py.std.time.time() - self._sessionstarttime

        self.eta_logger.save_session(self.time_taken, session_duration)

        print "\nResults (%.2fs):" % round(session_duration,2)
        if self.count('passed') > 0:
            self.write_line(
                "   %d passed" % self.count('passed') +
                bcolors.ENDC
            )

        if self.count('failed') > 0:
            self.write_line(
                bcolors.FAIL+
                "   %d failed" % self.count('failed') +
                bcolors.ENDC
            )
            for report in self.reports:
                if report.outcome == 'failed':
                    print "      - %s" % (
                        self._getcrashline(report)
                    )

        if self.count('skipped') > 0:
            self.write_line(
                bcolors.GRAY+
                "   %d skipped" % self.count('skipped') +
                bcolors.ENDC
            )

        if self.count('deselected') > 0:
            self.write_line(
                bcolors.GRAY+
                "   %d deselected" % self.count('deselected') +
                bcolors.ENDC
            )

    def summary_failures(self):
        # Prevent failure summary from being shown since we already
        # show the failure instantly after failure has occured.
        pass

    def summary_errors(self):
        # Prevent error summary from being shown since we already
        # show the error instantly after error has occured.
        pass

    def print_failure(self, report):
        if self.config.option.tbstyle != "no":
            if self.config.option.tbstyle == "line":
                line = self._getcrashline(report)
                self.write_line(line)
            else:
                msg = self._getfailureheadline(report)
                if not hasattr(report, 'when'):
                    msg = "ERROR collecting " + msg
                elif report.when == "setup":
                    msg = "ERROR at setup of " + msg
                elif report.when == "teardown":
                    msg = "ERROR at teardown of " + msg
                self.write_line('')
                self.write_sep(u"―", msg)
                self._outrep_summary(report)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
This example shows how to make and use a class. It mirrors my perl example
'moo.pl'
"""

class MyPerson(object):
    """Make empoyee person class with names, and title required."""

    def __init__(self, given_name, surname, title, salary=0):
        self.given_name =given_name
        self.surname    =surname
        self.title      =title
        self.salary     =salary

        if not isinstance( salary, ( int, long ) ):
            raise TypeError( "Error salary must be an integer" )

    def salary_pretty(self):
        """Return salary in pretty $000.00 format"""
        return '$'+ str(self.salary) +'.00'

person=MyPerson( given_name='neil', surname='watson', title='sysadmin')

print 'Name is '+ person.given_name +' '+ person.surname

# Python has no simple methods for making attributes read-only :(

print 'Title is '+ person.title
person.title='Manager'
print 'Title is now '+ person.title

person.salary=10000
print 'salary is', person.salary
print'Formatted salary is '+ person.salary_pretty()
<commit_msg>Python class example<commit_after>#!/usr/bin/env python

"""
This example shows how to make and use a class. It mirrors my perl example
'moo.pl'

BEWARE that read-only attributes are not possible or discouraged in python.
"""

class MyPerson(object):
    """Make empoyee person class with names, and title required."""

    def __init__(self, given_name, surname, title, salary=0):
        self.given_name =given_name
        self.surname    =surname
        self.title      =title
        self.salary     =salary

        if not isinstance( salary, ( int, long ) ):
            raise TypeError( "Error salary must be an integer" )

    def salary_pretty(self):
        """Return salary in pretty $000.00 format"""
        return '$'+ str(self.salary) +'.00'

person=MyPerson( given_name='neil', surname='watson', title='sysadmin')

print 'Name is '+ person.given_name +' '+ person.surname

# Python has no simple methods for making attributes read-only :(

print 'Title is '+ person.title
person.title='Manager'
print 'Title is now '+ person.title

person.salary=10000
print 'salary is', person.salary
print'Formatted salary is '+ person.salary_pretty()
<|endoftext|>


<commit_before>#!/usr/bin/env python2
(lambda _, __, ___, ____, _____, ______, _______, ________:
    getattr(
        __import__(True.__class__.__name__[_] + [].__class__.__name__[__]),
        ().__class__.__eq__.__class__.__name__[:__] +
        ().__iter__().__class__.__name__[_____:________]
    )(
        _, (lambda _, __, ___: _(_, __, ___))(
            lambda _, __, ___:
                chr(___ % __) + _(_, __, ___ // __) if ___ else
                (lambda: _).func_code.co_lnotab,
            _ << ________,
            (((_____ << ____) + _) << ((___ << _____) - ___)) + (((((___ << __)
            - _) << ___) + _) << ((_____ << ____) + (_ << _))) + (((_______ <<
            __) - _) << (((((_ << ___) + _)) << ___) + (_ << _))) + (((_______
            << ___) + _) << ((_ << ______) + _)) + (((_______ << ____) - _) <<
            ((_______ << ___))) + (((_ << ____) - _) << ((((___ << __) + _) <<
            __) - _)) - (_______ << ((((___ << __) - _) << __) + _)) + (_______
            << (((((_ << ___) + _)) << __))) - ((((((_ << ___) + _)) << __) +
            _) << ((((___ << __) + _) << _))) + (((_______ << __) - _) <<
            (((((_ << ___) + _)) << _))) + (((___ << ___) + _) << ((_____ <<
            _))) + (_____ << ______) + (_ << ___)
        )
    )
)(
    *(lambda _, __, ___: _(_, __, ___))(
        (lambda _, __, ___:
            [__(___[(lambda: _).func_code.co_nlocals])] +
            _(_, __, ___[(lambda _: _).func_code.co_nlocals:]) if ___ else []
        ),
        lambda _: _.func_code.co_argcount,
        (
            lambda _: _,
            lambda _, __: _,
            lambda _, __, ___: _,
            lambda _, __, ___, ____: _,
            lambda _, __, ___, ____, _____: _,
            lambda _, __, ___, ____, _____, ______: _,
            lambda _, __, ___, ____, _____, ______, _______: _,
            lambda _, __, ___, ____, _____, ______, _______, ________: _
        )
    )
)
<commit_msg>Fix that damn obfuscated python<commit_after>#!/usr/bin/env python2
print 'Hello, World!'
<|endoftext|>


<commit_before>#!/usr/bin/env python

print 'Python says: "Hi, how are you today?"'

# Function definition
def example():
    """ This is an example function. """
    pass  # Do nothing

def examine_artifact(artifact):
    from os import path
    return path.basename(artifact)

# File I/O
with open("virus.txt", "r") as infectedFile:
    virus = infectedFile.readlines()

# For loops over objects
for a in virus:
    print a

# Iterative for loop
for i in range(0, 5):
    print i**2

# List comprehension
huh = [i for i in range(0, 500, 100)]
print huh

# String slicing (aka magic)
string = "Hackers <3 Python"
print string[0]
print string[-1]
print string[:7]
print string[:4]
print string[8:10]
print string[3:14:3]

lol = "eminem"
print lol
print lol[::-1]

# You don't need a main. They are a good practice, however, to prevent globbing your global namespace
def main():

    # Function calls
    for artifact in virus:
        print examine_artifact(artifact)

    # Function pointers
    a_function = examine_artifact
    for artifact in virus:
        print a_function(artifact)

    # Docstrings
    print example.__doc__

    # Method inspection
    print example.__name__

    # Eval
    with open("python_virus.txt", "r") as seriousFile:
        temp = seriousFile.readlines()
        for t in temp:
            print eval(t)

if __name__ == '__main__':
    main()
<commit_msg>Update intro.py<commit_after>#!/usr/bin/env python

print 'Python says: "Hi, how are you today?"'

# Function definition
def example():
    """ This is an example function. """
    pass  # Do nothing, equivilent to { } in C/C++

# def           Tells python you're defining a function
# example       Name of the function
# :             Start of the statement (in this case, a function definition statement)
# ()            Arguments to the function (no arguments in this case)
# """ blah """  Multi-line comment. When placed inside the function (there is a tab or 4 spaces), it is the functions docstring
# pass          Expression that does nothing

def examine_artifact(artifact):
    from os import path  # From the OS library, import the path sub-module
    return path.basename(artifact)  # Function call to os.path.basename, with the argument artifact

# File I/O
with open("virus.txt", "r") as infectedFile:
    virus = infectedFile.readlines()  # Reads lines of text from the file, returning a list object with the lines

# with          Ensures the file object is closed when scope of the statement is left
# open          Opens the file "virus.txt" as read-only ("r")
# as            Assigns the result of the open to the variable infectedFile

# For loops over objects
for a in virus:
    print a

# Iterative for loop
for i in range(0, 5):
    print i**2

# List comprehension
a = [i for i in range(0, 500, 100)]  # Generates a list of numbers using the for loop
print a

# The previous code was rather pointless however, when you can do this...
b = range(0, 100, 10)
print b

# Adding two lists to create a new list
huh = a + b
print huh

# Print a copy of the list object that is sorted
print sorted(huh)

# Sort the list itself
huh.sort()
print huh

# String slicing (aka magic)
string = "Hackers <3 Python"
print string[0]
print string[-1]
print string[:7]
print string[:4]
print string[8:10]
print string[3:14:3]

# Printing a string backward
lol = "eminem"
print lol
print lol[::-1]

# Strings are lists of characters, so you can sort them!
print sorted(lol)

# You don't need a main. They are a good practice, however, to prevent overriding ("globbing") your global namespace
def main():

    # Function call on every object (in this case, a string) in a list
    for artifact in virus:
        print examine_artifact(artifact)

    # Function pointers
    a_function = examine_artifact  # Note the lack of ()
    for artifact in virus:
        print a_function(artifact)

    # Docstrings
    print example.__doc__
    print __doc__

    # Method inspection
    print example.__name__

    # Eval evaluates strings as Python expressions.
    # So, if you have a file with python code or data types, you can run it as if it were a part of your program.
    with open("python_virus.txt", "r") as seriousFile:
        temp = seriousFile.readlines()
        for t in temp:
            print eval(t)

# __name__ is the label for the currently executing file.
# If it is the file that we begin executing from, that name is __main__
# Otherwise, it would be intro
# This is useful for testing classes/functions defined in other files independantly from the rest of the code
if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from setuptools import setup, find_packages

setup(
    name='fusion-engine-client',
    version='v1.9.0',
    packages=find_packages(where='.'),
    install_requires=[
        'gpstime @ https://github.com/PointOneNav/gpstime/archive/f9e2ab58a8beeeafee992d87a0eafc50887ba849.zip#egg=gpstime',
        'numpy>=1.16.0',
        'construct>=2.10.0',
    ],
    extras_require={
        'analysis': [
            'argparse-formatter>=1.4',
            'plotly>=4.0.0',
            'pymap3d>=2.4.3',
        ],
    },
)
<commit_msg>Updated version number to 1.11.1.<commit_after>from setuptools import setup, find_packages

setup(
    name='fusion-engine-client',
    version='v1.11.1',
    packages=find_packages(where='.'),
    install_requires=[
        'gpstime @ https://github.com/PointOneNav/gpstime/archive/f9e2ab58a8beeeafee992d87a0eafc50887ba849.zip#egg=gpstime',
        'numpy>=1.16.0',
        'construct>=2.10.0',
    ],
    extras_require={
        'analysis': [
            'argparse-formatter>=1.4',
            'plotly>=4.0.0',
            'pymap3d>=2.4.3',
        ],
    },
)
<|endoftext|>


<commit_before>from django.apps import AppConfig

class PGPTablesConfig(AppConfig):
    name = 'pgp_tables'
<commit_msg>apps.verbose_name<commit_after>from django.apps import AppConfig

class PGPTablesConfig(AppConfig):
    name = 'pgp_tables'
    verbose_name = 'PGP Tables'
<|endoftext|>


<commit_before>#! /usr/bin/python
#
# See README for usage instructions.
import sys
import os
import subprocess

# We must use setuptools, not distutils, because we need to use the
# namespace_packages option for the "google" package.
try:
  from setuptools import setup, Extension
except ImportError:
  try:
    from ez_setup import use_setuptools
    use_setuptools()
    from setuptools import setup, Extension
  except ImportError:
    sys.stderr.write(
        "Could not import setuptools; make sure you have setuptools or "
        "ez_setup installed.\n")
    raise
from distutils.command.clean import clean as _clean
from distutils.command.build_py import build_py as _build_py
from distutils.spawn import find_executable

maintainer_email = "protobuf@googlegroups.com"

# Find the Protocol Compiler.
if os.path.exists("../src/protoc"):
  protoc = "../src/protoc"
elif os.path.exists("../src/protoc.exe"):
  protoc = "../src/protoc.exe"
elif os.path.exists("../vsprojects/Debug/protoc.exe"):
  protoc = "../vsprojects/Debug/protoc.exe"
elif os.path.exists("../vsprojects/Release/protoc.exe"):
  protoc = "../vsprojects/Release/protoc.exe"
else:
  protoc = find_executable("protoc")

def generate_proto(source):
  """Invokes the Protocol Compiler to generate a _pb2.py from the given
  .proto file.  Does nothing if the output already exists and is newer than
  the input."""

  output = source.replace(".proto", "_pb2.py").replace("../src/", "")

  if (not os.path.exists(output) or
      (os.path.exists(source) and
       os.path.getmtime(source) > os.path.getmtime(output))):
    print "Generating %s..." % output

    if not os.path.exists(source):
      sys.stderr.write("Can't find required file: %s\n" % source)
      sys.exit(-1)

    if protoc == None:
      sys.stderr.write(
          "protoc is not installed nor found in ../src.  Please compile it "
          "or install the binary package.\n")
      sys.exit(-1)

    protoc_command = [ protoc, "-I../src", "-I.", "--python_out=.", source ]
    if subprocess.call(protoc_command) != 0:
      sys.exit(-1)

def GenerateUnittestProtos():
  generate_proto("../src/google/protobuf/unittest.proto")
  generate_proto("../src/google/protobuf/unittest_custom_options.proto")
  generate_proto("../src/google/protobuf/unittest_import.proto")
  generate_proto("../src/google/protobuf/unittest_import_public.proto")
  generate_proto("../src/google/protobuf/unittest_mset.proto")
  generate_proto("../src/google/protobuf/unittest_no_generic_services.proto")
  generate_proto("google/protobuf/internal/test_bad_identifiers.proto")
  generate_proto("google/protobuf/internal/more_extensions.proto")
  generate_proto("google/protobuf/internal/more_extensions_dynamic.proto")
  generate_proto("google/protobuf/internal/more_messages.proto")
  generate_proto("google/protobuf/internal/factory_test1.proto")
  generate_proto("google/protobuf/internal/factory_test2.proto")

def MakeTestSuite():
  # This is apparently needed on some systems to make sure that the tests
  # work even if a previous version is already installed.
  if 'google' in sys.modules:
    del sys.modules['google']
  GenerateUnittestProtos()

  import unittest
  import google.protobuf.internal.generator_test     as generator_test
  import google.protobuf.internal.descriptor_test    as descriptor_test
  import google.protobuf.internal.reflection_test    as reflection_test
  import google.protobuf.internal.service_reflection_test \
    as service_reflection_test
  import google.protobuf.internal.text_format_test   as text_format_test
  import google.protobuf.internal.wire_format_test   as wire_format_test
  import google.protobuf.internal.unknown_fields_test as unknown_fields_test
  import google.protobuf.internal.descriptor_database_test \
      as descriptor_database_test
  import google.protobuf.internal.descriptor_pool_test as descriptor_pool_test
  import google.protobuf.internal.message_factory_test as message_factory_test
  import google.protobuf.internal.message_cpp_test as message_cpp_test
  import google.protobuf.internal.reflection_cpp_generated_test \
      as reflection_cpp_generated_test

  loader = unittest.defaultTestLoader
  suite = unittest.TestSuite()
  for test in [ generator_test,
                descriptor_test,
                reflection_test,
                service_reflection_test,
                text_format_test,
                wire_format_test ]:
    suite.addTest(loader.loadTestsFromModule(test))

  return suite

class clean(_clean):
  def run(self):
    # Delete generated files in the code tree.
    for (dirpath, dirnames, filenames) in os.walk("."):
      for filename in filenames:
        filepath = os.path.join(dirpath, filename)
        if filepath.endswith("_pb2.py") or filepath.endswith(".pyc") or \
          filepath.endswith(".so") or filepath.endswith(".o") or \
          filepath.endswith('google/protobuf/compiler/__init__.py'):
          os.remove(filepath)
    # _clean is an old-style class, so super() doesn't work.
    _clean.run(self)

class build_py(_build_py):
  def run(self):
    # Generate necessary .proto file if it doesn't exist.
    generate_proto("../src/google/protobuf/descriptor.proto")
    generate_proto("../src/google/protobuf/compiler/plugin.proto")

    GenerateUnittestProtos()
    # Make sure google.protobuf.compiler is a valid package.
    open('google/protobuf/compiler/__init__.py', 'a').close()
    # _build_py is an old-style class, so super() doesn't work.
    _build_py.run(self)

if __name__ == '__main__':
  ext_module_list = []

  # C++ implementation extension
  if os.getenv("PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION", "python") == "cpp":
    print "Using EXPERIMENTAL C++ Implmenetation."
    ext_module_list.append(Extension(
        "google.protobuf.internal._net_proto2___python",
        [ "google/protobuf/pyext/python_descriptor.cc",
          "google/protobuf/pyext/python_protobuf.cc",
          "google/protobuf/pyext/python-proto2.cc" ],
        include_dirs = [ "." ],
        libraries = [ "protobuf" ]))

  setup(name = 'protobuf',
        version = '2.5.0-pre',
        packages = [ 'google' ],
        namespace_packages = [ 'google' ],
        test_suite = 'setup.MakeTestSuite',
        # Must list modules explicitly so that we don't install tests.
        py_modules = [
          'google.protobuf.internal.api_implementation',
          'google.protobuf.internal.containers',
          'google.protobuf.internal.cpp_message',
          'google.protobuf.internal.decoder',
          'google.protobuf.internal.encoder',
          'google.protobuf.internal.message_listener',
          'google.protobuf.internal.python_message',
          'google.protobuf.internal.type_checkers',
          'google.protobuf.internal.wire_format',
          'google.protobuf.descriptor',
          'google.protobuf.descriptor_pb2',
          'google.protobuf.compiler.plugin_pb2',
          'google.protobuf.message',
          'google.protobuf.descriptor_database',
          'google.protobuf.descriptor_pool',
          'google.protobuf.message_factory',
          'google.protobuf.reflection',
          'google.protobuf.service',
          'google.protobuf.service_reflection',
          'google.protobuf.text_format' ],
        cmdclass = { 'clean': clean, 'build_py': build_py },
        install_requires = ['setuptools'],
        ext_modules = ext_module_list,
        url = 'http://code.google.com/p/protobuf/',
        maintainer = maintainer_email,
        maintainer_email = 'protobuf@googlegroups.com',
        license = 'New BSD License',
        description = 'Protocol Buffers',
        long_description =
          "Protocol Buffers are Google's data interchange format.",
        )
<commit_msg>Find the protoc from environ first for python.<commit_after>#! /usr/bin/python
#
# See README for usage instructions.
import sys
import os
import subprocess

# We must use setuptools, not distutils, because we need to use the
# namespace_packages option for the "google" package.
try:
  from setuptools import setup, Extension
except ImportError:
  try:
    from ez_setup import use_setuptools
    use_setuptools()
    from setuptools import setup, Extension
  except ImportError:
    sys.stderr.write(
        "Could not import setuptools; make sure you have setuptools or "
        "ez_setup installed.\n")
    raise
from distutils.command.clean import clean as _clean
from distutils.command.build_py import build_py as _build_py
from distutils.spawn import find_executable

maintainer_email = "protobuf@googlegroups.com"

# Find the Protocol Compiler.
if 'PROTOC' in os.environ and os.path.exists(os.environ['PROTOC']):
  protoc = os.environ['PROTOC']
elif os.path.exists("../src/protoc"):
  protoc = "../src/protoc"
elif os.path.exists("../src/protoc.exe"):
  protoc = "../src/protoc.exe"
elif os.path.exists("../vsprojects/Debug/protoc.exe"):
  protoc = "../vsprojects/Debug/protoc.exe"
elif os.path.exists("../vsprojects/Release/protoc.exe"):
  protoc = "../vsprojects/Release/protoc.exe"
else:
  protoc = find_executable("protoc")

def generate_proto(source):
  """Invokes the Protocol Compiler to generate a _pb2.py from the given
  .proto file.  Does nothing if the output already exists and is newer than
  the input."""

  output = source.replace(".proto", "_pb2.py").replace("../src/", "")

  if (not os.path.exists(output) or
      (os.path.exists(source) and
       os.path.getmtime(source) > os.path.getmtime(output))):
    print "Generating %s..." % output

    if not os.path.exists(source):
      sys.stderr.write("Can't find required file: %s\n" % source)
      sys.exit(-1)

    if protoc == None:
      sys.stderr.write(
          "protoc is not installed nor found in ../src.  Please compile it "
          "or install the binary package.\n")
      sys.exit(-1)

    protoc_command = [ protoc, "-I../src", "-I.", "--python_out=.", source ]
    if subprocess.call(protoc_command) != 0:
      sys.exit(-1)

def GenerateUnittestProtos():
  generate_proto("../src/google/protobuf/unittest.proto")
  generate_proto("../src/google/protobuf/unittest_custom_options.proto")
  generate_proto("../src/google/protobuf/unittest_import.proto")
  generate_proto("../src/google/protobuf/unittest_import_public.proto")
  generate_proto("../src/google/protobuf/unittest_mset.proto")
  generate_proto("../src/google/protobuf/unittest_no_generic_services.proto")
  generate_proto("google/protobuf/internal/test_bad_identifiers.proto")
  generate_proto("google/protobuf/internal/more_extensions.proto")
  generate_proto("google/protobuf/internal/more_extensions_dynamic.proto")
  generate_proto("google/protobuf/internal/more_messages.proto")
  generate_proto("google/protobuf/internal/factory_test1.proto")
  generate_proto("google/protobuf/internal/factory_test2.proto")

def MakeTestSuite():
  # This is apparently needed on some systems to make sure that the tests
  # work even if a previous version is already installed.
  if 'google' in sys.modules:
    del sys.modules['google']
  GenerateUnittestProtos()

  import unittest
  import google.protobuf.internal.generator_test     as generator_test
  import google.protobuf.internal.descriptor_test    as descriptor_test
  import google.protobuf.internal.reflection_test    as reflection_test
  import google.protobuf.internal.service_reflection_test \
    as service_reflection_test
  import google.protobuf.internal.text_format_test   as text_format_test
  import google.protobuf.internal.wire_format_test   as wire_format_test
  import google.protobuf.internal.unknown_fields_test as unknown_fields_test
  import google.protobuf.internal.descriptor_database_test \
      as descriptor_database_test
  import google.protobuf.internal.descriptor_pool_test as descriptor_pool_test
  import google.protobuf.internal.message_factory_test as message_factory_test
  import google.protobuf.internal.message_cpp_test as message_cpp_test
  import google.protobuf.internal.reflection_cpp_generated_test \
      as reflection_cpp_generated_test

  loader = unittest.defaultTestLoader
  suite = unittest.TestSuite()
  for test in [ generator_test,
                descriptor_test,
                reflection_test,
                service_reflection_test,
                text_format_test,
                wire_format_test ]:
    suite.addTest(loader.loadTestsFromModule(test))

  return suite

class clean(_clean):
  def run(self):
    # Delete generated files in the code tree.
    for (dirpath, dirnames, filenames) in os.walk("."):
      for filename in filenames:
        filepath = os.path.join(dirpath, filename)
        if filepath.endswith("_pb2.py") or filepath.endswith(".pyc") or \
          filepath.endswith(".so") or filepath.endswith(".o") or \
          filepath.endswith('google/protobuf/compiler/__init__.py'):
          os.remove(filepath)
    # _clean is an old-style class, so super() doesn't work.
    _clean.run(self)

class build_py(_build_py):
  def run(self):
    # Generate necessary .proto file if it doesn't exist.
    generate_proto("../src/google/protobuf/descriptor.proto")
    generate_proto("../src/google/protobuf/compiler/plugin.proto")

    GenerateUnittestProtos()
    # Make sure google.protobuf.compiler is a valid package.
    open('google/protobuf/compiler/__init__.py', 'a').close()
    # _build_py is an old-style class, so super() doesn't work.
    _build_py.run(self)

if __name__ == '__main__':
  ext_module_list = []

  # C++ implementation extension
  if os.getenv("PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION", "python") == "cpp":
    print "Using EXPERIMENTAL C++ Implmenetation."
    ext_module_list.append(Extension(
        "google.protobuf.internal._net_proto2___python",
        [ "google/protobuf/pyext/python_descriptor.cc",
          "google/protobuf/pyext/python_protobuf.cc",
          "google/protobuf/pyext/python-proto2.cc" ],
        include_dirs = [ "." ],
        libraries = [ "protobuf" ]))

  setup(name = 'protobuf',
        version = '2.5.0-pre',
        packages = [ 'google' ],
        namespace_packages = [ 'google' ],
        test_suite = 'setup.MakeTestSuite',
        # Must list modules explicitly so that we don't install tests.
        py_modules = [
          'google.protobuf.internal.api_implementation',
          'google.protobuf.internal.containers',
          'google.protobuf.internal.cpp_message',
          'google.protobuf.internal.decoder',
          'google.protobuf.internal.encoder',
          'google.protobuf.internal.message_listener',
          'google.protobuf.internal.python_message',
          'google.protobuf.internal.type_checkers',
          'google.protobuf.internal.wire_format',
          'google.protobuf.descriptor',
          'google.protobuf.descriptor_pb2',
          'google.protobuf.compiler.plugin_pb2',
          'google.protobuf.message',
          'google.protobuf.descriptor_database',
          'google.protobuf.descriptor_pool',
          'google.protobuf.message_factory',
          'google.protobuf.reflection',
          'google.protobuf.service',
          'google.protobuf.service_reflection',
          'google.protobuf.text_format' ],
        cmdclass = { 'clean': clean, 'build_py': build_py },
        install_requires = ['setuptools'],
        ext_modules = ext_module_list,
        url = 'http://code.google.com/p/protobuf/',
        maintainer = maintainer_email,
        maintainer_email = 'protobuf@googlegroups.com',
        license = 'New BSD License',
        description = 'Protocol Buffers',
        long_description =
          "Protocol Buffers are Google's data interchange format.",
        )
<|endoftext|>


<commit_before>import numpy as np
from sklearn.neighbors import NearestNeighbors
import multiprocessing
import ctypes
from scipy import sparse as sp
import subprocess
import time
import re
import os
import sys

def find_neighbors(data, k=30, metric='minkowski', p=2, n_jobs=-1):
    """
    Wraps sklearn.neighbors.NearestNeighbors
    Find k nearest neighbors of every point in data and delete self-distances

    :param data: n-by-d data matrix
    :param k: number for nearest neighbors search
    :param metric: string naming distance metric used to define neighbors
    :param p: if metric == "minkowski", p=2 --> euclidean, p=1 --> manhattan; otherwise ignored.
    :param n_jobs:
    :return d: n-by-k matrix of distances
    :return idx: n-by-k matrix of neighbor indices
    """
    if metric.lower() == "euclidean":
        metric = "minkowski"
        p = 2
    if metric.lower() == "manhattan":
        metric = "minkowski"
        p = 1
    if metric.lower() == "minkowski":
        algorithm = "kd_tree"
    elif metric.lower() == "cosine" or metric.lower() == "correlation":
        algorithm = "brute"
    else:
        algorithm = "auto"

    print("Finding {} nearest neighbors using {} metric and '{}' algorithm".format(k, metric, algorithm),
          flush=True)
    nbrs = NearestNeighbors(n_neighbors=k+1,        # k+1 because results include self
                            n_jobs=n_jobs,              # use multiple cores if possible
                            metric=metric,          # primary metric
                            p=p,                    # if metric == "minkowski", 2 --> euclidean, 1 --> manhattan
                            algorithm=algorithm     # kd_tree is fastest for minkowski metrics
                            ).fit(data)
    d, idx = nbrs.kneighbors(data)
    # Remove self-distances if these are in fact included
    if idx[0, 0] == 0:
        idx = np.delete(idx, 0, axis=1)
        d = np.delete(d, 0, axis=1)
    else:  # Otherwise delete the _last_ column of d and idx
        idx = np.delete(idx, -1, axis=1)
        d = np.delete(d, -1, axis=1)
    return d, idx

def neighbor_graph(kernel, kernelargs):
    """
    Apply kernel (i.e. affinity function) to kernelargs (containing information about the data)
    and return graph as a sparse COO matrix

    :param kernel: affinity function
    :param kernelargs: dictionary of keyword arguments for kernel
    :return graph: n-by-n COO sparse matrix
    """
    i, j, s = kernel(**kernelargs)
    n, k = kernelargs['idx'].shape
    graph = sp.coo_matrix((s, (i, j)), shape=(n, n))
    return graph

def gaussian_kernel(idx, d, sigma):
    """
    For truncated list of k-nearest distances, apply Gaussian kernel
    Assume distances in d are Euclidean
    :param idx:
    :param d:
    :param sigma:
    :return:
    """
    n, k = idx.shape
    i = [np.tile(x, (k,)) for x in range(n)]
    i = np.concatenate(np.array(i))
    j = np.concatenate(idx)
    d = np.concatenate(d)
    f = np.vectorize(lambda x: 1/(sigma * (2 * np.pi) ** .5) * np.exp(-.5 * (x / sigma) ** 2))
    # apply vectorized gaussian function
    p = f(d)
    return i, j, p

def jaccard_kernel(idx):
    """
    Compute Jaccard coefficient between nearest-neighbor sets
    :param idx: numpy array of nearest-neighbor indices
    :return (i, j, s): tuple of indices and jaccard coefficients, suitable for constructing COO matrix
    """
    n, k = idx.shape
    s = list()
    for i in range(n):
        shared_neighbors = np.fromiter((len(set(idx[i]).intersection(set(idx[j]))) for j in idx[i]), dtype=float)
        s.extend(shared_neighbors / (2 * k - shared_neighbors))
    i = np.concatenate(np.array([np.tile(x, (k, )) for x in range(n)]))
    j = np.concatenate(idx)
    return i, j, s

def calc_jaccard(i):
    """
    For shared array idx, compute jaccard coefficient between point i and i's direct neighbors
    Function must be defined at module level to work with multiprocessing.Pool
    :param i:
    :return idx[i]: Indices of i's neighbors
    :return coefficients: Jaccard coefficients for each of i's neighbors
    """
    coefficients = np.fromiter((len(set(idx[i]).intersection(set(idx[j]))) for j in idx[i]), dtype=float)
    coefficients /= (2 * idx.shape[1] - coefficients)
    return idx[i], coefficients

def parallel_jaccard_kernel(idx):

    """
    Compute Jaccard coefficient between nearest-neighbor sets
    via multiprocessing.Pool
    :param idx:
    :return (i, j, s):
    """

    shared_array_base = multiprocessing.Array(ctypes.c_int, idx.size, lock=False)
    shared_idx = np.frombuffer(shared_array_base, dtype=ctypes.c_int)
    shared_idx = shared_idx.reshape(idx.shape)
    np.copyto(shared_idx, idx)
    assert shared_idx.base.base == shared_array_base
    setattr(sys.modules[__name__], "idx", shared_idx)

    n = idx.shape[0]
    pool = multiprocessing.Pool()
    jaccard_values = pool.map(calc_jaccard, range(n))
    assert len(jaccard_values) == n
    del shared_idx
    pool.terminate()

    graph = sp.lil_matrix((n, n), dtype=float)
    for i, tup in enumerate(jaccard_values):
        graph.rows[i] = tup[0]
        graph.data[i] = tup[1]

    i, j = graph.nonzero()
    s = graph.tocoo().data
    return i, j, s[s > 0]

def graph2binary(filename, graph):
    """
    Write (weighted) graph to binary file filename.bin
    :param filename:
    :param graph:
    :return None: graph is written to filename.bin
    """
    tic = time.time()
    # Unpack values in graph
    i, j = graph.nonzero()
    s = graph.data
    # place i and j in single array as edge list
    ij = np.hstack((i[:, np.newaxis], j[:, np.newaxis]))
    # add dummy self-edges for vertices at the END of the list with no neighbors
    ijmax = np.union1d(i, j).max()
    n = graph.shape[0]
    missing = np.arange(ijmax+1, n)
    for q in missing:
        ij = np.append(ij, [[q, q]], axis=0)
        s = np.append(s, [0.], axis=0)
    # Check data types: int32 for indices, float64 for weights
    if ij.dtype != np.int32:
        ij = ij.astype('int32')
    if s.dtype != np.float64:
        s = s.astype('float64')
    # write to file (NB f.writelines is ~10x faster than np.tofile(f))
    with open(filename + '.bin', 'w+b') as f:
        f.writelines([e for t in zip(ij, s) for e in t])
    print("Wrote graph to binary file in {} seconds".format(time.time() - tic))

def runlouvain(filename, max_runs=100, time_limit=300, tol=1e-3):
    """
    From binary graph file filename.bin, optimize modularity by running multiple random re-starts of
    the Louvain C++ code.

    Louvain is run repeatedly until modularity has not increased in some number (20) of runs
    or if the total number of runs exceeds some larger number (max_runs) OR if a time limit (time_limit) is exceeded

    :param filename: *.bin file generated by graph2binary
    :param max_runs: maximum number of times to repeat Louvain before ending iterations and taking best result
    :param time_limit: maxumum number of seconds to repeat Louvain before ending iteratins and taking best result
    :param tol: precision for evaluating modularity increase
    :return communities: community assignments
    :return Q: modularity score corresponding to `communities`
    """
    def get_modularity(msg):
        pattern = re.compile('modularity increased from -*0.\d+ to 0.\d+')
        matches = pattern.findall(msg.decode())
        q = list()
        for line in matches:
            q.append(line.split(sep=" ")[-1])
        return list(map(float, q))

    print('Running Louvain modularity optimization', flush=True)

    # Use package location to find Louvain code
    # lpath = os.path.abspath(resource_filename(Requirement.parse("PhenoGraph"), 'louvain'))
    lpath = os.path.join(os.path.dirname(__file__), 'louvain')
    try:
        assert os.path.isdir(lpath)
    except AssertionError:
        print("Could not find Louvain code, tried: {}".format(lpath), flush=True)

    # Determine if we're using Windows, Mac, or Linux
    if sys.platform == "win32" or sys.platform == "cygwin":
        convert_binary = "convert.exe"
        community_binary = "community.exe"
        hierarchy_binary = "hierarchy.exe"
    elif sys.platform.startswith("linux"):
        convert_binary = "linux-convert"
        community_binary = "linux-community"
        hierarchy_binary = "linux-hierarchy"
    elif sys.platform == "darwin":
        convert_binary = "convert"
        community_binary = "community"
        hierarchy_binary = "hierarchy"
    else:
        raise RuntimeError("Operating system could not be determined or is not supported. "
                           "sys.platform == {}".format(sys.platform), flush=True)
    # Prepend appropriate path separator
    convert_binary = os.path.sep + convert_binary
    community_binary = os.path.sep + community_binary
    hierarchy_binary = os.path.sep + hierarchy_binary

    tic = time.time()

    # run convert
    args = [lpath + convert_binary, '-i', filename + '.bin', '-o',
            filename + '_graph.bin', '-w', filename + '_graph.weights']
    p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = p.communicate()
    # check for errors from convert
    if bool(out) or bool(err):
        print("stdout from convert: {}".format(out.decode()))
        print("stderr from convert: {}".format(err.decode()))

    Q = 0
    run = 0
    updated = 0
    while run - updated < 20 and run < max_runs and (time.time() - tic) < time_limit:

        # run community
        fout = open(filename + '.tree', 'w')
        args = [lpath + community_binary, filename + '_graph.bin', '-l', '-1', '-v', '-w', filename + '_graph.weights']
        p = subprocess.Popen(args, stdout=fout, stderr=subprocess.PIPE)
        # Here, we print communities to filename.tree and retain the modularity scores reported piped to stderr
        _, msg = p.communicate()
        fout.close()
        # get modularity from err msg
        q = get_modularity(msg)
        run += 1

        # continue only if we've reached a higher modularity than before
        if q[-1] - Q > tol:

            Q = q[-1]
            updated = run

            # run hierarchy
            args = [lpath + hierarchy_binary, filename + '.tree']
            p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out, err = p.communicate()
            # find number of levels in hierarchy and number of nodes in graph
            nlevels = int(re.findall('\d+', out.decode())[0])
            nnodes = int(re.findall('level 0: \d+', out.decode())[0].split(sep=" ")[-1])

            # get community assignments at each level in hierarchy
            hierarchy = np.empty((nnodes, nlevels), dtype='int')
            for level in range(nlevels):
                    args = [lpath + hierarchy_binary, filename + '.tree', '-l', str(level)]
                    p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                    out, err = p.communicate()
                    h = np.empty((nnodes,))
                    for i, line in enumerate(out.decode().splitlines()):
                        h[i] = int(line.split(sep=' ')[-1])
                    hierarchy[:, level] = h

            communities = hierarchy[:, nlevels-1]

            print("After {} runs, maximum modularity is Q = {}".format(run, Q), flush=True)

    print("Louvain completed {} runs in {} seconds".format(run, time.time() - tic), flush=True)

    return communities, Q
<commit_msg>Change default neighbors search algorithm from euclidean from `kd_tree` to `auto`<commit_after>import numpy as np
from sklearn.neighbors import NearestNeighbors
import multiprocessing
import ctypes
from scipy import sparse as sp
import subprocess
import time
import re
import os
import sys

def find_neighbors(data, k=30, metric='minkowski', p=2, n_jobs=-1):
    """
    Wraps sklearn.neighbors.NearestNeighbors
    Find k nearest neighbors of every point in data and delete self-distances

    :param data: n-by-d data matrix
    :param k: number for nearest neighbors search
    :param metric: string naming distance metric used to define neighbors
    :param p: if metric == "minkowski", p=2 --> euclidean, p=1 --> manhattan; otherwise ignored.
    :param n_jobs:
    :return d: n-by-k matrix of distances
    :return idx: n-by-k matrix of neighbor indices
    """
    if metric.lower() == "euclidean":
        metric = "minkowski"
        p = 2
    if metric.lower() == "manhattan":
        metric = "minkowski"
        p = 1
    if metric.lower() == "minkowski":
        algorithm = "auto"
    elif metric.lower() == "cosine" or metric.lower() == "correlation":
        algorithm = "brute"
    else:
        algorithm = "auto"

    print("Finding {} nearest neighbors using {} metric and '{}' algorithm".format(k, metric, algorithm),
          flush=True)
    nbrs = NearestNeighbors(n_neighbors=k+1,        # k+1 because results include self
                            n_jobs=n_jobs,              # use multiple cores if possible
                            metric=metric,          # primary metric
                            p=p,                    # if metric == "minkowski", 2 --> euclidean, 1 --> manhattan
                            algorithm=algorithm     # kd_tree is fastest for minkowski metrics
                            ).fit(data)
    d, idx = nbrs.kneighbors(data)
    # Remove self-distances if these are in fact included
    if idx[0, 0] == 0:
        idx = np.delete(idx, 0, axis=1)
        d = np.delete(d, 0, axis=1)
    else:  # Otherwise delete the _last_ column of d and idx
        idx = np.delete(idx, -1, axis=1)
        d = np.delete(d, -1, axis=1)
    return d, idx

def neighbor_graph(kernel, kernelargs):
    """
    Apply kernel (i.e. affinity function) to kernelargs (containing information about the data)
    and return graph as a sparse COO matrix

    :param kernel: affinity function
    :param kernelargs: dictionary of keyword arguments for kernel
    :return graph: n-by-n COO sparse matrix
    """
    i, j, s = kernel(**kernelargs)
    n, k = kernelargs['idx'].shape
    graph = sp.coo_matrix((s, (i, j)), shape=(n, n))
    return graph

def gaussian_kernel(idx, d, sigma):
    """
    For truncated list of k-nearest distances, apply Gaussian kernel
    Assume distances in d are Euclidean
    :param idx:
    :param d:
    :param sigma:
    :return:
    """
    n, k = idx.shape
    i = [np.tile(x, (k,)) for x in range(n)]
    i = np.concatenate(np.array(i))
    j = np.concatenate(idx)
    d = np.concatenate(d)
    f = np.vectorize(lambda x: 1/(sigma * (2 * np.pi) ** .5) * np.exp(-.5 * (x / sigma) ** 2))
    # apply vectorized gaussian function
    p = f(d)
    return i, j, p

def jaccard_kernel(idx):
    """
    Compute Jaccard coefficient between nearest-neighbor sets
    :param idx: numpy array of nearest-neighbor indices
    :return (i, j, s): tuple of indices and jaccard coefficients, suitable for constructing COO matrix
    """
    n, k = idx.shape
    s = list()
    for i in range(n):
        shared_neighbors = np.fromiter((len(set(idx[i]).intersection(set(idx[j]))) for j in idx[i]), dtype=float)
        s.extend(shared_neighbors / (2 * k - shared_neighbors))
    i = np.concatenate(np.array([np.tile(x, (k, )) for x in range(n)]))
    j = np.concatenate(idx)
    return i, j, s

def calc_jaccard(i):
    """
    For shared array idx, compute jaccard coefficient between point i and i's direct neighbors
    Function must be defined at module level to work with multiprocessing.Pool
    :param i:
    :return idx[i]: Indices of i's neighbors
    :return coefficients: Jaccard coefficients for each of i's neighbors
    """
    coefficients = np.fromiter((len(set(idx[i]).intersection(set(idx[j]))) for j in idx[i]), dtype=float)
    coefficients /= (2 * idx.shape[1] - coefficients)
    return idx[i], coefficients

def parallel_jaccard_kernel(idx):

    """
    Compute Jaccard coefficient between nearest-neighbor sets
    via multiprocessing.Pool
    :param idx:
    :return (i, j, s):
    """

    shared_array_base = multiprocessing.Array(ctypes.c_int, idx.size, lock=False)
    shared_idx = np.frombuffer(shared_array_base, dtype=ctypes.c_int)
    shared_idx = shared_idx.reshape(idx.shape)
    np.copyto(shared_idx, idx)
    assert shared_idx.base.base == shared_array_base
    setattr(sys.modules[__name__], "idx", shared_idx)

    n = idx.shape[0]
    pool = multiprocessing.Pool()
    jaccard_values = pool.map(calc_jaccard, range(n))
    assert len(jaccard_values) == n
    del shared_idx
    pool.terminate()

    graph = sp.lil_matrix((n, n), dtype=float)
    for i, tup in enumerate(jaccard_values):
        graph.rows[i] = tup[0]
        graph.data[i] = tup[1]

    i, j = graph.nonzero()
    s = graph.tocoo().data
    return i, j, s[s > 0]

def graph2binary(filename, graph):
    """
    Write (weighted) graph to binary file filename.bin
    :param filename:
    :param graph:
    :return None: graph is written to filename.bin
    """
    tic = time.time()
    # Unpack values in graph
    i, j = graph.nonzero()
    s = graph.data
    # place i and j in single array as edge list
    ij = np.hstack((i[:, np.newaxis], j[:, np.newaxis]))
    # add dummy self-edges for vertices at the END of the list with no neighbors
    ijmax = np.union1d(i, j).max()
    n = graph.shape[0]
    missing = np.arange(ijmax+1, n)
    for q in missing:
        ij = np.append(ij, [[q, q]], axis=0)
        s = np.append(s, [0.], axis=0)
    # Check data types: int32 for indices, float64 for weights
    if ij.dtype != np.int32:
        ij = ij.astype('int32')
    if s.dtype != np.float64:
        s = s.astype('float64')
    # write to file (NB f.writelines is ~10x faster than np.tofile(f))
    with open(filename + '.bin', 'w+b') as f:
        f.writelines([e for t in zip(ij, s) for e in t])
    print("Wrote graph to binary file in {} seconds".format(time.time() - tic))

def runlouvain(filename, max_runs=100, time_limit=300, tol=1e-3):
    """
    From binary graph file filename.bin, optimize modularity by running multiple random re-starts of
    the Louvain C++ code.

    Louvain is run repeatedly until modularity has not increased in some number (20) of runs
    or if the total number of runs exceeds some larger number (max_runs) OR if a time limit (time_limit) is exceeded

    :param filename: *.bin file generated by graph2binary
    :param max_runs: maximum number of times to repeat Louvain before ending iterations and taking best result
    :param time_limit: maxumum number of seconds to repeat Louvain before ending iteratins and taking best result
    :param tol: precision for evaluating modularity increase
    :return communities: community assignments
    :return Q: modularity score corresponding to `communities`
    """
    def get_modularity(msg):
        # pattern = re.compile('modularity increased from -*0.\d+ to 0.\d+')
        pattern = re.compile('modularity increased from -*\d.\d+e*-*\d+ to \d.\d+')
        matches = pattern.findall(msg.decode())
        q = list()
        for line in matches:
            q.append(line.split(sep=" ")[-1])
        return list(map(float, q))

    print('Running Louvain modularity optimization', flush=True)

    # Use package location to find Louvain code
    # lpath = os.path.abspath(resource_filename(Requirement.parse("PhenoGraph"), 'louvain'))
    lpath = os.path.join(os.path.dirname(__file__), 'louvain')
    try:
        assert os.path.isdir(lpath)
    except AssertionError:
        print("Could not find Louvain code, tried: {}".format(lpath), flush=True)

    # Determine if we're using Windows, Mac, or Linux
    if sys.platform == "win32" or sys.platform == "cygwin":
        convert_binary = "convert.exe"
        community_binary = "community.exe"
        hierarchy_binary = "hierarchy.exe"
    elif sys.platform.startswith("linux"):
        convert_binary = "linux-convert"
        community_binary = "linux-community"
        hierarchy_binary = "linux-hierarchy"
    elif sys.platform == "darwin":
        convert_binary = "convert"
        community_binary = "community"
        hierarchy_binary = "hierarchy"
    else:
        raise RuntimeError("Operating system could not be determined or is not supported. "
                           "sys.platform == {}".format(sys.platform), flush=True)
    # Prepend appropriate path separator
    convert_binary = os.path.sep + convert_binary
    community_binary = os.path.sep + community_binary
    hierarchy_binary = os.path.sep + hierarchy_binary

    tic = time.time()

    # run convert
    args = [lpath + convert_binary, '-i', filename + '.bin', '-o',
            filename + '_graph.bin', '-w', filename + '_graph.weights']
    p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = p.communicate()
    # check for errors from convert
    if bool(out) or bool(err):
        print("stdout from convert: {}".format(out.decode()))
        print("stderr from convert: {}".format(err.decode()))

    Q = 0
    run = 0
    updated = 0
    while run - updated < 20 and run < max_runs and (time.time() - tic) < time_limit:

        # run community
        fout = open(filename + '.tree', 'w')
        args = [lpath + community_binary, filename + '_graph.bin', '-l', '-1', '-v', '-w', filename + '_graph.weights']
        p = subprocess.Popen(args, stdout=fout, stderr=subprocess.PIPE)
        # Here, we print communities to filename.tree and retain the modularity scores reported piped to stderr
        _, msg = p.communicate()
        fout.close()
        # get modularity from err msg
        q = get_modularity(msg)
        run += 1

        # continue only if we've reached a higher modularity than before
        if q[-1] - Q > tol:

            Q = q[-1]
            updated = run

            # run hierarchy
            args = [lpath + hierarchy_binary, filename + '.tree']
            p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out, err = p.communicate()
            # find number of levels in hierarchy and number of nodes in graph
            nlevels = int(re.findall('\d+', out.decode())[0])
            nnodes = int(re.findall('level 0: \d+', out.decode())[0].split(sep=" ")[-1])

            # get community assignments at each level in hierarchy
            hierarchy = np.empty((nnodes, nlevels), dtype='int')
            for level in range(nlevels):
                    args = [lpath + hierarchy_binary, filename + '.tree', '-l', str(level)]
                    p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                    out, err = p.communicate()
                    h = np.empty((nnodes,))
                    for i, line in enumerate(out.decode().splitlines()):
                        h[i] = int(line.split(sep=' ')[-1])
                    hierarchy[:, level] = h

            communities = hierarchy[:, nlevels-1]

            print("After {} runs, maximum modularity is Q = {}".format(run, Q), flush=True)

    print("Louvain completed {} runs in {} seconds".format(run, time.time() - tic), flush=True)

    return communities, Q
<|endoftext|>


<commit_before>#!/usr/bin/env python

from setuptools import setup

setup(name='thunder',
    version='0.1',
    description='Neural data analysis in Spark',
    author='The Freeman Lab',
    author_email='the.freeman.lab@gmail.com',
    url='https://github.com/freeman-lab/thunder',
    packages=['thunder','thunder.regression','thunder.factorization','thunder.clustering','thunder.summary','thunder.util'],
    )<commit_msg>Updated ver number in setup<commit_after>#!/usr/bin/env python

from setuptools import setup

setup(name='thunder',
    version='0.1.0',
    description='Large-scale neural data analysis in Spark',
    author='The Freeman Lab',
    author_email='the.freeman.lab@gmail.com',
    url='https://github.com/freeman-lab/thunder',
    packages=['thunder','thunder.regression','thunder.factorization','thunder.clustering','thunder.sigprocessing','thunder.util'],
    )<|endoftext|>


<commit_before># stdlib
import shutil
import sys
import os.path
import subprocess

# internal
from . import utils
from . import tasks
from . import documents

# set up logging
import phial.loggers
log = phial.loggers.get_logger(__name__)

@utils.public
def pipeline(foreach=[], binary_mode=True, task_queue=tasks.global_queue, depends_on=None):
    def real_decorator(function):
        task_queue.enqueue(PipelineTask(function, foreach, binary_mode, function, depends_on))
        return function

    return real_decorator

class PipelineTask(tasks.Task):
    def __init__(self, function, foreach, binary_mode, id, depends_on=None):
        self.function = function
        self.foreach = foreach
        self.binary_mode = binary_mode
        self.files = None
        super(PipelineTask, self).__init__(id=id, depends_on=depends_on)

    def run(self, config):
        if self.binary_mode:
            def open_file(path):
                return open(path, "rb")
        else:
            open_file = documents.open_file

        # Glob and open all of the files the user specified
        globbed_foreach = utils.glob_foreach_list(self.foreach)
        files = [open_file(path) for path in globbed_foreach]

        result = self.function(PipelineSource(files))
        log.info("Pipe function {0!r} yielded {1!s} files.", self.function, len(result.contents))

        self.files = list(result.contents)
        for i in self.files:
            output_path = os.path.join(config["output"], i.name)
            if not utils.is_path_under_directory(output_path, config["output"]):
                log.fatal(
                    "Target path must be relative and under the output directory. Did you begin "
                    "the path with a / or .. ?")

            # Ensure that the target directory exists
            utils.makedirs(os.path.dirname(output_path))

            if self.binary_mode:
                output_file = open(output_path, "wb")
            else:
                output_file = documents.unicodify_file_object(open(output_path, "w"))

            shutil.copyfileobj(i, output_file)

class PipelineSource(object):
    def prepare_contents(self):
        for i in self.contents:
            assert hasattr(i, "name"), "{0!r} does not have name attribute.".format(i)
            i.seek(0)

    def __init__(self, contents):
        self.contents = contents
        self.prepare_contents()

    def pipe(self, transform):
        self.contents = transform(self.contents)
        self.prepare_contents()
        return self

    def __or__(self, transform):
        return self.pipe(transform)

    def __repr__(self):
        return self.__class__.__name__ + "(" + repr(self.contents) + ")"

class run(object):
    def __init__(self, output_name, args, popen_kwargs=None):
        self.args = args
        self.output_name = output_name
        self.popen_kwargs = popen_kwargs or {}

    def __call__(self, contents):
        p = subprocess.Popen(self.args, stdout=subprocess.PIPE, stdin=subprocess.PIPE,
                             **self.popen_kwargs)
        stdout = p.communicate("".join(i.read() for i in contents))[0]

        result = documents.file(name=self.output_name)
        result.write(stdout)
        return [result]

@utils.public
class concat(object):
    def __init__(self, output_name=None):
        self.output_name = output_name

    def __call__(self, contents):
        result = documents.file(name=self.output_name)
        for i in contents:
            shutil.copyfileobj(i, result)
        return [result]

@utils.public
class cout(object):
    def __init__(self, out=sys.stdout):
        self.out = sys.stdout

    def __call__(self, contents):
        for i in contents:
            shutil.copyfileobj(i, self.out)
        return contents

@utils.public
class move(object):
    def __init__(self, to):
        self.to = to

    def __call__(self, contents):
        for i in contents:
            i = os.path.join(self.to, i)
        return contents

@utils.public
class map(object):
    def __init__(self, func):
        self.func = func

    def __call__(self, contents):
        return [self.func(i) for i in contents]
<commit_msg>Prune out None values from pipelines.<commit_after># stdlib
import shutil
import sys
import os.path
import subprocess

# internal
from . import utils
from . import tasks
from . import documents

# set up logging
import phial.loggers
log = phial.loggers.get_logger(__name__)

@utils.public
def pipeline(foreach=[], binary_mode=True, task_queue=tasks.global_queue, depends_on=None):
    def real_decorator(function):
        task_queue.enqueue(PipelineTask(function, foreach, binary_mode, function, depends_on))
        return function

    return real_decorator

class PipelineTask(tasks.Task):
    def __init__(self, function, foreach, binary_mode, id, depends_on=None):
        self.function = function
        self.foreach = foreach
        self.binary_mode = binary_mode
        self.files = None
        super(PipelineTask, self).__init__(id=id, depends_on=depends_on)

    def run(self, config):
        if self.binary_mode:
            def open_file(path):
                return open(path, "rb")
        else:
            open_file = documents.open_file

        # Glob and open all of the files the user specified
        globbed_foreach = utils.glob_foreach_list(self.foreach)
        files = [open_file(path) for path in globbed_foreach]

        result = self.function(PipelineSource(files))
        log.info("Pipe function {0!r} yielded {1!s} files.", self.function, len(result.contents))

        self.files = list(result.contents)
        for i in self.files:
            output_path = os.path.join(config["output"], i.name)
            if not utils.is_path_under_directory(output_path, config["output"]):
                log.fatal(
                    "Target path must be relative and under the output directory. Did you begin "
                    "the path with a / or .. ?")

            # Ensure that the target directory exists
            utils.makedirs(os.path.dirname(output_path))

            if self.binary_mode:
                output_file = open(output_path, "wb")
            else:
                output_file = documents.unicodify_file_object(open(output_path, "w"))

            shutil.copyfileobj(i, output_file)

class PipelineSource(object):
    def prepare_contents(self):
        # We always prune out None values as if they were never there
        self.contents = [i for i in self.contents if i is not None]

        for i in self.contents:
            assert hasattr(i, "name"), "{0!r} does not have name attribute.".format(i)
            i.seek(0)

    def __init__(self, contents):
        self.contents = contents
        self.prepare_contents()

    def pipe(self, transform):
        self.contents = transform(self.contents)
        self.prepare_contents()
        return self

    def __or__(self, transform):
        return self.pipe(transform)

    def __repr__(self):
        return self.__class__.__name__ + "(" + repr(self.contents) + ")"

class run(object):
    def __init__(self, output_name, args, popen_kwargs=None):
        self.args = args
        self.output_name = output_name
        self.popen_kwargs = popen_kwargs or {}

    def __call__(self, contents):
        p = subprocess.Popen(self.args, stdout=subprocess.PIPE, stdin=subprocess.PIPE,
                             **self.popen_kwargs)
        stdout = p.communicate("".join(i.read() for i in contents))[0]

        result = documents.file(name=self.output_name)
        result.write(stdout)
        return [result]

@utils.public
class concat(object):
    def __init__(self, output_name=None):
        self.output_name = output_name

    def __call__(self, contents):
        result = documents.file(name=self.output_name)
        for i in contents:
            shutil.copyfileobj(i, result)
        return [result]

@utils.public
class cout(object):
    def __init__(self, out=sys.stdout):
        self.out = sys.stdout

    def __call__(self, contents):
        for i in contents:
            shutil.copyfileobj(i, self.out)
        return contents

@utils.public
class move(object):
    def __init__(self, to):
        self.to = to

    def __call__(self, contents):
        for i in contents:
            i = os.path.join(self.to, i)
        return contents

@utils.public
class map(object):
    def __init__(self, func):
        self.func = func

    def __call__(self, contents):
        return [self.func(i) for i in contents]
<|endoftext|>


<commit_before>"""
fast5 setup.py
(c) 2017: Matei David, Ontario Institute for Cancer Research
MIT License
"""

import os
import sys

from setuptools import setup, Extension
from Cython.Build import cythonize

# check HDF5 include and lib dirs
hdf5_dir = os.environ.get('HDF5_DIR', '/usr')
hdf5_include_dir = os.environ.get('HDF5_INCLUDE_DIR', os.path.join(hdf5_dir, 'include'))
hdf5_lib_dir = os.environ.get('HDF5_LIB_DIR', os.path.join(hdf5_dir, 'lib'))
hdf5_lib = os.environ.get('HDF5_LIB', 'hdf5')
if not os.path.isfile(os.path.join(hdf5_include_dir, 'H5pubconf.h')):
    sys.exit(hdf5_include_dir + ': could not find HDF5 header files; use HDF5_DIR or HDF5_INCLUDE_DIR')
if (not os.path.isfile(os.path.join(hdf5_lib_dir, 'lib' + hdf5_lib + '.so'))
    and not os.path.isfile(os.path.join(hdf5_lib_dir, 'lib' + hdf5_lib + '.a'))):
    sys.exit(hdf5_lib_dir + ': could not find HDF5 library file; use HDF5_DIR or HDF5_LIB_DIR/HDF5_LIB')

fast5_dir = os.environ.get('FAST5_DIR', '..')
fast5_src_dir = os.path.join(fast5_dir, 'src')
fast5_version = open(os.path.join(fast5_dir, 'VERSION')).readline().strip()

extra_compile_args = [
    '-std=c++11',
    '-Wall', '-Wextra', '-Wpedantic',
]
# don't indiscriminately add /usr/include to work around bug:
# https://lists.fedoraproject.org/archives/list/devel@lists.fedoraproject.org/thread/Q5SWCUUMWQ4EMS7CU2CBOZHV3WZYOOTT/
for d in [hdf5_include_dir]:
    if d != '/usr/include':
        extra_compile_args += ['-isystem', d]
#extra_compile_args += ['-O0', '-g3', '-ggdb', '-fno-eliminate-unused-debug-types', '-v']

extra_link_args = []
#extra_link_args += ['-v']

#if sys.platform == 'darwin':
#    extra_compile_args.append('-mmacosx-version-min=10.7')

ext = Extension(
    'fast5',
    language='c++',
    sources=['fast5.pyx'],
    include_dirs=[fast5_src_dir],
    library_dirs=[hdf5_lib_dir],
    runtime_library_dirs=[hdf5_lib_dir],
    libraries=[hdf5_lib],
    extra_compile_args=extra_compile_args,
    extra_link_args=extra_link_args,
)

setup(
    name='fast5',
    description='Fast5 file interface.',
    version=fast5_version,
    author='Matei David, Ontario Institute for Cancer Research',
    author_email='matei.david at oicr.on.ca',
    license='MIT',
    url='https://github.com/mateidavid/fast5',
    ext_modules=cythonize(ext),
)
<commit_msg>disable cython recompilation by default<commit_after>"""
fast5 setup.py
(c) 2017: Matei David, Ontario Institute for Cancer Research
MIT License
"""

import os
import sys

from setuptools import setup, Extension

use_cython = os.environ.get('USE_CYTHON', '') != ''

# check HDF5 include and lib dirs
hdf5_dir = os.environ.get('HDF5_DIR', '/usr')
hdf5_include_dir = os.environ.get('HDF5_INCLUDE_DIR', os.path.join(hdf5_dir, 'include'))
hdf5_lib_dir = os.environ.get('HDF5_LIB_DIR', os.path.join(hdf5_dir, 'lib'))
hdf5_lib = os.environ.get('HDF5_LIB', 'hdf5')
if not os.path.isfile(os.path.join(hdf5_include_dir, 'H5pubconf.h')):
    sys.exit(hdf5_include_dir + ': could not find HDF5 header files; use HDF5_DIR or HDF5_INCLUDE_DIR')
if (not os.path.isfile(os.path.join(hdf5_lib_dir, 'lib' + hdf5_lib + '.so'))
    and not os.path.isfile(os.path.join(hdf5_lib_dir, 'lib' + hdf5_lib + '.a'))):
    sys.exit(hdf5_lib_dir + ': could not find HDF5 library file; use HDF5_DIR or HDF5_LIB_DIR/HDF5_LIB')

fast5_dir = os.environ.get('FAST5_DIR', '..')
fast5_src_dir = os.path.join(fast5_dir, 'src')
fast5_version = open(os.path.join(fast5_dir, 'VERSION')).readline().strip()

extra_compile_args = [
    '-std=c++11',
    '-Wall', '-Wextra', '-Wpedantic',
]
# don't indiscriminately add /usr/include to work around bug:
# https://lists.fedoraproject.org/archives/list/devel@lists.fedoraproject.org/thread/Q5SWCUUMWQ4EMS7CU2CBOZHV3WZYOOTT/
for d in [hdf5_include_dir]:
    if d != '/usr/include':
        extra_compile_args += ['-isystem', d]
#extra_compile_args += ['-O0', '-g3', '-ggdb', '-fno-eliminate-unused-debug-types', '-v']

extra_link_args = []
#extra_link_args += ['-v']

#if sys.platform == 'darwin':
#    extra_compile_args.append('-mmacosx-version-min=10.7')

extensions = [
    Extension(
        'fast5',
        language='c++',
        sources=['fast5.' + ['cpp', 'pyx'][use_cython]],
        include_dirs=[fast5_src_dir],
        library_dirs=[hdf5_lib_dir],
        runtime_library_dirs=[hdf5_lib_dir],
        libraries=[hdf5_lib],
        extra_compile_args=extra_compile_args,
        extra_link_args=extra_link_args,
    ),
]

if use_cython:
    from Cython.Build import cythonize
    extensions = cythonize(extensions)

setup(
    name='fast5',
    description='Fast5 file interface.',
    version=fast5_version,
    author='Matei David, Ontario Institute for Cancer Research',
    author_email='matei.david at oicr.on.ca',
    license='MIT',
    url='https://github.com/mateidavid/fast5',
    ext_modules=extensions,
)
<|endoftext|>