<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Miner

TODO:
    Finish docs

"""

from __future__ import absolute_import, print_function, unicode_literals
from csv import DictWriter
from json import dump, dumps, load
from os import path

from bs4 import BeautifulSoup
from tqdm import trange

from .patterns import MONEY_PAT, TITLE_SPLIT_PAT, ZIP_PAT, filter_addr
from .settings import HTML_FILE, NO_CASE
from .settings import FEATURES, FIELDS, INTERNAL_FIELDS

features = [i + ':' for i in FEATURES]

class Miner(object):

    def __init__(self, responses, output, gui=False):

        self.responses = responses
        self.output = output
        self.gui = gui
        self.dataset = []

    @staticmethod
    def scrape(self, html_data):
        """Scrapes the desired features

        Args:
            html_data: <str>, source HTML

        Returns:
            scraped_features: <dict>, features scraped and mapped from content
        """

        soup = BeautifulSoup(html_data, "html.parser")
        tr_list = soup.find_all("tr")

        feature_list = []
        for tag in tr_list:
            try:
                tag = [j.string for j in tag.findAll("span")]
                if set(tuple(tag)) & set(features):
                    try:
                        tag = [i for i in tag if "(each" not in i.lower()]
                    except AttributeError:
                        continue
                    feature_list.append(tag)

            except IndexError:
                continue

        try:
            # flatten multidimensional list
            feature_list = [item.replace(':', '')
                            for sublist in feature_list for item in sublist]

        except Exception as e:
            print(e, feature_list)

        return feature_list

    @staticmethod
    def distribute(self, case_num, feature_list):

        # break up elements with n-tuples greater than 2
        # then convert list of tuples to dict for faster lookup
        raw_business = [
            tuple(feature_list[i:i + 2])
            for i in range(0, len(feature_list), 2)
            if any(x in feature_list[i:i + 2][0] for x in INTERNAL_FIELDS)
        ]

        feature_list = dict([
            tuple(feature_list[i:i + 2])
            for i in range(0, len(feature_list), 2)
            if feature_list[i:i + 2][0] in FEATURES
        ])

        filtered_business = []

        for label, value in enumerate(raw_business):
            try:
                if value[1].upper() == "PROPERTY ADDRESS" and \
                        raw_business[label + 1][0].upper() == \
                        "BUSINESS OR ORGANIZATION NAME":
                    filtered_business.append(raw_business[label + 1])

            except IndexError:
                print("Party Type issue at Case", feature_list["Case Number"])

        scraped_features = []

        for address in filtered_business:

            str_address = filter_addr(str(address[-1]))

            temp_features = {
                key: value for key, value in feature_list.items()
                if key in ["Title",
                           "Case Type",
                           "Case Number",
                           "Filing Date"]
            }

            # break up Title feature into Plaintiff and Defendant
            try:
                temp_features["Plaintiff"], temp_features["Defendant"] = \
                    TITLE_SPLIT_PAT.split(temp_features["Title"])

            except ValueError:
                temp_features["Plaintiff"], temp_features["Defendant"] = \
                    ('', '')

            if temp_features["Case Type"].upper() == "FORECLOSURE":
                temp_features["Case Type"] = "Mortgage"

            temp_features["Address"] = \
                str_address if str_address else address[-1]

            temp_features["Zip Code"] = ''.join(ZIP_PAT.findall(address[-1]))

            temp_features["Partial Cost"] = ''.join(
                MONEY_PAT.findall(address[-1])
            )

            scraped_features.append(temp_features)
            temp_features = {}

        if not scraped_features:

            if not path.isfile(NO_CASE):
                with open(NO_CASE, 'w') as no_case_file:
                    dump([], no_case_file)

            with open(NO_CASE, 'r+') as no_case_file:
                no_case_data = load(no_case_file)
                no_case_data.append(str(case_num[:-5]))
                no_case_file.seek(0)
                no_case_file.write(dumps(sorted(list(set(no_case_data)))))
                no_case_file.truncate()

        return scraped_features

    def scan_files(self):

        case_range = trange(len(self.responses), desc='Mining', leave=True) \
            if not self.gui else range(len(self.responses))

        for file_name in case_range:
            with open(
                HTML_FILE.format(case=self.responses[file_name]), 'r'
            ) as html_src:

                if not self.gui:
                    case_range.set_description(
                        "Mining {}".format(self.responses[file_name])
                    )

                feature_list = self.scrape(html_src.read())
                row = self.distribute(self.responses[file_name], feature_list)

                self.dataset.extend(row)

    def export(self):

        file_exists = path.isfile(self.output)

        with open(self.output, 'a') as csv_file:
            writer = DictWriter(
                csv_file,
                fieldnames=[
                    col for col in FIELDS if col not in INTERNAL_FIELDS
                ]
            )

            if not file_exists:
                writer.writeheader()

            for row in self.dataset:
                writer.writerow(row)
<commit_msg>refactoring miner<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Miner

TODO:
    Finish docs

"""

from __future__ import absolute_import, print_function, unicode_literals
from csv import DictWriter
from json import dump, dumps, load
from os import path

from bs4 import BeautifulSoup
from tqdm import trange

from .patterns import MONEY_PAT, TITLE_SPLIT_PAT, ZIP_PAT, filter_addr
from .settings import HTML_FILE, NO_CASE
from .settings import FEATURES, FIELDS, INTERNAL_FIELDS

features = [i + ':' for i in FEATURES]

class Miner(object):

    def __init__(self, responses, output, gui=False):

        self.responses = responses
        self.output = output
        self.gui = gui
        self.dataset = []

    @staticmethod
    def scrape(html_data):
        """Scrapes the desired features

        Args:
            html_data: <str>, source HTML

        Returns:
            scraped_features: <dict>, features scraped and mapped from content
        """

        soup = BeautifulSoup(html_data, "html.parser")
        tr_list = soup.find_all("tr")

        feature_list = []
        for tag in tr_list:
            try:
                tag = [j.string for j in tag.findAll("span")]
                if set(tuple(tag)) & set(features):
                    try:
                        tag = [i for i in tag if "(each" not in i.lower()]
                    except AttributeError:
                        continue
                    feature_list.append(tag)

            except IndexError:
                continue

        try:
            # flatten multidimensional list
            feature_list = [item.replace(':', '')
                            for sublist in feature_list for item in sublist]

        except Exception as e:
            print(e, feature_list)

        return feature_list

    @staticmethod
    def distribute(case_num, feature_list):

        # break up elements with n-tuples greater than 2
        # then convert list of tuples to dict for faster lookup
        raw_business = [
            tuple(feature_list[i:i + 2])
            for i in range(0, len(feature_list), 2)
            if any(x in feature_list[i:i + 2][0] for x in INTERNAL_FIELDS)
        ]

        feature_list = dict([
            tuple(feature_list[i:i + 2])
            for i in range(0, len(feature_list), 2)
            if feature_list[i:i + 2][0] in FEATURES
        ])

        filtered_business = []

        for label, value in enumerate(raw_business):
            try:
                if value[1].upper() == "PROPERTY ADDRESS" and \
                        raw_business[label + 1][0].upper() == \
                        "BUSINESS OR ORGANIZATION NAME":
                    filtered_business.append(raw_business[label + 1])

            except IndexError:
                print("Party Type issue at Case", feature_list["Case Number"])

        scraped_features = []

        for address in filtered_business:

            str_address = filter_addr(str(address[-1]))

            temp_features = {
                key: value for key, value in feature_list.items()
                if key in ["Title",
                           "Case Type",
                           "Case Number",
                           "Filing Date"]
            }

            # break up Title feature into Plaintiff and Defendant
            try:
                temp_features["Plaintiff"], temp_features["Defendant"] = \
                    TITLE_SPLIT_PAT.split(temp_features["Title"])

            except ValueError:
                temp_features["Plaintiff"], temp_features["Defendant"] = \
                    ('', '')

            if temp_features["Case Type"].upper() == "FORECLOSURE":
                temp_features["Case Type"] = "Mortgage"

            temp_features["Address"] = \
                str_address if str_address else address[-1]

            temp_features["Zip Code"] = ''.join(ZIP_PAT.findall(address[-1]))

            temp_features["Partial Cost"] = ''.join(
                MONEY_PAT.findall(address[-1])
            )

            scraped_features.append(temp_features)
            temp_features = {}

        if not scraped_features:

            if not path.isfile(NO_CASE):
                with open(NO_CASE, 'w') as no_case_file:
                    dump([], no_case_file)

            with open(NO_CASE, 'r+') as no_case_file:
                no_case_data = load(no_case_file)
                no_case_data.append(str(case_num[:-5]))
                no_case_file.seek(0)
                no_case_file.write(dumps(sorted(list(set(no_case_data)))))
                no_case_file.truncate()

        return scraped_features

    def scan_files(self):

        case_range = trange(len(self.responses), desc='Mining', leave=True) \
            if not self.gui else range(len(self.responses))

        for file_name in case_range:
            with open(
                HTML_FILE.format(case=self.responses[file_name]), 'r'
            ) as html_src:

                if not self.gui:
                    case_range.set_description(
                        "Mining {}".format(self.responses[file_name])
                    )

                feature_list = self.scrape(html_src.read())
                row = self.distribute(self.responses[file_name], feature_list)

                self.dataset.extend(row)

    def export(self):

        file_exists = path.isfile(self.output)

        with open(self.output, 'a') as csv_file:
            writer = DictWriter(
                csv_file,
                fieldnames=[
                    col for col in FIELDS if col not in INTERNAL_FIELDS
                ]
            )

            if not file_exists:
                writer.writeheader()

            for row in self.dataset:
                writer.writerow(row)
<|endoftext|>


<commit_before>import importlib
import inspect
import logging
import pkgutil
from collections import defaultdict

from cloudbridge.cloud import providers
from cloudbridge.cloud.interfaces import CloudProvider
from cloudbridge.cloud.interfaces import TestMockHelperMixin

log = logging.getLogger(__name__)

class ProviderList(object):
    AWS = 'aws'
    AZURE = 'azure'
    GCE = 'gce'
    OPENSTACK = 'openstack'

class CloudProviderFactory(object):

    """
    Get info and handle on the available cloud provider implementations.
    """

    def __init__(self):
        self.provider_list = defaultdict(dict)
        log.debug("Providers List: %s", self.provider_list)

    def register_provider_class(self, cls):
        """
        Registers a provider class with the factory. The class must
        inherit from cloudbridge.cloud.interfaces.CloudProvider
        and also have a class attribute named PROVIDER_ID.

        The PROVIDER_ID is a user friendly name for the cloud provider,
        such as 'aws'. The PROVIDER_ID must also be included in the
        cloudbridge.factory.ProviderList.

        :type  cls: class
        :param cls: A class implementing the CloudProvider interface.
                    Mock providers must also implement
                    :py:class:`cloudbridge.cloud.base.helpers.
                    TestMockHelperMixin`.
        """
        if isinstance(cls, type) and issubclass(cls, CloudProvider):
            if hasattr(cls, "PROVIDER_ID"):
                provider_id = getattr(cls, "PROVIDER_ID")
                if issubclass(cls, TestMockHelperMixin):
                    if self.provider_list.get(provider_id, {}).get(
                            'mock_class'):
                        log.warning("Mock provider with id: %s is already "
                                    "registered. Overriding with class: %s",
                                    provider_id, cls)
                    self.provider_list[provider_id]['mock_class'] = cls
                else:
                    if self.provider_list.get(provider_id, {}).get('class'):
                        log.warning("Provider with id: %s is already "
                                    "registered. Overriding with class: %s",
                                    provider_id, cls)
                    self.provider_list[provider_id]['class'] = cls
            else:
                log.warning("Provider class: %s implements CloudProvider but"
                            " does not define PROVIDER_ID. Ignoring...", cls)
        else:
            log.debug("Class: %s does not implement the CloudProvider"
                      "  interface. Ignoring...", cls)

    def discover_providers(self):
        """
        Discover all available providers within the
        ``cloudbridge.cloud.providers`` package.
        Note that this methods does not guard against a failed import.
        """
        for _, modname, _ in pkgutil.iter_modules(providers.__path__):
            log.debug("Importing provider: %s", modname)
            self._import_provider(modname)

    def _import_provider(self, module_name):
        """
        Imports and registers providers from the given module name.
        Raises an ImportError if the import does not succeed.
        """
        log.debug("Importing providers from %s", module_name)
        module = importlib.import_module(
            "{0}.{1}".format(providers.__name__,
                             module_name))
        classes = inspect.getmembers(module, inspect.isclass)
        for _, cls in classes:
            log.debug("Registering the provider: %s", cls)
            self.register_provider_class(cls)

    def list_providers(self):
        """
        Get a list of available providers.

        It uses a simple automatic discovery system by iterating through all
        submodules in cloudbridge.cloud.providers.

        :rtype: dict
        :return: A dict of available providers and their implementations in the
                 following format::
                 {'aws': {'class': aws.provider.AWSCloudProvider,
                          'mock_class': aws.provider.MockAWSCloudProvider},
                  'openstack': {'class': openstack.provider.OpenStackCloudProvi
                                         der}
                 }
        """
        if not self.provider_list:
            self.discover_providers()
        log.debug("List of available providers: %s", self.provider_list)
        return self.provider_list

    def create_provider(self, name, config):
        """
        Searches all available providers for a CloudProvider interface with the
        given name, and instantiates it based on the given config dictionary,
        where the config dictionary is a dictionary understood by that
        cloud provider.

        :type name: str
        :param name: Cloud provider name: one of ``aws``, ``openstack``,
        ``azure``.

        :type config: :class:`dict`
        :param config: A dictionary or an iterable of key/value pairs (as tuples
                       or other iterables of length two). See specific provider
                       implementation for the required fields.

        :return:  a concrete provider instance
        :rtype: ``object`` of :class:`.CloudProvider`
        """
        log.info("Creating '%s' provider", name)
        provider_class = self.get_provider_class(name)
        if provider_class is None:
            log.exception("A provider with the name %s could not "
                          "be found", name)
            raise NotImplementedError(
                'A provider with name {0} could not be'
                ' found'.format(name))
        log.debug("Created '%s' provider", name)
        return provider_class(config)

    def get_provider_class(self, name, get_mock=False):
        """
        Return a class for the requested provider.

        :type get_mock: ``bool``
        :param get_mock: If True, returns a mock version of the provider
        if available, or the real version if not.

        :rtype: provider class or ``None``
        :return: A class corresponding to the requested provider or ``None``
                 if the provider was not found.
        """
        log.debug("Returning a class for the %s provider", name)
        impl = self.list_providers().get(name)
        if impl:
            if get_mock and impl.get("mock_class"):
                log.debug("param get_mock set to True, returning "
                          "a mock version of the provider %s", name)
                return impl["mock_class"]
            else:
                log.debug("Returning the real version of %s", name)
                return impl["class"]
        else:
            log.debug("Provider with the name: %s not found", name)
            return None

    def get_all_provider_classes(self, get_mock=False):
        """
        Returns a list of classes for all available provider implementations

        :type get_mock: ``bool``
        :param get_mock: If True, returns a mock version of the provider
        if available, or the real version if not.

        :rtype: type ``class`` or ``None``
        :return: A list of all available provider classes or an empty list
        if none found.
        """
        all_providers = []
        for impl in self.list_providers().values():
            if get_mock and impl.get("mock_class"):
                log.debug("param get_mock set to True, appending "
                          "a mock version of the provider %s", impl)
                all_providers.append(impl["mock_class"])
            else:
                all_providers.append(impl["class"])
        log.info("List of provider classes: %s", all_providers)
        return all_providers
<commit_msg>Fixed flake8 error<commit_after>import importlib
import inspect
import logging
import pkgutil
from collections import defaultdict

from cloudbridge.cloud import providers
from cloudbridge.cloud.interfaces import CloudProvider
from cloudbridge.cloud.interfaces import TestMockHelperMixin

log = logging.getLogger(__name__)

class ProviderList(object):
    AWS = 'aws'
    AZURE = 'azure'
    GCE = 'gce'
    OPENSTACK = 'openstack'

class CloudProviderFactory(object):

    """
    Get info and handle on the available cloud provider implementations.
    """

    def __init__(self):
        self.provider_list = defaultdict(dict)
        log.debug("Providers List: %s", self.provider_list)

    def register_provider_class(self, cls):
        """
        Registers a provider class with the factory. The class must
        inherit from cloudbridge.cloud.interfaces.CloudProvider
        and also have a class attribute named PROVIDER_ID.

        The PROVIDER_ID is a user friendly name for the cloud provider,
        such as 'aws'. The PROVIDER_ID must also be included in the
        cloudbridge.factory.ProviderList.

        :type  cls: class
        :param cls: A class implementing the CloudProvider interface.
                    Mock providers must also implement
                    :py:class:`cloudbridge.cloud.base.helpers.
                    TestMockHelperMixin`.
        """
        if isinstance(cls, type) and issubclass(cls, CloudProvider):
            if hasattr(cls, "PROVIDER_ID"):
                provider_id = getattr(cls, "PROVIDER_ID")
                if issubclass(cls, TestMockHelperMixin):
                    if self.provider_list.get(provider_id, {}).get(
                            'mock_class'):
                        log.warning("Mock provider with id: %s is already "
                                    "registered. Overriding with class: %s",
                                    provider_id, cls)
                    self.provider_list[provider_id]['mock_class'] = cls
                else:
                    if self.provider_list.get(provider_id, {}).get('class'):
                        log.warning("Provider with id: %s is already "
                                    "registered. Overriding with class: %s",
                                    provider_id, cls)
                    self.provider_list[provider_id]['class'] = cls
            else:
                log.warning("Provider class: %s implements CloudProvider but"
                            " does not define PROVIDER_ID. Ignoring...", cls)
        else:
            log.debug("Class: %s does not implement the CloudProvider"
                      "  interface. Ignoring...", cls)

    def discover_providers(self):
        """
        Discover all available providers within the
        ``cloudbridge.cloud.providers`` package.
        Note that this methods does not guard against a failed import.
        """
        for _, modname, _ in pkgutil.iter_modules(providers.__path__):
            log.debug("Importing provider: %s", modname)
            self._import_provider(modname)

    def _import_provider(self, module_name):
        """
        Imports and registers providers from the given module name.
        Raises an ImportError if the import does not succeed.
        """
        log.debug("Importing providers from %s", module_name)
        module = importlib.import_module(
            "{0}.{1}".format(providers.__name__,
                             module_name))
        classes = inspect.getmembers(module, inspect.isclass)
        for _, cls in classes:
            log.debug("Registering the provider: %s", cls)
            self.register_provider_class(cls)

    def list_providers(self):
        """
        Get a list of available providers.

        It uses a simple automatic discovery system by iterating through all
        submodules in cloudbridge.cloud.providers.

        :rtype: dict
        :return: A dict of available providers and their implementations in the
                 following format::
                 {'aws': {'class': aws.provider.AWSCloudProvider,
                          'mock_class': aws.provider.MockAWSCloudProvider},
                  'openstack': {'class': openstack.provider.OpenStackCloudProvi
                                         der}
                 }
        """
        if not self.provider_list:
            self.discover_providers()
        log.debug("List of available providers: %s", self.provider_list)
        return self.provider_list

    def create_provider(self, name, config):
        """
        Searches all available providers for a CloudProvider interface with the
        given name, and instantiates it based on the given config dictionary,
        where the config dictionary is a dictionary understood by that
        cloud provider.

        :type name: str
        :param name: Cloud provider name: one of ``aws``, ``openstack``,
        ``azure``.

        :type config: :class:`dict`
        :param config: A dictionary or an iterable of key/value pairs (as
                       tuples or other iterables of length two). See specific
                       provider implementation for the required fields.

        :return:  a concrete provider instance
        :rtype: ``object`` of :class:`.CloudProvider`
        """
        log.info("Creating '%s' provider", name)
        provider_class = self.get_provider_class(name)
        if provider_class is None:
            log.exception("A provider with the name %s could not "
                          "be found", name)
            raise NotImplementedError(
                'A provider with name {0} could not be'
                ' found'.format(name))
        log.debug("Created '%s' provider", name)
        return provider_class(config)

    def get_provider_class(self, name, get_mock=False):
        """
        Return a class for the requested provider.

        :type get_mock: ``bool``
        :param get_mock: If True, returns a mock version of the provider
        if available, or the real version if not.

        :rtype: provider class or ``None``
        :return: A class corresponding to the requested provider or ``None``
                 if the provider was not found.
        """
        log.debug("Returning a class for the %s provider", name)
        impl = self.list_providers().get(name)
        if impl:
            if get_mock and impl.get("mock_class"):
                log.debug("param get_mock set to True, returning "
                          "a mock version of the provider %s", name)
                return impl["mock_class"]
            else:
                log.debug("Returning the real version of %s", name)
                return impl["class"]
        else:
            log.debug("Provider with the name: %s not found", name)
            return None

    def get_all_provider_classes(self, get_mock=False):
        """
        Returns a list of classes for all available provider implementations

        :type get_mock: ``bool``
        :param get_mock: If True, returns a mock version of the provider
        if available, or the real version if not.

        :rtype: type ``class`` or ``None``
        :return: A list of all available provider classes or an empty list
        if none found.
        """
        all_providers = []
        for impl in self.list_providers().values():
            if get_mock and impl.get("mock_class"):
                log.debug("param get_mock set to True, appending "
                          "a mock version of the provider %s", impl)
                all_providers.append(impl["mock_class"])
            else:
                all_providers.append(impl["class"])
        log.info("List of provider classes: %s", all_providers)
        return all_providers
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""Qt dock window."""

# -----------------------------------------------------------------------------
# Imports
# -----------------------------------------------------------------------------

import sys

from PyQt4 import QtCore, QtGui
from vispy import app

from ._misc import _is_interactive

# -----------------------------------------------------------------------------
# Dock main window
# -----------------------------------------------------------------------------

class DockWindow(QtGui.QMainWindow):
    def __init__(self,
                 position=(1100, 200),
                 size=(800, 600),
                 title=None,
                 ):
        super(DockWindow, self).__init__()
        if title is None:
            title = 'phy'
        self.setWindowTitle(title)
        self.move(position[0], position[1])
        self.resize(QtCore.QSize(size[0], size[1]))
        self.setObjectName("MainWindow")
        QtCore.QMetaObject.connectSlotsByName(self)
        self.setDockOptions(QtGui.QMainWindow.AllowTabbedDocks |
                            QtGui.QMainWindow.AllowNestedDocks |
                            QtGui.QMainWindow.AnimatedDocks
                            )

    def add_action(self,
                   text,
                   callback=None,
                   shortcut=None,
                   checkable=False,
                   checked=False,
                   ):
        """Add an action with a keyboard shortcut."""
        action = QtGui.QAction(text, self)
        action.triggered.connect(callback)
        action.setShortcut(shortcut)
        action.setCheckable(checkable)
        action.setChecked(checked)
        self.addAction(action)
        return action

    def add_view(self,
                 view,
                 title='view',
                 position='right',
                 closable=True,
                 floatable=True,
                 floating=None,
                 **kwargs):
        """Add a widget to the main window."""

        if isinstance(view, app.Canvas):
            view = view.native

        # Create the dock widget.
        dockwidget = QtGui.QDockWidget(self)
        dockwidget.setWindowTitle(title)
        dockwidget.setWidget(view)

        # Set dock widget options.
        options = QtGui.QDockWidget.DockWidgetMovable
        if closable:
            options = options | QtGui.QDockWidget.DockWidgetClosable
        if floatable:
            options = options | QtGui.QDockWidget.DockWidgetFloatable

        dockwidget.setFeatures(options)
        dockwidget.setAllowedAreas(QtCore.Qt.LeftDockWidgetArea |
                                   QtCore.Qt.RightDockWidgetArea |
                                   QtCore.Qt.TopDockWidgetArea |
                                   QtCore.Qt.BottomDockWidgetArea
                                   )

        q_position = {
            'left': QtCore.Qt.LeftDockWidgetArea,
            'right': QtCore.Qt.RightDockWidgetArea,
            'top': QtCore.Qt.TopDockWidgetArea,
            'bottom': QtCore.Qt.BottomDockWidgetArea,
        }[position]
        self.addDockWidget(q_position, dockwidget)
        if floating is not None:
            dockwidget.setFloating(floating)
        dockwidget.show()
        return dockwidget

    def list_views(self, title=''):
        """List all views which title start with a given string."""
        children = self.findChildren(QtGui.QWidget)
        return [child for child in children
                if child.windowTitle().startswith(title)]

    def shortcut(self, text, key):
        """Decorator to add a global keyboard shortcut."""
        def wrap(func):
            self.add_action(text, shortcut=key, callback=func)
        return wrap

# -----------------------------------------------------------------------------
# Qt app and event loop integration with IPython
# -----------------------------------------------------------------------------

_APP = None

def _close_qt_after(window, duration):
    """Close a Qt window after a given duration."""
    def callback():
        window.close()
    QtCore.QTimer.singleShot(int(1000 * duration), callback)

def _try_enable_ipython_qt():
    """Try to enable IPython Qt event loop integration.

    Returns True in the following cases:

    * python -i test.py
    * ipython -i test.py
    * ipython and %run test.py

    Returns False in the following cases:

    * python test.py
    * ipython test.py

    """
    try:
        from IPython import get_ipython
        ip = get_ipython()
    except ImportError:
        return False
    if not _is_interactive():
        return False
    if ip:
        qt_app = ip.enable_gui('qt')
        if qt_app:
            return qt_app
    return False

def start_qt_app():
    """Start a Qt application if necessary.

    If a new Qt application is created, this function returns it.
    If no new application is created, the function returns None.

    """
    # Only start a Qt application if there is no
    # IPython event loop integration.
    global _APP
    if _try_enable_ipython_qt():
        return
    if QtGui.QApplication.instance():
        _APP = QtGui.QApplication.instance()
        return
    if _APP:
        return
    app.use_app("pyqt4")
    _APP = QtGui.QApplication(sys.argv)
    return _APP

def run_qt_app():
    """Start the Qt application's event loop."""
    if _APP is not None:
        _APP.exec_()
<commit_msg>Fix for Python 2.<commit_after># -*- coding: utf-8 -*-

"""Qt dock window."""

# -----------------------------------------------------------------------------
# Imports
# -----------------------------------------------------------------------------

import sys

from PyQt4 import QtCore, QtGui
from QtGui import QMainWindow
from vispy import app

from ._misc import _is_interactive

# -----------------------------------------------------------------------------
# Dock main window
# -----------------------------------------------------------------------------

class DockWindow(QMainWindow):
    def __init__(self,
                 position=(1100, 200),
                 size=(800, 600),
                 title=None,
                 ):
        super(DockWindow, self).__init__()
        if title is None:
            title = 'phy'
        self.setWindowTitle(title)
        self.move(position[0], position[1])
        self.resize(QtCore.QSize(size[0], size[1]))
        self.setObjectName("MainWindow")
        QtCore.QMetaObject.connectSlotsByName(self)
        self.setDockOptions(QtGui.QMainWindow.AllowTabbedDocks |
                            QtGui.QMainWindow.AllowNestedDocks |
                            QtGui.QMainWindow.AnimatedDocks
                            )

    def add_action(self,
                   text,
                   callback=None,
                   shortcut=None,
                   checkable=False,
                   checked=False,
                   ):
        """Add an action with a keyboard shortcut."""
        action = QtGui.QAction(text, self)
        action.triggered.connect(callback)
        action.setShortcut(shortcut)
        action.setCheckable(checkable)
        action.setChecked(checked)
        self.addAction(action)
        return action

    def add_view(self,
                 view,
                 title='view',
                 position='right',
                 closable=True,
                 floatable=True,
                 floating=None,
                 **kwargs):
        """Add a widget to the main window."""

        if isinstance(view, app.Canvas):
            view = view.native

        # Create the dock widget.
        dockwidget = QtGui.QDockWidget(self)
        dockwidget.setWindowTitle(title)
        dockwidget.setWidget(view)

        # Set dock widget options.
        options = QtGui.QDockWidget.DockWidgetMovable
        if closable:
            options = options | QtGui.QDockWidget.DockWidgetClosable
        if floatable:
            options = options | QtGui.QDockWidget.DockWidgetFloatable

        dockwidget.setFeatures(options)
        dockwidget.setAllowedAreas(QtCore.Qt.LeftDockWidgetArea |
                                   QtCore.Qt.RightDockWidgetArea |
                                   QtCore.Qt.TopDockWidgetArea |
                                   QtCore.Qt.BottomDockWidgetArea
                                   )

        q_position = {
            'left': QtCore.Qt.LeftDockWidgetArea,
            'right': QtCore.Qt.RightDockWidgetArea,
            'top': QtCore.Qt.TopDockWidgetArea,
            'bottom': QtCore.Qt.BottomDockWidgetArea,
        }[position]
        self.addDockWidget(q_position, dockwidget)
        if floating is not None:
            dockwidget.setFloating(floating)
        dockwidget.show()
        return dockwidget

    def list_views(self, title=''):
        """List all views which title start with a given string."""
        children = self.findChildren(QtGui.QWidget)
        return [child for child in children
                if str(child.windowTitle()).startswith(title)]

    def shortcut(self, text, key):
        """Decorator to add a global keyboard shortcut."""
        def wrap(func):
            self.add_action(text, shortcut=key, callback=func)
        return wrap

# -----------------------------------------------------------------------------
# Qt app and event loop integration with IPython
# -----------------------------------------------------------------------------

_APP = None

def _close_qt_after(window, duration):
    """Close a Qt window after a given duration."""
    def callback():
        window.close()
    QtCore.QTimer.singleShot(int(1000 * duration), callback)

def _try_enable_ipython_qt():
    """Try to enable IPython Qt event loop integration.

    Returns True in the following cases:

    * python -i test.py
    * ipython -i test.py
    * ipython and %run test.py

    Returns False in the following cases:

    * python test.py
    * ipython test.py

    """
    try:
        from IPython import get_ipython
        ip = get_ipython()
    except ImportError:
        return False
    if not _is_interactive():
        return False
    if ip:
        qt_app = ip.enable_gui('qt')
        if qt_app:
            return qt_app
    return False

def start_qt_app():
    """Start a Qt application if necessary.

    If a new Qt application is created, this function returns it.
    If no new application is created, the function returns None.

    """
    # Only start a Qt application if there is no
    # IPython event loop integration.
    global _APP
    if _try_enable_ipython_qt():
        return
    if QtGui.QApplication.instance():
        _APP = QtGui.QApplication.instance()
        return
    if _APP:
        return
    app.use_app("pyqt4")
    _APP = QtGui.QApplication(sys.argv)
    return _APP

def run_qt_app():
    """Start the Qt application's event loop."""
    if _APP is not None:
        _APP.exec_()
<|endoftext|>


<commit_before>import unittest

import io

class test_LoadFile(unittest.TestCase):

	def test_loadFile_filenameType(self):
		with self.assertRaises(TypeError):
			io.loadFile(42)

	def test_loadFile_verboseType(self):
		with self.assertRaises(TypeError):
			io.loadFile("filename", 42)

	def test_loadFile_nonExistentFile(self):
		with self.assertRaises(IOError):
			io.loadFile("file_does_not_exist")

	def test_loadFile_directoryNotFile(self):
		with self.assertRaises(IOError):
			io.loadFile("examples")

if __name__ == '__main__':
	unittest.main()
<commit_msg>added tests for key extraction & sweeping<commit_after>import unittest

import io

class test_LoadFile(unittest.TestCase):

	def test_loadFile_filenameType(self):
		with self.assertRaises(TypeError):
			io.loadFile(42)

	def test_loadFile_verboseType(self):
		with self.assertRaises(TypeError):
			io.loadFile("filename", 42)

	def test_loadFile_nonExistentFile(self):
		with self.assertRaises(IOError):
			io.loadFile("file_does_not_exist")

	def test_loadFile_directoryNotFile(self):
		with self.assertRaises(IOError):
			io.loadFile("examples")

	def test_loadFile_exampleValues(self):
		# check the length
		pdDF = io.loadFile("examples/example[exampleSample]/IMG_Data/example.phylodist")
		self.assertEquals(len(pdDF.index), 59476)
		# spot check row 98's locus_tag
		self.assertEquals(pdDF.at[97,'locus_tag'], "Ga0066528_1383641")

class test_defaultKeyExtractionFunction(unittest.TestCase):

	def test_defaultKeyExtractionFunction_value(self):
		self.assertEquals(io.defaultKeyExtractionFunction("examples/example[exampleSample]/IMG_Data/example.phylodist"), "exampleSample")

class test_sweepFiles(unittest.TestCase):

	def test_sweepFiles_fileNotDirectory(self):
		with self.assertRaises(IOError):
			io.sweepFiles("phylodist/test.py")

	def test_sweepFiles_nonExistentDir(self):
		with self.assertRaises(IOError):
			io.sweepFiles("directory_does_not_exist")

	def test_sweepFiles_directoryType(self):
		with self.assertRaises(TypeError):
			io.sweepFiles(42)

	def test_sweepFiles_values(self):
		pdSD = io.sweepFiles("examples");
		# check the length of the dictionary
		self.assertEquals(len(pdSD), 2)
		# verify the sample key names are correct
		for key in pdSD.keys():
			self.assertIn(key, ["exampleSample", "anotherExampleSample"])
		# spot check specific values in the dictionary's data frame
		self.assertEquals(pdSD['exampleSample'].at[97, 'locus_tag'], "Ga0066528_1383641")
		self.assertEquals(pdSD['anotherExampleSample'].at[1013, 'locus_tag'], "Ga0066521_1490921")

if __name__ == '__main__':
	unittest.main()
<|endoftext|>


<commit_before>"""
physt
=====

P(i/y)thon h(i/y)stograms. Inspired (and based on) numpy.histogram,
but designed for humans(TM) on steroids(TM).

(C) Jan Pipek, 2016-7, MIT licence
See https://github.com/janpipek/physt
"""

from __future__ import absolute_import
from . import binnings

__version__ = str('0.3.33')

def histogram(data, bins=None, *args, **kwargs):
    """Facade function to create 1D histograms.

    This proceeds in three steps:
    1) Based on magical parameter bins, construct bins for the histogram
    2) Calculate frequencies for the bins
    3) Construct the histogram object itself

    *Guiding principle:* parameters understood by numpy.histogram should be
    understood also by physt.histogram as well and should result in a Histogram1D
    object with (h.numpy_bins, h.frequencies) same as the numpy.histogram
    output. Additional functionality is a bonus.

    This function is also aliased as "h1".

    Parameters
    ----------
    data : array_like, optional
        Container of all the values (tuple, list, np.ndarray, pd.Series)
    bins: int or sequence of scalars or callable or str, optional
        If iterable => the bins themselves
        If int => number of bins for default binning
        If callable => use binning method (+ args, kwargs)
        If string => use named binning method (+ args, kwargs)
    weights: array_like, optional
        (as numpy.histogram)
    keep_missed: Optional[bool]
        store statistics about how many values were lower than limits
        and how many higher than limits (default: True)
    dropna: bool
        whether to clear data from nan's before histogramming
    name: str
        name of the histogram
    axis_name: str
        name of the variable on x axis
    adaptive: bool
        whether we want the bins to be modifiable
        (useful for continuous filling of a priori unknown data)
    dtype: type
        customize underlying data type: default int64 (without weight) or float (with weights)

    Other numpy.histogram parameters are excluded, see the methods of the Histogram1D class itself.

    Returns
    -------
    physt.histogram1d.Histogram1D

    See Also
    --------
    numpy.histogram
    """
    import numpy as np
    from .histogram1d import Histogram1D, calculate_frequencies
    from .binnings import calculate_bins

    adaptive = kwargs.pop("adaptive", False)
    dtype = kwargs.pop("dtype", None)

    if isinstance(data, tuple) and isinstance(data[0], str):    # Works for groupby DataSeries
        return histogram(data[1], bins, *args, name=data[0], **kwargs)
    elif type(data).__name__ == "DataFrame":
        raise RuntimeError("Cannot create histogram from a pandas DataFrame. Use Series.")

    # Collect arguments (not to send them to binning algorithms)
    dropna = kwargs.pop("dropna", False)
    weights = kwargs.pop("weights", None)
    keep_missed = kwargs.pop("keep_missed", True)
    name = kwargs.pop("name", None)
    axis_name = kwargs.pop("axis_name", None)

    # Convert to array
    if data is not None:
        array = np.asarray(data) #.flatten()
        if dropna:
            array = array[~np.isnan(array)]
    else:
        array = None

    # Get binning
    binning = calculate_bins(array, bins, *args,
                             check_nan=not dropna and array is not None,
                             adaptive=adaptive, **kwargs)
    # bins = binning.bins

    # Get frequencies
    if array is not None:
        (frequencies, errors2, underflow, overflow, stats) =\
            calculate_frequencies(array, binning=binning,
                                  weights=weights, dtype=dtype)
    else:
        frequencies = None
        errors2 = None
        underflow = 0
        overflow = 0
        stats = {"sum": 0.0, "sum2": 0.0}

    # Construct the object
    if not keep_missed:
        underflow = 0
        overflow = 0
    if not axis_name:
        if hasattr(data, "name"):
            axis_name = data.name
        elif hasattr(data, "fields") and len(data.fields) == 1 and isinstance(data.fields[0], str):
            # Case of dask fields (examples)
            axis_name = data.fields[0]
    return Histogram1D(binning=binning, frequencies=frequencies,
                       errors2=errors2, overflow=overflow,
                       underflow=underflow, stats=stats, dtype=dtype,
                       keep_missed=keep_missed, name=name, axis_name=axis_name)

def histogram2d(data1, data2, bins=10, *args, **kwargs):
    """Facade function to create 2D histograms.

    For implementation and parameters, see histogramdd.

    This function is also aliased as "h2".

    Returns
    -------
    physt.histogram_nd.Histogram2D

    See Also
    --------
    numpy.histogram2d
    histogramdd
    """
    import numpy as np

    # guess axis names
    if "axis_names" not in kwargs:
        if hasattr(data1, "name") and hasattr(data2, "name"):
            kwargs["axis_names"] = [data1.name, data2.name]
    if data1 is not None and data2 is not None:
        data1 = np.asarray(data1)
        data2 = np.asarray(data2)
        data = np.concatenate([data1[:, np.newaxis],
                               data2[:, np.newaxis]], axis=1)
    else:
        data = None
    return histogramdd(data, bins, *args, dim=2, **kwargs)

def histogramdd(data, bins=10, *args, **kwargs):
    """Facade function to create n-dimensional histograms.

    3D variant of this function is also aliased as "h3".

    Parameters
    ----------
    data : array_like
        Container of all the values
    bins: Any
    weights: array_like, optional
        (as numpy.histogram)
    dropna: bool
        whether to clear data from nan's before histogramming
    name: str
        name of the histogram
    axis_names: Iterable[str]
        names of the variable on x axis
    adaptive:
        whether the bins should be updated when new non-fitting value are filled
    dtype: Optional[type]
        Underlying type for the histogram.
        If weights are specified, default is float. Otherwise int64
    dim: int
        Dimension - necessary if you are creating an empty adaptive histogram

    Returns
    -------
    physt.histogram_nd.HistogramND

    See Also
    --------
    numpy.histogramdd
    """
    import numpy as np
    from . import histogram_nd
    from .binnings import calculate_bins_nd

    adaptive = kwargs.pop("adaptive", False)
    dropna = kwargs.pop("dropna", False)
    name = kwargs.pop("name", None)
    dim = kwargs.pop("dim", None)
    axis_names = kwargs.pop("axis_names", None)

    # pandas - guess axis names
    if not "axis_names" in kwargs:
        if hasattr(data, "columns"):
            try:
                kwargs["axis_names"] = tuple(data.columns)
            except:
                pass # Perhaps columns has different meaning here.

    # Prepare and check data
    # Convert to array
    if data is not None:
        data = np.asarray(data)
        if data.ndim != 2:
            raise RuntimeError("Array must have shape (n, d)")
        if dim is not None and dim != data.shape[1]:
            raise RuntimeError("Dimension mismatch: {0}!={1}".format(dim, data.shape[1]))
        _, dim = data.shape
        if dropna:
            data = data[~np.isnan(data).any(axis=1)]
        check_nan = not dropna
    else:
        if dim is None:
            raise RuntimeError("You have to specify either data or its dimension.")
        data = np.zeros((0, dim))
        check_nan = False

    # Prepare bins
    bin_schemas = calculate_bins_nd(data, bins, *args, check_nan=check_nan, adaptive=adaptive,
                                    **kwargs)
    #bins = [binning.bins for binning in bin_schemas]

    # Prepare remaining data
    weights = kwargs.pop("weights", None)
    frequencies, errors2, missed = histogram_nd.calculate_frequencies(data, ndim=dim,
                                                                      binnings=bin_schemas,
                                                                      weights=weights)

    kwargs["name"] = name
    if axis_names:
        kwargs["axis_names"] = axis_names
    if dim == 2:
        return histogram_nd.Histogram2D(binnings=bin_schemas, frequencies=frequencies,
                                        errors2=errors2, **kwargs)
    else:
        return histogram_nd.HistogramND(dimension=dim, binnings=bin_schemas,
                                        frequencies=frequencies, errors2=errors2, **kwargs)

# Aliases
h1 = histogram
h2 = histogram2d
h = histogramdd

def h3(data, *args, **kwargs):
    """Facade function to create 3D histograms.

    Parameters
    ----------
    data : array_like or list[array_like] or tuple[array_like]
        Can be a single array (with three columns) or three different arrays
        (for each component)

    Returns
    -------
    physt.histogram_nd.HistogramND
    """
    import numpy as np

    if data is not None and isinstance(data, (list, tuple)) and not np.isscalar(data[0]):
        if "axis_names" not in kwargs:
            kwargs["axis_names"] = [(column.name if hasattr(column, "name") else None) for column in data]
        data = np.concatenate([item[:, np.newaxis] for item in data], axis=1)
    elif data is not None:
        data = np.asarray(data)
        n, dim = data.shape
        if dim != 3:
            raise RuntimeError("Array must have shape (n, 3)")
    else:
        kwargs["dim"] = 3
    return histogramdd(data, *args, **kwargs)

from .special import polar_histogram
<commit_msg>dropna by default<commit_after>"""
physt
=====

P(i/y)thon h(i/y)stograms. Inspired (and based on) numpy.histogram,
but designed for humans(TM) on steroids(TM).

(C) Jan Pipek, 2016-7, MIT licence
See https://github.com/janpipek/physt
"""

from __future__ import absolute_import
from . import binnings

__version__ = str('0.3.33')

def histogram(data, bins=None, *args, **kwargs):
    """Facade function to create 1D histograms.

    This proceeds in three steps:
    1) Based on magical parameter bins, construct bins for the histogram
    2) Calculate frequencies for the bins
    3) Construct the histogram object itself

    *Guiding principle:* parameters understood by numpy.histogram should be
    understood also by physt.histogram as well and should result in a Histogram1D
    object with (h.numpy_bins, h.frequencies) same as the numpy.histogram
    output. Additional functionality is a bonus.

    This function is also aliased as "h1".

    Parameters
    ----------
    data : array_like, optional
        Container of all the values (tuple, list, np.ndarray, pd.Series)
    bins: int or sequence of scalars or callable or str, optional
        If iterable => the bins themselves
        If int => number of bins for default binning
        If callable => use binning method (+ args, kwargs)
        If string => use named binning method (+ args, kwargs)
    weights: array_like, optional
        (as numpy.histogram)
    keep_missed: Optional[bool]
        store statistics about how many values were lower than limits
        and how many higher than limits (default: True)
    dropna: bool
        whether to clear data from nan's before histogramming
    name: str
        name of the histogram
    axis_name: str
        name of the variable on x axis
    adaptive: bool
        whether we want the bins to be modifiable
        (useful for continuous filling of a priori unknown data)
    dtype: type
        customize underlying data type: default int64 (without weight) or float (with weights)

    Other numpy.histogram parameters are excluded, see the methods of the Histogram1D class itself.

    Returns
    -------
    physt.histogram1d.Histogram1D

    See Also
    --------
    numpy.histogram
    """
    import numpy as np
    from .histogram1d import Histogram1D, calculate_frequencies
    from .binnings import calculate_bins

    adaptive = kwargs.pop("adaptive", False)
    dtype = kwargs.pop("dtype", None)

    if isinstance(data, tuple) and isinstance(data[0], str):    # Works for groupby DataSeries
        return histogram(data[1], bins, *args, name=data[0], **kwargs)
    elif type(data).__name__ == "DataFrame":
        raise RuntimeError("Cannot create histogram from a pandas DataFrame. Use Series.")

    # Collect arguments (not to send them to binning algorithms)
    dropna = kwargs.pop("dropna", True)
    weights = kwargs.pop("weights", None)
    keep_missed = kwargs.pop("keep_missed", True)
    name = kwargs.pop("name", None)
    axis_name = kwargs.pop("axis_name", None)

    # Convert to array
    if data is not None:
        array = np.asarray(data) #.flatten()
        if dropna:
            array = array[~np.isnan(array)]
    else:
        array = None

    # Get binning
    binning = calculate_bins(array, bins, *args,
                             check_nan=not dropna and array is not None,
                             adaptive=adaptive, **kwargs)
    # bins = binning.bins

    # Get frequencies
    if array is not None:
        (frequencies, errors2, underflow, overflow, stats) =\
            calculate_frequencies(array, binning=binning,
                                  weights=weights, dtype=dtype)
    else:
        frequencies = None
        errors2 = None
        underflow = 0
        overflow = 0
        stats = {"sum": 0.0, "sum2": 0.0}

    # Construct the object
    if not keep_missed:
        underflow = 0
        overflow = 0
    if not axis_name:
        if hasattr(data, "name"):
            axis_name = data.name
        elif hasattr(data, "fields") and len(data.fields) == 1 and isinstance(data.fields[0], str):
            # Case of dask fields (examples)
            axis_name = data.fields[0]
    return Histogram1D(binning=binning, frequencies=frequencies,
                       errors2=errors2, overflow=overflow,
                       underflow=underflow, stats=stats, dtype=dtype,
                       keep_missed=keep_missed, name=name, axis_name=axis_name)

def histogram2d(data1, data2, bins=10, *args, **kwargs):
    """Facade function to create 2D histograms.

    For implementation and parameters, see histogramdd.

    This function is also aliased as "h2".

    Returns
    -------
    physt.histogram_nd.Histogram2D

    See Also
    --------
    numpy.histogram2d
    histogramdd
    """
    import numpy as np

    # guess axis names
    if "axis_names" not in kwargs:
        if hasattr(data1, "name") and hasattr(data2, "name"):
            kwargs["axis_names"] = [data1.name, data2.name]
    if data1 is not None and data2 is not None:
        data1 = np.asarray(data1)
        data2 = np.asarray(data2)
        data = np.concatenate([data1[:, np.newaxis],
                               data2[:, np.newaxis]], axis=1)
    else:
        data = None
    return histogramdd(data, bins, *args, dim=2, **kwargs)

def histogramdd(data, bins=10, *args, **kwargs):
    """Facade function to create n-dimensional histograms.

    3D variant of this function is also aliased as "h3".

    Parameters
    ----------
    data : array_like
        Container of all the values
    bins: Any
    weights: array_like, optional
        (as numpy.histogram)
    dropna: bool
        whether to clear data from nan's before histogramming
    name: str
        name of the histogram
    axis_names: Iterable[str]
        names of the variable on x axis
    adaptive:
        whether the bins should be updated when new non-fitting value are filled
    dtype: Optional[type]
        Underlying type for the histogram.
        If weights are specified, default is float. Otherwise int64
    dim: int
        Dimension - necessary if you are creating an empty adaptive histogram

    Returns
    -------
    physt.histogram_nd.HistogramND

    See Also
    --------
    numpy.histogramdd
    """
    import numpy as np
    from . import histogram_nd
    from .binnings import calculate_bins_nd

    adaptive = kwargs.pop("adaptive", False)
    dropna = kwargs.pop("dropna", True)
    name = kwargs.pop("name", None)
    dim = kwargs.pop("dim", None)
    axis_names = kwargs.pop("axis_names", None)

    # pandas - guess axis names
    if not "axis_names" in kwargs:
        if hasattr(data, "columns"):
            try:
                kwargs["axis_names"] = tuple(data.columns)
            except:
                pass # Perhaps columns has different meaning here.

    # Prepare and check data
    # Convert to array
    if data is not None:
        data = np.asarray(data)
        if data.ndim != 2:
            raise RuntimeError("Array must have shape (n, d)")
        if dim is not None and dim != data.shape[1]:
            raise RuntimeError("Dimension mismatch: {0}!={1}".format(dim, data.shape[1]))
        _, dim = data.shape
        if dropna:
            data = data[~np.isnan(data).any(axis=1)]
        check_nan = not dropna
    else:
        if dim is None:
            raise RuntimeError("You have to specify either data or its dimension.")
        data = np.zeros((0, dim))
        check_nan = False

    # Prepare bins
    bin_schemas = calculate_bins_nd(data, bins, *args, check_nan=check_nan, adaptive=adaptive,
                                    **kwargs)
    #bins = [binning.bins for binning in bin_schemas]

    # Prepare remaining data
    weights = kwargs.pop("weights", None)
    frequencies, errors2, missed = histogram_nd.calculate_frequencies(data, ndim=dim,
                                                                      binnings=bin_schemas,
                                                                      weights=weights)

    kwargs["name"] = name
    if axis_names:
        kwargs["axis_names"] = axis_names
    if dim == 2:
        return histogram_nd.Histogram2D(binnings=bin_schemas, frequencies=frequencies,
                                        errors2=errors2, **kwargs)
    else:
        return histogram_nd.HistogramND(dimension=dim, binnings=bin_schemas,
                                        frequencies=frequencies, errors2=errors2, **kwargs)

# Aliases
h1 = histogram
h2 = histogram2d
h = histogramdd

def h3(data, *args, **kwargs):
    """Facade function to create 3D histograms.

    Parameters
    ----------
    data : array_like or list[array_like] or tuple[array_like]
        Can be a single array (with three columns) or three different arrays
        (for each component)

    Returns
    -------
    physt.histogram_nd.HistogramND
    """
    import numpy as np

    if data is not None and isinstance(data, (list, tuple)) and not np.isscalar(data[0]):
        if "axis_names" not in kwargs:
            kwargs["axis_names"] = [(column.name if hasattr(column, "name") else None) for column in data]
        data = np.concatenate([item[:, np.newaxis] for item in data], axis=1)
    elif data is not None:
        data = np.asarray(data)
        n, dim = data.shape
        if dim != 3:
            raise RuntimeError("Array must have shape (n, 3)")
    else:
        kwargs["dim"] = 3
    return histogramdd(data, *args, **kwargs)

from .special import polar_histogram
<|endoftext|>


<commit_before>"""
    picasso.render
    ~~~~~~~~~~~~~~

    Render single molecule localizations to a super-resolution image

    :author: Joerg Schnitzbauer, 2015
    :copyright: Copyright (c) 2015 Jungmann Lab, MPI of Biochemistry
"""
import numpy as _np
import numba as _numba
import scipy.signal as _signal
from tqdm import trange as _trange

_DRAW_MAX_SIGMA = 3

def render(
    locs,
    info=None,
    oversampling=1,
    viewport=None,
    blur_method=None,
    min_blur_width=0,
):
    if viewport is None:
        try:
            viewport = [(0, 0), (info[0]["Height"], info[0]["Width"])]
        except TypeError:
            raise ValueError("Need info if no viewport is provided.")
    (y_min, x_min), (y_max, x_max) = viewport
    if blur_method is None:
        return render_hist(locs, oversampling, y_min, x_min, y_max, x_max)
    elif blur_method == "gaussian":
        return render_gaussian(
            locs, oversampling, y_min, x_min, y_max, x_max, min_blur_width
        )
    elif blur_method == "gaussian_iso":
        return render_gaussian_iso(
            locs, oversampling, y_min, x_min, y_max, x_max, min_blur_width
        )
    elif blur_method == "smooth":
        return render_smooth(locs, oversampling, y_min, x_min, y_max, x_max)
    elif blur_method == "convolve":
        return render_convolve(
            locs, oversampling, y_min, x_min, y_max, x_max, min_blur_width
        )
    else:
        raise Exception("blur_method not understood.")

@_numba.jit(nopython=True, nogil=True)
def _render_setup(locs, oversampling, y_min, x_min, y_max, x_max):
    n_pixel_y = int(_np.ceil(oversampling * (y_max - y_min)))
    n_pixel_x = int(_np.ceil(oversampling * (x_max - x_min)))
    x = locs.x
    y = locs.y
    in_view = (x > x_min) & (y > y_min) & (x < x_max) & (y < y_max)
    x = x[in_view]
    y = y[in_view]
    x = oversampling * (x - x_min)
    y = oversampling * (y - y_min)
    image = _np.zeros((n_pixel_y, n_pixel_x), dtype=_np.float32)
    return image, n_pixel_y, n_pixel_x, x, y, in_view

@_numba.jit(nopython=True, nogil=True)
def _render_setup3d(
    locs, oversampling, y_min, x_min, y_max, x_max, z_min, z_max, pixelsize
):
    n_pixel_y = int(_np.ceil(oversampling * (y_max - y_min)))
    n_pixel_x = int(_np.ceil(oversampling * (x_max - x_min)))
    n_pixel_z = int(_np.ceil(oversampling * (z_max - z_min) / pixelsize))
    x = locs.x
    y = locs.y
    z = locs.z
    in_view = (
        (x > x_min)
        & (y > y_min)
        & (z > z_min)
        & (x < x_max)
        & (y < y_max)
        & (z < z_max)
    )
    x = x[in_view]
    y = y[in_view]
    z = z[in_view]
    x = oversampling * (x - x_min)
    y = oversampling * (y - y_min)
    z = oversampling * (z - z_min) / pixelsize
    image = _np.zeros((n_pixel_y, n_pixel_x, n_pixel_z), dtype=_np.float32)
    return image, n_pixel_y, n_pixel_x, n_pixel_z, x, y, z, in_view

@_numba.jit(nopython=True, nogil=True)
def _render_setupz(locs, oversampling, x_min, z_min, x_max, z_max, pixelsize):
    n_pixel_x = int(_np.ceil(oversampling * (x_max - x_min)))
    n_pixel_z = int(_np.ceil(oversampling * (z_max - z_min) / pixelsize))
    x = locs.x
    z = locs.z
    in_view = (x > x_min) & (z > z_min) & (x < x_max) & (z < z_max)
    x = x[in_view]
    z = z[in_view]
    x = oversampling * (x - x_min)
    z = oversampling * (z - z_min) / pixelsize
    image = _np.zeros((n_pixel_x, n_pixel_z), dtype=_np.float32)
    return image, n_pixel_z, n_pixel_x, x, z, in_view

@_numba.jit(nopython=True, nogil=True)
def _fill(image, x, y):
    x = x.astype(_np.int32)
    y = y.astype(_np.int32)
    for i, j in zip(x, y):
        image[j, i] += 1

@_numba.jit(nopython=True, nogil=True)
def _fill3d(image, x, y, z):
    x = x.astype(_np.int32)
    y = y.astype(_np.int32)
    z = z.astype(_np.int32)
    for i, j, k in zip(x, y, z):
        image[j, i, k] += 1
    return image

@_numba.jit(nopython=True, nogil=True)
def render_hist(locs, oversampling, y_min, x_min, y_max, x_max):
    image, n_pixel_y, n_pixel_x, x, y, in_view = _render_setup(
        locs, oversampling, y_min, x_min, y_max, x_max
    )
    _fill(image, x, y)
    return len(x), image

@_numba.jit(nopython=True, nogil=True)
def render_histz(locs, oversampling, x_min, z_min, x_max, z_max, pixelsize):
    image, n_pixel_z, n_pixel_x, x, z, in_view = _render_setupz(
        locs, oversampling, x_min, z_min, x_max, z_max, pixelsize
    )
    _fill(image, z, x)
    return len(x), image

@_numba.jit(nopython=True, nogil=True)
def render_hist3d(
    locs, oversampling, y_min, x_min, y_max, x_max, z_min, z_max, pixelsize
):
    image, n_pixel_y, n_pixel_x, n_pixel_z, x, y, z, in_view = _render_setup3d(
        locs,
        oversampling,
        y_min,
        x_min,
        y_max,
        x_max,
        z_min,
        z_max,
        pixelsize,
    )
    _fill3d(image, x, y, z)
    return len(x), image

@_numba.jit(nopython=True, nogil=True)
def render_gaussian(
    locs, oversampling, y_min, x_min, y_max, x_max, min_blur_width
):
    image, n_pixel_y, n_pixel_x, x, y, in_view = _render_setup(
        locs, oversampling, y_min, x_min, y_max, x_max
    )
    blur_width = oversampling * _np.maximum(locs.lpx, min_blur_width)
    blur_height = oversampling * _np.maximum(locs.lpy, min_blur_width)
    sy = blur_height[in_view]
    sx = blur_width[in_view]
    for x_, y_, sx_, sy_ in zip(x, y, sx, sy):
        max_y = _DRAW_MAX_SIGMA * sy_
        i_min = _np.int32(y_ - max_y)
        if i_min < 0:
            i_min = 0
        i_max = _np.int32(y_ + max_y + 1)
        if i_max > n_pixel_y:
            i_max = n_pixel_y
        max_x = _DRAW_MAX_SIGMA * sx_
        j_min = _np.int32(x_ - max_x)
        if j_min < 0:
            j_min = 0
        j_max = _np.int32(x_ + max_x) + 1
        if j_max > n_pixel_x:
            j_max = n_pixel_x
        for i in range(i_min, i_max):
            for j in range(j_min, j_max):
                image[i, j] += _np.exp(
                    -(
                        (j - x_ + 0.5) ** 2 / (2 * sx_ ** 2)
                        + (i - y_ + 0.5) ** 2 / (2 * sy_ ** 2)
                    )
                ) / (2 * _np.pi * sx_ * sy_)
    return len(x), image

@_numba.jit(nopython=True, nogil=True)
def render_gaussian_iso(
    locs, oversampling, y_min, x_min, y_max, x_max, min_blur_width
):
    image, n_pixel_y, n_pixel_x, x, y, in_view = _render_setup(
        locs, oversampling, y_min, x_min, y_max, x_max
    )
    blur_width = oversampling * _np.maximum(locs.lpx, min_blur_width)
    blur_height = oversampling * _np.maximum(locs.lpy, min_blur_width)
    sy = (blur_height[in_view] + blur_width[in_view]) / 2
    sx = sy
    for x_, y_, sx_, sy_ in zip(x, y, sx, sy):
        max_y = _DRAW_MAX_SIGMA * sy_
        i_min = _np.int32(y_ - max_y)
        if i_min < 0:
            i_min = 0
        i_max = _np.int32(y_ + max_y + 1)
        if i_max > n_pixel_y:
            i_max = n_pixel_y
        max_x = _DRAW_MAX_SIGMA * sx_
        j_min = _np.int32(x_ - max_x)
        if j_min < 0:
            j_min = 0
        j_max = _np.int32(x_ + max_x) + 1
        if j_max > n_pixel_x:
            j_max = n_pixel_x
        for i in range(i_min, i_max):
            for j in range(j_min, j_max):
                image[i, j] += _np.exp(
                    -(
                        (j - x_ + 0.5) ** 2 / (2 * sx_ ** 2)
                        + (i - y_ + 0.5) ** 2 / (2 * sy_ ** 2)
                    )
                ) / (2 * _np.pi * sx_ * sy_)
    return len(x), image

def render_convolve(
    locs, oversampling, y_min, x_min, y_max, x_max, min_blur_width
):
    image, n_pixel_y, n_pixel_x, x, y, in_view = _render_setup(
        locs, oversampling, y_min, x_min, y_max, x_max
    )
    _fill(image, x, y)
    n = len(x)
    if n == 0:
        return 0, image
    else:
        blur_width = oversampling * max(
            _np.median(locs.lpx[in_view]), min_blur_width
        )
        blur_height = oversampling * max(
            _np.median(locs.lpy[in_view]), min_blur_width
        )
        return n, _fftconvolve(image, blur_width, blur_height)

def render_smooth(locs, oversampling, y_min, x_min, y_max, x_max):
    image, n_pixel_y, n_pixel_x, x, y, in_view = _render_setup(
        locs, oversampling, y_min, x_min, y_max, x_max
    )
    _fill(image, x, y)
    n = len(x)
    if n == 0:
        return 0, image
    else:
        return n, _fftconvolve(image, 1, 1)

def _fftconvolve(image, blur_width, blur_height):
    kernel_width = 10 * int(_np.round(blur_width)) + 1
    kernel_height = 10 * int(_np.round(blur_height)) + 1
    kernel_y = _signal.gaussian(kernel_height, blur_height)
    kernel_x = _signal.gaussian(kernel_width, blur_width)
    kernel = _np.outer(kernel_y, kernel_x)
    kernel /= kernel.sum()
    return _signal.fftconvolve(image, kernel, mode="same")

def segment(locs, info, segmentation, kwargs={}, callback=None):
    Y = info[0]["Height"]
    X = info[0]["Width"]
    n_frames = info[0]["Frames"]
    n_seg = n_segments(info, segmentation)
    bounds = _np.linspace(0, n_frames - 1, n_seg + 1, dtype=_np.uint32)
    segments = _np.zeros((n_seg, Y, X))
    if callback is not None:
        callback(0)
    for i in _trange(n_seg, desc="Generating segments", unit="segments"):
        segment_locs = locs[
            (locs.frame >= bounds[i]) & (locs.frame < bounds[i + 1])
        ]
        _, segments[i] = render(segment_locs, info, **kwargs)
        if callback is not None:
            callback(i + 1)
    return bounds, segments

def n_segments(info, segmentation):
    n_frames = info[0]["Frames"]
    return int(_np.round(n_frames / segmentation))
<commit_msg>black commit<commit_after>"""
    picasso.render
    ~~~~~~~~~~~~~~

    Render single molecule localizations to a super-resolution image

    :author: Joerg Schnitzbauer, 2015
    :copyright: Copyright (c) 2015 Jungmann Lab, MPI of Biochemistry
"""
import numpy as _np
import numba as _numba
import scipy.signal as _signal
from tqdm import trange as _trange

_DRAW_MAX_SIGMA = 3

def render(
    locs, info=None, oversampling=1, viewport=None, blur_method=None, min_blur_width=0
):
    if viewport is None:
        try:
            viewport = [(0, 0), (info[0]["Height"], info[0]["Width"])]
        except TypeError:
            raise ValueError("Need info if no viewport is provided.")
    (y_min, x_min), (y_max, x_max) = viewport
    if blur_method is None:
        return render_hist(locs, oversampling, y_min, x_min, y_max, x_max)
    elif blur_method == "gaussian":
        return render_gaussian(
            locs, oversampling, y_min, x_min, y_max, x_max, min_blur_width
        )
    elif blur_method == "gaussian_iso":
        return render_gaussian_iso(
            locs, oversampling, y_min, x_min, y_max, x_max, min_blur_width
        )
    elif blur_method == "smooth":
        return render_smooth(locs, oversampling, y_min, x_min, y_max, x_max)
    elif blur_method == "convolve":
        return render_convolve(
            locs, oversampling, y_min, x_min, y_max, x_max, min_blur_width
        )
    else:
        raise Exception("blur_method not understood.")

@_numba.jit(nopython=True, nogil=True)
def _render_setup(locs, oversampling, y_min, x_min, y_max, x_max):
    n_pixel_y = int(_np.ceil(oversampling * (y_max - y_min)))
    n_pixel_x = int(_np.ceil(oversampling * (x_max - x_min)))
    x = locs.x
    y = locs.y
    in_view = (x > x_min) & (y > y_min) & (x < x_max) & (y < y_max)
    x = x[in_view]
    y = y[in_view]
    x = oversampling * (x - x_min)
    y = oversampling * (y - y_min)
    image = _np.zeros((n_pixel_y, n_pixel_x), dtype=_np.float32)
    return image, n_pixel_y, n_pixel_x, x, y, in_view

@_numba.jit(nopython=True, nogil=True)
def _render_setup3d(
    locs, oversampling, y_min, x_min, y_max, x_max, z_min, z_max, pixelsize
):
    n_pixel_y = int(_np.ceil(oversampling * (y_max - y_min)))
    n_pixel_x = int(_np.ceil(oversampling * (x_max - x_min)))
    n_pixel_z = int(_np.ceil(oversampling * (z_max - z_min) / pixelsize))
    x = locs.x
    y = locs.y
    z = locs.z
    in_view = (
        (x > x_min)
        & (y > y_min)
        & (z > z_min)
        & (x < x_max)
        & (y < y_max)
        & (z < z_max)
    )
    x = x[in_view]
    y = y[in_view]
    z = z[in_view]
    x = oversampling * (x - x_min)
    y = oversampling * (y - y_min)
    z = oversampling * (z - z_min) / pixelsize
    image = _np.zeros((n_pixel_y, n_pixel_x, n_pixel_z), dtype=_np.float32)
    return image, n_pixel_y, n_pixel_x, n_pixel_z, x, y, z, in_view

@_numba.jit(nopython=True, nogil=True)
def _render_setupz(locs, oversampling, x_min, z_min, x_max, z_max, pixelsize):
    n_pixel_x = int(_np.ceil(oversampling * (x_max - x_min)))
    n_pixel_z = int(_np.ceil(oversampling * (z_max - z_min) / pixelsize))
    x = locs.x
    z = locs.z
    in_view = (x > x_min) & (z > z_min) & (x < x_max) & (z < z_max)
    x = x[in_view]
    z = z[in_view]
    x = oversampling * (x - x_min)
    z = oversampling * (z - z_min) / pixelsize
    image = _np.zeros((n_pixel_x, n_pixel_z), dtype=_np.float32)
    return image, n_pixel_z, n_pixel_x, x, z, in_view

@_numba.jit(nopython=True, nogil=True)
def _fill(image, x, y):
    x = x.astype(_np.int32)
    y = y.astype(_np.int32)
    for i, j in zip(x, y):
        image[j, i] += 1

@_numba.jit(nopython=True, nogil=True)
def _fill3d(image, x, y, z):
    x = x.astype(_np.int32)
    y = y.astype(_np.int32)
    z = z.astype(_np.int32)
    for i, j, k in zip(x, y, z):
        image[j, i, k] += 1
    return image

@_numba.jit(nopython=True, nogil=True)
def render_hist(locs, oversampling, y_min, x_min, y_max, x_max):
    image, n_pixel_y, n_pixel_x, x, y, in_view = _render_setup(
        locs, oversampling, y_min, x_min, y_max, x_max
    )
    _fill(image, x, y)
    return len(x), image

@_numba.jit(nopython=True, nogil=True)
def render_histz(locs, oversampling, x_min, z_min, x_max, z_max, pixelsize):
    image, n_pixel_z, n_pixel_x, x, z, in_view = _render_setupz(
        locs, oversampling, x_min, z_min, x_max, z_max, pixelsize
    )
    _fill(image, z, x)
    return len(x), image

@_numba.jit(nopython=True, nogil=True)
def render_hist3d(
    locs, oversampling, y_min, x_min, y_max, x_max, z_min, z_max, pixelsize
):
    image, n_pixel_y, n_pixel_x, n_pixel_z, x, y, z, in_view = _render_setup3d(
        locs, oversampling, y_min, x_min, y_max, x_max, z_min, z_max, pixelsize
    )
    _fill3d(image, x, y, z)
    return len(x), image

@_numba.jit(nopython=True, nogil=True)
def render_gaussian(locs, oversampling, y_min, x_min, y_max, x_max, min_blur_width):
    image, n_pixel_y, n_pixel_x, x, y, in_view = _render_setup(
        locs, oversampling, y_min, x_min, y_max, x_max
    )
    blur_width = oversampling * _np.maximum(locs.lpx, min_blur_width)
    blur_height = oversampling * _np.maximum(locs.lpy, min_blur_width)
    sy = blur_height[in_view]
    sx = blur_width[in_view]
    for x_, y_, sx_, sy_ in zip(x, y, sx, sy):
        max_y = _DRAW_MAX_SIGMA * sy_
        i_min = _np.int32(y_ - max_y)
        if i_min < 0:
            i_min = 0
        i_max = _np.int32(y_ + max_y + 1)
        if i_max > n_pixel_y:
            i_max = n_pixel_y
        max_x = _DRAW_MAX_SIGMA * sx_
        j_min = _np.int32(x_ - max_x)
        if j_min < 0:
            j_min = 0
        j_max = _np.int32(x_ + max_x) + 1
        if j_max > n_pixel_x:
            j_max = n_pixel_x
        for i in range(i_min, i_max):
            for j in range(j_min, j_max):
                image[i, j] += _np.exp(
                    -(
                        (j - x_ + 0.5) ** 2 / (2 * sx_ ** 2)
                        + (i - y_ + 0.5) ** 2 / (2 * sy_ ** 2)
                    )
                ) / (2 * _np.pi * sx_ * sy_)
    return len(x), image

@_numba.jit(nopython=True, nogil=True)
def render_gaussian_iso(locs, oversampling, y_min, x_min, y_max, x_max, min_blur_width):
    image, n_pixel_y, n_pixel_x, x, y, in_view = _render_setup(
        locs, oversampling, y_min, x_min, y_max, x_max
    )
    blur_width = oversampling * _np.maximum(locs.lpx, min_blur_width)
    blur_height = oversampling * _np.maximum(locs.lpy, min_blur_width)
    sy = (blur_height[in_view] + blur_width[in_view]) / 2
    sx = sy
    for x_, y_, sx_, sy_ in zip(x, y, sx, sy):
        max_y = _DRAW_MAX_SIGMA * sy_
        i_min = _np.int32(y_ - max_y)
        if i_min < 0:
            i_min = 0
        i_max = _np.int32(y_ + max_y + 1)
        if i_max > n_pixel_y:
            i_max = n_pixel_y
        max_x = _DRAW_MAX_SIGMA * sx_
        j_min = _np.int32(x_ - max_x)
        if j_min < 0:
            j_min = 0
        j_max = _np.int32(x_ + max_x) + 1
        if j_max > n_pixel_x:
            j_max = n_pixel_x
        for i in range(i_min, i_max):
            for j in range(j_min, j_max):
                image[i, j] += _np.exp(
                    -(
                        (j - x_ + 0.5) ** 2 / (2 * sx_ ** 2)
                        + (i - y_ + 0.5) ** 2 / (2 * sy_ ** 2)
                    )
                ) / (2 * _np.pi * sx_ * sy_)
    return len(x), image

def render_convolve(locs, oversampling, y_min, x_min, y_max, x_max, min_blur_width):
    image, n_pixel_y, n_pixel_x, x, y, in_view = _render_setup(
        locs, oversampling, y_min, x_min, y_max, x_max
    )
    _fill(image, x, y)
    n = len(x)
    if n == 0:
        return 0, image
    else:
        blur_width = oversampling * max(_np.median(locs.lpx[in_view]), min_blur_width)
        blur_height = oversampling * max(_np.median(locs.lpy[in_view]), min_blur_width)
        return n, _fftconvolve(image, blur_width, blur_height)

def render_smooth(locs, oversampling, y_min, x_min, y_max, x_max):
    image, n_pixel_y, n_pixel_x, x, y, in_view = _render_setup(
        locs, oversampling, y_min, x_min, y_max, x_max
    )
    _fill(image, x, y)
    n = len(x)
    if n == 0:
        return 0, image
    else:
        return n, _fftconvolve(image, 1, 1)

def _fftconvolve(image, blur_width, blur_height):
    kernel_width = 10 * int(_np.round(blur_width)) + 1
    kernel_height = 10 * int(_np.round(blur_height)) + 1
    kernel_y = _signal.gaussian(kernel_height, blur_height)
    kernel_x = _signal.gaussian(kernel_width, blur_width)
    kernel = _np.outer(kernel_y, kernel_x)
    kernel /= kernel.sum()
    return _signal.fftconvolve(image, kernel, mode="same")

def segment(locs, info, segmentation, kwargs={}, callback=None):
    Y = info[0]["Height"]
    X = info[0]["Width"]
    n_frames = info[0]["Frames"]
    n_seg = n_segments(info, segmentation)
    bounds = _np.linspace(0, n_frames - 1, n_seg + 1, dtype=_np.uint32)
    segments = _np.zeros((n_seg, Y, X))
    if callback is not None:
        callback(0)
    for i in _trange(n_seg, desc="Generating segments", unit="segments"):
        segment_locs = locs[(locs.frame >= bounds[i]) & (locs.frame < bounds[i + 1])]
        _, segments[i] = render(segment_locs, info, **kwargs)
        if callback is not None:
            callback(i + 1)
    return bounds, segments

def n_segments(info, segmentation):
    n_frames = info[0]["Frames"]
    return int(_np.round(n_frames / segmentation))
<|endoftext|>


<commit_before>import asyncio
import logging

import hangups
import hangups.auth

from . import irc, util

logger = logging.getLogger(__name__)

class Server(object):

    def __init__(self, host='localhost', port='6667', cookies=None):
        self.clients = {}
        self._hangups = hangups.Client(cookies)
        self._hangups.on_connect.add_observer(self._on_hangups_connect)

    def run(self, host, port):
        loop = asyncio.get_event_loop()
        loop.run_until_complete(
            asyncio.start_server(self._on_client_connect, host=host, port=port)
        )
        logger.info('Waiting for hangups to connect...')
        loop.run_until_complete(self._hangups.connect())

    # Hangups Callbacks

    def _on_hangups_connect(self, initial_data):
        """Called when hangups successfully auths with hangouts."""
        self._user_list = hangups.UserList(
            self._hangups, initial_data.self_entity, initial_data.entities,
            initial_data.conversation_participants
        )
        self._conv_list = hangups.ConversationList(
            self._hangups, initial_data.conversation_states, self._user_list,
            initial_data.sync_timestamp
        )
        self._conv_list.on_event.add_observer(self._on_hangups_event)
        logger.info('Hangups connected. Connect your IRC clients!')

    def _on_hangups_event(self, conv_event):
        """Called when a hangups conversation event occurs."""
        if isinstance(conv_event, hangups.ChatMessageEvent):
            conv = self._conv_list.get(conv_event.conversation_id)
            user = conv.get_user(conv_event.user_id)
            sender = util.get_nick(user)
            hostmask = util.get_hostmask(user)
            channel = util.conversation_to_channel(conv)
            message = conv_event.text
            print(hostmask+' -> '+channel+' : '+conv_event.text)
            if len(conv.users) < 3:
                return

            for client in self.clients.values():
                if message in client.sent_messages and sender == client.nickname:
                    client.sent_messages.remove(message)
                    client.privmsg(hostmask, channel, conv_event.text)
                else:
                    client.privmsg(hostmask, channel, conv_event.text)

    # Client Callbacks

    def _on_client_connect(self, client_reader, client_writer):
        """Called when an IRC client connects."""
        client = irc.Client(client_reader, client_writer)
        task = asyncio.Task(self._handle_client(client))
        self.clients[task] = client
        logger.info("New Connection")
        task.add_done_callback(self._on_client_lost)

    def _on_client_lost(self, task):
        """Called when an IRC client disconnects."""
        self.clients[task].writer.close()
        del self.clients[task]
        logger.info("End Connection")

    @asyncio.coroutine
    def _handle_client(self, client):
        username = None
        welcomed = False

        while True:
            line = yield from client.readline()
            line = line.decode('utf-8').strip('\r\n')

            if not line:
                logger.info("Connection lost")
                break
            logger.info('Received: %r', line)

            if line.startswith('NICK'):
                client.nickname = line.split(' ', 1)[1]
            elif line.startswith('USER'):
                username = line.split(' ', 1)[1]
            elif line.startswith('LIST'):
                info = (
                    (util.conversation_to_channel(conv), len(conv.users),
                     util.get_topic(conv)) for conv in self._conv_list.get_all()
                )
                client.list_channels(info)
            elif line.startswith('PRIVMSG'):
                channel, message = line.split(' ', 2)[1:]
                conv = util.channel_to_conversation(channel, self._conv_list)
                client.sent_messages.append(message[1:])
                segments = hangups.ChatMessageSegment.from_str(message[1:])
                asyncio.async(conv.send_message(segments))
            elif line.startswith('JOIN'):
                channel = line.split(' ')[1]
                conv = util.channel_to_conversation(channel, self._conv_list)
                # If a JOIN is successful, the user receives a JOIN message as
                # confirmation and is then sent the channel's topic (using
                # RPL_TOPIC) and the list of users who are on the channel (using
                # RPL_NAMREPLY), which MUST include the user joining.
                client.write(util.get_nick(self._user_list._self_user),
                             'JOIN', channel)
                client.topic(channel, util.get_topic(conv))
                client.list_nicks(channel,
                                  (util.get_nick(user) for user in conv.users))
            elif line.startswith('WHO'):
                query = line.split(' ')[1]
                if query.startswith('#'):
                    conv = util.channel_to_conversation(channel,
                                                         self._conv_list)
                    responses = [{
                        'channel': query,
                        'user': util.get_name(user),
                        'nick': util.get_nick(user),
                        'real_name': user.full_name,
                    } for user in conv.users]
                    client.who(query, responses)
            elif line.startswith('PING'):
                client.pong()
            elif line.startswith('QUIT'):
                client.close()

            if not welcomed and client.nickname and username:
                welcomed = True
                client.swrite(irc.RPL_WELCOME, ':Welcome to pickups!')
                client.tell_nick(util.get_nick(self._user_list._self_user))

                # Sending the MOTD seems be required for Pidgin to connect.
                client.swrite(irc.RPL_MOTDSTART,
                              ':- pickups Message of the Day - ')
                client.swrite(irc.RPL_MOTD, ':- insert MOTD here')
                client.swrite(irc.RPL_ENDOFMOTD, ':End of MOTD command')
<commit_msg>Allow joining multiple rooms in one join command<commit_after>import asyncio
import logging

import hangups
import hangups.auth

from . import irc, util

logger = logging.getLogger(__name__)

class Server(object):

    def __init__(self, host='localhost', port='6667', cookies=None):
        self.clients = {}
        self._hangups = hangups.Client(cookies)
        self._hangups.on_connect.add_observer(self._on_hangups_connect)

    def run(self, host, port):
        loop = asyncio.get_event_loop()
        loop.run_until_complete(
            asyncio.start_server(self._on_client_connect, host=host, port=port)
        )
        logger.info('Waiting for hangups to connect...')
        loop.run_until_complete(self._hangups.connect())

    # Hangups Callbacks

    def _on_hangups_connect(self, initial_data):
        """Called when hangups successfully auths with hangouts."""
        self._user_list = hangups.UserList(
            self._hangups, initial_data.self_entity, initial_data.entities,
            initial_data.conversation_participants
        )
        self._conv_list = hangups.ConversationList(
            self._hangups, initial_data.conversation_states, self._user_list,
            initial_data.sync_timestamp
        )
        self._conv_list.on_event.add_observer(self._on_hangups_event)
        logger.info('Hangups connected. Connect your IRC clients!')

    def _on_hangups_event(self, conv_event):
        """Called when a hangups conversation event occurs."""
        if isinstance(conv_event, hangups.ChatMessageEvent):
            conv = self._conv_list.get(conv_event.conversation_id)
            user = conv.get_user(conv_event.user_id)
            sender = util.get_nick(user)
            hostmask = util.get_hostmask(user)
            channel = util.conversation_to_channel(conv)
            message = conv_event.text
            print(hostmask+' -> '+channel+' : '+conv_event.text)
            if len(conv.users) < 3:
                return

            for client in self.clients.values():
                if message in client.sent_messages and sender == client.nickname:
                    client.sent_messages.remove(message)
                    client.privmsg(hostmask, channel, conv_event.text)
                else:
                    client.privmsg(hostmask, channel, conv_event.text)

    # Client Callbacks

    def _on_client_connect(self, client_reader, client_writer):
        """Called when an IRC client connects."""
        client = irc.Client(client_reader, client_writer)
        task = asyncio.Task(self._handle_client(client))
        self.clients[task] = client
        logger.info("New Connection")
        task.add_done_callback(self._on_client_lost)

    def _on_client_lost(self, task):
        """Called when an IRC client disconnects."""
        self.clients[task].writer.close()
        del self.clients[task]
        logger.info("End Connection")

    @asyncio.coroutine
    def _handle_client(self, client):
        username = None
        welcomed = False

        while True:
            line = yield from client.readline()
            line = line.decode('utf-8').strip('\r\n')

            if not line:
                logger.info("Connection lost")
                break
            logger.info('Received: %r', line)

            if line.startswith('NICK'):
                client.nickname = line.split(' ', 1)[1]
            elif line.startswith('USER'):
                username = line.split(' ', 1)[1]
            elif line.startswith('LIST'):
                info = (
                    (util.conversation_to_channel(conv), len(conv.users),
                     util.get_topic(conv)) for conv in self._conv_list.get_all()
                )
                client.list_channels(info)
            elif line.startswith('PRIVMSG'):
                channel, message = line.split(' ', 2)[1:]
                conv = util.channel_to_conversation(channel, self._conv_list)
                client.sent_messages.append(message[1:])
                segments = hangups.ChatMessageSegment.from_str(message[1:])
                asyncio.async(conv.send_message(segments))
            elif line.startswith('JOIN'):
                channels = line.split(' ')[1]
                channels = channels.split(',')
                for channel in channels:
                    print("Joining '"+channel+"'")
                    conv = util.channel_to_conversation(channel, self._conv_list)
                    # If a JOIN is successful, the user receives a JOIN message as
                    # confirmation and is then sent the channel's topic (using
                    # RPL_TOPIC) and the list of users who are on the channel (using
                    # RPL_NAMREPLY), which MUST include the user joining.
                    client.write(util.get_nick(self._user_list._self_user),
                                 'JOIN', channel)
                    client.topic(channel, util.get_topic(conv))
                    client.list_nicks(channel,
                                      (util.get_nick(user) for user in conv.users))
            elif line.startswith('WHO'):
                query = line.split(' ')[1]
                if query.startswith('#'):
                    conv = util.channel_to_conversation(channel,
                                                         self._conv_list)
                    responses = [{
                        'channel': query,
                        'user': util.get_name(user),
                        'nick': util.get_nick(user),
                        'real_name': user.full_name,
                    } for user in conv.users]
                    client.who(query, responses)
            elif line.startswith('PING'):
                client.pong()
            elif line.startswith('QUIT'):
                client.close()

            if not welcomed and client.nickname and username:
                welcomed = True
                client.swrite(irc.RPL_WELCOME, ':Welcome to pickups!')
                client.tell_nick(util.get_nick(self._user_list._self_user))

                # Sending the MOTD seems be required for Pidgin to connect.
                client.swrite(irc.RPL_MOTDSTART,
                              ':- pickups Message of the Day - ')
                client.swrite(irc.RPL_MOTD, ':- insert MOTD here')
                client.swrite(irc.RPL_ENDOFMOTD, ':End of MOTD command')
<|endoftext|>