<commit_before>"""Support for Hangouts."""
import logging

from hangups.auth import GoogleAuthError
import voluptuous as vol

from homeassistant import config_entries
from homeassistant.components.conversation.util import create_matcher
from homeassistant.const import EVENT_HOMEASSISTANT_STOP
from homeassistant.helpers import dispatcher, intent
import homeassistant.helpers.config_validation as cv

# We need an import from .config_flow, without it .config_flow is never loaded.
from .config_flow import HangoutsFlowHandler  # noqa: F401
from .const import (
    CONF_BOT,
    CONF_DEFAULT_CONVERSATIONS,
    CONF_ERROR_SUPPRESSED_CONVERSATIONS,
    CONF_INTENTS,
    CONF_MATCHERS,
    CONF_REFRESH_TOKEN,
    CONF_SENTENCES,
    DOMAIN,
    EVENT_HANGOUTS_CONNECTED,
    EVENT_HANGOUTS_CONVERSATIONS_CHANGED,
    EVENT_HANGOUTS_CONVERSATIONS_RESOLVED,
    INTENT_HELP,
    INTENT_SCHEMA,
    MESSAGE_SCHEMA,
    SERVICE_RECONNECT,
    SERVICE_SEND_MESSAGE,
    SERVICE_UPDATE,
    TARGETS_SCHEMA,
)
from .hangouts_bot import HangoutsBot
from .intents import HelpIntent

_LOGGER = logging.getLogger(__name__)

CONFIG_SCHEMA = vol.Schema(
    {
        DOMAIN: vol.Schema(
            {
                vol.Optional(CONF_INTENTS, default={}): vol.Schema(
                    {cv.string: INTENT_SCHEMA}
                ),
                vol.Optional(CONF_DEFAULT_CONVERSATIONS, default=[]): [TARGETS_SCHEMA],
                vol.Optional(CONF_ERROR_SUPPRESSED_CONVERSATIONS, default=[]): [
                    TARGETS_SCHEMA
                ],
            }
        )
    },
    extra=vol.ALLOW_EXTRA,
)

async def async_setup(hass, config):
    """Set up the Hangouts bot component."""
    config = config.get(DOMAIN)
    if config is None:
        hass.data[DOMAIN] = {
            CONF_INTENTS: {},
            CONF_DEFAULT_CONVERSATIONS: [],
            CONF_ERROR_SUPPRESSED_CONVERSATIONS: [],
        }
        return True

    hass.data[DOMAIN] = {
        CONF_INTENTS: config[CONF_INTENTS],
        CONF_DEFAULT_CONVERSATIONS: config[CONF_DEFAULT_CONVERSATIONS],
        CONF_ERROR_SUPPRESSED_CONVERSATIONS: config[
            CONF_ERROR_SUPPRESSED_CONVERSATIONS
        ],
    }

    if (
        hass.data[DOMAIN][CONF_INTENTS]
        and INTENT_HELP not in hass.data[DOMAIN][CONF_INTENTS]
    ):
        hass.data[DOMAIN][CONF_INTENTS][INTENT_HELP] = {CONF_SENTENCES: ["HELP"]}

    for data in hass.data[DOMAIN][CONF_INTENTS].values():
        matchers = []
        for sentence in data[CONF_SENTENCES]:
            matchers.append(create_matcher(sentence))

        data[CONF_MATCHERS] = matchers

    hass.async_create_task(
        hass.config_entries.flow.async_init(
            DOMAIN, context={"source": config_entries.SOURCE_IMPORT}
        )
    )

    return True

async def async_setup_entry(hass, config):
    """Set up a config entry."""
    try:
        bot = HangoutsBot(
            hass,
            config.data.get(CONF_REFRESH_TOKEN),
            hass.data[DOMAIN][CONF_INTENTS],
            hass.data[DOMAIN][CONF_DEFAULT_CONVERSATIONS],
            hass.data[DOMAIN][CONF_ERROR_SUPPRESSED_CONVERSATIONS],
        )
        hass.data[DOMAIN][CONF_BOT] = bot
    except GoogleAuthError as exception:
        _LOGGER.error("Hangouts failed to log in: %s", str(exception))
        return False

    dispatcher.async_dispatcher_connect(
        hass, EVENT_HANGOUTS_CONNECTED, bot.async_handle_update_users_and_conversations
    )

    dispatcher.async_dispatcher_connect(
        hass, EVENT_HANGOUTS_CONVERSATIONS_CHANGED, bot.async_resolve_conversations
    )

    dispatcher.async_dispatcher_connect(
        hass,
        EVENT_HANGOUTS_CONVERSATIONS_RESOLVED,
        bot.async_update_conversation_commands,
    )

    hass.bus.async_listen_once(EVENT_HOMEASSISTANT_STOP, bot.async_handle_hass_stop)

    await bot.async_connect()

    hass.services.async_register(
        DOMAIN,
        SERVICE_SEND_MESSAGE,
        bot.async_handle_send_message,
        schema=MESSAGE_SCHEMA,
    )
    hass.services.async_register(
        DOMAIN,
        SERVICE_UPDATE,
        bot.async_handle_update_users_and_conversations,
        schema=vol.Schema({}),
    )

    hass.services.async_register(
        DOMAIN, SERVICE_RECONNECT, bot.async_handle_reconnect, schema=vol.Schema({})
    )

    intent.async_register(hass, HelpIntent(hass))

    return True

async def async_unload_entry(hass, _):
    """Unload a config entry."""
    bot = hass.data[DOMAIN].pop(CONF_BOT)
    await bot.async_disconnect()
    return True
<commit_msg>Fix memory leak in hangouts (#49471)<commit_after>"""Support for Hangouts."""
import logging

from hangups.auth import GoogleAuthError
import voluptuous as vol

from homeassistant import config_entries
from homeassistant.components.conversation.util import create_matcher
from homeassistant.const import EVENT_HOMEASSISTANT_STOP
from homeassistant.helpers import dispatcher, intent
import homeassistant.helpers.config_validation as cv

# We need an import from .config_flow, without it .config_flow is never loaded.
from .config_flow import HangoutsFlowHandler  # noqa: F401
from .const import (
    CONF_BOT,
    CONF_DEFAULT_CONVERSATIONS,
    CONF_ERROR_SUPPRESSED_CONVERSATIONS,
    CONF_INTENTS,
    CONF_MATCHERS,
    CONF_REFRESH_TOKEN,
    CONF_SENTENCES,
    DOMAIN,
    EVENT_HANGOUTS_CONNECTED,
    EVENT_HANGOUTS_CONVERSATIONS_CHANGED,
    EVENT_HANGOUTS_CONVERSATIONS_RESOLVED,
    INTENT_HELP,
    INTENT_SCHEMA,
    MESSAGE_SCHEMA,
    SERVICE_RECONNECT,
    SERVICE_SEND_MESSAGE,
    SERVICE_UPDATE,
    TARGETS_SCHEMA,
)
from .hangouts_bot import HangoutsBot
from .intents import HelpIntent

_LOGGER = logging.getLogger(__name__)

CONFIG_SCHEMA = vol.Schema(
    {
        DOMAIN: vol.Schema(
            {
                vol.Optional(CONF_INTENTS, default={}): vol.Schema(
                    {cv.string: INTENT_SCHEMA}
                ),
                vol.Optional(CONF_DEFAULT_CONVERSATIONS, default=[]): [TARGETS_SCHEMA],
                vol.Optional(CONF_ERROR_SUPPRESSED_CONVERSATIONS, default=[]): [
                    TARGETS_SCHEMA
                ],
            }
        )
    },
    extra=vol.ALLOW_EXTRA,
)

async def async_setup(hass, config):
    """Set up the Hangouts bot component."""
    config = config.get(DOMAIN)
    if config is None:
        hass.data[DOMAIN] = {
            CONF_INTENTS: {},
            CONF_DEFAULT_CONVERSATIONS: [],
            CONF_ERROR_SUPPRESSED_CONVERSATIONS: [],
        }
        return True

    hass.data[DOMAIN] = {
        CONF_INTENTS: config[CONF_INTENTS],
        CONF_DEFAULT_CONVERSATIONS: config[CONF_DEFAULT_CONVERSATIONS],
        CONF_ERROR_SUPPRESSED_CONVERSATIONS: config[
            CONF_ERROR_SUPPRESSED_CONVERSATIONS
        ],
    }

    if (
        hass.data[DOMAIN][CONF_INTENTS]
        and INTENT_HELP not in hass.data[DOMAIN][CONF_INTENTS]
    ):
        hass.data[DOMAIN][CONF_INTENTS][INTENT_HELP] = {CONF_SENTENCES: ["HELP"]}

    for data in hass.data[DOMAIN][CONF_INTENTS].values():
        matchers = []
        for sentence in data[CONF_SENTENCES]:
            matchers.append(create_matcher(sentence))

        data[CONF_MATCHERS] = matchers

    hass.async_create_task(
        hass.config_entries.flow.async_init(
            DOMAIN, context={"source": config_entries.SOURCE_IMPORT}
        )
    )

    return True

async def async_setup_entry(hass, config):
    """Set up a config entry."""
    try:
        bot = HangoutsBot(
            hass,
            config.data.get(CONF_REFRESH_TOKEN),
            hass.data[DOMAIN][CONF_INTENTS],
            hass.data[DOMAIN][CONF_DEFAULT_CONVERSATIONS],
            hass.data[DOMAIN][CONF_ERROR_SUPPRESSED_CONVERSATIONS],
        )
        hass.data[DOMAIN][CONF_BOT] = bot
    except GoogleAuthError as exception:
        _LOGGER.error("Hangouts failed to log in: %s", str(exception))
        return False

    dispatcher.async_dispatcher_connect(
        hass, EVENT_HANGOUTS_CONNECTED, bot.async_handle_update_users_and_conversations
    )

    dispatcher.async_dispatcher_connect(
        hass, EVENT_HANGOUTS_CONVERSATIONS_CHANGED, bot.async_resolve_conversations
    )

    dispatcher.async_dispatcher_connect(
        hass,
        EVENT_HANGOUTS_CONVERSATIONS_RESOLVED,
        bot.async_update_conversation_commands,
    )

    config.async_on_unload(
        hass.bus.async_listen_once(EVENT_HOMEASSISTANT_STOP, bot.async_handle_hass_stop)
    )

    await bot.async_connect()

    hass.services.async_register(
        DOMAIN,
        SERVICE_SEND_MESSAGE,
        bot.async_handle_send_message,
        schema=MESSAGE_SCHEMA,
    )
    hass.services.async_register(
        DOMAIN,
        SERVICE_UPDATE,
        bot.async_handle_update_users_and_conversations,
        schema=vol.Schema({}),
    )

    hass.services.async_register(
        DOMAIN, SERVICE_RECONNECT, bot.async_handle_reconnect, schema=vol.Schema({})
    )

    intent.async_register(hass, HelpIntent(hass))

    return True

async def async_unload_entry(hass, _):
    """Unload a config entry."""
    bot = hass.data[DOMAIN].pop(CONF_BOT)
    await bot.async_disconnect()
    return True
<|endoftext|>


<commit_before>__version_info__ = {
    'major': 0,
    'minor': 4,
    'micro': 9,
    'releaselevel': 'final',
    'serial': 0
}

def get_version():
    vers = ["%(major)i.%(minor)i" % __version_info__, ]

    if __version_info__['micro']:
        vers.append(".%(micro)i" % __version_info__)
    if __version_info__['releaselevel'] != 'final':
        vers.append('%(releaselevel)s%(serial)i' % __version_info__)
    return ''.join(vers)

__version__ = get_version()

class AlreadyRegistered(Exception):
    """
    An attempt was made to register a model more than once.
    """
    pass

registry = []

def register(model, tag_descriptor_attr='supertags',
             tagged_item_manager_attr='supertagged'):
    """
    Sets the given model class up for working with supertags.
    """

    from supertagging.managers import ModelTaggedItemManager, TagDescriptor

    if model in registry:
        raise AlreadyRegistered("The model '%s' has already been "
            "registered." % model._meta.object_name)
    if hasattr(model, tag_descriptor_attr):
        raise AttributeError("'%s' already has an attribute '%s'. You must "
            "provide a custom tag_descriptor_attr to register." % (
                model._meta.object_name,
                tag_descriptor_attr,
            )
        )
    if hasattr(model, tagged_item_manager_attr):
        raise AttributeError("'%s' already has an attribute '%s'. You must "
            "provide a custom tagged_item_manager_attr to register." % (
                model._meta.object_name,
                tagged_item_manager_attr,
            )
        )

    # Add tag descriptor
    setattr(model, tag_descriptor_attr, TagDescriptor())

    # Add custom manager
    ModelTaggedItemManager().contribute_to_class(model, tagged_item_manager_attr)

    # Finally register in registry
    registry.append(model)<commit_msg>Bumped version to 0.4.10<commit_after>__version_info__ = {
    'major': 0,
    'minor': 4,
    'micro': 10,
    'releaselevel': 'final',
    'serial': 0
}

def get_version():
    vers = ["%(major)i.%(minor)i" % __version_info__, ]

    if __version_info__['micro']:
        vers.append(".%(micro)i" % __version_info__)
    if __version_info__['releaselevel'] != 'final':
        vers.append('%(releaselevel)s%(serial)i' % __version_info__)
    return ''.join(vers)

__version__ = get_version()

class AlreadyRegistered(Exception):
    """
    An attempt was made to register a model more than once.
    """
    pass

registry = []

def register(model, tag_descriptor_attr='supertags',
             tagged_item_manager_attr='supertagged'):
    """
    Sets the given model class up for working with supertags.
    """

    from supertagging.managers import ModelTaggedItemManager, TagDescriptor

    if model in registry:
        raise AlreadyRegistered("The model '%s' has already been "
            "registered." % model._meta.object_name)
    if hasattr(model, tag_descriptor_attr):
        raise AttributeError("'%s' already has an attribute '%s'. You must "
            "provide a custom tag_descriptor_attr to register." % (
                model._meta.object_name,
                tag_descriptor_attr,
            )
        )
    if hasattr(model, tagged_item_manager_attr):
        raise AttributeError("'%s' already has an attribute '%s'. You must "
            "provide a custom tagged_item_manager_attr to register." % (
                model._meta.object_name,
                tagged_item_manager_attr,
            )
        )

    # Add tag descriptor
    setattr(model, tag_descriptor_attr, TagDescriptor())

    # Add custom manager
    ModelTaggedItemManager().contribute_to_class(model, tagged_item_manager_attr)

    # Finally register in registry
    registry.append(model)<|endoftext|>


<commit_before>"""Support for sending data to an Influx database."""
import logging
import re
import queue
import threading
import time
import math

import requests.exceptions
import voluptuous as vol

from homeassistant.const import (
    CONF_DOMAINS, CONF_ENTITIES, CONF_EXCLUDE, CONF_HOST, CONF_INCLUDE,
    CONF_PASSWORD, CONF_PORT, CONF_SSL, CONF_USERNAME, CONF_VERIFY_SSL,
    EVENT_STATE_CHANGED, EVENT_HOMEASSISTANT_STOP, STATE_UNAVAILABLE,
    STATE_UNKNOWN)
from homeassistant.helpers import state as state_helper, event as event_helper
import homeassistant.helpers.config_validation as cv
from homeassistant.helpers.entity_values import EntityValues

_LOGGER = logging.getLogger(__name__)

CONF_DB_NAME = 'database'
CONF_TAGS = 'tags'
CONF_DEFAULT_MEASUREMENT = 'default_measurement'
CONF_OVERRIDE_MEASUREMENT = 'override_measurement'
CONF_TAGS_ATTRIBUTES = 'tags_attributes'
CONF_COMPONENT_CONFIG = 'component_config'
CONF_COMPONENT_CONFIG_GLOB = 'component_config_glob'
CONF_COMPONENT_CONFIG_DOMAIN = 'component_config_domain'
CONF_RETRY_COUNT = 'max_retries'

DEFAULT_DATABASE = 'home_assistant'
DEFAULT_VERIFY_SSL = True
DOMAIN = 'influxdb'

TIMEOUT = 5
RETRY_DELAY = 20
QUEUE_BACKLOG_SECONDS = 30
RETRY_INTERVAL = 60  # seconds

BATCH_TIMEOUT = 1
BATCH_BUFFER_SIZE = 100

COMPONENT_CONFIG_SCHEMA_ENTRY = vol.Schema({
    vol.Optional(CONF_OVERRIDE_MEASUREMENT): cv.string,
})

CONFIG_SCHEMA = vol.Schema({
    DOMAIN: vol.All(vol.Schema({
        vol.Optional(CONF_HOST): cv.string,
        vol.Inclusive(CONF_USERNAME, 'authentication'): cv.string,
        vol.Inclusive(CONF_PASSWORD, 'authentication'): cv.string,
        vol.Optional(CONF_EXCLUDE, default={}): vol.Schema({
            vol.Optional(CONF_ENTITIES, default=[]): cv.entity_ids,
            vol.Optional(CONF_DOMAINS, default=[]):
                vol.All(cv.ensure_list, [cv.string])
        }),
        vol.Optional(CONF_INCLUDE, default={}): vol.Schema({
            vol.Optional(CONF_ENTITIES, default=[]): cv.entity_ids,
            vol.Optional(CONF_DOMAINS, default=[]):
                vol.All(cv.ensure_list, [cv.string])
        }),
        vol.Optional(CONF_DB_NAME, default=DEFAULT_DATABASE): cv.string,
        vol.Optional(CONF_PORT): cv.port,
        vol.Optional(CONF_SSL): cv.boolean,
        vol.Optional(CONF_RETRY_COUNT, default=0): cv.positive_int,
        vol.Optional(CONF_DEFAULT_MEASUREMENT): cv.string,
        vol.Optional(CONF_OVERRIDE_MEASUREMENT): cv.string,
        vol.Optional(CONF_TAGS, default={}):
            vol.Schema({cv.string: cv.string}),
        vol.Optional(CONF_TAGS_ATTRIBUTES, default=[]):
            vol.All(cv.ensure_list, [cv.string]),
        vol.Optional(CONF_VERIFY_SSL, default=DEFAULT_VERIFY_SSL): cv.boolean,
        vol.Optional(CONF_COMPONENT_CONFIG, default={}):
            vol.Schema({cv.entity_id: COMPONENT_CONFIG_SCHEMA_ENTRY}),
        vol.Optional(CONF_COMPONENT_CONFIG_GLOB, default={}):
            vol.Schema({cv.string: COMPONENT_CONFIG_SCHEMA_ENTRY}),
        vol.Optional(CONF_COMPONENT_CONFIG_DOMAIN, default={}):
            vol.Schema({cv.string: COMPONENT_CONFIG_SCHEMA_ENTRY}),
    })),
}, extra=vol.ALLOW_EXTRA)

RE_DIGIT_TAIL = re.compile(r'^[^\.]*\d+\.?\d+[^\.]*$')
RE_DECIMAL = re.compile(r'[^\d.]+')

def setup(hass, config):
    """Set up the InfluxDB component."""
    from influxdb import InfluxDBClient, exceptions

    conf = config[DOMAIN]

    kwargs = {
        'database': conf[CONF_DB_NAME],
        'verify_ssl': conf[CONF_VERIFY_SSL],
        'timeout': TIMEOUT
    }

    if CONF_HOST in conf:
        kwargs['host'] = conf[CONF_HOST]

    if CONF_PORT in conf:
        kwargs['port'] = conf[CONF_PORT]

    if CONF_USERNAME in conf:
        kwargs['username'] = conf[CONF_USERNAME]

    if CONF_PASSWORD in conf:
        kwargs['password'] = conf[CONF_PASSWORD]

    if CONF_SSL in conf:
        kwargs['ssl'] = conf[CONF_SSL]

    include = conf.get(CONF_INCLUDE, {})
    exclude = conf.get(CONF_EXCLUDE, {})
    whitelist_e = set(include.get(CONF_ENTITIES, []))
    whitelist_d = set(include.get(CONF_DOMAINS, []))
    blacklist_e = set(exclude.get(CONF_ENTITIES, []))
    blacklist_d = set(exclude.get(CONF_DOMAINS, []))
    tags = conf.get(CONF_TAGS)
    tags_attributes = conf.get(CONF_TAGS_ATTRIBUTES)
    default_measurement = conf.get(CONF_DEFAULT_MEASUREMENT)
    override_measurement = conf.get(CONF_OVERRIDE_MEASUREMENT)
    component_config = EntityValues(
        conf[CONF_COMPONENT_CONFIG],
        conf[CONF_COMPONENT_CONFIG_DOMAIN],
        conf[CONF_COMPONENT_CONFIG_GLOB])
    max_tries = conf.get(CONF_RETRY_COUNT)

    try:
        influx = InfluxDBClient(**kwargs)
        influx.write_points([])
    except (exceptions.InfluxDBClientError,
            requests.exceptions.ConnectionError) as exc:
        _LOGGER.warning(
            "Database host is not accessible due to '%s', please "
            "check your entries in the configuration file (host, "
            "port, etc.) and verify that the database exists and is "
            "READ/WRITE. Retrying again in %s seconds.", exc, RETRY_INTERVAL
        )
        event_helper.call_later(
            hass, RETRY_INTERVAL, lambda _: setup(hass, config)
        )
        return True

    def event_to_json(event):
        """Add an event to the outgoing Influx list."""
        state = event.data.get('new_state')
        if state is None or state.state in (
                STATE_UNKNOWN, '', STATE_UNAVAILABLE) or \
                state.entity_id in blacklist_e or state.domain in blacklist_d:
            return

        try:
            if ((whitelist_e or whitelist_d)
                    and state.entity_id not in whitelist_e
                    and state.domain not in whitelist_d):
                return

            _include_state = _include_value = False

            _state_as_value = float(state.state)
            _include_value = True
        except ValueError:
            try:
                _state_as_value = float(state_helper.state_as_number(state))
                _include_state = _include_value = True
            except ValueError:
                _include_state = True

        include_uom = True
        measurement = component_config.get(state.entity_id).get(
            CONF_OVERRIDE_MEASUREMENT)
        if measurement in (None, ''):
            if override_measurement:
                measurement = override_measurement
            else:
                measurement = state.attributes.get('unit_of_measurement')
                if measurement in (None, ''):
                    if default_measurement:
                        measurement = default_measurement
                    else:
                        measurement = state.entity_id
                else:
                    include_uom = False

        json = {
            'measurement': measurement,
            'tags': {
                'domain': state.domain,
                'entity_id': state.object_id,
            },
            'time': event.time_fired,
            'fields': {}
        }
        if _include_state:
            json['fields']['state'] = state.state
        if _include_value:
            json['fields']['value'] = _state_as_value

        for key, value in state.attributes.items():
            if key in tags_attributes:
                json['tags'][key] = value
            elif key != 'unit_of_measurement' or include_uom:
                # If the key is already in fields
                if key in json['fields']:
                    key = key + "_"
                # Prevent column data errors in influxDB.
                # For each value we try to cast it as float
                # But if we can not do it we store the value
                # as string add "_str" postfix to the field key
                try:
                    json['fields'][key] = float(value)
                except (ValueError, TypeError):
                    new_key = "{}_str".format(key)
                    new_value = str(value)
                    json['fields'][new_key] = new_value

                    if RE_DIGIT_TAIL.match(new_value):
                        json['fields'][key] = float(
                            RE_DECIMAL.sub('', new_value))

                # Infinity and NaN are not valid floats in InfluxDB
                try:
                    if not math.isfinite(json['fields'][key]):
                        del json['fields'][key]
                except (KeyError, TypeError):
                    pass

        json['tags'].update(tags)

        return json

    instance = hass.data[DOMAIN] = InfluxThread(
        hass, influx, event_to_json, max_tries)
    instance.start()

    def shutdown(event):
        """Shut down the thread."""
        instance.queue.put(None)
        instance.join()
        influx.close()

    hass.bus.listen_once(EVENT_HOMEASSISTANT_STOP, shutdown)

    return True

class InfluxThread(threading.Thread):
    """A threaded event handler class."""

    def __init__(self, hass, influx, event_to_json, max_tries):
        """Initialize the listener."""
        threading.Thread.__init__(self, name='InfluxDB')
        self.queue = queue.Queue()
        self.influx = influx
        self.event_to_json = event_to_json
        self.max_tries = max_tries
        self.write_errors = 0
        self.shutdown = False
        hass.bus.listen(EVENT_STATE_CHANGED, self._event_listener)

    def _event_listener(self, event):
        """Listen for new messages on the bus and queue them for Influx."""
        item = (time.monotonic(), event)
        self.queue.put(item)

    @staticmethod
    def batch_timeout():
        """Return number of seconds to wait for more events."""
        return BATCH_TIMEOUT

    def get_events_json(self):
        """Return a batch of events formatted for writing."""
        queue_seconds = QUEUE_BACKLOG_SECONDS + self.max_tries*RETRY_DELAY

        count = 0
        json = []

        dropped = 0

        try:
            while len(json) < BATCH_BUFFER_SIZE and not self.shutdown:
                timeout = None if count == 0 else self.batch_timeout()
                item = self.queue.get(timeout=timeout)
                count += 1

                if item is None:
                    self.shutdown = True
                else:
                    timestamp, event = item
                    age = time.monotonic() - timestamp

                    if age < queue_seconds:
                        event_json = self.event_to_json(event)
                        if event_json:
                            json.append(event_json)
                    else:
                        dropped += 1

        except queue.Empty:
            pass

        if dropped:
            _LOGGER.warning("Catching up, dropped %d old events", dropped)

        return count, json

    def write_to_influxdb(self, json):
        """Write preprocessed events to influxdb, with retry."""
        from influxdb import exceptions

        for retry in range(self.max_tries+1):
            try:
                self.influx.write_points(json)

                if self.write_errors:
                    _LOGGER.error("Resumed, lost %d events", self.write_errors)
                    self.write_errors = 0

                _LOGGER.debug("Wrote %d events", len(json))
                break
            except (exceptions.InfluxDBClientError, IOError):
                if retry < self.max_tries:
                    time.sleep(RETRY_DELAY)
                else:
                    if not self.write_errors:
                        _LOGGER.exception("Write error")
                    self.write_errors += len(json)

    def run(self):
        """Process incoming events."""
        while not self.shutdown:
            count, json = self.get_events_json()
            if json:
                self.write_to_influxdb(json)
            for _ in range(count):
                self.queue.task_done()

    def block_till_done(self):
        """Block till all events processed."""
        self.queue.join()
<commit_msg>Catch RequestException in influxdb writer (#23508)<commit_after>"""Support for sending data to an Influx database."""
import logging
import re
import queue
import threading
import time
import math

import requests.exceptions
import voluptuous as vol

from homeassistant.const import (
    CONF_DOMAINS, CONF_ENTITIES, CONF_EXCLUDE, CONF_HOST, CONF_INCLUDE,
    CONF_PASSWORD, CONF_PORT, CONF_SSL, CONF_USERNAME, CONF_VERIFY_SSL,
    EVENT_STATE_CHANGED, EVENT_HOMEASSISTANT_STOP, STATE_UNAVAILABLE,
    STATE_UNKNOWN)
from homeassistant.helpers import state as state_helper, event as event_helper
import homeassistant.helpers.config_validation as cv
from homeassistant.helpers.entity_values import EntityValues

_LOGGER = logging.getLogger(__name__)

CONF_DB_NAME = 'database'
CONF_TAGS = 'tags'
CONF_DEFAULT_MEASUREMENT = 'default_measurement'
CONF_OVERRIDE_MEASUREMENT = 'override_measurement'
CONF_TAGS_ATTRIBUTES = 'tags_attributes'
CONF_COMPONENT_CONFIG = 'component_config'
CONF_COMPONENT_CONFIG_GLOB = 'component_config_glob'
CONF_COMPONENT_CONFIG_DOMAIN = 'component_config_domain'
CONF_RETRY_COUNT = 'max_retries'

DEFAULT_DATABASE = 'home_assistant'
DEFAULT_VERIFY_SSL = True
DOMAIN = 'influxdb'

TIMEOUT = 5
RETRY_DELAY = 20
QUEUE_BACKLOG_SECONDS = 30
RETRY_INTERVAL = 60  # seconds

BATCH_TIMEOUT = 1
BATCH_BUFFER_SIZE = 100

COMPONENT_CONFIG_SCHEMA_ENTRY = vol.Schema({
    vol.Optional(CONF_OVERRIDE_MEASUREMENT): cv.string,
})

CONFIG_SCHEMA = vol.Schema({
    DOMAIN: vol.All(vol.Schema({
        vol.Optional(CONF_HOST): cv.string,
        vol.Inclusive(CONF_USERNAME, 'authentication'): cv.string,
        vol.Inclusive(CONF_PASSWORD, 'authentication'): cv.string,
        vol.Optional(CONF_EXCLUDE, default={}): vol.Schema({
            vol.Optional(CONF_ENTITIES, default=[]): cv.entity_ids,
            vol.Optional(CONF_DOMAINS, default=[]):
                vol.All(cv.ensure_list, [cv.string])
        }),
        vol.Optional(CONF_INCLUDE, default={}): vol.Schema({
            vol.Optional(CONF_ENTITIES, default=[]): cv.entity_ids,
            vol.Optional(CONF_DOMAINS, default=[]):
                vol.All(cv.ensure_list, [cv.string])
        }),
        vol.Optional(CONF_DB_NAME, default=DEFAULT_DATABASE): cv.string,
        vol.Optional(CONF_PORT): cv.port,
        vol.Optional(CONF_SSL): cv.boolean,
        vol.Optional(CONF_RETRY_COUNT, default=0): cv.positive_int,
        vol.Optional(CONF_DEFAULT_MEASUREMENT): cv.string,
        vol.Optional(CONF_OVERRIDE_MEASUREMENT): cv.string,
        vol.Optional(CONF_TAGS, default={}):
            vol.Schema({cv.string: cv.string}),
        vol.Optional(CONF_TAGS_ATTRIBUTES, default=[]):
            vol.All(cv.ensure_list, [cv.string]),
        vol.Optional(CONF_VERIFY_SSL, default=DEFAULT_VERIFY_SSL): cv.boolean,
        vol.Optional(CONF_COMPONENT_CONFIG, default={}):
            vol.Schema({cv.entity_id: COMPONENT_CONFIG_SCHEMA_ENTRY}),
        vol.Optional(CONF_COMPONENT_CONFIG_GLOB, default={}):
            vol.Schema({cv.string: COMPONENT_CONFIG_SCHEMA_ENTRY}),
        vol.Optional(CONF_COMPONENT_CONFIG_DOMAIN, default={}):
            vol.Schema({cv.string: COMPONENT_CONFIG_SCHEMA_ENTRY}),
    })),
}, extra=vol.ALLOW_EXTRA)

RE_DIGIT_TAIL = re.compile(r'^[^\.]*\d+\.?\d+[^\.]*$')
RE_DECIMAL = re.compile(r'[^\d.]+')

def setup(hass, config):
    """Set up the InfluxDB component."""
    from influxdb import InfluxDBClient, exceptions

    conf = config[DOMAIN]

    kwargs = {
        'database': conf[CONF_DB_NAME],
        'verify_ssl': conf[CONF_VERIFY_SSL],
        'timeout': TIMEOUT
    }

    if CONF_HOST in conf:
        kwargs['host'] = conf[CONF_HOST]

    if CONF_PORT in conf:
        kwargs['port'] = conf[CONF_PORT]

    if CONF_USERNAME in conf:
        kwargs['username'] = conf[CONF_USERNAME]

    if CONF_PASSWORD in conf:
        kwargs['password'] = conf[CONF_PASSWORD]

    if CONF_SSL in conf:
        kwargs['ssl'] = conf[CONF_SSL]

    include = conf.get(CONF_INCLUDE, {})
    exclude = conf.get(CONF_EXCLUDE, {})
    whitelist_e = set(include.get(CONF_ENTITIES, []))
    whitelist_d = set(include.get(CONF_DOMAINS, []))
    blacklist_e = set(exclude.get(CONF_ENTITIES, []))
    blacklist_d = set(exclude.get(CONF_DOMAINS, []))
    tags = conf.get(CONF_TAGS)
    tags_attributes = conf.get(CONF_TAGS_ATTRIBUTES)
    default_measurement = conf.get(CONF_DEFAULT_MEASUREMENT)
    override_measurement = conf.get(CONF_OVERRIDE_MEASUREMENT)
    component_config = EntityValues(
        conf[CONF_COMPONENT_CONFIG],
        conf[CONF_COMPONENT_CONFIG_DOMAIN],
        conf[CONF_COMPONENT_CONFIG_GLOB])
    max_tries = conf.get(CONF_RETRY_COUNT)

    try:
        influx = InfluxDBClient(**kwargs)
        influx.write_points([])
    except (exceptions.InfluxDBClientError,
            requests.exceptions.ConnectionError) as exc:
        _LOGGER.warning(
            "Database host is not accessible due to '%s', please "
            "check your entries in the configuration file (host, "
            "port, etc.) and verify that the database exists and is "
            "READ/WRITE. Retrying again in %s seconds.", exc, RETRY_INTERVAL
        )
        event_helper.call_later(
            hass, RETRY_INTERVAL, lambda _: setup(hass, config)
        )
        return True

    def event_to_json(event):
        """Add an event to the outgoing Influx list."""
        state = event.data.get('new_state')
        if state is None or state.state in (
                STATE_UNKNOWN, '', STATE_UNAVAILABLE) or \
                state.entity_id in blacklist_e or state.domain in blacklist_d:
            return

        try:
            if ((whitelist_e or whitelist_d)
                    and state.entity_id not in whitelist_e
                    and state.domain not in whitelist_d):
                return

            _include_state = _include_value = False

            _state_as_value = float(state.state)
            _include_value = True
        except ValueError:
            try:
                _state_as_value = float(state_helper.state_as_number(state))
                _include_state = _include_value = True
            except ValueError:
                _include_state = True

        include_uom = True
        measurement = component_config.get(state.entity_id).get(
            CONF_OVERRIDE_MEASUREMENT)
        if measurement in (None, ''):
            if override_measurement:
                measurement = override_measurement
            else:
                measurement = state.attributes.get('unit_of_measurement')
                if measurement in (None, ''):
                    if default_measurement:
                        measurement = default_measurement
                    else:
                        measurement = state.entity_id
                else:
                    include_uom = False

        json = {
            'measurement': measurement,
            'tags': {
                'domain': state.domain,
                'entity_id': state.object_id,
            },
            'time': event.time_fired,
            'fields': {}
        }
        if _include_state:
            json['fields']['state'] = state.state
        if _include_value:
            json['fields']['value'] = _state_as_value

        for key, value in state.attributes.items():
            if key in tags_attributes:
                json['tags'][key] = value
            elif key != 'unit_of_measurement' or include_uom:
                # If the key is already in fields
                if key in json['fields']:
                    key = key + "_"
                # Prevent column data errors in influxDB.
                # For each value we try to cast it as float
                # But if we can not do it we store the value
                # as string add "_str" postfix to the field key
                try:
                    json['fields'][key] = float(value)
                except (ValueError, TypeError):
                    new_key = "{}_str".format(key)
                    new_value = str(value)
                    json['fields'][new_key] = new_value

                    if RE_DIGIT_TAIL.match(new_value):
                        json['fields'][key] = float(
                            RE_DECIMAL.sub('', new_value))

                # Infinity and NaN are not valid floats in InfluxDB
                try:
                    if not math.isfinite(json['fields'][key]):
                        del json['fields'][key]
                except (KeyError, TypeError):
                    pass

        json['tags'].update(tags)

        return json

    instance = hass.data[DOMAIN] = InfluxThread(
        hass, influx, event_to_json, max_tries)
    instance.start()

    def shutdown(event):
        """Shut down the thread."""
        instance.queue.put(None)
        instance.join()
        influx.close()

    hass.bus.listen_once(EVENT_HOMEASSISTANT_STOP, shutdown)

    return True

class InfluxThread(threading.Thread):
    """A threaded event handler class."""

    def __init__(self, hass, influx, event_to_json, max_tries):
        """Initialize the listener."""
        threading.Thread.__init__(self, name='InfluxDB')
        self.queue = queue.Queue()
        self.influx = influx
        self.event_to_json = event_to_json
        self.max_tries = max_tries
        self.write_errors = 0
        self.shutdown = False
        hass.bus.listen(EVENT_STATE_CHANGED, self._event_listener)

    def _event_listener(self, event):
        """Listen for new messages on the bus and queue them for Influx."""
        item = (time.monotonic(), event)
        self.queue.put(item)

    @staticmethod
    def batch_timeout():
        """Return number of seconds to wait for more events."""
        return BATCH_TIMEOUT

    def get_events_json(self):
        """Return a batch of events formatted for writing."""
        queue_seconds = QUEUE_BACKLOG_SECONDS + self.max_tries*RETRY_DELAY

        count = 0
        json = []

        dropped = 0

        try:
            while len(json) < BATCH_BUFFER_SIZE and not self.shutdown:
                timeout = None if count == 0 else self.batch_timeout()
                item = self.queue.get(timeout=timeout)
                count += 1

                if item is None:
                    self.shutdown = True
                else:
                    timestamp, event = item
                    age = time.monotonic() - timestamp

                    if age < queue_seconds:
                        event_json = self.event_to_json(event)
                        if event_json:
                            json.append(event_json)
                    else:
                        dropped += 1

        except queue.Empty:
            pass

        if dropped:
            _LOGGER.warning("Catching up, dropped %d old events", dropped)

        return count, json

    def write_to_influxdb(self, json):
        """Write preprocessed events to influxdb, with retry."""
        from influxdb import exceptions

        for retry in range(self.max_tries+1):
            try:
                self.influx.write_points(json)

                if self.write_errors:
                    _LOGGER.error("Resumed, lost %d events", self.write_errors)
                    self.write_errors = 0

                _LOGGER.debug("Wrote %d events", len(json))
                break
            except (exceptions.InfluxDBClientError,
                    requests.exceptions.RequestException,
                    IOError):
                if retry < self.max_tries:
                    time.sleep(RETRY_DELAY)
                else:
                    if not self.write_errors:
                        _LOGGER.exception("Write error")
                    self.write_errors += len(json)

    def run(self):
        """Process incoming events."""
        while not self.shutdown:
            count, json = self.get_events_json()
            if json:
                self.write_to_influxdb(json)
            for _ in range(count):
                self.queue.task_done()

    def block_till_done(self):
        """Block till all events processed."""
        self.queue.join()
<|endoftext|>


<commit_before>"""noseplugin.py - start and stop JVM when running unit tests

python-javabridge is licensed under the BSD license.  See the
accompanying file LICENSE for details.

Copyright (c) 2003-2009 Massachusetts Institute of Technology
Copyright (c) 2009-2013 Broad Institute
All rights reserved.

"""

import logging
from nose.plugins import Plugin
import os
import numpy as np
np.seterr(all='ignore')
import sys

import javabridge

log = logging.getLogger(__name__)

class JavabridgePlugin(Plugin):
    '''Javabridge nose test plugin

    This plugin starts the JVM before running tests and kills it when
    the tests are done. The plugin is necessary because the JVM cannot
    be restarted once it is killed, so unittest's setUp() and
    tearDown() methods cannot be used to start and stop the JVM.
    '''
    enabled = False
    name = "javabridge"
    score = 100
    extra_jvm_args = []

    def begin(self):
        javabridge.start_vm(self.extra_jvm_args,
                            class_path=self.class_path.split(os.pathsep),
                            run_headless=self.headless,
                            max_heap_size=self.max_heap_size)

    def options(self, parser, env=os.environ):
        super(JavabridgePlugin, self).options(parser, env=env)
        parser.add_option("--classpath", action="store",
                          default=env.get('NOSE_CLASSPATH'),
                          metavar="PATH",
                          dest="classpath",
                          help="Additional class path for JVM [NOSE_CLASSPATH]")
        parser.add_option("--no-headless", action="store_true",
                          default=bool(env.get('NOSE_NO_HEADLESS')),
                          dest="no_headless",
                          help="Set Java environment variable, java.awt.headless to false to allow AWT calls [NOSE_NO_HEADLESS]")
        parser.add_option("--max-heap-size", action="store",
                          default=env.get('NOSE_MAX_HEAP_SIZE'),
                          dest="max_heap_size",
                          help="Set the maximum heap size argument to the JVM as in the -Xmx command-line argument [NOSE_MAX_HEAP_SIZE]")

    def configure(self, options, conf):
        super(JavabridgePlugin, self).configure(options, conf)
        self.class_path = os.pathsep.join(javabridge.JARS)
        if options.classpath:
            self.class_path = os.pathsep.join([options.classpath, self.class_path])
        self.headless = not options.no_headless
        self.max_heap_size = options.max_heap_size

    def prepareTestRunner(self, testRunner):
        '''Need to make the test runner call finalize if in Wing

        Wing IDE's XML test runner fails to call finalize, so we
        wrap it and add that function here
        '''
        if (getattr(testRunner, "__module__","unknown") ==
            "wingtest_common"):
            outer_self = self
            class TestRunnerProxy(object):
                def run(self, test):
                    result = testRunner.run(test)
                    outer_self.finalize(testRunner.result)
                    return result

                @property
                def result(self):
                    return testRunner.result
            return TestRunnerProxy()

    def finalize(self, result):
        javabridge.kill_vm()
<commit_msg>Postpone importing javabridge<commit_after>"""noseplugin.py - start and stop JVM when running unit tests

python-javabridge is licensed under the BSD license.  See the
accompanying file LICENSE for details.

Copyright (c) 2003-2009 Massachusetts Institute of Technology
Copyright (c) 2009-2013 Broad Institute
All rights reserved.

"""

import logging
from nose.plugins import Plugin
import os
import numpy as np
np.seterr(all='ignore')
import sys

log = logging.getLogger(__name__)

class JavabridgePlugin(Plugin):
    '''Javabridge nose test plugin

    This plugin starts the JVM before running tests and kills it when
    the tests are done. The plugin is necessary because the JVM cannot
    be restarted once it is killed, so unittest's setUp() and
    tearDown() methods cannot be used to start and stop the JVM.
    '''
    enabled = False
    name = "javabridge"
    score = 100
    extra_jvm_args = []

    def begin(self):
        import javabridge

        javabridge.start_vm(self.extra_jvm_args,
                            class_path=self.class_path.split(os.pathsep),
                            run_headless=self.headless,
                            max_heap_size=self.max_heap_size)

    def options(self, parser, env=os.environ):
        super(JavabridgePlugin, self).options(parser, env=env)
        parser.add_option("--classpath", action="store",
                          default=env.get('NOSE_CLASSPATH'),
                          metavar="PATH",
                          dest="classpath",
                          help="Additional class path for JVM [NOSE_CLASSPATH]")
        parser.add_option("--no-headless", action="store_true",
                          default=bool(env.get('NOSE_NO_HEADLESS')),
                          dest="no_headless",
                          help="Set Java environment variable, java.awt.headless to false to allow AWT calls [NOSE_NO_HEADLESS]")
        parser.add_option("--max-heap-size", action="store",
                          default=env.get('NOSE_MAX_HEAP_SIZE'),
                          dest="max_heap_size",
                          help="Set the maximum heap size argument to the JVM as in the -Xmx command-line argument [NOSE_MAX_HEAP_SIZE]")

    def configure(self, options, conf):
        import javabridge
        super(JavabridgePlugin, self).configure(options, conf)
        self.class_path = os.pathsep.join(javabridge.JARS)
        if options.classpath:
            self.class_path = os.pathsep.join([options.classpath, self.class_path])
        self.headless = not options.no_headless
        self.max_heap_size = options.max_heap_size

    def prepareTestRunner(self, testRunner):
        '''Need to make the test runner call finalize if in Wing

        Wing IDE's XML test runner fails to call finalize, so we
        wrap it and add that function here
        '''
        if (getattr(testRunner, "__module__","unknown") ==
            "wingtest_common"):
            outer_self = self
            class TestRunnerProxy(object):
                def run(self, test):
                    result = testRunner.run(test)
                    outer_self.finalize(testRunner.result)
                    return result

                @property
                def result(self):
                    return testRunner.result
            return TestRunnerProxy()

    def finalize(self, result):
        import javabridge
        javabridge.kill_vm()
<|endoftext|>


<commit_before>import json
from copy import deepcopy
from ipaddress import ip_interface, ip_network

from ...utils import sorted_dict
from ..base import BaseRenderer
from .timezones import timezones

class NetworkRenderer(BaseRenderer):
    """
    Renders content importable with:
        uci import network
    """
    def _get_interfaces(self):
        """
        converts interfaces object to UCI interface directives
        """
        interfaces = self.config.get('interfaces', [])
        # this line ensures interfaces are not entirely
        # ignored if they do not contain any address
        default_address = [{'proto': 'none'}]
        # results container
        uci_interfaces = []
        for interface in interfaces:
            counter = 1
            is_bridge = False
            # determine uci logical interface name
            network = interface.get('network')
            uci_name = interface['name'] if not network else network
            # convert dot and dashes to underscore
            uci_name = uci_name.replace('.', '_').replace('-', '_')
            # determine if must be type bridge
            if interface.get('type') == 'bridge':
                is_bridge = True
                bridge_members = ' '.join(interface['bridge_members'])
            # ensure address list is not never empty, even when 'addresses' is []
            address_list = interface.get('addresses')
            if not address_list:
                address_list = default_address
            # address list defaults to empty list
            for address in address_list:
                # prepare new UCI interface directive
                uci_interface = deepcopy(interface)
                if network:
                    del uci_interface['network']
                if uci_interface.get('autostart'):
                    uci_interface['auto'] = interface['autostart']
                    del uci_interface['autostart']
                if uci_interface.get('disabled'):
                    uci_interface['enabled'] = not interface['disabled']
                    del uci_interface['disabled']
                if 'addresses' in uci_interface:
                    del uci_interface['addresses']
                if 'type' in uci_interface:
                    del uci_interface['type']
                if 'wireless' in uci_interface:
                    del uci_interface['wireless']
                # default values
                address_key = None
                address_value = None
                # proto defaults to static
                proto = address.get('proto', 'static')
                # add suffix if there is more than one config block
                if counter > 1:
                    name = '{name}_{counter}'.format(name=uci_name, counter=counter)
                else:
                    name = uci_name
                if address.get('family') == 'ipv4':
                    address_key = 'ipaddr'
                elif address.get('family') == 'ipv6':
                    address_key = 'ip6addr'
                    proto = proto.replace('dhcp', 'dhcpv6')
                if address.get('address') and address.get('mask'):
                    address_value = '{address}/{mask}'.format(**address)
                # update interface dict
                uci_interface.update({
                    'name': name,
                    'ifname': interface['name'],
                    'proto': proto,
                    'dns': self.__get_dns_servers(),
                    'dns_search': self.__get_dns_search()
                })
                # bridging
                if is_bridge:
                    uci_interface['type'] = 'bridge'
                    # put bridge members in ifname attribute
                    if bridge_members:
                        uci_interface['ifname'] = bridge_members
                    # if no members, this is an empty bridge
                    else:
                        uci_interface['bridge_empty'] = True
                        del uci_interface['ifname']
                    # ensure type "bridge" is only given to one logical interface
                    is_bridge = False
                # bridge has already been defined
                # but we need to add more references to it
                elif interface.get('type') == 'bridge':
                    # openwrt adds "br-"" prefix to bridge interfaces
                    # we need to take this into account when referring
                    # to these physical names
                    uci_interface['ifname'] = 'br-{0}'.format(interface['name'])
                # delete bridge_members attribtue if bridge is empty
                if uci_interface.get('bridge_members') is not None:
                    del uci_interface['bridge_members']
                # add address if any (with correct option name)
                if address_key and address_value:
                    uci_interface[address_key] = address_value
                # merge additional address fields (discard default ones first)
                address_copy = address.copy()
                for key in ['address', 'mask', 'proto', 'family']:
                    if key in address_copy:
                        del address_copy[key]
                uci_interface.update(address_copy)
                # append to interface list
                uci_interfaces.append(sorted_dict(uci_interface))
                counter += 1
        return uci_interfaces

    def _get_routes(self):
        routes = self.config.get('routes', [])
        # results container
        uci_routes = []
        counter = 1
        # build uci_routes
        for route in routes:
            # prepare UCI route directive
            uci_route = route.copy()
            del uci_route['device']
            del uci_route['next']
            del uci_route['destination']
            del uci_route['cost']
            network = ip_interface(route['destination'])
            version = 'route' if network.version == 4 else 'route6'
            target = network.ip if network.version == 4 else network.network
            uci_route.update({
                'version': version,
                'name': 'route{0}'.format(counter),
                'interface': route['device'],
                'target': str(target),
                'gateway': route['next'],
                'metric': route['cost'],
                'source': route.get('source')
            })
            if network.version == 4:
                uci_route['netmask'] = str(network.netmask)
            uci_routes.append(sorted_dict(uci_route))
            counter += 1
        return uci_routes

    def _get_ip_rules(self):
        rules = self.config.get('ip_rules', [])
        uci_rules = []
        for rule in rules:
            uci_rule = rule.copy()
            src_net = None
            dest_net = None
            family = 4
            if rule.get('src'):
                src_net = ip_network(rule['src'])
            if rule.get('dest'):
                dest_net = ip_network(rule['dest'])
            if dest_net or src_net:
                family = dest_net.version if dest_net else src_net.version
            uci_rule['block_name'] = 'rule{0}'.format(family).replace('4', '')
            uci_rules.append(sorted_dict(uci_rule))
        return uci_rules

    def __get_dns_servers(self):
        dns = self.config.get('dns_servers', None)
        if dns:
            return ' '.join(dns)

    def __get_dns_search(self):
        dns_search = self.config.get('dns_search', None)
        if dns_search:
            return ' '.join(dns_search)

    def _get_switches(self):
        uci_switches = []
        for switch in self.config.get('switch', []):
            uci_switch = sorted_dict(deepcopy(switch))
            uci_switch['vlan'] = [sorted_dict(vlan) for vlan in uci_switch['vlan']]
            uci_switches.append(uci_switch)
        return uci_switches

    def _get_globals(self):
        ula_prefix = self.config.get('general', {}).get('ula_prefix', None)
        if ula_prefix:
            return {'ula_prefix': ula_prefix}
        return {}

class SystemRenderer(BaseRenderer):
    """
    Renders content importable with:
        uci import system
    """
    def _get_system(self):
        general = self.config.get('general', {}).copy()
        # ula_prefix is not related to system
        if 'ula_prefix' in general:
            del general['ula_prefix']
        if general:
            timezone_human = general.get('timezone', 'UTC')
            timezone_value = timezones[timezone_human]
            general.update({
                'hostname': general.get('hostname', 'OpenWRT'),
                'timezone': timezone_value,
            })
        return sorted_dict(general)

    def _get_ntp(self):
        return sorted_dict(self.config.get('ntp', {}))

    def _get_leds(self):
        uci_leds = []
        for led in self.config.get('led', []):
            uci_leds.append(sorted_dict(led))
        return uci_leds

class WirelessRenderer(BaseRenderer):
    """
    Renders content importable with:
        uci import wireless
    """
    def _get_radios(self):
        radios = self.config.get('radios', [])
        uci_radios = []
        for radio in radios:
            uci_radio = radio.copy()
            # rename tx_power to txpower
            if 'tx_power' in radio:
                uci_radio['txpower'] = radio['tx_power']
                del uci_radio['tx_power']
            # rename driver to type
            uci_radio['type'] = radio['driver']
            del uci_radio['driver']
            # determine hwmode option
            uci_radio['hwmode'] = self.__get_hwmode(radio)
            del uci_radio['protocol']
            # determine channel width
            if radio['driver'] == 'mac80211':
                uci_radio['htmode'] = self.__get_htmode(radio)
            del uci_radio['channel_width']
            # ensure country is uppercase
            if uci_radio.get('country'):
                uci_radio['country'] = uci_radio['country'].upper()
            # append sorted dict
            uci_radios.append(sorted_dict(uci_radio))
        return uci_radios

    def __get_hwmode(self, radio):
        """
        possible return values are: 11a, 11b, 11g
        """
        protocol = radio['protocol']
        if protocol not in ['802.11n', '802.11ac']:
            return protocol.replace('802.', '')
        elif protocol == '802.11n' and radio['channel'] <= 13:
            return '11g'
        return '11a'

    def __get_htmode(self, radio):
        """
        only for mac80211 driver
        """
        if radio['protocol'] == '802.11n':
            return 'HT{0}'.format(radio['channel_width'])
        elif radio['protocol'] == '802.11ac':
            return 'VHT{0}'.format(radio['channel_width'])
        # disables n
        return 'NONE'

    def _get_wifi_interfaces(self):
        # select interfaces that have type == "wireless"
        wifi_interfaces = [i for i in self.config.get('interfaces', [])
                           if 'wireless' in i]
        # results container
        uci_wifi_ifaces = []
        for wifi_interface in wifi_interfaces:
            wireless = wifi_interface['wireless']
            # prepare UCI wifi-iface directive
            uci_wifi = wireless.copy()
            # inherit "disabled" attribute if present
            uci_wifi['disabled'] = wifi_interface.get('disabled')
            # add ifname
            uci_wifi['ifname'] = wifi_interface['name']
            # rename radio to device
            uci_wifi['device'] = wireless['radio']
            del uci_wifi['radio']
            # map netjson wifi modes to uci wifi modes
            modes = {
                'access_point': 'ap',
                'station': 'sta',
                'adhoc': 'adhoc',
                'monitor': 'monitor',
                '802.11s': 'mesh'
            }
            uci_wifi['mode'] = modes[wireless['mode']]
            # map advanced 802.11 netjson attributes to UCI
            wifi_options = {
                'ack_distance': 'distance',
                'rts_threshold': 'rts',
                'frag_threshold': 'frag'
            }
            for netjson_key, uci_key in wifi_options.items():
                value = wireless.get(netjson_key)
                if value is not None:
                    # ignore if 0 (autogenerated UIs might use 0 as default value)
                    if value > 0:
                        uci_wifi[uci_key] = value
                    del uci_wifi[netjson_key]
            # determine encryption for wifi
            if uci_wifi.get('encryption'):
                del uci_wifi['encryption']
                uci_encryption = self.__get_encryption(wireless)
                uci_wifi.update(uci_encryption)
            # attached networks (openwrt specific)
            # by default the wifi interface is attached
            # to its defining interface
            # but this behaviour can be overridden
            if not uci_wifi.get('network'):
                # get network, default to ifname
                network = wifi_interface.get('network', wifi_interface['name'])
                uci_wifi['network'] = [network]
            uci_wifi['network'] = ' '.join(uci_wifi['network'])\
                                     .replace('.', '_')\
                                     .replace('-', '_')
            uci_wifi_ifaces.append(sorted_dict(uci_wifi))
        return uci_wifi_ifaces

    def __get_encryption(self, wireless):
        encryption = wireless.get('encryption', {})
        disabled = encryption.get('disabled', False)
        uci = {}
        encryption_map = {
            'wep_open': 'wep-open',
            'wep_shared': 'wep-shared',
            'wpa_personal': 'psk',
            'wpa2_personal': 'psk2',
            'wpa_personal_mixed': 'psk-mixed',
            'wpa_enterprise': 'wpa',
            'wpa2_enterprise': 'wpa2',
            'wpa_enterprise_mixed': 'wpa-mixed',
            'wps': 'psk'
        }
        # if encryption disabled return empty dict
        if not encryption or disabled:
            return uci
        # otherwise configure encryption
        protocol = encryption['protocol']
        # default to protocol raw value in order
        # to allow customization by child classes
        uci['encryption'] = encryption_map.get(protocol, protocol)
        if protocol.startswith('wep'):
            uci['key'] = '1'
            uci['key1'] = encryption['key']
            # tell hostapd/wpa_supplicant key is not hex format
            if protocol == 'wep_open':
                uci['key1'] = 's:{0}'.format(uci['key1'])
        else:
            uci['key'] = encryption['key']
        # add ciphers
        if encryption.get('ciphers'):
            uci['encryption'] += '+{0}'.format('+'.join(encryption['ciphers']))
        return uci

class DefaultRenderer(BaseRenderer):
    """
    Default OpenWrt Renderer
    Allows great flexibility in defining UCI configuration in JSON format
    """
    def _get_custom_packages(self):
        # determine config keys to ignore
        ignore_list = list(self.backend.schema['properties'].keys())
        ignore_list += self.backend.get_packages()
        # determine custom packages
        custom_packages = {}
        for key, value in self.config.items():
            if key not in ignore_list:
                block_list = []
                # sort each config block
                if isinstance(value, list):
                    for block in value[:]:
                        # config block must be a dict
                        # with a key named "config_name"
                        # otherwise it's skipped with a warning
                        if not isinstance(block, dict) or 'config_name' not in block:
                            json_block = json.dumps(block, indent=4)
                            print('Unrecognized config block was skipped:\n\n'
                                  '{0}\n\n'.format(json_block))
                            continue
                        block_list.append(sorted_dict(block))
                # if not a list just skip
                else:  # pragma: nocover
                    continue
                custom_packages[key] = block_list
        # sort custom packages
        return sorted_dict(custom_packages)
<commit_msg>[OpenWrt] Refactored __get_hwmode method of WirelessRenderer<commit_after>import json
from copy import deepcopy
from ipaddress import ip_interface, ip_network

from ...utils import sorted_dict
from ..base import BaseRenderer
from .timezones import timezones

class NetworkRenderer(BaseRenderer):
    """
    Renders content importable with:
        uci import network
    """
    def _get_interfaces(self):
        """
        converts interfaces object to UCI interface directives
        """
        interfaces = self.config.get('interfaces', [])
        # this line ensures interfaces are not entirely
        # ignored if they do not contain any address
        default_address = [{'proto': 'none'}]
        # results container
        uci_interfaces = []
        for interface in interfaces:
            counter = 1
            is_bridge = False
            # determine uci logical interface name
            network = interface.get('network')
            uci_name = interface['name'] if not network else network
            # convert dot and dashes to underscore
            uci_name = uci_name.replace('.', '_').replace('-', '_')
            # determine if must be type bridge
            if interface.get('type') == 'bridge':
                is_bridge = True
                bridge_members = ' '.join(interface['bridge_members'])
            # ensure address list is not never empty, even when 'addresses' is []
            address_list = interface.get('addresses')
            if not address_list:
                address_list = default_address
            # address list defaults to empty list
            for address in address_list:
                # prepare new UCI interface directive
                uci_interface = deepcopy(interface)
                if network:
                    del uci_interface['network']
                if uci_interface.get('autostart'):
                    uci_interface['auto'] = interface['autostart']
                    del uci_interface['autostart']
                if uci_interface.get('disabled'):
                    uci_interface['enabled'] = not interface['disabled']
                    del uci_interface['disabled']
                if 'addresses' in uci_interface:
                    del uci_interface['addresses']
                if 'type' in uci_interface:
                    del uci_interface['type']
                if 'wireless' in uci_interface:
                    del uci_interface['wireless']
                # default values
                address_key = None
                address_value = None
                # proto defaults to static
                proto = address.get('proto', 'static')
                # add suffix if there is more than one config block
                if counter > 1:
                    name = '{name}_{counter}'.format(name=uci_name, counter=counter)
                else:
                    name = uci_name
                if address.get('family') == 'ipv4':
                    address_key = 'ipaddr'
                elif address.get('family') == 'ipv6':
                    address_key = 'ip6addr'
                    proto = proto.replace('dhcp', 'dhcpv6')
                if address.get('address') and address.get('mask'):
                    address_value = '{address}/{mask}'.format(**address)
                # update interface dict
                uci_interface.update({
                    'name': name,
                    'ifname': interface['name'],
                    'proto': proto,
                    'dns': self.__get_dns_servers(),
                    'dns_search': self.__get_dns_search()
                })
                # bridging
                if is_bridge:
                    uci_interface['type'] = 'bridge'
                    # put bridge members in ifname attribute
                    if bridge_members:
                        uci_interface['ifname'] = bridge_members
                    # if no members, this is an empty bridge
                    else:
                        uci_interface['bridge_empty'] = True
                        del uci_interface['ifname']
                    # ensure type "bridge" is only given to one logical interface
                    is_bridge = False
                # bridge has already been defined
                # but we need to add more references to it
                elif interface.get('type') == 'bridge':
                    # openwrt adds "br-"" prefix to bridge interfaces
                    # we need to take this into account when referring
                    # to these physical names
                    uci_interface['ifname'] = 'br-{0}'.format(interface['name'])
                # delete bridge_members attribtue if bridge is empty
                if uci_interface.get('bridge_members') is not None:
                    del uci_interface['bridge_members']
                # add address if any (with correct option name)
                if address_key and address_value:
                    uci_interface[address_key] = address_value
                # merge additional address fields (discard default ones first)
                address_copy = address.copy()
                for key in ['address', 'mask', 'proto', 'family']:
                    if key in address_copy:
                        del address_copy[key]
                uci_interface.update(address_copy)
                # append to interface list
                uci_interfaces.append(sorted_dict(uci_interface))
                counter += 1
        return uci_interfaces

    def _get_routes(self):
        routes = self.config.get('routes', [])
        # results container
        uci_routes = []
        counter = 1
        # build uci_routes
        for route in routes:
            # prepare UCI route directive
            uci_route = route.copy()
            del uci_route['device']
            del uci_route['next']
            del uci_route['destination']
            del uci_route['cost']
            network = ip_interface(route['destination'])
            version = 'route' if network.version == 4 else 'route6'
            target = network.ip if network.version == 4 else network.network
            uci_route.update({
                'version': version,
                'name': 'route{0}'.format(counter),
                'interface': route['device'],
                'target': str(target),
                'gateway': route['next'],
                'metric': route['cost'],
                'source': route.get('source')
            })
            if network.version == 4:
                uci_route['netmask'] = str(network.netmask)
            uci_routes.append(sorted_dict(uci_route))
            counter += 1
        return uci_routes

    def _get_ip_rules(self):
        rules = self.config.get('ip_rules', [])
        uci_rules = []
        for rule in rules:
            uci_rule = rule.copy()
            src_net = None
            dest_net = None
            family = 4
            if rule.get('src'):
                src_net = ip_network(rule['src'])
            if rule.get('dest'):
                dest_net = ip_network(rule['dest'])
            if dest_net or src_net:
                family = dest_net.version if dest_net else src_net.version
            uci_rule['block_name'] = 'rule{0}'.format(family).replace('4', '')
            uci_rules.append(sorted_dict(uci_rule))
        return uci_rules

    def __get_dns_servers(self):
        dns = self.config.get('dns_servers', None)
        if dns:
            return ' '.join(dns)

    def __get_dns_search(self):
        dns_search = self.config.get('dns_search', None)
        if dns_search:
            return ' '.join(dns_search)

    def _get_switches(self):
        uci_switches = []
        for switch in self.config.get('switch', []):
            uci_switch = sorted_dict(deepcopy(switch))
            uci_switch['vlan'] = [sorted_dict(vlan) for vlan in uci_switch['vlan']]
            uci_switches.append(uci_switch)
        return uci_switches

    def _get_globals(self):
        ula_prefix = self.config.get('general', {}).get('ula_prefix', None)
        if ula_prefix:
            return {'ula_prefix': ula_prefix}
        return {}

class SystemRenderer(BaseRenderer):
    """
    Renders content importable with:
        uci import system
    """
    def _get_system(self):
        general = self.config.get('general', {}).copy()
        # ula_prefix is not related to system
        if 'ula_prefix' in general:
            del general['ula_prefix']
        if general:
            timezone_human = general.get('timezone', 'UTC')
            timezone_value = timezones[timezone_human]
            general.update({
                'hostname': general.get('hostname', 'OpenWRT'),
                'timezone': timezone_value,
            })
        return sorted_dict(general)

    def _get_ntp(self):
        return sorted_dict(self.config.get('ntp', {}))

    def _get_leds(self):
        uci_leds = []
        for led in self.config.get('led', []):
            uci_leds.append(sorted_dict(led))
        return uci_leds

class WirelessRenderer(BaseRenderer):
    """
    Renders content importable with:
        uci import wireless
    """
    def _get_radios(self):
        radios = self.config.get('radios', [])
        uci_radios = []
        for radio in radios:
            uci_radio = radio.copy()
            # rename tx_power to txpower
            if 'tx_power' in radio:
                uci_radio['txpower'] = radio['tx_power']
                del uci_radio['tx_power']
            # rename driver to type
            uci_radio['type'] = radio['driver']
            del uci_radio['driver']
            # determine hwmode option
            uci_radio['hwmode'] = self.__get_hwmode(radio)
            del uci_radio['protocol']
            # determine channel width
            if radio['driver'] == 'mac80211':
                uci_radio['htmode'] = self.__get_htmode(radio)
            del uci_radio['channel_width']
            # ensure country is uppercase
            if uci_radio.get('country'):
                uci_radio['country'] = uci_radio['country'].upper()
            # append sorted dict
            uci_radios.append(sorted_dict(uci_radio))
        return uci_radios

    def __get_hwmode(self, radio):
        """
        possible return values are: 11a, 11b, 11g
        """
        protocol = radio['protocol']
        if protocol in ['802.11a', '802.11b', '802.11g']:
            # return 11a, 11b or 11g
            return protocol[4:]
        # determine hwmode depending on channel used
        if radio['channel'] <= 13:
            return '11g'
        else:
            return '11a'

    def __get_htmode(self, radio):
        """
        only for mac80211 driver
        """
        if radio['protocol'] == '802.11n':
            return 'HT{0}'.format(radio['channel_width'])
        elif radio['protocol'] == '802.11ac':
            return 'VHT{0}'.format(radio['channel_width'])
        # disables n
        return 'NONE'

    def _get_wifi_interfaces(self):
        # select interfaces that have type == "wireless"
        wifi_interfaces = [i for i in self.config.get('interfaces', [])
                           if 'wireless' in i]
        # results container
        uci_wifi_ifaces = []
        for wifi_interface in wifi_interfaces:
            wireless = wifi_interface['wireless']
            # prepare UCI wifi-iface directive
            uci_wifi = wireless.copy()
            # inherit "disabled" attribute if present
            uci_wifi['disabled'] = wifi_interface.get('disabled')
            # add ifname
            uci_wifi['ifname'] = wifi_interface['name']
            # rename radio to device
            uci_wifi['device'] = wireless['radio']
            del uci_wifi['radio']
            # map netjson wifi modes to uci wifi modes
            modes = {
                'access_point': 'ap',
                'station': 'sta',
                'adhoc': 'adhoc',
                'monitor': 'monitor',
                '802.11s': 'mesh'
            }
            uci_wifi['mode'] = modes[wireless['mode']]
            # map advanced 802.11 netjson attributes to UCI
            wifi_options = {
                'ack_distance': 'distance',
                'rts_threshold': 'rts',
                'frag_threshold': 'frag'
            }
            for netjson_key, uci_key in wifi_options.items():
                value = wireless.get(netjson_key)
                if value is not None:
                    # ignore if 0 (autogenerated UIs might use 0 as default value)
                    if value > 0:
                        uci_wifi[uci_key] = value
                    del uci_wifi[netjson_key]
            # determine encryption for wifi
            if uci_wifi.get('encryption'):
                del uci_wifi['encryption']
                uci_encryption = self.__get_encryption(wireless)
                uci_wifi.update(uci_encryption)
            # attached networks (openwrt specific)
            # by default the wifi interface is attached
            # to its defining interface
            # but this behaviour can be overridden
            if not uci_wifi.get('network'):
                # get network, default to ifname
                network = wifi_interface.get('network', wifi_interface['name'])
                uci_wifi['network'] = [network]
            uci_wifi['network'] = ' '.join(uci_wifi['network'])\
                                     .replace('.', '_')\
                                     .replace('-', '_')
            uci_wifi_ifaces.append(sorted_dict(uci_wifi))
        return uci_wifi_ifaces

    def __get_encryption(self, wireless):
        encryption = wireless.get('encryption', {})
        disabled = encryption.get('disabled', False)
        uci = {}
        encryption_map = {
            'wep_open': 'wep-open',
            'wep_shared': 'wep-shared',
            'wpa_personal': 'psk',
            'wpa2_personal': 'psk2',
            'wpa_personal_mixed': 'psk-mixed',
            'wpa_enterprise': 'wpa',
            'wpa2_enterprise': 'wpa2',
            'wpa_enterprise_mixed': 'wpa-mixed',
            'wps': 'psk'
        }
        # if encryption disabled return empty dict
        if not encryption or disabled:
            return uci
        # otherwise configure encryption
        protocol = encryption['protocol']
        # default to protocol raw value in order
        # to allow customization by child classes
        uci['encryption'] = encryption_map.get(protocol, protocol)
        if protocol.startswith('wep'):
            uci['key'] = '1'
            uci['key1'] = encryption['key']
            # tell hostapd/wpa_supplicant key is not hex format
            if protocol == 'wep_open':
                uci['key1'] = 's:{0}'.format(uci['key1'])
        else:
            uci['key'] = encryption['key']
        # add ciphers
        if encryption.get('ciphers'):
            uci['encryption'] += '+{0}'.format('+'.join(encryption['ciphers']))
        return uci

class DefaultRenderer(BaseRenderer):
    """
    Default OpenWrt Renderer
    Allows great flexibility in defining UCI configuration in JSON format
    """
    def _get_custom_packages(self):
        # determine config keys to ignore
        ignore_list = list(self.backend.schema['properties'].keys())
        ignore_list += self.backend.get_packages()
        # determine custom packages
        custom_packages = {}
        for key, value in self.config.items():
            if key not in ignore_list:
                block_list = []
                # sort each config block
                if isinstance(value, list):
                    for block in value[:]:
                        # config block must be a dict
                        # with a key named "config_name"
                        # otherwise it's skipped with a warning
                        if not isinstance(block, dict) or 'config_name' not in block:
                            json_block = json.dumps(block, indent=4)
                            print('Unrecognized config block was skipped:\n\n'
                                  '{0}\n\n'.format(json_block))
                            continue
                        block_list.append(sorted_dict(block))
                # if not a list just skip
                else:  # pragma: nocover
                    continue
                custom_packages[key] = block_list
        # sort custom packages
        return sorted_dict(custom_packages)
<|endoftext|>


<commit_before># coding: utf-8
# Copyright 2015
# Jerry Zhou <jerryzhou@hotmail.ca> and Christian Fobel <christian@fobel.net>
import re
import warnings

import pandas as pd
import numpy as np

from . import INKSCAPE_NSMAP
from .draw import draw_lines_svg_layer as _draw_lines_svg_layer

def extend_shapes(df_shapes, axis, distance):
    '''
    Extend shape/polygon outline away from polygon center point by absolute
    distance.
    '''
    df_shapes_i = df_shapes.copy()
    offsets = df_shapes_i[axis + '_center_offset'].copy()
    offsets[offsets < 0] -= distance
    offsets[offsets >= 0] += distance
    df_shapes_i[axis] = df_shapes_i[axis + '_center'] + offsets
    return df_shapes_i

def extract_adjacent_shapes(df_shapes, shape_i_column, extend=.5):
    '''
    Generate list of connections between "adjacent" polygon shapes based on
    geometrical "closeness".

    Parameters
    ----------
    df_shapes : pandas.DataFrame
        Table of polygon shape vertices (one row per vertex).

        Table rows with the same value in the :data:`shape_i_column` column
        are grouped together as a polygon.
    shape_i_column : str or list[str]
        Column name(s) that identify the polygon each row belongs to.
    extend : float, optional
        Extend ``x``/``y`` coords by the specified number of absolute units
        from the center point of each polygon.
        Each polygon is stretched independently in the ``x`` and ``y`` direction.
        In each direction, a polygon considered adjacent to all polygons that
        are overlapped by the extended shape.

    Returns
    -------
    pandas.DataFrame
        Adjacency list as a frame containing the columns ``source`` and
        ``target``.

        The ``source`` and ``target`` of each adjacency connection is ordered
        such that the ``source`` is less than the ``target``.
    '''
    # Find corners of each solid shape outline.
    # Extend x coords by abs units
    df_scaled_x = extend_shapes(df_shapes, 'x', extend)
    # Extend y coords by abs units
    df_scaled_y = extend_shapes(df_shapes, 'y', extend)

    df_corners = df_shapes.groupby(shape_i_column).agg({'x': ['min', 'max'],
                                                        'y': ['min', 'max']})

    # Find adjacent electrodes
    row_list = []

    for shapeNumber in df_shapes[shape_i_column].drop_duplicates():
        df_stretched = df_scaled_x[df_scaled_x[shape_i_column]
                                   .isin([shapeNumber])]
        xmin_x, xmax_x, ymin_x, ymax_x = (df_stretched.x.min(),
                                          df_stretched.x.max(),
                                          df_stretched.y.min(),
                                          df_stretched.y.max())
        df_stretched = df_scaled_y[df_scaled_y[shape_i_column]
                                   .isin([shapeNumber])]
        xmin_y, xmax_y, ymin_y, ymax_y = (df_stretched.x.min(),
                                          df_stretched.x.max(),
                                          df_stretched.y.min(),
                                          df_stretched.y.max())

        #Some conditions unnecessary if it is assumed that electrodes don't overlap
        adjacent = df_corners[
            ((df_corners.x['min'] < xmax_x) & (df_corners.x['max'] >= xmax_x)
            # Check in x stretched direction
            |(df_corners.x['min'] < xmin_x) & (df_corners.x['max'] >= xmin_x))
            # Check if y is within bounds
            & (df_corners.y['min'] < ymax_x) & (df_corners.y['max'] > ymin_x) |

            #maybe do ymax_x - df_corners.y['min'] > threshold &
            #  df_corners.y['max'] - ymin_x > threshold

            ((df_corners.y['min'] < ymax_y) & (df_corners.y['max'] >= ymax_y)
             # Checks in y stretched direction
             |(df_corners.y['min'] < ymin_y) & (df_corners.y['max'] >= ymin_y))
             # Check if x in within bounds
            & ((df_corners.x['min'] < xmax_y) & (df_corners.x['max'] > xmin_y))
        ].index.values

        for shape in adjacent:
            temp_dict = {}
            reverse_dict = {}

            temp_dict ['source'] = shapeNumber
            reverse_dict['source'] = shape
            temp_dict ['target'] = shape
            reverse_dict['target'] = shapeNumber

            if(reverse_dict not in row_list):
                row_list.append(temp_dict)

    df_connected = (pd.DataFrame(row_list)[['source', 'target']]
                    .sort_index(axis=1, ascending=True)
                    .sort_values(['source', 'target']))
    return df_connected

def get_adjacency_matrix(df_connected):
    '''
    Return matrix where $a_{i,j} = 1$ indicates polygon $i$ is connected to
    polygon $j$.

    Also, return mapping (and reverse mapping) from original keys in
    `df_connected` to zero-based integer index used for matrix rows and
    columns.
    '''
    sorted_path_keys = np.sort(np.unique(df_connected[['source', 'target']]
                                         .values.ravel()))
    indexed_paths = pd.Series(sorted_path_keys)
    path_indexes = pd.Series(indexed_paths.index, index=sorted_path_keys)

    adjacency_matrix = np.zeros((path_indexes.shape[0], ) * 2, dtype=int)
    for i_key, j_key in df_connected[['source', 'target']].values:
        i, j = path_indexes.loc[[i_key, j_key]]
        adjacency_matrix[i, j] = 1
        adjacency_matrix[j, i] = 1
    return adjacency_matrix, indexed_paths, path_indexes

def extract_connections(svg_source, shapes_canvas, line_layer='Connections',
                        line_xpath=None, path_xpath=None, namespaces=None):
    '''
    Load all ``<svg:line>`` elements and ``<svg:path>`` elements from a layer
    of an SVG source.  For each element, if endpoints overlap distinct shapes
    in :data:`shapes_canvas`, add connection between overlapped shapes.

    .. versionchanged:: 0.6.post1
        Allow both ``<svg:line>`` *and* ``<svg:path>`` instances to denote
        connected/adjacent shapes.

    .. versionadded:: 0.6.post1
        :data:`path_xpath`

    Parameters
    ----------
    svg_source : filepath
        Input SVG file containing connection lines.
    shapes_canvas : shapes_canvas.ShapesCanvas
        Shapes canvas containing shapes to compare against connection
        endpoints.
    line_layer : str
        Name of layer in SVG containing connection lines.
    line_xpath : str
        XPath string to iterate through connection lines.
    path_xpath : str
        XPath string to iterate through connection paths.
    namespaces : dict
        SVG namespaces (compatible with :func:`etree.parse`).

    Returns
    -------
    pandas.DataFrame
        Each row corresponds to connection between two shapes in
        :data:`shapes_canvas`, denoted ``source`` and ``target``.
    '''
    from lxml import etree

    if namespaces is None:
        namespaces = INKSCAPE_NSMAP

    e_root = etree.parse(svg_source)
    frames = []

    if line_xpath is None:
        line_xpath = ("//svg:g[@inkscape:label='%s']/svg:line" % line_layer)
    coords_columns = ['x1', 'y1', 'x2', 'y2']

    for line_i in e_root.xpath(line_xpath, namespaces=namespaces):
        line_i_dict = dict(line_i.items())
        values = ([line_i_dict.get('id', None)] +
                  [float(line_i_dict[k]) for k in coords_columns])
        frames.append(values)

    cre_path_ends = re.compile(r'^\s*M\s*(?P<start_x>\d+(\.\d+)?),\s*'
                               r'(?P<start_y>\d+(\.\d+)?).*L\s*'
                               r'(?P<end_x>\d+(\.\d+)?),\s*'
                               r'(?P<end_y>\d+(\.\d+)?)\D*$')

    if path_xpath is None:
        path_xpath = ("//svg:g[@inkscape:label='%s']/svg:path" % line_layer)

    for path_i in e_root.xpath(path_xpath, namespaces=namespaces):
        path_i_dict = dict(path_i.items())
        match_i = cre_path_ends.match(path_i_dict['d'])
        if match_i:
            frames.append([path_i_dict['id']] + map(float,
                                                    (match_i.group('start_x'),
                                                     match_i.group('start_y'),
                                                     match_i.group('end_x'),
                                                     match_i.group('end_y'))))

    if not frames:
        return pd.DataFrame(None, columns=['source', 'target'])

    df_connection_lines = pd.DataFrame(frames, columns=['id'] + coords_columns)

    df_shape_connections_i = pd.DataFrame([[shapes_canvas.find_shape(x1, y1),
                                            shapes_canvas.find_shape(x2, y2)]
                                           for i, (x1, y1, x2, y2) in
                                           df_connection_lines[coords_columns]
                                           .iterrows()],
                                          columns=['source', 'target'])
    df_shape_connections_i.sort_index(axis=1, inplace=True)
    df_shape_connections_i['line_id'] = df_connection_lines['id']
    return df_shape_connections_i.dropna()

def draw_lines_svg_layer(df_endpoints, layer_name='Connections'):
    warnings.warn('`draw_lines_svg_layer` has been moved to `svg_model.draw`')
    return _draw_lines_svg_layer(df_endpoints, layer_name=layer_name)
<commit_msg>[FIX] Support H/V endpoint in SVG connection paths<commit_after># coding: utf-8
# Copyright 2015
# Jerry Zhou <jerryzhou@hotmail.ca> and Christian Fobel <christian@fobel.net>
import re
import warnings

import pandas as pd
import numpy as np

from . import INKSCAPE_NSMAP
from .draw import draw_lines_svg_layer as _draw_lines_svg_layer

def extend_shapes(df_shapes, axis, distance):
    '''
    Extend shape/polygon outline away from polygon center point by absolute
    distance.
    '''
    df_shapes_i = df_shapes.copy()
    offsets = df_shapes_i[axis + '_center_offset'].copy()
    offsets[offsets < 0] -= distance
    offsets[offsets >= 0] += distance
    df_shapes_i[axis] = df_shapes_i[axis + '_center'] + offsets
    return df_shapes_i

def extract_adjacent_shapes(df_shapes, shape_i_column, extend=.5):
    '''
    Generate list of connections between "adjacent" polygon shapes based on
    geometrical "closeness".

    Parameters
    ----------
    df_shapes : pandas.DataFrame
        Table of polygon shape vertices (one row per vertex).

        Table rows with the same value in the :data:`shape_i_column` column
        are grouped together as a polygon.
    shape_i_column : str or list[str]
        Column name(s) that identify the polygon each row belongs to.
    extend : float, optional
        Extend ``x``/``y`` coords by the specified number of absolute units
        from the center point of each polygon.
        Each polygon is stretched independently in the ``x`` and ``y`` direction.
        In each direction, a polygon considered adjacent to all polygons that
        are overlapped by the extended shape.

    Returns
    -------
    pandas.DataFrame
        Adjacency list as a frame containing the columns ``source`` and
        ``target``.

        The ``source`` and ``target`` of each adjacency connection is ordered
        such that the ``source`` is less than the ``target``.
    '''
    # Find corners of each solid shape outline.
    # Extend x coords by abs units
    df_scaled_x = extend_shapes(df_shapes, 'x', extend)
    # Extend y coords by abs units
    df_scaled_y = extend_shapes(df_shapes, 'y', extend)

    df_corners = df_shapes.groupby(shape_i_column).agg({'x': ['min', 'max'],
                                                        'y': ['min', 'max']})

    # Find adjacent electrodes
    row_list = []

    for shapeNumber in df_shapes[shape_i_column].drop_duplicates():
        df_stretched = df_scaled_x[df_scaled_x[shape_i_column]
                                   .isin([shapeNumber])]
        xmin_x, xmax_x, ymin_x, ymax_x = (df_stretched.x.min(),
                                          df_stretched.x.max(),
                                          df_stretched.y.min(),
                                          df_stretched.y.max())
        df_stretched = df_scaled_y[df_scaled_y[shape_i_column]
                                   .isin([shapeNumber])]
        xmin_y, xmax_y, ymin_y, ymax_y = (df_stretched.x.min(),
                                          df_stretched.x.max(),
                                          df_stretched.y.min(),
                                          df_stretched.y.max())

        #Some conditions unnecessary if it is assumed that electrodes don't overlap
        adjacent = df_corners[
            ((df_corners.x['min'] < xmax_x) & (df_corners.x['max'] >= xmax_x)
            # Check in x stretched direction
            |(df_corners.x['min'] < xmin_x) & (df_corners.x['max'] >= xmin_x))
            # Check if y is within bounds
            & (df_corners.y['min'] < ymax_x) & (df_corners.y['max'] > ymin_x) |

            #maybe do ymax_x - df_corners.y['min'] > threshold &
            #  df_corners.y['max'] - ymin_x > threshold

            ((df_corners.y['min'] < ymax_y) & (df_corners.y['max'] >= ymax_y)
             # Checks in y stretched direction
             |(df_corners.y['min'] < ymin_y) & (df_corners.y['max'] >= ymin_y))
             # Check if x in within bounds
            & ((df_corners.x['min'] < xmax_y) & (df_corners.x['max'] > xmin_y))
        ].index.values

        for shape in adjacent:
            temp_dict = {}
            reverse_dict = {}

            temp_dict ['source'] = shapeNumber
            reverse_dict['source'] = shape
            temp_dict ['target'] = shape
            reverse_dict['target'] = shapeNumber

            if(reverse_dict not in row_list):
                row_list.append(temp_dict)

    df_connected = (pd.DataFrame(row_list)[['source', 'target']]
                    .sort_index(axis=1, ascending=True)
                    .sort_values(['source', 'target']))
    return df_connected

def get_adjacency_matrix(df_connected):
    '''
    Return matrix where $a_{i,j} = 1$ indicates polygon $i$ is connected to
    polygon $j$.

    Also, return mapping (and reverse mapping) from original keys in
    `df_connected` to zero-based integer index used for matrix rows and
    columns.
    '''
    sorted_path_keys = np.sort(np.unique(df_connected[['source', 'target']]
                                         .values.ravel()))
    indexed_paths = pd.Series(sorted_path_keys)
    path_indexes = pd.Series(indexed_paths.index, index=sorted_path_keys)

    adjacency_matrix = np.zeros((path_indexes.shape[0], ) * 2, dtype=int)
    for i_key, j_key in df_connected[['source', 'target']].values:
        i, j = path_indexes.loc[[i_key, j_key]]
        adjacency_matrix[i, j] = 1
        adjacency_matrix[j, i] = 1
    return adjacency_matrix, indexed_paths, path_indexes

def extract_connections(svg_source, shapes_canvas, line_layer='Connections',
                        line_xpath=None, path_xpath=None, namespaces=None):
    '''
    Load all ``<svg:line>`` elements and ``<svg:path>`` elements from a layer
    of an SVG source.  For each element, if endpoints overlap distinct shapes
    in :data:`shapes_canvas`, add connection between overlapped shapes.

    .. versionchanged:: 0.6.post1
        Allow both ``<svg:line>`` *and* ``<svg:path>`` instances to denote
        connected/adjacent shapes.

    .. versionadded:: 0.6.post1
        :data:`path_xpath`

    Parameters
    ----------
    svg_source : filepath
        Input SVG file containing connection lines.
    shapes_canvas : shapes_canvas.ShapesCanvas
        Shapes canvas containing shapes to compare against connection
        endpoints.
    line_layer : str
        Name of layer in SVG containing connection lines.
    line_xpath : str
        XPath string to iterate through connection lines.
    path_xpath : str
        XPath string to iterate through connection paths.
    namespaces : dict
        SVG namespaces (compatible with :func:`etree.parse`).

    Returns
    -------
    pandas.DataFrame
        Each row corresponds to connection between two shapes in
        :data:`shapes_canvas`, denoted ``source`` and ``target``.
    '''
    from lxml import etree

    if namespaces is None:
        # Inkscape namespace is required to match SVG elements as well as
        # Inkscape-specific SVG tags and attributes (e.g., `inkscape:label`).
        namespaces = INKSCAPE_NSMAP

    # Parse SVG source.
    e_root = etree.parse(svg_source)
    # List to hold records of form: `[<id>, <x1>, <y1>, <x2>, <y2>]`.
    frames = []

    if line_xpath is None:
        # Define query to look for `svg:line` elements in top level of layer of
        # SVG specified to contain connections.
        line_xpath = ("//svg:g[@inkscape:label='%s']/svg:line" % line_layer)
    coords_columns = ['x1', 'y1', 'x2', 'y2']

    for line_i in e_root.xpath(line_xpath, namespaces=namespaces):
        # Extract start and end coordinate from `svg:line` element.
        line_i_dict = dict(line_i.items())
        values = ([line_i_dict.get('id', None)] +
                  [float(line_i_dict[k]) for k in coords_columns])
        # Append record for end points of current line.
        frames.append(values)

    # Regular expression pattern to match start and end coordinates of
    # connection `svg:path` element.
    cre_path_ends = re.compile(r'^\s*M\s*(?P<start_x>\d+(\.\d+)?),\s*'
                               r'(?P<start_y>\d+(\.\d+)?).*'
                               # Diagonal line...
                               r'((L\s*(?P<end_x>\d+(\.\d+)?),\s*'
                               r'(?P<end_y>\d+(\.\d+)?))|'
                               # or Vertical line...
                               r'(V\s*(?P<end_vy>\d+(\.\d+)?))|'
                               # or Horizontal line
                               r'(H\s*(?P<end_hx>\d+(\.\d+)?))'
                               r')\D*'
                               r'$')

    if path_xpath is None:
        # Define query to look for `svg:path` elements in top level of layer of
        # SVG specified to contain connections.
        path_xpath = ("//svg:g[@inkscape:label='%s']/svg:path" % line_layer)

    for path_i in e_root.xpath(path_xpath, namespaces=namespaces):
        path_i_dict = dict(path_i.items())
        match_i = cre_path_ends.match(path_i_dict['d'])
        if match_i:
            # Connection `svg:path` matched required format.  Extract start and
            # end coordinates.
            match_dict_i = match_i.groupdict()
            if match_dict_i['end_vy']:
                # Path ended with vertical line
                match_dict_i['end_x'] = match_dict_i['start_x']
                match_dict_i['end_y'] = match_dict_i['end_vy']
            if match_dict_i['end_hx']:
                # Path ended with horizontal line
                match_dict_i['end_x'] = match_dict_i['end_hx']
                match_dict_i['end_y'] = match_dict_i['start_y']
            # Append record for end points of current path.
            frames.append([path_i_dict['id']] + map(float,
                                                    (match_dict_i['start_x'],
                                                     match_dict_i['start_y'],
                                                     match_dict_i['end_x'],
                                                     match_dict_i['end_y'])))

    if not frames:
        return pd.DataFrame(None, columns=['source', 'target'])

    df_connection_lines = pd.DataFrame(frames, columns=['id'] + coords_columns)

    # Use `shapes_canvas.find_shape` to determine shapes overlapped by end
    # points of each `svg:path` or `svg:line`.
    df_shape_connections_i = pd.DataFrame([[shapes_canvas.find_shape(x1, y1),
                                            shapes_canvas.find_shape(x2, y2)]
                                           for i, (x1, y1, x2, y2) in
                                           df_connection_lines[coords_columns]
                                           .iterrows()],
                                          columns=['source', 'target'])
    # Order the source and target of each row so the source shape identifier is
    # always the lowest.
    df_shape_connections_i.sort_index(axis=1, inplace=True)
    # Tag each shape connection with the corresponding `svg:line`/`svg:path`
    # identifier.  May be useful, e.g., in debugging.
    df_shape_connections_i['line_id'] = df_connection_lines['id']
    # Remove connections where source or target shape was not matched (e.g., if
    # one or more end points does not overlap with a shape).
    return df_shape_connections_i.dropna()

def draw_lines_svg_layer(df_endpoints, layer_name='Connections'):
    warnings.warn('`draw_lines_svg_layer` has been moved to `svg_model.draw`')
    return _draw_lines_svg_layer(df_endpoints, layer_name=layer_name)
<|endoftext|>