<commit_before>"""
Support for KNX components.

For more details about this component, please refer to the documentation at
https://home-assistant.io/components/knx/
"""
import logging

import voluptuous as vol

import homeassistant.helpers.config_validation as cv
from homeassistant.const import (
    EVENT_HOMEASSISTANT_STOP, CONF_HOST, CONF_PORT)
from homeassistant.helpers.entity import Entity

REQUIREMENTS = ['knxip==0.3.3']

_LOGGER = logging.getLogger(__name__)

DEFAULT_HOST = '0.0.0.0'
DEFAULT_PORT = 3671
DOMAIN = 'knx'

EVENT_KNX_FRAME_RECEIVED = 'knx_frame_received'

KNXTUNNEL = None

CONFIG_SCHEMA = vol.Schema({
    DOMAIN: vol.Schema({
        vol.Optional(CONF_HOST, default=DEFAULT_HOST): cv.string,
        vol.Optional(CONF_PORT, default=DEFAULT_PORT): cv.port,
    }),
}, extra=vol.ALLOW_EXTRA)

def setup(hass, config):
    """Set up the connection to the KNX IP interface."""
    global KNXTUNNEL

    from knxip.ip import KNXIPTunnel
    from knxip.core import KNXException

    host = config[DOMAIN].get(CONF_HOST)
    port = config[DOMAIN].get(CONF_PORT)

    if host is '0.0.0.0':
        _LOGGER.debug("Will try to auto-detect KNX/IP gateway")

    KNXTUNNEL = KNXIPTunnel(host, port)
    try:
        res = KNXTUNNEL.connect()
        _LOGGER.debug("Res = %s", res)
        if not res:
            _LOGGER.exception("Could not connect to KNX/IP interface %s", host)
            return False

    except KNXException as ex:
        _LOGGER.exception("Can't connect to KNX/IP interface: %s", ex)
        KNXTUNNEL = None
        return False

    _LOGGER.info("KNX IP tunnel to %s:%i established", host, port)

    hass.bus.listen_once(EVENT_HOMEASSISTANT_STOP, close_tunnel)
    return True

def close_tunnel(_data):
    """Close the NKX tunnel connection on shutdown."""
    global KNXTUNNEL

    KNXTUNNEL.disconnect()
    KNXTUNNEL = None

class KNXConfig(object):
    """Handle the fetching of configuration from the config file."""

    def __init__(self, config):
        """Initialize the configuration."""
        from knxip.core import parse_group_address

        self.config = config
        self.should_poll = config.get('poll', True)
        if config.get('address'):
            self._address = parse_group_address(config.get('address'))
        else:
            self._address = None
        if self.config.get('state_address'):
            self._state_address = parse_group_address(
                self.config.get('state_address'))
        else:
            self._state_address = None

    @property
    def name(self):
        """Return the name given to the entity."""
        return self.config['name']

    @property
    def address(self):
        """Return the address of the device as an integer value.

        3 types of addresses are supported:
        integer - 0-65535
        2 level - a/b
        3 level - a/b/c
        """
        return self._address

    @property
    def state_address(self):
        """Return the group address the device sends its current state to.

        Some KNX devices can send the current state to a seperate
        group address. This makes send e.g. when an actuator can
        be switched but also have a timer functionality.
        """
        return self._state_address

class KNXGroupAddress(Entity):
    """Representation of devices connected to a KNX group address."""

    def __init__(self, hass, config):
        """Initialize the device."""
        self._config = config
        self._state = False
        self._data = None
        _LOGGER.debug("Initalizing KNX group address %s", self.address)

        def handle_knx_message(addr, data):
            """Handle an incoming KNX frame.

            Handle an incoming frame and update our status if it contains
            information relating to this device.
            """
            if (addr == self.state_address) or (addr == self.address):
                self._state = data[0]
                self.schedule_update_ha_state()

        KNXTUNNEL.register_listener(self.address, handle_knx_message)
        if self.state_address:
            KNXTUNNEL.register_listener(self.state_address, handle_knx_message)

    @property
    def name(self):
        """Return the entity's display name."""
        return self._config.name

    @property
    def config(self):
        """Return the entity's configuration."""
        return self._config

    @property
    def should_poll(self):
        """Return the state of the polling, if needed."""
        return self._config.should_poll

    @property
    def is_on(self):
        """Return True if the value is not 0 is on, else False."""
        return self._state != 0

    @property
    def address(self):
        """Return the KNX group address."""
        return self._config.address

    @property
    def state_address(self):
        """Return the KNX group address."""
        return self._config.state_address

    @property
    def cache(self):
        """Return the name given to the entity."""
        return self._config.config.get('cache', True)

    def group_write(self, value):
        """Write to the group address."""
        KNXTUNNEL.group_write(self.address, [value])

    def update(self):
        """Get the state from KNX bus or cache."""
        from knxip.core import KNXException

        try:
            if self.state_address:
                res = KNXTUNNEL.group_read(
                    self.state_address, use_cache=self.cache)
            else:
                res = KNXTUNNEL.group_read(self.address, use_cache=self.cache)

            if res:
                self._state = res[0]
                self._data = res
            else:
                _LOGGER.debug(
                    "Unable to read from KNX address: %s (None)", self.address)

        except KNXException:
            _LOGGER.exception(
                "Unable to read from KNX address: %s", self.address)
            return False

class KNXMultiAddressDevice(Entity):
    """Representation of devices connected to a multiple KNX group address.

    This is needed for devices like dimmers or shutter actuators as they have
    to be controlled by multiple group addresses.
    """

    def __init__(self, hass, config, required, optional=None):
        """Initialize the device.

        The namelist argument lists the required addresses. E.g. for a dimming
        actuators, the namelist might look like:
        onoff_address: 0/0/1
        brightness_address: 0/0/2
        """
        from knxip.core import parse_group_address, KNXException

        self.names = {}
        self.values = {}

        self._config = config
        self._state = False
        self._data = None
        _LOGGER.debug("Initalizing KNX multi address device")

        settings = self._config.config
        # parse required addresses
        for name in required:
            _LOGGER.info(name)
            paramname = '{}{}'.format(name, '_address')
            addr = settings.get(paramname)
            if addr is None:
                _LOGGER.exception(
                    "Required KNX group address %s missing", paramname)
                raise KNXException(
                    "Group address for %s missing in configuration", paramname)
            _LOGGER.debug("%s: %s=%s", settings.get('name'), paramname, addr)
            addr = parse_group_address(addr)
            self.names[addr] = name

        # parse optional addresses
        for name in optional:
            paramname = '{}{}'.format(name, '_address')
            addr = settings.get(paramname)
            _LOGGER.debug("%s: %s=%s", settings.get('name'), paramname, addr)
            if addr:
                try:
                    addr = parse_group_address(addr)
                except KNXException:
                    _LOGGER.exception("Cannot parse group address %s", addr)
                self.names[addr] = name

    @property
    def name(self):
        """Return the entity's display name."""
        return self._config.name

    @property
    def config(self):
        """Return the entity's configuration."""
        return self._config

    @property
    def should_poll(self):
        """Return the state of the polling, if needed."""
        return self._config.should_poll

    @property
    def cache(self):
        """Return the name given to the entity."""
        return self._config.config.get('cache', True)

    def has_attribute(self, name):
        """Check if the attribute with the given name is defined.

        This is mostly important for optional addresses.
        """
        for attributename, dummy_attribute in self.names.items():
            if attributename == name:
                return True
        return False

    def set_percentage(self, name, percentage):
        """Set a percentage in knx for a given attribute.

        DPT_Scaling / DPT 5.001 is a single byte scaled percentage
        """
        percentage = abs(percentage)  # only accept positive values
        scaled_value = percentage * 255 / 100
        value = min(255, scaled_value)
        self.set_int_value(name, value)

    def get_percentage(self, name):
        """Get a percentage from knx for a given attribute.

        DPT_Scaling / DPT 5.001 is a single byte scaled percentage
        """
        value = self.get_int_value(name)
        percentage = round(value * 100 / 255)
        return percentage

    def set_int_value(self, name, value, num_bytes=1):
        """Set an integer value for a given attribute."""
        # KNX packets are big endian
        value = round(value)      # only accept integers
        b_value = value.to_bytes(num_bytes, byteorder='big')
        self.set_value(name, list(b_value))

    def get_int_value(self, name):
        """Get an integer value for a given attribute."""
        # KNX packets are big endian
        summed_value = 0
        raw_value = self.value(name)
        try:
            # convert raw value in bytes
            for val in raw_value:
                summed_value *= 256
                summed_value += val
        except TypeError:
            # pknx returns a non-iterable type for unsuccessful reads
            pass

        return summed_value

    def value(self, name):
        """Return the value to a given named attribute."""
        from knxip.core import KNXException

        addr = None
        for attributeaddress, attributename in self.names.items():
            if attributename == name:
                addr = attributeaddress

        if addr is None:
            _LOGGER.exception("Attribute %s undefined", name)
            return False

        try:
            res = KNXTUNNEL.group_read(addr, use_cache=self.cache)
        except KNXException:
            _LOGGER.exception("Unable to read from KNX address: %s", addr)
            return False

        return res

    def set_value(self, name, value):
        """Set the value of a given named attribute."""
        from knxip.core import KNXException

        addr = None
        for attributeaddress, attributename in self.names.items():
            if attributename == name:
                addr = attributeaddress

        if addr is None:
            _LOGGER.exception("Attribute %s undefined", name)
            return False

        try:
            KNXTUNNEL.group_write(addr, value)
        except KNXException:
            _LOGGER.exception("Unable to write to KNX address: %s", addr)
            return False

        return True
<commit_msg>components.knx - KNXMultiAddressDevice corrections (#8275)<commit_after>"""
Support for KNX components.

For more details about this component, please refer to the documentation at
https://home-assistant.io/components/knx/
"""
import logging

import voluptuous as vol

import homeassistant.helpers.config_validation as cv
from homeassistant.const import (
    EVENT_HOMEASSISTANT_STOP, CONF_HOST, CONF_PORT)
from homeassistant.helpers.entity import Entity

REQUIREMENTS = ['knxip==0.3.3']

_LOGGER = logging.getLogger(__name__)

DEFAULT_HOST = '0.0.0.0'
DEFAULT_PORT = 3671
DOMAIN = 'knx'

EVENT_KNX_FRAME_RECEIVED = 'knx_frame_received'

KNXTUNNEL = None

CONFIG_SCHEMA = vol.Schema({
    DOMAIN: vol.Schema({
        vol.Optional(CONF_HOST, default=DEFAULT_HOST): cv.string,
        vol.Optional(CONF_PORT, default=DEFAULT_PORT): cv.port,
    }),
}, extra=vol.ALLOW_EXTRA)

def setup(hass, config):
    """Set up the connection to the KNX IP interface."""
    global KNXTUNNEL

    from knxip.ip import KNXIPTunnel
    from knxip.core import KNXException

    host = config[DOMAIN].get(CONF_HOST)
    port = config[DOMAIN].get(CONF_PORT)

    if host is '0.0.0.0':
        _LOGGER.debug("Will try to auto-detect KNX/IP gateway")

    KNXTUNNEL = KNXIPTunnel(host, port)
    try:
        res = KNXTUNNEL.connect()
        _LOGGER.debug("Res = %s", res)
        if not res:
            _LOGGER.error("Could not connect to KNX/IP interface %s", host)
            return False

    except KNXException as ex:
        _LOGGER.exception("Can't connect to KNX/IP interface: %s", ex)
        KNXTUNNEL = None
        return False

    _LOGGER.info("KNX IP tunnel to %s:%i established", host, port)

    hass.bus.listen_once(EVENT_HOMEASSISTANT_STOP, close_tunnel)
    return True

def close_tunnel(_data):
    """Close the NKX tunnel connection on shutdown."""
    global KNXTUNNEL

    KNXTUNNEL.disconnect()
    KNXTUNNEL = None

class KNXConfig(object):
    """Handle the fetching of configuration from the config file."""

    def __init__(self, config):
        """Initialize the configuration."""
        from knxip.core import parse_group_address

        self.config = config
        self.should_poll = config.get('poll', True)
        if config.get('address'):
            self._address = parse_group_address(config.get('address'))
        else:
            self._address = None
        if self.config.get('state_address'):
            self._state_address = parse_group_address(
                self.config.get('state_address'))
        else:
            self._state_address = None

    @property
    def name(self):
        """Return the name given to the entity."""
        return self.config['name']

    @property
    def address(self):
        """Return the address of the device as an integer value.

        3 types of addresses are supported:
        integer - 0-65535
        2 level - a/b
        3 level - a/b/c
        """
        return self._address

    @property
    def state_address(self):
        """Return the group address the device sends its current state to.

        Some KNX devices can send the current state to a seperate
        group address. This makes send e.g. when an actuator can
        be switched but also have a timer functionality.
        """
        return self._state_address

class KNXGroupAddress(Entity):
    """Representation of devices connected to a KNX group address."""

    def __init__(self, hass, config):
        """Initialize the device."""
        self._config = config
        self._state = False
        self._data = None
        _LOGGER.debug(
            "Initalizing KNX group address for %s (%s)",
            self.name, self.address
        )

        def handle_knx_message(addr, data):
            """Handle an incoming KNX frame.

            Handle an incoming frame and update our status if it contains
            information relating to this device.
            """
            if (addr == self.state_address) or (addr == self.address):
                self._state = data[0]
                self.schedule_update_ha_state()

        KNXTUNNEL.register_listener(self.address, handle_knx_message)
        if self.state_address:
            KNXTUNNEL.register_listener(self.state_address, handle_knx_message)

    @property
    def name(self):
        """Return the entity's display name."""
        return self._config.name

    @property
    def config(self):
        """Return the entity's configuration."""
        return self._config

    @property
    def should_poll(self):
        """Return the state of the polling, if needed."""
        return self._config.should_poll

    @property
    def is_on(self):
        """Return True if the value is not 0 is on, else False."""
        return self._state != 0

    @property
    def address(self):
        """Return the KNX group address."""
        return self._config.address

    @property
    def state_address(self):
        """Return the KNX group address."""
        return self._config.state_address

    @property
    def cache(self):
        """Return the name given to the entity."""
        return self._config.config.get('cache', True)

    def group_write(self, value):
        """Write to the group address."""
        KNXTUNNEL.group_write(self.address, [value])

    def update(self):
        """Get the state from KNX bus or cache."""
        from knxip.core import KNXException

        try:
            if self.state_address:
                res = KNXTUNNEL.group_read(
                    self.state_address, use_cache=self.cache)
            else:
                res = KNXTUNNEL.group_read(self.address, use_cache=self.cache)

            if res:
                self._state = res[0]
                self._data = res
            else:
                _LOGGER.debug(
                    "%s: unable to read from KNX address: %s (None)",
                    self.name, self.address
                )

        except KNXException:
            _LOGGER.exception(
                "%s: unable to read from KNX address: %s",
                self.name, self.address
            )
            return False

class KNXMultiAddressDevice(Entity):
    """Representation of devices connected to a multiple KNX group address.

    This is needed for devices like dimmers or shutter actuators as they have
    to be controlled by multiple group addresses.
    """

    def __init__(self, hass, config, required, optional=None):
        """Initialize the device.

        The namelist argument lists the required addresses. E.g. for a dimming
        actuators, the namelist might look like:
        onoff_address: 0/0/1
        brightness_address: 0/0/2
        """
        from knxip.core import parse_group_address, KNXException

        self.names = {}
        self.values = {}

        self._config = config
        self._state = False
        self._data = None
        _LOGGER.debug(
            "%s: initalizing KNX multi address device",
            self.name
        )

        settings = self._config.config
        if config.address:
            _LOGGER.debug(
                "%s: base address: address=%s",
                self.name, settings.get('address')
            )
            self.names[config.address] = 'base'
        if config.state_address:
            _LOGGER.debug(
                "%s, state address: state_address=%s",
                self.name, settings.get('state_address')
            )
            self.names[config.state_address] = 'state'

        # parse required addresses
        for name in required:
            paramname = '{}{}'.format(name, '_address')
            addr = settings.get(paramname)
            if addr is None:
                _LOGGER.error(
                    "%s: Required KNX group address %s missing",
                    self.name, paramname
                )
                raise KNXException(
                    "%s: Group address for {} missing in "
                    "configuration for {}".format(
                        self.name, paramname
                    )
                )
            _LOGGER.debug(
                "%s: (required parameter) %s=%s",
                self.name, paramname, addr
            )
            addr = parse_group_address(addr)
            self.names[addr] = name

        # parse optional addresses
        for name in optional:
            paramname = '{}{}'.format(name, '_address')
            addr = settings.get(paramname)
            _LOGGER.debug(
                "%s: (optional parameter) %s=%s",
                self.name, paramname, addr
            )
            if addr:
                try:
                    addr = parse_group_address(addr)
                except KNXException:
                    _LOGGER.exception(
                        "%s: cannot parse group address %s",
                        self.name, addr
                    )
                self.names[addr] = name

    @property
    def name(self):
        """Return the entity's display name."""
        return self._config.name

    @property
    def config(self):
        """Return the entity's configuration."""
        return self._config

    @property
    def should_poll(self):
        """Return the state of the polling, if needed."""
        return self._config.should_poll

    @property
    def cache(self):
        """Return the name given to the entity."""
        return self._config.config.get('cache', True)

    def has_attribute(self, name):
        """Check if the attribute with the given name is defined.

        This is mostly important for optional addresses.
        """
        for attributename in self.names.values():
            if attributename == name:
                return True
        return False

    def set_percentage(self, name, percentage):
        """Set a percentage in knx for a given attribute.

        DPT_Scaling / DPT 5.001 is a single byte scaled percentage
        """
        percentage = abs(percentage)  # only accept positive values
        scaled_value = percentage * 255 / 100
        value = min(255, scaled_value)
        return self.set_int_value(name, value)

    def get_percentage(self, name):
        """Get a percentage from knx for a given attribute.

        DPT_Scaling / DPT 5.001 is a single byte scaled percentage
        """
        value = self.get_int_value(name)
        percentage = round(value * 100 / 255)
        return percentage

    def set_int_value(self, name, value, num_bytes=1):
        """Set an integer value for a given attribute."""
        # KNX packets are big endian
        value = round(value)      # only accept integers
        b_value = value.to_bytes(num_bytes, byteorder='big')
        return self.set_value(name, list(b_value))

    def get_int_value(self, name):
        """Get an integer value for a given attribute."""
        # KNX packets are big endian
        summed_value = 0
        raw_value = self.value(name)
        try:
            # convert raw value in bytes
            for val in raw_value:
                summed_value *= 256
                summed_value += val
        except TypeError:
            # pknx returns a non-iterable type for unsuccessful reads
            pass

        return summed_value

    def value(self, name):
        """Return the value to a given named attribute."""
        from knxip.core import KNXException

        addr = None
        for attributeaddress, attributename in self.names.items():
            if attributename == name:
                addr = attributeaddress

        if addr is None:
            _LOGGER.error("%s: attribute '%s' undefined",
                          self.name, name)
            _LOGGER.debug(
                "%s: defined attributes: %s",
                self.name, str(self.names)
            )
            return False

        try:
            res = KNXTUNNEL.group_read(addr, use_cache=self.cache)
        except KNXException:
            _LOGGER.exception(
                "%s: unable to read from KNX address: %s",
                self.name, addr
            )
            return False

        return res

    def set_value(self, name, value):
        """Set the value of a given named attribute."""
        from knxip.core import KNXException

        addr = None
        for attributeaddress, attributename in self.names.items():
            if attributename == name:
                addr = attributeaddress

        if addr is None:
            _LOGGER.error("%s: attribute '%s' undefined",
                          self.name, name)
            _LOGGER.debug(
                "%s: defined attributes: %s",
                self.name, str(self.names)
            )
            return False

        try:
            KNXTUNNEL.group_write(addr, value)
        except KNXException:
            _LOGGER.exception(
                "%s: unable to write to KNX address: %s",
                self.name, addr
            )
            return False

        return True
<|endoftext|>


<commit_before># Copyright (C) 2009  Renato Lima - Akretion
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

{
    "name": "Brazilian Localization Sale",
    "category": "Localisation",
    "license": "AGPL-3",
    "author":
        'Akretion, '
        'Odoo Community Association (OCA)',
    "website": "http://github.com/OCA/l10n-brazil",
    "version": "12.0.5.1.0",
    "depends": ["sale_management", "l10n_br_account"],
    "data": [
        # Data
        "data/company.xml",

        # Security
        "security/ir.model.access.csv",
        "security/l10n_br_sale_security.xml",

        # View
        "views/res_config_settings_view.xml",
        "views/res_company_view.xml",
        "views/sale_view.xml",

        # Report
        "report/sale_report_view.xml",
        "report/sale_report_templates.xml",
    ],
    "demo": [
        # Demo
        "demo/product.xml",
        "demo/company.xml",
        "demo/l10n_br_sale.xml",
    ],
    "installable": True,
    "auto_install": True,
    "post_init_hook": "post_init_hook",
    "development_status": "Production/Stable",
    "maintainers": ["renatonlima"],
}
<commit_msg>l10n_br_sale 12.0.5.2.0<commit_after># Copyright (C) 2009  Renato Lima - Akretion
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

{
    "name": "Brazilian Localization Sale",
    "category": "Localisation",
    "license": "AGPL-3",
    "author":
        'Akretion, '
        'Odoo Community Association (OCA)',
    "website": "http://github.com/OCA/l10n-brazil",
    "version": "12.0.5.2.0",
    "depends": ["sale_management", "l10n_br_account"],
    "data": [
        # Data
        "data/company.xml",

        # Security
        "security/ir.model.access.csv",
        "security/l10n_br_sale_security.xml",

        # View
        "views/res_config_settings_view.xml",
        "views/res_company_view.xml",
        "views/sale_view.xml",

        # Report
        "report/sale_report_view.xml",
        "report/sale_report_templates.xml",
    ],
    "demo": [
        # Demo
        "demo/product.xml",
        "demo/company.xml",
        "demo/l10n_br_sale.xml",
    ],
    "installable": True,
    "auto_install": True,
    "post_init_hook": "post_init_hook",
    "development_status": "Production/Stable",
    "maintainers": ["renatonlima"],
}
<|endoftext|>


<commit_before>import sys
from os import walk
import cv2
import imutils
import numpy as np

from .logger import Logger
from .settings import ImageProcessingClassifierSettings

from PIL import Image

class ColorClassifier(object):
	"""
	Step 4: Autonomously classify shape color and alphanumeric color.
	"""

	def __init__(self, img=None):
		"""
		Constructor.

		:param img:		Relative path to image.
		:type img:		String

		:return:		Nothing.
		"""
		self.img = img

	def get_color(self):
		"""
		Gets the color of the background and text inside a target.

		If no img is passed to get_color, then this function assumes to use the
		PIL image passed in the constructor.

		See section 3 for code source:
			https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_ml/py_kmeans/
			py_kmeans_opencv/py_kmeans_opencv.html

		:return:		Two element list, where the first element is the shape
						color, and the second element is the text color. Ex:
							["red", "green"]
						or, simply:
							[<background color>, <text color>]
		"""
		# preconditions
		if self.img is None:
			raise Exception("classify_color.py: No image was passed in the constructor.")

		# initialize
		opened_img = Image.open(self.img)
		lines_to_grab = 6

		Logger.log("classify_color.py: Starting work on new image.")

		# convert pil to opencv2
		img = np.array(opened_img)
		img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

		Z = img.reshape((-1,3))

		# convert to np.float32
		Z = np.float32(Z)

		# define criteria, number of clusters(K) and apply kmeans()
		criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 10, 1.0)
		K = 3
		ret,label,center = cv2.kmeans(Z, K, None, criteria, 9, cv2.KMEANS_RANDOM_CENTERS)

		# now convert back into uint8, and make original image
		center = np.uint8(center)
		flattened_img = center[label.flatten()]
		reshaped_img = flattened_img.reshape((img.shape))

		# grab colors
		corrected_img = cv2.cvtColor(reshaped_img, cv2.COLOR_RGB2BGR)
		colors_from_img = set(tuple(v) for m2d in corrected_img for v in m2d)

		Logger.log("classify_color.py: Raw colors from image: " + str(colors_from_img))

		color_names_from_img = []

		for pixel_from_img in colors_from_img:
			color_names_from_img.append(self.convert_rgba_to_color_name(list(pixel_from_img)))

		if "white" in color_names_from_img:
			color_names_from_img.remove("white") # remove background

		# figure out which color comes first, reading from the vertical center row, outwards
		height, width, channels = img.shape
		row_factor = int(round(height/(float(lines_to_grab) + 1.0)))

		shape_color = None
		text_color = None
		row = 0

		im = opened_img
		im_width, im_height = im.size
		pixels = list(im.getdata())

		try:
			count = {
				list(colors_from_img)[0]: 0,
				list(colors_from_img)[1]: 0,
				list(colors_from_img)[2]: 0
			}
		except:
			count = {
				list(colors_from_img)[0]: 0,
				list(colors_from_img)[1]: 0
			}

		Logger.log("classify_color.py: Count before: ")
		Logger.log("classify_color.py: " + str(count))

		for line in range(lines_to_grab):
			row = row + row_factor

			Logger.log("classify_color.py: Line collection #" + str(line + 1) + " start at height: " + str(row))

			for column in range(1, width):
				# check original image to see if alpha is 0
				if pixels[(im_width * row) + column] == (255, 255, 255, 0):
					pass # empty, transparent background pixel
				else:
					count[tuple(corrected_img.tolist()[row][column])] += 1

		Logger.log("classify_color.py: Count results: ")
		Logger.log("classify_color.py: " + str(count))

		new_count = {}

		for i in range(len(count.keys())):
			new_count[self.convert_rgba_to_color_name(list(count.keys()[i]))] = count[count.keys()[i]]

			Logger.log("classify_color.py: Discovered color: " + str(list(count.keys()[i])) + " as " + self.convert_rgba_to_color_name(list(count.keys()[i])))

		Logger.log("classify_color.py: Count color rgb convert to names: ")
		Logger.log("classify_color.py: " + str(new_count))

		# determine shape color based on max and min
		maximum = 0
		minimum = 0

		if len(new_count.keys()) == 3:
			maximum = max(new_count, key=new_count.get)

			del new_count[min(new_count, key=new_count.get)]

			minimum = min(new_count, key=new_count.get)
		else: # if background color is white
			if new_count[new_count.keys()[0]] >= new_count[new_count.keys()[1]]:
				maximum = new_count.keys()[0]
				minimum = new_count.keys()[1]
			else:
				maximum = new_count.keys()[1]
				minimum = new_count.keys()[0]

		Logger.log("classify_color.py: Determined shape color to be '" + maximum + "'")
		Logger.log("classify_color.py: Assumed text color to be '" + minimum + "'")
		Logger.log("classify_color.py: Checking text color...")

		# confirm the text color is, indeed, the text color
		original_text_color_pixels = []

		for x in range(1, im_width): # grab pixel locations of assumed text color
			for y in range(1, im_height):
				if self.convert_rgba_to_color_name(list(corrected_img[x, y])) == minimum:
					original_text_color_pixels.append([x, y])

		filtered_pixels = []

		for x in range(1, opened_img.size[0]): # find every pixel that is surrounded by similar pixels
			for y in range(1, opened_img.size[1]): # [0] is width, [1] is height
				if [x, y] in original_text_color_pixels:
					if [x, y + 1] in original_text_color_pixels: # above
						if [x + 1, y] in original_text_color_pixels: # right
							if [x, y - 1] in original_text_color_pixels: # below
								if [x - 1, y] in original_text_color_pixels: # left
									filtered_pixels.append(list(opened_img.getpixel((y, x))))

		minimum = self.convert_rgba_to_color_name( list( np.mean(np.array(filtered_pixels), axis=0) ) )

		result = [maximum, minimum] # shape color, text color

		return result

	"""
	Converts a standard [X, Y] pixel location to OpenCVs one
	dimensional pixel list.

	:param width:	The width of the OpenCV image.
	:param x:		Column position.
	:param y:		Row position:

	:type width:	Integer.
	:type x:		Integer.
	:type y:		Integer.

	:return:		Integer.
	"""
	def x_y_to_single(self, width, x, y):
		return (width * y) + x

	def three_dimesional_distance(self, p1, p2):
		"""
		Calculates the distance between two three dimesional points.

		p1 or p2 should be a list of three elements:
			[x, y, z]

		:param p1:	point one with x, y, z.
		:param p2:	point two with x, y, z.

		:type p1:	three element list.
		:type p2:	three element list.

		:return:	float distance.

		See page 3 of:
			http://www.math.usm.edu/lambers/mat169/fall09/lecture17.pdf
		"""
		return np.sqrt((p2[0] - p1[0])**2 + (p2[1] - p1[1])**2 + (p2[2] - p1[2])**2)

	def convert_rgba_to_color_name(self, rgba):
		"""
		Using min-distance approximation, this function converts the rgba value,
		which is a list of [r, g, b, a] into it's closest color name, which are
		as follows:

			"white": [255, 255, 255, 255]
			"black": [0, 0, 0, 255]
			"gray": [128, 128, 128, 255]
			"red": [255, 0, 0, 255]
			"blue": [0, 0, 255, 255]
			"green": [0, 255, 0, 255]
			"yellow": [255, 255, 0, 255]
			"purple": [128, 0, 128, 255]
			"brown": [165, 42, 42, 255]
			"orange": [255, 165, 0, 255]

		:param rgba:	color component in the format [r, g, b, a].
		:type rgba:		four element list.

		:return:		"white", "black", etc.
		"""
		if isinstance(rgba, list) is False:
			raise TypeError("Expected a list, was passed a " + str(type(rgba)))

		distances = []
		i = 0

		for color in ImageProcessingClassifierSettings.COLORS.keys():
			distances.append(0)

			c = ImageProcessingClassifierSettings.COLORS[color]

			distances[i] = self.three_dimesional_distance(
				[rgba[0], rgba[1], rgba[2]],
				[c[0], c[1], c[2]]
			)

			i += 1

		# approximate color name based on min-distance
		d = sys.maxint
		i = 0
		j = 0

		for distance in distances:
			if distance < d:
				d = distance
				i = j

			j += 1

		return ImageProcessingClassifierSettings.COLORS.keys()[i]

	def convert_color_name_to_rgba(self, color_name):
		"""
		Takes in a cardinal color name and converts it to an rgba value with
		100 percent alpha.

		:param color_name:	Cardinal name of color.

							Either "white", "black",
							"gray", "red", "blue", "green", "yellow", "purple",
							"brown", or "orange".

		:type color_name:	String.

		:return:			rgba version of color.
		"""
		return ImageProcessingClassifierSettings.COLORS[color_name]
<commit_msg>fixed array out of bounds error in color classifier<commit_after>import sys
from os import walk
import cv2
import imutils
import numpy as np

from .logger import Logger
from .settings import ImageProcessingClassifierSettings

from PIL import Image

class ColorClassifier(object):
	"""
	Step 4: Autonomously classify shape color and alphanumeric color.
	"""

	def __init__(self, img=None):
		"""
		Constructor.

		:param img:		Relative path to image.
		:type img:		String

		:return:		Nothing.
		"""
		self.img = img

	def get_color(self):
		"""
		Gets the color of the background and text inside a target.

		If no img is passed to get_color, then this function assumes to use the
		PIL image passed in the constructor.

		See section 3 for code source:
			https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_ml/py_kmeans/
			py_kmeans_opencv/py_kmeans_opencv.html

		:return:		Two element list, where the first element is the shape
						color, and the second element is the text color. Ex:
							["red", "green"]
						or, simply:
							[<background color>, <text color>]
		"""
		# preconditions
		if self.img is None:
			raise Exception("classify_color.py: No image was passed in the constructor.")

		# initialize
		opened_img = Image.open(self.img)
		lines_to_grab = 6

		Logger.log("classify_color.py: Starting work on new image.")

		# convert pil to opencv2
		img = np.array(opened_img)
		img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

		Z = img.reshape((-1,3))

		# convert to np.float32
		Z = np.float32(Z)

		# define criteria, number of clusters(K) and apply kmeans()
		criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 10, 1.0)
		K = 3
		ret,label,center = cv2.kmeans(Z, K, None, criteria, 9, cv2.KMEANS_RANDOM_CENTERS)

		# now convert back into uint8, and make original image
		center = np.uint8(center)
		flattened_img = center[label.flatten()]
		reshaped_img = flattened_img.reshape((img.shape))

		# grab colors
		corrected_img = cv2.cvtColor(reshaped_img, cv2.COLOR_RGB2BGR)
		colors_from_img = set(tuple(v) for m2d in corrected_img for v in m2d)

		Logger.log("classify_color.py: Raw colors from image: " + str(colors_from_img))

		color_names_from_img = []

		for pixel_from_img in colors_from_img:
			color_names_from_img.append(self.convert_rgba_to_color_name(list(pixel_from_img)))

		if "white" in color_names_from_img:
			color_names_from_img.remove("white") # remove background

		# figure out which color comes first, reading from the vertical center row, outwards
		height, width, channels = img.shape
		row_factor = int(round(height/(float(lines_to_grab) + 1.0)))

		shape_color = None
		text_color = None
		row = 0

		im = opened_img
		im_width, im_height = im.size
		pixels = list(im.getdata())

		try:
			count = {
				list(colors_from_img)[0]: 0,
				list(colors_from_img)[1]: 0,
				list(colors_from_img)[2]: 0
			}
		except:
			count = {
				list(colors_from_img)[0]: 0,
				list(colors_from_img)[1]: 0
			}

		Logger.log("classify_color.py: Count before: ")
		Logger.log("classify_color.py: " + str(count))

		for line in range(lines_to_grab):
			row = row + row_factor

			Logger.log("classify_color.py: Line collection #" + str(line + 1) + " start at height: " + str(row))

			for column in range(1, width):
				# check original image to see if alpha is 0
				if pixels[(im_width * row) + column] == (255, 255, 255, 0):
					pass # empty, transparent background pixel
				else:
					count[tuple(corrected_img.tolist()[row][column])] += 1

		Logger.log("classify_color.py: Count results: ")
		Logger.log("classify_color.py: " + str(count))

		new_count = {}

		for i in range(len(count.keys())):
			new_count[self.convert_rgba_to_color_name(list(count.keys()[i]))] = count[count.keys()[i]]

			Logger.log("classify_color.py: Discovered color: " + str(list(count.keys()[i])) + " as " + self.convert_rgba_to_color_name(list(count.keys()[i])))

		Logger.log("classify_color.py: Count color rgb convert to names: ")
		Logger.log("classify_color.py: " + str(new_count))

		# determine shape color based on max and min
		maximum = 0
		minimum = 0

		if len(new_count.keys()) == 3:
			maximum = max(new_count, key=new_count.get)

			del new_count[min(new_count, key=new_count.get)]

			minimum = min(new_count, key=new_count.get)
		else: # if background color is white
			if new_count[new_count.keys()[0]] >= new_count[new_count.keys()[1]]:
				maximum = new_count.keys()[0]
				minimum = new_count.keys()[1]
			else:
				maximum = new_count.keys()[1]
				minimum = new_count.keys()[0]

		Logger.log("classify_color.py: Determined shape color to be '" + maximum + "'")
		Logger.log("classify_color.py: Assumed text color to be '" + minimum + "'")
		Logger.log("classify_color.py: Checking text color...")

		# confirm the text color is, indeed, the text color
		original_text_color_pixels = []

		im_width, im_height = corrected_img.shape[:2]

		for x in range(1, im_width): # grab pixel locations of assumed text color
			for y in range(1, im_height):
				if self.convert_rgba_to_color_name(list(corrected_img[x, y])) == minimum:
					original_text_color_pixels.append([x, y])

		filtered_pixels = []

		for x in range(1, opened_img.size[0]): # find every pixel that is surrounded by similar pixels
			for y in range(1, opened_img.size[1]): # [0] is width, [1] is height
				if [x, y] in original_text_color_pixels:
					if [x, y + 1] in original_text_color_pixels: # above
						if [x + 1, y] in original_text_color_pixels: # right
							if [x, y - 1] in original_text_color_pixels: # below
								if [x - 1, y] in original_text_color_pixels: # left
									filtered_pixels.append(list(opened_img.getpixel((y, x))))

		if len(filtered_pixels) == 0:
			Logger.log("classify_color.py: Found no suitable pixels to double-check text color.")
			Logger.log("classify_color.py: Assumed text color of '" + minimum + "' is assumed to be correct")
		else:
			minimum = self.convert_rgba_to_color_name( [float(sum(col))/len(col) for col in zip(*filtered_pixels)] )

		result = [maximum, minimum] # shape color, text color

		return result

	"""
	Converts a standard [X, Y] pixel location to OpenCVs one
	dimensional pixel list.

	:param width:	The width of the OpenCV image.
	:param x:		Column position.
	:param y:		Row position:

	:type width:	Integer.
	:type x:		Integer.
	:type y:		Integer.

	:return:		Integer.
	"""
	def x_y_to_single(self, width, x, y):
		return (width * y) + x

	def three_dimesional_distance(self, p1, p2):
		"""
		Calculates the distance between two three dimesional points.

		p1 or p2 should be a list of three elements:
			[x, y, z]

		:param p1:	point one with x, y, z.
		:param p2:	point two with x, y, z.

		:type p1:	three element list.
		:type p2:	three element list.

		:return:	float distance.

		See page 3 of:
			http://www.math.usm.edu/lambers/mat169/fall09/lecture17.pdf
		"""
		return np.sqrt((p2[0] - p1[0])**2 + (p2[1] - p1[1])**2 + (p2[2] - p1[2])**2)

	def convert_rgba_to_color_name(self, rgba):
		"""
		Using min-distance approximation, this function converts the rgba value,
		which is a list of [r, g, b, a] into it's closest color name, which are
		as follows:

			"white": [255, 255, 255, 255]
			"black": [0, 0, 0, 255]
			"gray": [128, 128, 128, 255]
			"red": [255, 0, 0, 255]
			"blue": [0, 0, 255, 255]
			"green": [0, 255, 0, 255]
			"yellow": [255, 255, 0, 255]
			"purple": [128, 0, 128, 255]
			"brown": [165, 42, 42, 255]
			"orange": [255, 165, 0, 255]

		:param rgba:	color component in the format [r, g, b, a].
		:type rgba:		four element list.

		:return:		"white", "black", etc.
		"""
		#print("HELPELPELPEPLELPELPE")
		#print("convert_rgba_to_color_name: was passed rgba -> ")
		#print(str(rgba))
		#print("it's type is -> ")
		#print(str(type(rgba)))
		#print("it's length is -> ")
		#print(str(len(rgba)))

		if isinstance(rgba, list) is False:
			raise TypeError("Expected a list, was passed a " + str(type(rgba)))

		distances = []
		i = 0

		for color in ImageProcessingClassifierSettings.COLORS.keys():
			distances.append(0)

			c = ImageProcessingClassifierSettings.COLORS[color]

			distances[i] = self.three_dimesional_distance(
				[rgba[0], rgba[1], rgba[2]],
				[c[0], c[1], c[2]]
			)

			i += 1

		# approximate color name based on min-distance
		d = sys.maxint
		i = 0
		j = 0

		for distance in distances:
			if distance < d:
				d = distance
				i = j

			j += 1

		return ImageProcessingClassifierSettings.COLORS.keys()[i]

	def convert_color_name_to_rgba(self, color_name):
		"""
		Takes in a cardinal color name and converts it to an rgba value with
		100 percent alpha.

		:param color_name:	Cardinal name of color.

							Either "white", "black",
							"gray", "red", "blue", "green", "yellow", "purple",
							"brown", or "orange".

		:type color_name:	String.

		:return:			rgba version of color.
		"""
		return ImageProcessingClassifierSettings.COLORS[color_name]
<|endoftext|>


<commit_before># Copyright 2004-2011 Luis Manuel Angueira Blanco - Pexego
# Copyright 2013-2019 Ignacio Ibeas - Acysos S.L. (http://acysos.com)
# Copyright 2015 Ainara Galdona <agaldona@avanzosc.com>
# Copyright 2016 Antonio Espinosa <antonio.espinosa@tecnativa.com>
# Copyright 2018 Juan Vicente Pascual <jvpascual@puntsistemes.es>
# Copyright 2019 Tecnativa - Carlos Dauden
# Copyright 2013-2021 Tecnativa - Pedro M. Baeza
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl

{
    "name": "AEAT Base",
    "summary": "Modulo base para declaraciones de la AEAT",
    "version": "14.0.1.2.0",
    "author": "Pexego, "
    "Acysos S.L., "
    "AvanzOSC, "
    "Tecnativa, "
    "Odoo Community Association (OCA)",
    "license": "AGPL-3",
    "website": "https://github.com/OCA/l10n-spain",
    "category": "Accounting & Finance",
    "depends": ["l10n_es", "account_tax_balance"],
    "external_dependencies": {"python": ["unidecode", "cryptography"]},
    "data": [
        "security/aeat_security.xml",
        "security/ir.model.access.csv",
        "data/aeat_partner.xml",
        "data/ir_config_parameter.xml",
        "data/aeat_tax_agency_data.xml",
        "wizard/export_to_boe_wizard.xml",
        "wizard/compare_boe_file_views.xml",
        "wizard/aeat_certificate_password_view.xml",
        "views/aeat_menuitem.xml",
        "views/aeat_report_view.xml",
        "views/aeat_tax_agency_view.xml",
        "views/aeat_tax_line_view.xml",
        "views/aeat_export_configuration_view.xml",
        "views/aeat_tax_code_mapping_view.xml",
        "views/account_move_line_view.xml",
        "views/res_company_view.xml",
        "views/res_partner_view.xml",
        "views/aeat_certificate_view.xml",
    ],
    "installable": True,
}
<commit_msg>l10n_es_aeat 14.0.1.3.0<commit_after># Copyright 2004-2011 Luis Manuel Angueira Blanco - Pexego
# Copyright 2013-2019 Ignacio Ibeas - Acysos S.L. (http://acysos.com)
# Copyright 2015 Ainara Galdona <agaldona@avanzosc.com>
# Copyright 2016 Antonio Espinosa <antonio.espinosa@tecnativa.com>
# Copyright 2018 Juan Vicente Pascual <jvpascual@puntsistemes.es>
# Copyright 2019 Tecnativa - Carlos Dauden
# Copyright 2013-2021 Tecnativa - Pedro M. Baeza
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl

{
    "name": "AEAT Base",
    "summary": "Modulo base para declaraciones de la AEAT",
    "version": "14.0.1.3.0",
    "author": "Pexego, "
    "Acysos S.L., "
    "AvanzOSC, "
    "Tecnativa, "
    "Odoo Community Association (OCA)",
    "license": "AGPL-3",
    "website": "https://github.com/OCA/l10n-spain",
    "category": "Accounting & Finance",
    "depends": ["l10n_es", "account_tax_balance"],
    "external_dependencies": {"python": ["unidecode", "cryptography"]},
    "data": [
        "security/aeat_security.xml",
        "security/ir.model.access.csv",
        "data/aeat_partner.xml",
        "data/ir_config_parameter.xml",
        "data/aeat_tax_agency_data.xml",
        "wizard/export_to_boe_wizard.xml",
        "wizard/compare_boe_file_views.xml",
        "wizard/aeat_certificate_password_view.xml",
        "views/aeat_menuitem.xml",
        "views/aeat_report_view.xml",
        "views/aeat_tax_agency_view.xml",
        "views/aeat_tax_line_view.xml",
        "views/aeat_export_configuration_view.xml",
        "views/aeat_tax_code_mapping_view.xml",
        "views/account_move_line_view.xml",
        "views/res_company_view.xml",
        "views/res_partner_view.xml",
        "views/aeat_certificate_view.xml",
    ],
    "installable": True,
}
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (c) 2009 Zikzakmedia S.L. (http://zikzakmedia.com) All Rights Reserved.
#                       Jordi Esteve <jesteve@zikzakmedia.com>
#    $Id$
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import l10n_ES_toponyms<commit_msg>[FIX] l10n_es_toponyms: Corregimos el import del __init__<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (c) 2009 Zikzakmedia S.L. (http://zikzakmedia.com) All Rights Reserved.
#                       Jordi Esteve <jesteve@zikzakmedia.com>
#    $Id$
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import l10n_es_toponyms<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- encoding: utf-8 -*-

# Copyright 2011-2017, Nigel Small
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

from re import compile as re_compile
from unicodedata import category

from py2neo.compat import number, string, unicode, unichr
from py2neo.types import Node, Relationship, Walkable, LabelSetView, PropertyDictView, PropertySelector

NULL = u"null"
TRUE = u"true"
FALSE = u"false"

ID_START = {u"_"} | {unichr(x) for x in range(0xFFFF)
                     if category(unichr(x)) in ("LC", "Ll", "Lm", "Lo", "Lt", "Lu", "Nl")}
ID_CONTINUE = ID_START | {unichr(x) for x in range(0xFFFF)
                          if category(unichr(x)) in ("Mn", "Mc", "Nd", "Pc", "Sc")}

DOUBLE_QUOTE = u'"'
SINGLE_QUOTE = u"'"

ESCAPED_DOUBLE_QUOTE = u'\\"'
ESCAPED_SINGLE_QUOTE = u"\\'"

X_ESCAPE = re_compile(r"(\\x([0-9a-f]{2}))")
DOUBLE_QUOTED_SAFE = re_compile(r"([ -!#-\[\]-~]+)")
SINGLE_QUOTED_SAFE = re_compile(r"([ -&(-\[\]-~]+)")

class CypherEncoder(object):

    __default_instance = None

    def __new__(cls, *args, **kwargs):
        if not kwargs:
            if cls.__default_instance is None:
                cls.__default_instance = super(CypherEncoder, cls).__new__(cls)
            return cls.__default_instance
        return super(CypherEncoder, cls).__new__(cls)

    encoding = "utf-8"
    quote = None
    sequence_separator = u", "
    key_value_separator = u": "
    node_template = u"{labels} {properties}"
    related_node_template = u"{property.name}"
    relationship_template = u"{type} {properties}"

    def __init__(self, encoding=None, quote=None, sequence_separator=None, key_value_separator=None,
                 node_template=None, related_node_template=None, relationship_template=None):
        if encoding:
            self.encoding = encoding
        if quote:
            self.quote = quote
        if sequence_separator:
            self.sequence_separator = sequence_separator
        if key_value_separator:
            self.key_value_separator = key_value_separator
        if node_template:
            self.node_template = node_template
        if related_node_template:
            self.related_node_template = related_node_template
        if relationship_template:
            self.relationship_template = relationship_template

    def encode_key(self, key):
        if isinstance(key, bytes):
            key = key.decode(self.encoding)
        assert isinstance(key, unicode)
        if not key:
            raise ValueError("Keys cannot be empty")
        if key[0] in ID_START and all(key[i] in ID_CONTINUE for i in range(1, len(key))):
            return key
        else:
            return u"`" + key.replace(u"`", u"``") + u"`"

    def encode_value(self, value):
        if value is None:
            return NULL
        if value is True:
            return TRUE
        if value is False:
            return FALSE
        if isinstance(value, number):
            return unicode(value)
        if isinstance(value, string):
            return self.encode_string(value)
        if isinstance(value, Node):
            return self.encode_node(value)
        if isinstance(value, Relationship):
            return self.encode_relationship(value)
        if isinstance(value, Walkable):
            return self.encode_path(value)
        if isinstance(value, list):
            return self.encode_list(value)
        if isinstance(value, dict):
            return self.encode_map(value)
        raise TypeError("Values of type %s are not supported" % value.__class__.__name__)

    def encode_string(self, value):
        if isinstance(value, bytes):
            value = value.decode(self.encoding)
        assert isinstance(value, unicode)

        quote = self.quote
        if quote is None:
            quote = DOUBLE_QUOTE if SINGLE_QUOTE in value and DOUBLE_QUOTE not in value else SINGLE_QUOTE

        if quote == SINGLE_QUOTE:
            escaped_quote = ESCAPED_SINGLE_QUOTE
            safe = SINGLE_QUOTED_SAFE
        elif quote == DOUBLE_QUOTE:
            escaped_quote = ESCAPED_DOUBLE_QUOTE
            safe = DOUBLE_QUOTED_SAFE
        else:
            raise ValueError("Unsupported quote character %r" % quote)

        if not value:
            return quote + quote

        parts = safe.split(value)
        for i in range(0, len(parts), 2):
            parts[i] = (X_ESCAPE.sub(u"\\u00\\2", parts[i].encode("unicode-escape")).
                        replace(quote, escaped_quote).replace(u"\\u0008", u"\\b").replace(u"\\u000c", u"\\f"))
        return quote + u"".join(parts) + quote

    def encode_list(self, values):
        return u"[" + self.sequence_separator.join(map(self.encode_value, values)) + u"]"

    def encode_map(self, values):
        return u"{" + self.sequence_separator.join(self.encode_key(key) + self.key_value_separator + self.encode_value(value)
                                                   for key, value in values.items()) + u"}"

    def encode_node(self, node):
        return self._encode_node(node, self.node_template)

    def encode_relationship(self, relationship):
        return u"{}-{}->{}".format(
            self._encode_node(relationship.start_node(), self.related_node_template),
            self._encode_relationship_detail(relationship, self.relationship_template),
            self._encode_node(relationship.end_node(), self.related_node_template),
        )

    def encode_path(self, path):
        last_node = path.start_node()
        encoded = [self._encode_node(last_node, self.related_node_template)]
        append = encoded.append
        for relationship in path.relationships():
            if relationship.start_node() == last_node:
                append(u"-")
                append(self._encode_relationship_detail(relationship, self.relationship_template))
                append(u"->")
                last_node = relationship.end_node()
            else:
                append(u"<-")
                append(self._encode_relationship_detail(relationship, self.relationship_template))
                append(u"-")
                last_node = relationship.start_node()
            append(self._encode_node(last_node, self.related_node_template))
        return u"".join(encoded)

    def _encode_node(self, node, template):
        return u"(" + template.format(
            labels=LabelSetView(node.labels(), encoding=self.encoding, quote=self.quote),
            properties=PropertyDictView(node, encoding=self.encoding, quote=self.quote),
            property=PropertySelector(node, u""),
        ).strip() + u")"

    def _encode_relationship_detail(self, relationship, template):
        return u"[" + template.format(
            type=u":" + relationship.type(),
            properties=PropertyDictView(relationship, encoding=self.encoding, quote=self.quote),
            property=PropertySelector(relationship, u""),
        ).strip() + u"]"
<commit_msg>toothpicks<commit_after>#!/usr/bin/env python
# -*- encoding: utf-8 -*-

# Copyright 2011-2017, Nigel Small
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

from re import compile as re_compile
from unicodedata import category

from py2neo.compat import number, string, unicode, unichr
from py2neo.types import Node, Relationship, Walkable, LabelSetView, PropertyDictView, PropertySelector

NULL = u"null"
TRUE = u"true"
FALSE = u"false"

ID_START = {u"_"} | {unichr(x) for x in range(0xFFFF)
                     if category(unichr(x)) in ("LC", "Ll", "Lm", "Lo", "Lt", "Lu", "Nl")}
ID_CONTINUE = ID_START | {unichr(x) for x in range(0xFFFF)
                          if category(unichr(x)) in ("Mn", "Mc", "Nd", "Pc", "Sc")}

DOUBLE_QUOTE = u'"'
SINGLE_QUOTE = u"'"

ESCAPED_DOUBLE_QUOTE = u'\\"'
ESCAPED_SINGLE_QUOTE = u"\\'"

X_ESCAPE = re_compile(r"(\\x([0-9a-f]{2}))")
DOUBLE_QUOTED_SAFE = re_compile(r"([ -!#-\[\]-~]+)")
SINGLE_QUOTED_SAFE = re_compile(r"([ -&(-\[\]-~]+)")

class CypherEncoder(object):

    __default_instance = None

    def __new__(cls, *args, **kwargs):
        if not kwargs:
            if cls.__default_instance is None:
                cls.__default_instance = super(CypherEncoder, cls).__new__(cls)
            return cls.__default_instance
        return super(CypherEncoder, cls).__new__(cls)

    encoding = "utf-8"
    quote = None
    sequence_separator = u", "
    key_value_separator = u": "
    node_template = u"{labels} {properties}"
    related_node_template = u"{property.name}"
    relationship_template = u"{type} {properties}"

    def __init__(self, encoding=None, quote=None, sequence_separator=None, key_value_separator=None,
                 node_template=None, related_node_template=None, relationship_template=None):
        if encoding:
            self.encoding = encoding
        if quote:
            self.quote = quote
        if sequence_separator:
            self.sequence_separator = sequence_separator
        if key_value_separator:
            self.key_value_separator = key_value_separator
        if node_template:
            self.node_template = node_template
        if related_node_template:
            self.related_node_template = related_node_template
        if relationship_template:
            self.relationship_template = relationship_template

    def encode_key(self, key):
        if isinstance(key, bytes):
            key = key.decode(self.encoding)
        assert isinstance(key, unicode)
        if not key:
            raise ValueError("Keys cannot be empty")
        if key[0] in ID_START and all(key[i] in ID_CONTINUE for i in range(1, len(key))):
            return key
        else:
            return u"`" + key.replace(u"`", u"``") + u"`"

    def encode_value(self, value):
        if value is None:
            return NULL
        if value is True:
            return TRUE
        if value is False:
            return FALSE
        if isinstance(value, number):
            return unicode(value)
        if isinstance(value, string):
            return self.encode_string(value)
        if isinstance(value, Node):
            return self.encode_node(value)
        if isinstance(value, Relationship):
            return self.encode_relationship(value)
        if isinstance(value, Walkable):
            return self.encode_path(value)
        if isinstance(value, list):
            return self.encode_list(value)
        if isinstance(value, dict):
            return self.encode_map(value)
        raise TypeError("Values of type %s are not supported" % value.__class__.__name__)

    def encode_string(self, value):
        if isinstance(value, bytes):
            value = value.decode(self.encoding)
        assert isinstance(value, unicode)

        quote = self.quote
        if quote is None:
            quote = DOUBLE_QUOTE if SINGLE_QUOTE in value and DOUBLE_QUOTE not in value else SINGLE_QUOTE

        if quote == SINGLE_QUOTE:
            escaped_quote = ESCAPED_SINGLE_QUOTE
            safe = SINGLE_QUOTED_SAFE
        elif quote == DOUBLE_QUOTE:
            escaped_quote = ESCAPED_DOUBLE_QUOTE
            safe = DOUBLE_QUOTED_SAFE
        else:
            raise ValueError("Unsupported quote character %r" % quote)

        if not value:
            return quote + quote

        parts = safe.split(value)
        for i in range(0, len(parts), 2):
            parts[i] = (X_ESCAPE.sub(u"\\\\u00\\2", parts[i].encode("unicode-escape").decode("utf-8")).
                        replace(quote, escaped_quote).replace(u"\\u0008", u"\\b").replace(u"\\u000c", u"\\f"))
        return quote + u"".join(parts) + quote

    def encode_list(self, values):
        return u"[" + self.sequence_separator.join(map(self.encode_value, values)) + u"]"

    def encode_map(self, values):
        return u"{" + self.sequence_separator.join(self.encode_key(key) + self.key_value_separator + self.encode_value(value)
                                                   for key, value in values.items()) + u"}"

    def encode_node(self, node):
        return self._encode_node(node, self.node_template)

    def encode_relationship(self, relationship):
        return u"{}-{}->{}".format(
            self._encode_node(relationship.start_node(), self.related_node_template),
            self._encode_relationship_detail(relationship, self.relationship_template),
            self._encode_node(relationship.end_node(), self.related_node_template),
        )

    def encode_path(self, path):
        last_node = path.start_node()
        encoded = [self._encode_node(last_node, self.related_node_template)]
        append = encoded.append
        for relationship in path.relationships():
            if relationship.start_node() == last_node:
                append(u"-")
                append(self._encode_relationship_detail(relationship, self.relationship_template))
                append(u"->")
                last_node = relationship.end_node()
            else:
                append(u"<-")
                append(self._encode_relationship_detail(relationship, self.relationship_template))
                append(u"-")
                last_node = relationship.start_node()
            append(self._encode_node(last_node, self.related_node_template))
        return u"".join(encoded)

    def _encode_node(self, node, template):
        return u"(" + template.format(
            labels=LabelSetView(node.labels(), encoding=self.encoding, quote=self.quote),
            properties=PropertyDictView(node, encoding=self.encoding, quote=self.quote),
            property=PropertySelector(node, u""),
        ).strip() + u")"

    def _encode_relationship_detail(self, relationship, template):
        return u"[" + template.format(
            type=u":" + relationship.type(),
            properties=PropertyDictView(relationship, encoding=self.encoding, quote=self.quote),
            property=PropertySelector(relationship, u""),
        ).strip() + u"]"
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    pyClanSphere.database
    ~~~~~~~~~~~~~~~~~~~~~

    This module is a rather complex layer on top of SQLAlchemy 0.4.
    Basically you will never use the `pyClanSphere.database` module except you
    are a core developer, but always the high level
    :mod:`~pyClanSphere.database.db` module which you can import from the
    :mod:`pyClanSphere.api` module.

    :copyright: (c) 2009 by the pyClanSphere Team, see AUTHORS for more details.
    :license: BSD, see LICENSE for more details.
"""
import re
import os
import sys
import time
import urlparse
from os import path
from datetime import datetime, timedelta
from types import ModuleType
from copy import deepcopy

import sqlalchemy
from sqlalchemy import orm
from sqlalchemy.engine.url import make_url, URL
from sqlalchemy.exc import ArgumentError
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.interfaces import ConnectionProxy
from sqlalchemy.orm.collections import attribute_mapped_collection
from sqlalchemy.orm.interfaces import AttributeExtension
from sqlalchemy.sql import func
from sqlalchemy.util import to_list
from sqlalchemy.types import MutableType

from werkzeug import url_decode
from werkzeug.exceptions import NotFound

from pyClanSphere.utils import local_manager

if sys.platform == 'win32':
    _timer = time.clock
else:
    _timer = time.time

_sqlite_re = re.compile(r'sqlite:(?:(?://(.*?))|memory)(?:\?(.*))?$')

def get_engine():
    """Return the active database engine (the database engine of the active
    application).  If no application is enabled this has an undefined behavior.
    If you are not sure if the application is bound to the active thread, use
    :func:`~pyClanSphere.application.get_application` and check it for `None`.
    The database engine is stored on the application object as `database_engine`.
    """
    from pyClanSphere.application import get_application
    return get_application().database_engine

def create_engine(uri, relative_to=None, debug=False):
    """Create a new engine.  This works a bit like SQLAlchemy's
    `create_engine` with the difference that it automaticaly set's MySQL
    engines to 'utf-8', and paths for SQLite are relative to the path
    provided as `relative_to`. Also hooks in LookLively to catch MySQL's
    weird way of connection termination without termination.

    Furthermore the engine is created with `convert_unicode` by default.
    """
    # special case sqlite.  We want nicer urls for that one.
    if uri.startswith('sqlite:'):
        match = _sqlite_re.match(uri)
        if match is None:
            raise ArgumentError('Could not parse rfc1738 URL')
        database, query = match.groups()
        if database is None:
            database = ':memory:'
        elif relative_to is not None:
            database = path.join(relative_to, database)
        if query:
            query = url_decode(query).to_dict()
        else:
            query = {}
        info = URL('sqlite', database=database, query=query)

    else:
        info = make_url(uri)

        # if mysql is the database engine and no connection encoding is
        # provided we set it to utf-8
        if info.drivername == 'mysql':
            info.query.setdefault('charset', 'utf8')
            if 'listeners' not in options:
                options['listeners'] = [LookLively()]
            else:
                options['listeners'].append(LookLively())

    options = {'convert_unicode': True}

    # alternative pool sizes / recycle settings and more.  These are
    # interpreter wide and not from the config for the following reasons:
    #
    # - system administrators can set it independently from the webserver
    #   configuration via SetEnv and friends.
    # - this setting is deployment dependent should not affect a development
    #   server for the same instance or a development shell
    for key in 'pool_size', 'pool_recycle', 'pool_timeout':
        value = os.environ.get(key.upper())
        if value is not None:
            options[key] = int(value)

    # if debugging is enabled, hook the ConnectionDebugProxy in
    if debug:
        options['proxy'] = ConnectionDebugProxy()

    return sqlalchemy.create_engine(info, **options)

def secure_database_uri(uri):
    """Returns the database uri with confidental information stripped."""
    obj = make_url(uri)
    if obj.password:
        obj.password = '***'
    return unicode(obj).replace(u':%2A%2A%2A@', u':***@', 1)

def attribute_loaded(model, attribute):
    """Returns true if the attribute of the model was already loaded."""
    # XXX: this works but it relys on a specific implementation in
    # SQLAlchemy.  Figure out if SA provides a way to query that information.
    return attribute in model.__dict__

class ConnectionDebugProxy(ConnectionProxy):
    """Helps debugging the database."""

    def cursor_execute(self, execute, cursor, statement, parameters,
                       context, executemany):
        start = _timer()
        try:
            return execute(cursor, statement, parameters, context)
        finally:
            from pyClanSphere.application import get_request
            from pyClanSphere.utils.debug import find_calling_context
            request = get_request()
            if request is not None:
                request.queries.append((statement, parameters, start,
                                        _timer(), find_calling_context()))

class LookLively(object):
    """Ensures that MySQL connections checked out of the pool are alive.

    Specific to the MySQLdb DB-API.  Note that this can not totally
    guarantee live connections- the remote side can drop the connection
    in the time between ping and the connection reaching user code.

    This is a simplistic implementation.  If there's a lot of pool churn
    (i.e. implicit connections checking in and out all the time), one
    possible and easy optimization would be to add a timer check:

    1) On check-in, record the current time (integer part) into the
       connection record's .properties
    2) On check-out, compare the current integer time to the (possibly
       empty) record in .properties.  If it is still the same second as
       when the connection was last checked in, skip the ping.  The
       connection is probably fine.

    Something much like this logic will go into the SQLAlchemy core
    eventually.

    -jek
    """

    def checkout(self, dbapi_con, con_record, con_proxy):
        try:
            try:
                dbapi_con.ping(False)
            except TypeError:
                dbapi_con.ping()
        except dbapi_con.OperationalError, ex:
            if ex.args[0] in (2006, 2013, 2014, 2045, 2055):
                raise exc.DisconnectionError()
            else:
                raise

class Query(orm.Query):
    """Default query class."""

    def lightweight(self, deferred=None, lazy=None, eager=None):
        """Send a lightweight query which deferes some more expensive
        things such as comment queries or even text data yet also
         offers eagerloading externals as well.
        """

        args = map(db.lazyload, lazy or ()) + map(db.defer, deferred or ()) + map(db.eagerload, eager or ())
        return self.options(*args)

    def first(self, raise_if_missing=False):
        """Return the first result of this `Query` or None if the result
        doesn't contain any rows.  If `raise_if_missing` is set to `True`
        a `NotFound` exception is raised if no row is found.
        """
        rv = orm.Query.first(self)
        if rv is None and raise_if_missing:
            raise NotFound()
        return rv

session = orm.scoped_session(lambda: orm.create_session(get_engine(),
                             autoflush=True, autocommit=False),
                             local_manager.get_ident)

# Session.mapper is deprecated in SQL Alchemy 0.5.5 and above´
# New mapper doesn't auto-add, so put together a MapperExtension
# that allows emulating that behaviour (it's added automatically
# unless you specify extenion=... in your mapper() call)
class AutoAddExt(orm.MapperExtension):
    def init_instance(self, mapper, class_, oldinit, instance, args, kwargs):
        session = kwargs.pop('_sa_session', None)
        if session is None:
            session = db.session
        session.add(instance)
        return orm.EXT_CONTINUE

def session_mapper(scoped_session):
    def mapper(cls, *arg, **kw):
        if cls.__init__ is object.__init__:
            def __init__(self, **kwargs):
                for key, value in kwargs.items():
                    setattr(self, key, value)
            cls.__init__ = __init__
        if not hasattr(cls,'query'):
            cls.query = scoped_session.query_property()
        if not 'extension' in kw:
            kw['extension'] = AutoAddExt()
        return orm.mapper(cls, *arg, **kw)
    return mapper

#: create a new module for all the database related functions and objects
sys.modules['pyClanSphere.database.db'] = db = ModuleType('db')
key = value = mod = None
for mod in sqlalchemy, orm:
    for key, value in mod.__dict__.iteritems():
        if key in mod.__all__:
            setattr(db, key, value)
del key, mod, value

#: forward some session methods to the module as well
for name in 'delete', 'flush', 'execute', 'begin', 'mapper', \
            'commit', 'rollback', 'refresh', 'expire', \
            'query_property':
    setattr(db, name, getattr(session, name))

#: metadata for the core tables and the core table definitions
metadata = db.MetaData()

# configure a declarative base.  This is unused in the code but makes it easier
# for plugins to work with the database.
class ModelBase(object):
    """Internal baseclass for `Model`."""
Model = declarative_base(name='Model', metadata=metadata, cls=ModelBase, mapper=session_mapper(session))

#: and finally hook our own implementations of various objects in
db.Model = Model
db.Query = Query
db.AutoAddExt = AutoAddExt
db.get_engine = get_engine
db.create_engine = create_engine
db.session = session
db.mapper = session_mapper(session)
db.association_proxy = association_proxy
db.attribute_loaded = attribute_loaded
db.AttributeExtension = AttributeExtension
db.AutoAddExtension = AutoAddExt
db.EXT_CONTINUE = orm.EXT_CONTINUE
db.EXT_STOP = orm.EXT_STOP
db.attribute_mapped_collection = attribute_mapped_collection
db.func = func

#: called at the end of a request
cleanup_session = session.remove

users = db.Table('users', metadata,
    db.Column('user_id', db.Integer, primary_key=True),
    db.Column('username', db.String(30)),
    db.Column('real_name', db.String(180)),
    db.Column('display_name', db.String(180)),
    db.Column('gender_male', db.Boolean),
    db.Column('birthday', db.DateTime),
    db.Column('height', db.Integer),
    db.Column('address', db.String(250)),
    db.Column('zip', db.Integer),
    db.Column('city', db.String(250)),
    db.Column('country', db.String(200)),
    db.Column('pw_hash', db.String(70)),
    db.Column('email', db.String(250)),
    db.Column('www', db.String(200))
)

groups = db.Table('groups', metadata,
    db.Column('group_id', db.Integer, primary_key=True),
    db.Column('name', db.String(30))
)

group_users = db.Table('group_users', metadata,
    db.Column('group_id', db.Integer, db.ForeignKey('groups.group_id')),
    db.Column('user_id', db.Integer, db.ForeignKey('users.user_id'))
)

privileges = db.Table('privileges', metadata,
    db.Column('privilege_id', db.Integer, primary_key=True),
    db.Column('name', db.String(50), unique=True)
)

user_privileges = db.Table('user_privileges', metadata,
    db.Column('user_id', db.Integer, db.ForeignKey('users.user_id')),
    db.Column('privilege_id', db.Integer,
              db.ForeignKey('privileges.privilege_id'))
)

group_privileges = db.Table('group_privileges', metadata,
    db.Column('group_id', db.Integer, db.ForeignKey('groups.group_id')),
    db.Column('privilege_id', db.Integer,
              db.ForeignKey('privileges.privilege_id'))
)

redirects = db.Table('redirects', metadata,
    db.Column('redirect_id', db.Integer, primary_key=True),
    db.Column('original', db.String(200), unique=True),
    db.Column('new', db.String(200))
)

notification_subscriptions = db.Table('notification_subscriptions', metadata,
    db.Column('subscription_id', db.Integer, primary_key=True),
    db.Column('user_id', db.Integer, db.ForeignKey('users.user_id')),
    db.Column('notification_system', db.String(50)),
    db.Column('notification_id', db.String(100)),
    db.UniqueConstraint('user_id', 'notification_system', 'notification_id')
)

imaccounts = db.Table('imaccounts', metadata,
    db.Column('account_id', db.Integer, primary_key=True),
    db.Column('user_id', db.ForeignKey('users.user_id')),
    db.Column('service', db.Integer),
    db.Column('account', db.String(64)),
    db.UniqueConstraint('user_id', 'service', 'account')
)

passwordrequests = db.Table('passwordrequests', metadata,
    db.Column('req_id', db.String(36), primary_key=True),
    db.Column('user_id', db.ForeignKey('users.user_id')),
    db.Column('ip', db.String(64)),
    db.Column('requesttime', db.DateTime)
)

def init_database(engine):
    """This is called from the websetup which explains why it takes an engine
    and not a pyClanSphere application.
    """
    # XXX: consider using something like this for mysql:
    #   cx = engine.connect()
    #   cx.execute('set storage_engine=innodb')
    #   metadata.create_all(cx)
    metadata.create_all(engine)
<commit_msg>no need to make it conditional -.-<commit_after># -*- coding: utf-8 -*-
"""
    pyClanSphere.database
    ~~~~~~~~~~~~~~~~~~~~~

    This module is a rather complex layer on top of SQLAlchemy 0.4.
    Basically you will never use the `pyClanSphere.database` module except you
    are a core developer, but always the high level
    :mod:`~pyClanSphere.database.db` module which you can import from the
    :mod:`pyClanSphere.api` module.

    :copyright: (c) 2009 by the pyClanSphere Team, see AUTHORS for more details.
    :license: BSD, see LICENSE for more details.
"""
import re
import os
import sys
import time
import urlparse
from os import path
from datetime import datetime, timedelta
from types import ModuleType
from copy import deepcopy

import sqlalchemy
from sqlalchemy import orm
from sqlalchemy.engine.url import make_url, URL
from sqlalchemy.exc import ArgumentError
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.interfaces import ConnectionProxy
from sqlalchemy.orm.collections import attribute_mapped_collection
from sqlalchemy.orm.interfaces import AttributeExtension
from sqlalchemy.sql import func
from sqlalchemy.util import to_list
from sqlalchemy.types import MutableType

from werkzeug import url_decode
from werkzeug.exceptions import NotFound

from pyClanSphere.utils import local_manager

if sys.platform == 'win32':
    _timer = time.clock
else:
    _timer = time.time

_sqlite_re = re.compile(r'sqlite:(?:(?://(.*?))|memory)(?:\?(.*))?$')

def get_engine():
    """Return the active database engine (the database engine of the active
    application).  If no application is enabled this has an undefined behavior.
    If you are not sure if the application is bound to the active thread, use
    :func:`~pyClanSphere.application.get_application` and check it for `None`.
    The database engine is stored on the application object as `database_engine`.
    """
    from pyClanSphere.application import get_application
    return get_application().database_engine

def create_engine(uri, relative_to=None, debug=False):
    """Create a new engine.  This works a bit like SQLAlchemy's
    `create_engine` with the difference that it automaticaly set's MySQL
    engines to 'utf-8', and paths for SQLite are relative to the path
    provided as `relative_to`. Also hooks in LookLively to catch MySQL's
    weird way of connection termination without termination.

    Furthermore the engine is created with `convert_unicode` by default.
    """

    # This is a good idea in any case
    options = {'convert_unicode': True}

    # special case sqlite.  We want nicer urls for that one.
    if uri.startswith('sqlite:'):
        match = _sqlite_re.match(uri)
        if match is None:
            raise ArgumentError('Could not parse rfc1738 URL')
        database, query = match.groups()
        if database is None:
            database = ':memory:'
        elif relative_to is not None:
            database = path.join(relative_to, database)
        if query:
            query = url_decode(query).to_dict()
        else:
            query = {}
        info = URL('sqlite', database=database, query=query)

    else:
        info = make_url(uri)

        # if mysql is the database engine and no connection encoding is
        # provided we set it to utf-8
        if info.drivername == 'mysql':
            info.query.setdefault('charset', 'utf8')
            options['listeners'] = [LookLively()]

    # alternative pool sizes / recycle settings and more.  These are
    # interpreter wide and not from the config for the following reasons:
    #
    # - system administrators can set it independently from the webserver
    #   configuration via SetEnv and friends.
    # - this setting is deployment dependent should not affect a development
    #   server for the same instance or a development shell
    for key in 'pool_size', 'pool_recycle', 'pool_timeout':
        value = os.environ.get(key.upper())
        if value is not None:
            options[key] = int(value)

    # if debugging is enabled, hook the ConnectionDebugProxy in
    if debug:
        options['proxy'] = ConnectionDebugProxy()

    return sqlalchemy.create_engine(info, **options)

def secure_database_uri(uri):
    """Returns the database uri with confidental information stripped."""
    obj = make_url(uri)
    if obj.password:
        obj.password = '***'
    return unicode(obj).replace(u':%2A%2A%2A@', u':***@', 1)

def attribute_loaded(model, attribute):
    """Returns true if the attribute of the model was already loaded."""
    # XXX: this works but it relys on a specific implementation in
    # SQLAlchemy.  Figure out if SA provides a way to query that information.
    return attribute in model.__dict__

class ConnectionDebugProxy(ConnectionProxy):
    """Helps debugging the database."""

    def cursor_execute(self, execute, cursor, statement, parameters,
                       context, executemany):
        start = _timer()
        try:
            return execute(cursor, statement, parameters, context)
        finally:
            from pyClanSphere.application import get_request
            from pyClanSphere.utils.debug import find_calling_context
            request = get_request()
            if request is not None:
                request.queries.append((statement, parameters, start,
                                        _timer(), find_calling_context()))

class LookLively(object):
    """Ensures that MySQL connections checked out of the pool are alive.

    Specific to the MySQLdb DB-API.  Note that this can not totally
    guarantee live connections- the remote side can drop the connection
    in the time between ping and the connection reaching user code.

    This is a simplistic implementation.  If there's a lot of pool churn
    (i.e. implicit connections checking in and out all the time), one
    possible and easy optimization would be to add a timer check:

    1) On check-in, record the current time (integer part) into the
       connection record's .properties
    2) On check-out, compare the current integer time to the (possibly
       empty) record in .properties.  If it is still the same second as
       when the connection was last checked in, skip the ping.  The
       connection is probably fine.

    Something much like this logic will go into the SQLAlchemy core
    eventually.

    -jek
    """

    def checkout(self, dbapi_con, con_record, con_proxy):
        try:
            try:
                dbapi_con.ping(False)
            except TypeError:
                dbapi_con.ping()
        except dbapi_con.OperationalError, ex:
            if ex.args[0] in (2006, 2013, 2014, 2045, 2055):
                raise exc.DisconnectionError()
            else:
                raise

class Query(orm.Query):
    """Default query class."""

    def lightweight(self, deferred=None, lazy=None, eager=None):
        """Send a lightweight query which deferes some more expensive
        things such as comment queries or even text data yet also
         offers eagerloading externals as well.
        """

        args = map(db.lazyload, lazy or ()) + map(db.defer, deferred or ()) + map(db.eagerload, eager or ())
        return self.options(*args)

    def first(self, raise_if_missing=False):
        """Return the first result of this `Query` or None if the result
        doesn't contain any rows.  If `raise_if_missing` is set to `True`
        a `NotFound` exception is raised if no row is found.
        """
        rv = orm.Query.first(self)
        if rv is None and raise_if_missing:
            raise NotFound()
        return rv

session = orm.scoped_session(lambda: orm.create_session(get_engine(),
                             autoflush=True, autocommit=False),
                             local_manager.get_ident)

# Session.mapper is deprecated in SQL Alchemy 0.5.5 and above´
# New mapper doesn't auto-add, so put together a MapperExtension
# that allows emulating that behaviour (it's added automatically
# unless you specify extenion=... in your mapper() call)
class AutoAddExt(orm.MapperExtension):
    def init_instance(self, mapper, class_, oldinit, instance, args, kwargs):
        session = kwargs.pop('_sa_session', None)
        if session is None:
            session = db.session
        session.add(instance)
        return orm.EXT_CONTINUE

def session_mapper(scoped_session):
    def mapper(cls, *arg, **kw):
        if cls.__init__ is object.__init__:
            def __init__(self, **kwargs):
                for key, value in kwargs.items():
                    setattr(self, key, value)
            cls.__init__ = __init__
        if not hasattr(cls,'query'):
            cls.query = scoped_session.query_property()
        if not 'extension' in kw:
            kw['extension'] = AutoAddExt()
        return orm.mapper(cls, *arg, **kw)
    return mapper

#: create a new module for all the database related functions and objects
sys.modules['pyClanSphere.database.db'] = db = ModuleType('db')
key = value = mod = None
for mod in sqlalchemy, orm:
    for key, value in mod.__dict__.iteritems():
        if key in mod.__all__:
            setattr(db, key, value)
del key, mod, value

#: forward some session methods to the module as well
for name in 'delete', 'flush', 'execute', 'begin', 'mapper', \
            'commit', 'rollback', 'refresh', 'expire', \
            'query_property':
    setattr(db, name, getattr(session, name))

#: metadata for the core tables and the core table definitions
metadata = db.MetaData()

# configure a declarative base.  This is unused in the code but makes it easier
# for plugins to work with the database.
class ModelBase(object):
    """Internal baseclass for `Model`."""
Model = declarative_base(name='Model', metadata=metadata, cls=ModelBase, mapper=session_mapper(session))

#: and finally hook our own implementations of various objects in
db.Model = Model
db.Query = Query
db.AutoAddExt = AutoAddExt
db.get_engine = get_engine
db.create_engine = create_engine
db.session = session
db.mapper = session_mapper(session)
db.association_proxy = association_proxy
db.attribute_loaded = attribute_loaded
db.AttributeExtension = AttributeExtension
db.AutoAddExtension = AutoAddExt
db.EXT_CONTINUE = orm.EXT_CONTINUE
db.EXT_STOP = orm.EXT_STOP
db.attribute_mapped_collection = attribute_mapped_collection
db.func = func

#: called at the end of a request
cleanup_session = session.remove

users = db.Table('users', metadata,
    db.Column('user_id', db.Integer, primary_key=True),
    db.Column('username', db.String(30)),
    db.Column('real_name', db.String(180)),
    db.Column('display_name', db.String(180)),
    db.Column('gender_male', db.Boolean),
    db.Column('birthday', db.DateTime),
    db.Column('height', db.Integer),
    db.Column('address', db.String(250)),
    db.Column('zip', db.Integer),
    db.Column('city', db.String(250)),
    db.Column('country', db.String(200)),
    db.Column('pw_hash', db.String(70)),
    db.Column('email', db.String(250)),
    db.Column('www', db.String(200))
)

groups = db.Table('groups', metadata,
    db.Column('group_id', db.Integer, primary_key=True),
    db.Column('name', db.String(30))
)

group_users = db.Table('group_users', metadata,
    db.Column('group_id', db.Integer, db.ForeignKey('groups.group_id')),
    db.Column('user_id', db.Integer, db.ForeignKey('users.user_id'))
)

privileges = db.Table('privileges', metadata,
    db.Column('privilege_id', db.Integer, primary_key=True),
    db.Column('name', db.String(50), unique=True)
)

user_privileges = db.Table('user_privileges', metadata,
    db.Column('user_id', db.Integer, db.ForeignKey('users.user_id')),
    db.Column('privilege_id', db.Integer,
              db.ForeignKey('privileges.privilege_id'))
)

group_privileges = db.Table('group_privileges', metadata,
    db.Column('group_id', db.Integer, db.ForeignKey('groups.group_id')),
    db.Column('privilege_id', db.Integer,
              db.ForeignKey('privileges.privilege_id'))
)

redirects = db.Table('redirects', metadata,
    db.Column('redirect_id', db.Integer, primary_key=True),
    db.Column('original', db.String(200), unique=True),
    db.Column('new', db.String(200))
)

notification_subscriptions = db.Table('notification_subscriptions', metadata,
    db.Column('subscription_id', db.Integer, primary_key=True),
    db.Column('user_id', db.Integer, db.ForeignKey('users.user_id')),
    db.Column('notification_system', db.String(50)),
    db.Column('notification_id', db.String(100)),
    db.UniqueConstraint('user_id', 'notification_system', 'notification_id')
)

imaccounts = db.Table('imaccounts', metadata,
    db.Column('account_id', db.Integer, primary_key=True),
    db.Column('user_id', db.ForeignKey('users.user_id')),
    db.Column('service', db.Integer),
    db.Column('account', db.String(64)),
    db.UniqueConstraint('user_id', 'service', 'account')
)

passwordrequests = db.Table('passwordrequests', metadata,
    db.Column('req_id', db.String(36), primary_key=True),
    db.Column('user_id', db.ForeignKey('users.user_id')),
    db.Column('ip', db.String(64)),
    db.Column('requesttime', db.DateTime)
)

def init_database(engine):
    """This is called from the websetup which explains why it takes an engine
    and not a pyClanSphere application.
    """
    # XXX: consider using something like this for mysql:
    #   cx = engine.connect()
    #   cx.execute('set storage_engine=innodb')
    #   metadata.create_all(cx)
    metadata.create_all(engine)
<|endoftext|>