<commit_before># vim: set et sw=4 sts=4 fileencoding=utf-8:
#
# Copyright (c) 2013 Dave Hughes
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

"""
This module provides a target wrapper for SQL-based databases, which can
provide a powerful means of analyzing log data.

The :class:`SQLTarget` class accepts row objects in its
:meth:`~SQLTarget.write` method and automatically generates the required SQL
``INSERT`` statements to append records to the specified target table.

The implementation has been tested with SQLite3 (built into Python), and
PostgreSQL, but should work with any `PEP-249`_ (Python DB API 2.0) compatible
database cursor. A list of available Python database drives is maintained on
the Python wiki `DatabaseInterfaces`_ page.

Classes
=======

.. autoclass:: SQLTarget
   :members:

Exceptions
==========

.. autoexception:: SQLError
   :members:

.. autoexception:: SQLWarning

Examples
========

A typical example of working with the class is shown below::

    import io
    import sqlite3
    from www2csv import apache, sql

    connection = sqlite3.connect('apache.db', detect_types=sqlite3.PARSE_DECLTYPES)

    with io.open('/var/log/apache2/access.log', 'rb') as infile:
        with io.open('apache.csv', 'wb') as outfile:
            with apache.ApacheSource(infile) as source:
                with sql.SQLTarget(sqlite3, connection, 'log_entries', create_table=True) as target:
                    for row in source:
                        target.write(row)

.. _PEP-249: http://www.python.org/dev/peps/pep-0249/
.. _DatabaseInterfaces: http://wiki.python.org/moin/DatabaseInterfaces
"""

from __future__ import (
    unicode_literals,
    absolute_import,
    print_function,
    division,
    )

import warnings
import sqlite3
from datetime import date, time, datetime

from www2csv import datatypes

# Make Py2 str same as Py3
str = type('')

__all__ = [
# XXX    'SQLSource',
    'SQLTarget',
    'SQLError',
    'SQLWarning',
    ]

class SQLError(StandardError):
    """
    Base class for all fatal errors generated by classes in the sql module.
    """

class SQLWarning(Warning):
    """
    Raised when an error is encountered inserting a log row.
    """

class SQLSource(object):
    # XXX Code SQLSource
    pass

class SQLTarget(object):
    """
    Wraps a database connection to insert row tuples into an SQL database
    table.

    This wrapper provides a simple :meth:`write` method which can be used to
    insert row tuples into a specified table, which can optionally by created
    automatically by the wrapper before insertion of the first row. The wrapper
    must be passed a database connection object that confirms to the Python
    DB-API (version 2.0) as defined by `PEP-249`_.

    The *db_module* parameter must be passed the module that defines the
    database interface (this odd requirement is simply so that the wrapper
    can look up the parameter style that the interface uses).

    The *connection* parameter must be given an active database connection
    object (presumably belonging to the module passed to *db_module*).

    The *table* parameter is the final mandatory parameter which names the
    table that values are to be inserted into. If the table name requires
    quoting in the target SQL dialect, you should include such quoting in the
    *table* value (this class does not try and discern what database engine
    you are connecting to and thus has no idea about non-standard quoting
    styles like ```MySQL``` or ``[MS-SQL]``).

    The *commit* parameter controls how often a ``COMMIT`` statement is
    executed when inserting rows. By default, this is 1000 which is usually
    sufficient to provide decent performance but may (in certain database
    engines with fixed size transaction logs) cause errors, in which case you
    may wish to specify a lower value.

    If the *create_table* parameter is set to True (it defaults to False), when
    the :meth:`write` method is first called, the class will determine column
    names and types from the row passed in and will attempt to generate and
    execute a ``CREATE TABLE`` statement to set up the target table
    automatically. The database types that are used in the ``CREATE TABLE``
    statement are controlled by other optional parameters and are documented in
    the table below:

    +-----------------+-------------------------------------------------------+
    | Parameter       | Default Value (SQL)                                   |
    +=================+=======================================================+
    | *str_type*      | ``VARCHAR(1000)`` - typically used for URL fields.    |
    +-----------------+-------------------------------------------------------+
    | *int_type*      | ``INTEGER`` - used for fields like status and size.   |
    |                 | If your server is serving large binaries you may wish |
    |                 | to use a 64-bit type like ``BIGINT`` here instead.    |
    +-----------------+-------------------------------------------------------+
    | *fixed_type*    | ``DOUBLE`` - used for fields like time_taken. Some    |
    |                 | users may wish to change this an appropriate          |
    |                 | ``NUMERIC`` or ``DECIMAL`` specification for          |
    |                 | precision.                                            |
    +-----------------+-------------------------------------------------------+
    | *date_type*     | ``DATE``                                              |
    +-----------------+-------------------------------------------------------+
    | *time_type*     | ``TIME``                                              |
    +-----------------+-------------------------------------------------------+
    | *datetime_type* | ``TIMESTAMP`` - MS-SQL users will likely wish to      |
    |                 | change this to ``DATETIME`` or ``SMALLDATETIME``.     |
    |                 | MySQL users may wish to change this to ``DATETIME``,  |
    |                 | although ``TIMESTAMP`` is technically also supported  |
    |                 | (albeit with functional differences).                 |
    +-----------------+-------------------------------------------------------+
    | *ip_type*       | ``VARCHAR(53)`` - this is sufficient for storing all  |
    |                 | possible IP address and port combinations up and      |
    |                 | including an IPv6 v4-mapped address. If you are       |
    |                 | certain you will only need IPv4 support you may wish  |
    |                 | to use a length of 21 (with port) or 15 (no port).    |
    |                 | PostgreSQL users may wish to use the special ``inet`` |
    |                 | type instead as this is much more efficient but       |
    |                 | cannot store port information.                        |
    +-----------------+-------------------------------------------------------+
    | *hostname_type* | ``VARCHAR(255)``                                      |
    +-----------------+-------------------------------------------------------+
    | *path_type*     | ``VARCHAR(260)``                                      |
    +-----------------+-------------------------------------------------------+

    If the *drop_table* parameter is set to True (it defaults to False), the
    wrapper will first attempt to use ``DROP TABLE`` to destroy any existing
    table before attempting ``CREATE TABLE``. If *ignore_drop_errors* is
    True (which it is by default) then any errors encountered during the drop
    operation (e.g. if the table does not exist) will be ignored.
    """

    def __init__(
            self, db_module, connection, table, commit=1000,
            create_table=False, drop_table=False, ignore_drop_errors=True,
            str_type='VARCHAR(1000)', int_type='INTEGER', fixed_type='DOUBLE',
            date_type='DATE', time_type='TIME', datetime_type='TIMESTAMP',
            ip_type='VARCHAR(53)', hostname_type='VARCHAR(255)',
            path_type='VARCHAR(260)'):
        if not hasattr(db_module, 'paramstyle'):
            raise NameError('The database module has no "paramstyle" global')
        if not hasattr(db_module, 'Error'):
            raise NameError('The database module has no "Error" class')
        self.db_module = db_module
        self.connection = connection
        self.table = table
        if commit < 1:
            raise ValueError('commit must be 1 or more')
        self.commit = commit
        self.create_table = create_table
        self.drop_table = drop_table
        self.ignore_drop_errors = ignore_drop_errors
        self.type_map = {
            # Python base types
            str:                   str_type,
            int:                   int_type,
            float:                 fixed_type,
            date:                  date_type,
            time:                  time_type,
            datetime:              datetime_type,
            ipaddress.IPv4Address: ip_type,
            ipaddress.IPv6Address: ip_type,
            # www2csv types
            datatypes.Date:        date_type,
            datatypes.Time:        time_type,
            datatypes.DateTime:    datetime_type,
            datatypes.Url:         str_type,
            datatypes.IPv4Address: ip_type,
            datatypes.IPv6Address: ip_type,
            datatypes.IPv4Port:    ip_type,
            datatypes.IPv6Port:    ip_type,
            datatypes.Hostname:    hostname_type,
            datatypes.Path:        path_type,
            }
        self._first_row = None
        self._row_casts = None
        self._cursor = None
        self._insert = None
        self._counter = 0

    def __enter__(self):
        logging.debug('Entering SQL context')
        logging.debug('Constructing cursor')
        self._cursor = self.connection.cursor()
        return self

    def __exit__(self, exc_type, exc_value, exc_traceback):
        logging.debug('Exiting SQL context')
        logging.debug('Closing cursor')
        self._cursor.close()
        self._cursor = None
        self._first_row = None
        self._row_casts = None
        self._insert = None
        self._counter = 0
        logging.debug('COMMIT')
        self.connection.commit()

    def _create_table(self, row):
        logging.debug('Creating table %s' % self.table)
        sql = 'CREATE TABLE %(table)s (%(fields)s)' % {
            'table':  self.table,
            'fields': ', '.join([
                '%(name)s %(type)s' % {
                    'name': name,
                    'type': self.type_map[type(value)],
                    }
                for (name, value) in zip(
                    row._fields if hasattr(row, '_fields') else
                    ['field%d' % (i + 1) for i in range(len(row))],
                    row)
                ]),
            }
        logging.debug(sql)
        self._cursor.execute(sql)
        logging.debug('COMMIT')
        self.connection.commit()

    def _drop_table(self):
        logging.debug('Dropping table %s' % self.table)
        sql = 'DROP TABLE %s' % self.table
        logging.debug(sql)
        self._cursor.execute(sql)
        logging.debug('COMMIT')
        self.connection.commit()

    def _generate_statement(self, row):
        # Technically we ought to quote the table substitution below in the
        # case that self.table contains a keyword, or "unsafe" characters
        # in SQL. However, that means getting into what constitutes a
        # keyword in various engines, not to mention the myriad quoting
        # systems ([MS SQL], `MySQL`, "standard") that exist in SQL
        # implementations. Instead, we simply assume if the user wants
        # quoting, they can supply it themselves in the table parameter...
        #
        # The parameter bindings are constructed according to the provided
        # paramstyle, so here's the obligatory whinge about Python's crap
        # DB-API. Why do we have *FIVE* different paramstyles?! What's
        # wrong with the absolutely standard qmark (?) paramstyle which
        # *EVERY* database (yes, even MySQL!) supports?! Why do I have to
        # write cryptic garbage like this to construct SQL in Python?! Why
        # for that matter do I have to get the user to pass in paramstyle
        # to the constructor - why isn't it at least an attribute on the
        # connection object?! Eurgh - PEP-249 is garbage...
        logging.debug('Constructing INSERT statement')
        self._insert = 'INSERT INTO %(table)s VALUES (%(values)s)' % {
            'table':  self.table,
            'values': ', '.join([{
                'qmark':    '?',
                'numeric':  ':%d' % i,
                'named':    ':%s' % name,
                'format':   '%s',
                'pyformat': '%%(%s)s' % name,
                }[self.db_module.paramstyle]
                for (i, name) in enumerate(
                    row._fields if hasattr(row, '_fields') else
                    ['field%d' % (j + 1) for j in range(len(row))]
                    )
                ]),
            }
        logging.debug(self._insert)
        logging.debug('Constructing row casts')
        # Bit of a dirty hack, but it seems the most user-friendly way of
        # dealing with IP addresses depending on the type selected for the
        # target table
        ip_bases = (ipaddress.IPv4Address, ipaddress.IPv6Address)
        if self.type_map[datatypes.IPv4Address].upper().startswith('INT'):
            ip_cast = int
        else:
            ip_cast = str
        self._row_casts = [
            ip_cast if isinstance(value, ip_bases) else
            str if isinstance(value, datatypes.Url) else
            None
            for value in row
            ]

    def write(self, row):
        if self._first_row:
            if len(row) != len(self._first_row):
                raise TypeError('Rows must have the same number of elements')
        else:
            logging.debug('First row')
            self._first_row = row
            self._generate_statement(row)
            if self.drop_table:
                try:
                    self._drop_table()
                except self.db_module.Error as exc:
                    if not self.ignore_drop_errors:
                        raise SQLError(str(exc))
                    logging.debug('While dropping table %s occurred', str(exc))
            if self.create_table:
                self._create_table(row)
        # XXX What about paramstyles pyformat and named? Eurgh...
        cast_to_str = (datatypes.IPv4Address, datatypes.IPv6Address)
        params = [
            cast(value) if cast else value
            for (cast, value) in zip(self._row_casts, row)
            ]
        try:
            self._cursor.execute(self._insert, params)
        except self.db_module.Error as exc:
            warnings.warn(
                '%s while inserting row %s' % (str(exc), str(row)), SQLWarning)
        self._counter += 1
        if self._counter % self.commit == 0:
            logging.debug('COMMIT')
            self.connection.commit()
<commit_msg>Ooops - removed a bit too much...<commit_after># vim: set et sw=4 sts=4 fileencoding=utf-8:
#
# Copyright (c) 2013 Dave Hughes
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

"""
This module provides a target wrapper for SQL-based databases, which can
provide a powerful means of analyzing log data.

The :class:`SQLTarget` class accepts row objects in its
:meth:`~SQLTarget.write` method and automatically generates the required SQL
``INSERT`` statements to append records to the specified target table.

The implementation has been tested with SQLite3 (built into Python), and
PostgreSQL, but should work with any `PEP-249`_ (Python DB API 2.0) compatible
database cursor. A list of available Python database drives is maintained on
the Python wiki `DatabaseInterfaces`_ page.

Classes
=======

.. autoclass:: SQLTarget
   :members:

Exceptions
==========

.. autoexception:: SQLError
   :members:

.. autoexception:: SQLWarning

Examples
========

A typical example of working with the class is shown below::

    import io
    import sqlite3
    from www2csv import apache, sql

    connection = sqlite3.connect('apache.db', detect_types=sqlite3.PARSE_DECLTYPES)

    with io.open('/var/log/apache2/access.log', 'rb') as infile:
        with io.open('apache.csv', 'wb') as outfile:
            with apache.ApacheSource(infile) as source:
                with sql.SQLTarget(sqlite3, connection, 'log_entries', create_table=True) as target:
                    for row in source:
                        target.write(row)

.. _PEP-249: http://www.python.org/dev/peps/pep-0249/
.. _DatabaseInterfaces: http://wiki.python.org/moin/DatabaseInterfaces
"""

from __future__ import (
    unicode_literals,
    absolute_import,
    print_function,
    division,
    )

import warnings
import logging
import ipaddress
import sqlite3
from datetime import date, time, datetime

from www2csv import datatypes

# Make Py2 str same as Py3
str = type('')

__all__ = [
# XXX    'SQLSource',
    'SQLTarget',
    'SQLError',
    'SQLWarning',
    ]

class SQLError(StandardError):
    """
    Base class for all fatal errors generated by classes in the sql module.
    """

class SQLWarning(Warning):
    """
    Raised when an error is encountered inserting a log row.
    """

class SQLSource(object):
    # XXX Code SQLSource
    pass

class SQLTarget(object):
    """
    Wraps a database connection to insert row tuples into an SQL database
    table.

    This wrapper provides a simple :meth:`write` method which can be used to
    insert row tuples into a specified table, which can optionally by created
    automatically by the wrapper before insertion of the first row. The wrapper
    must be passed a database connection object that confirms to the Python
    DB-API (version 2.0) as defined by `PEP-249`_.

    The *db_module* parameter must be passed the module that defines the
    database interface (this odd requirement is simply so that the wrapper
    can look up the parameter style that the interface uses).

    The *connection* parameter must be given an active database connection
    object (presumably belonging to the module passed to *db_module*).

    The *table* parameter is the final mandatory parameter which names the
    table that values are to be inserted into. If the table name requires
    quoting in the target SQL dialect, you should include such quoting in the
    *table* value (this class does not try and discern what database engine
    you are connecting to and thus has no idea about non-standard quoting
    styles like ```MySQL``` or ``[MS-SQL]``).

    The *commit* parameter controls how often a ``COMMIT`` statement is
    executed when inserting rows. By default, this is 1000 which is usually
    sufficient to provide decent performance but may (in certain database
    engines with fixed size transaction logs) cause errors, in which case you
    may wish to specify a lower value.

    If the *create_table* parameter is set to True (it defaults to False), when
    the :meth:`write` method is first called, the class will determine column
    names and types from the row passed in and will attempt to generate and
    execute a ``CREATE TABLE`` statement to set up the target table
    automatically. The database types that are used in the ``CREATE TABLE``
    statement are controlled by other optional parameters and are documented in
    the table below:

    +-----------------+-------------------------------------------------------+
    | Parameter       | Default Value (SQL)                                   |
    +=================+=======================================================+
    | *str_type*      | ``VARCHAR(1000)`` - typically used for URL fields.    |
    +-----------------+-------------------------------------------------------+
    | *int_type*      | ``INTEGER`` - used for fields like status and size.   |
    |                 | If your server is serving large binaries you may wish |
    |                 | to use a 64-bit type like ``BIGINT`` here instead.    |
    +-----------------+-------------------------------------------------------+
    | *fixed_type*    | ``DOUBLE`` - used for fields like time_taken. Some    |
    |                 | users may wish to change this an appropriate          |
    |                 | ``NUMERIC`` or ``DECIMAL`` specification for          |
    |                 | precision.                                            |
    +-----------------+-------------------------------------------------------+
    | *date_type*     | ``DATE``                                              |
    +-----------------+-------------------------------------------------------+
    | *time_type*     | ``TIME``                                              |
    +-----------------+-------------------------------------------------------+
    | *datetime_type* | ``TIMESTAMP`` - MS-SQL users will likely wish to      |
    |                 | change this to ``DATETIME`` or ``SMALLDATETIME``.     |
    |                 | MySQL users may wish to change this to ``DATETIME``,  |
    |                 | although ``TIMESTAMP`` is technically also supported  |
    |                 | (albeit with functional differences).                 |
    +-----------------+-------------------------------------------------------+
    | *ip_type*       | ``VARCHAR(53)`` - this is sufficient for storing all  |
    |                 | possible IP address and port combinations up and      |
    |                 | including an IPv6 v4-mapped address. If you are       |
    |                 | certain you will only need IPv4 support you may wish  |
    |                 | to use a length of 21 (with port) or 15 (no port).    |
    |                 | PostgreSQL users may wish to use the special ``inet`` |
    |                 | type instead as this is much more efficient but       |
    |                 | cannot store port information.                        |
    +-----------------+-------------------------------------------------------+
    | *hostname_type* | ``VARCHAR(255)``                                      |
    +-----------------+-------------------------------------------------------+
    | *path_type*     | ``VARCHAR(260)``                                      |
    +-----------------+-------------------------------------------------------+

    If the *drop_table* parameter is set to True (it defaults to False), the
    wrapper will first attempt to use ``DROP TABLE`` to destroy any existing
    table before attempting ``CREATE TABLE``. If *ignore_drop_errors* is
    True (which it is by default) then any errors encountered during the drop
    operation (e.g. if the table does not exist) will be ignored.
    """

    def __init__(
            self, db_module, connection, table, commit=1000,
            create_table=False, drop_table=False, ignore_drop_errors=True,
            str_type='VARCHAR(1000)', int_type='INTEGER', fixed_type='DOUBLE',
            date_type='DATE', time_type='TIME', datetime_type='TIMESTAMP',
            ip_type='VARCHAR(53)', hostname_type='VARCHAR(255)',
            path_type='VARCHAR(260)'):
        if not hasattr(db_module, 'paramstyle'):
            raise NameError('The database module has no "paramstyle" global')
        if not hasattr(db_module, 'Error'):
            raise NameError('The database module has no "Error" class')
        self.db_module = db_module
        self.connection = connection
        self.table = table
        if commit < 1:
            raise ValueError('commit must be 1 or more')
        self.commit = commit
        self.create_table = create_table
        self.drop_table = drop_table
        self.ignore_drop_errors = ignore_drop_errors
        self.type_map = {
            # Python base types
            str:                   str_type,
            int:                   int_type,
            float:                 fixed_type,
            date:                  date_type,
            time:                  time_type,
            datetime:              datetime_type,
            ipaddress.IPv4Address: ip_type,
            ipaddress.IPv6Address: ip_type,
            # www2csv types
            datatypes.Date:        date_type,
            datatypes.Time:        time_type,
            datatypes.DateTime:    datetime_type,
            datatypes.Url:         str_type,
            datatypes.IPv4Address: ip_type,
            datatypes.IPv6Address: ip_type,
            datatypes.IPv4Port:    ip_type,
            datatypes.IPv6Port:    ip_type,
            datatypes.Hostname:    hostname_type,
            datatypes.Path:        path_type,
            }
        self._first_row = None
        self._row_casts = None
        self._cursor = None
        self._insert = None
        self._counter = 0

    def __enter__(self):
        logging.debug('Entering SQL context')
        logging.debug('Constructing cursor')
        self._cursor = self.connection.cursor()
        return self

    def __exit__(self, exc_type, exc_value, exc_traceback):
        logging.debug('Exiting SQL context')
        logging.debug('Closing cursor')
        self._cursor.close()
        self._cursor = None
        self._first_row = None
        self._row_casts = None
        self._insert = None
        self._counter = 0
        logging.debug('COMMIT')
        self.connection.commit()

    def _create_table(self, row):
        logging.debug('Creating table %s' % self.table)
        sql = 'CREATE TABLE %(table)s (%(fields)s)' % {
            'table':  self.table,
            'fields': ', '.join([
                '%(name)s %(type)s' % {
                    'name': name,
                    'type': self.type_map[type(value)],
                    }
                for (name, value) in zip(
                    row._fields if hasattr(row, '_fields') else
                    ['field%d' % (i + 1) for i in range(len(row))],
                    row)
                ]),
            }
        logging.debug(sql)
        self._cursor.execute(sql)
        logging.debug('COMMIT')
        self.connection.commit()

    def _drop_table(self):
        logging.debug('Dropping table %s' % self.table)
        sql = 'DROP TABLE %s' % self.table
        logging.debug(sql)
        self._cursor.execute(sql)
        logging.debug('COMMIT')
        self.connection.commit()

    def _generate_statement(self, row):
        # Technically we ought to quote the table substitution below in the
        # case that self.table contains a keyword, or "unsafe" characters
        # in SQL. However, that means getting into what constitutes a
        # keyword in various engines, not to mention the myriad quoting
        # systems ([MS SQL], `MySQL`, "standard") that exist in SQL
        # implementations. Instead, we simply assume if the user wants
        # quoting, they can supply it themselves in the table parameter...
        #
        # The parameter bindings are constructed according to the provided
        # paramstyle, so here's the obligatory whinge about Python's crap
        # DB-API. Why do we have *FIVE* different paramstyles?! What's
        # wrong with the absolutely standard qmark (?) paramstyle which
        # *EVERY* database (yes, even MySQL!) supports?! Why do I have to
        # write cryptic garbage like this to construct SQL in Python?! Why
        # for that matter do I have to get the user to pass in paramstyle
        # to the constructor - why isn't it at least an attribute on the
        # connection object?! Eurgh - PEP-249 is garbage...
        logging.debug('Constructing INSERT statement')
        self._insert = 'INSERT INTO %(table)s VALUES (%(values)s)' % {
            'table':  self.table,
            'values': ', '.join([{
                'qmark':    '?',
                'numeric':  ':%d' % i,
                'named':    ':%s' % name,
                'format':   '%s',
                'pyformat': '%%(%s)s' % name,
                }[self.db_module.paramstyle]
                for (i, name) in enumerate(
                    row._fields if hasattr(row, '_fields') else
                    ['field%d' % (j + 1) for j in range(len(row))]
                    )
                ]),
            }
        logging.debug(self._insert)
        logging.debug('Constructing row casts')
        # Bit of a dirty hack, but it seems the most user-friendly way of
        # dealing with IP addresses depending on the type selected for the
        # target table
        ip_bases = (ipaddress.IPv4Address, ipaddress.IPv6Address)
        if self.type_map[datatypes.IPv4Address].upper().startswith('INT'):
            ip_cast = int
        else:
            ip_cast = str
        self._row_casts = [
            ip_cast if isinstance(value, ip_bases) else
            str if isinstance(value, datatypes.Url) else
            None
            for value in row
            ]

    def write(self, row):
        if self._first_row:
            if len(row) != len(self._first_row):
                raise TypeError('Rows must have the same number of elements')
        else:
            logging.debug('First row')
            self._first_row = row
            self._generate_statement(row)
            if self.drop_table:
                try:
                    self._drop_table()
                except self.db_module.Error as exc:
                    if not self.ignore_drop_errors:
                        raise SQLError(str(exc))
                    logging.debug('While dropping table %s occurred', str(exc))
            if self.create_table:
                self._create_table(row)
        # XXX What about paramstyles pyformat and named? Eurgh...
        cast_to_str = (datatypes.IPv4Address, datatypes.IPv6Address)
        params = [
            cast(value) if cast else value
            for (cast, value) in zip(self._row_casts, row)
            ]
        try:
            self._cursor.execute(self._insert, params)
        except self.db_module.Error as exc:
            warnings.warn(
                '%s while inserting row %s' % (str(exc), str(row)), SQLWarning)
        self._counter += 1
        if self._counter % self.commit == 0:
            logging.debug('COMMIT')
            self.connection.commit()
<|endoftext|>


<commit_before>from __future__ import division, unicode_literals, print_function

"""
create nanoparticle using wulff construction
"""

from pymatgen.symmetry.analyzer import SpacegroupAnalyzer

from mpinterfaces import get_struct_from_mp
from mpinterfaces.nanopartcle import Nanoparticle

#-----------------------------------
#nanopartcle specifications
#-----------------------------------
#max radius in angstroms
rmax = 15
#surface families to be chopped off
hkl_family = [(1,0,0), (1,1,1)]
#surfac energies could be in any units, will be normalized
surface_energies = [28,25]

#-----------------------------------
#intial structure
#-----------------------------------
#caution: set the structure wrt which the the miller indices are
#specified. use your own key
structure = get_struct_from_mp('PbS', MAPI_KEY="")
#primitve ---> conventional cell
sa = SpacegroupAnalyzer(structure)
structure_conventional = sa.get_conventional_standard_structure()

#-----------------------------------
# create nanoparticle
#-----------------------------------
nanoparticle = Nanoparticle(structure_conventional, rmax=rmax,
                            hkl_family=hkl_family,
                            surface_energies=surface_energies)
nanoparticle.create()
nanoparticle.to(fmt='xyz', filename='nanoparticle.xyz')

<commit_msg>bug fix in examples<commit_after>from __future__ import division, unicode_literals, print_function

"""
create nanoparticle using wulff construction
"""

from pymatgen.symmetry.analyzer import SpacegroupAnalyzer

from mpinterfaces import get_struct_from_mp
from mpinterfaces.nanoparticle import Nanoparticle

#-----------------------------------
#nanopartcle specifications
#-----------------------------------
#max radius in angstroms
rmax = 15
#surface families to be chopped off
hkl_family = [(1,0,0), (1,1,1)]
#surfac energies could be in any units, will be normalized
surface_energies = [28,25]

#-----------------------------------
#intial structure
#-----------------------------------
#caution: set the structure wrt which the the miller indices are
#specified. use your own key
structure = get_struct_from_mp('PbS', MAPI_KEY="")
#primitve ---> conventional cell
sa = SpacegroupAnalyzer(structure)
structure_conventional = sa.get_conventional_standard_structure()

#-----------------------------------
# create nanoparticle
#-----------------------------------
nanoparticle = Nanoparticle(structure_conventional, rmax=rmax,
                            hkl_family=hkl_family,
                            surface_energies=surface_energies)
nanoparticle.create()
nanoparticle.to(fmt='xyz', filename='nanoparticle.xyz')

<|endoftext|>


<commit_before>import sys, os, re, tempfile, linecache
import gccxmlparser, typedesc
import subprocess # the subprocess module is required

try:
    set
except NameError:
    from sets import Set as set

if sys.platform == "win32":

    def _locate_gccxml():
        import _winreg
        for subkey, valuename in [
            (r"Software\Kitware\GCCXMLComplete 0.9.0", ""),
            (r"Software\gccxml", "loc"),
            (r"Software\Kitware\GCC_XML", "loc"),
            ]:
            for root in (_winreg.HKEY_CURRENT_USER, _winreg.HKEY_LOCAL_MACHINE):
                try:
                    hkey = _winreg.OpenKey(root, subkey, 0, _winreg.KEY_READ)
                except WindowsError, detail:
                    if detail.errno != 2:
                        raise
                else:
                    return _winreg.QueryValueEx(hkey, valuename)[0] + r"\bin"

    loc = _locate_gccxml()
    if loc:
        os.environ["PATH"] = loc

class CompilerError(Exception):
    pass

class IncludeParser(object):
    def __init__(self, options):
        """
        options must be an object having these attributes:
          verbose - integer
          flags - sequence of strings
          keep_temporary_files - true if temporary files should not be deleted
          cpp_symbols - whether to include preprocessor symbols in the XML file
          xml_file - pathname of output file (may be None)
        """
        self.options = options
        self.excluded = set()

    def create_source_file(self, lines, ext=".cpp"):
        "Create a temporary file, write lines to it, and return the filename"
        fd, fname = tempfile.mkstemp(ext, text=True)
        stream = os.fdopen(fd, "w")
        if lines:
            for line in lines:
                stream.write("%s\n" % line)
        stream.close()
        return fname

    def compile_and_dump(self, lines=None):
        """Create a temporary source file, dump preprocessor
        definitions, and remove the source file again."""
        fname = self.create_source_file(lines)
        try:
            args = ["gccxml", "--preprocess", "-dM", fname]
            if lines and self.options.flags:
                args.extend(self.options.flags)
            if self.options.verbose:
                print >> sys.stderr, "running:", " ".join(args)
            proc = subprocess.Popen(args,
                                    stdout=subprocess.PIPE,
                                    stdin=subprocess.PIPE)
            data, err = proc.communicate()
        finally:
            if not self.options.keep_temporary_files:
                os.remove(fname)
            else:
                print >> sys.stderr, "Info: file '%s' not removed" % fname
        return [line[len("#define "):]
                for line in data.splitlines()
                if line.startswith("#define ")]

    def create_xml(self, lines, xmlfile):
        """Create a temporary source file, 'compile' with gccxml to an
        xmlfile, and remove the source file again."""
        fname = self.create_source_file(lines)
        args = ["gccxml", fname]
        if xmlfile is not None:
            args.append("-fxml=%s" % xmlfile)
        if self.options.flags:
            args.extend(self.options.flags)
        try:
            if self.options.verbose:
                print >> sys.stderr, "running:", " ".join(args)
            proc = subprocess.Popen(args,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE,
                                    stdin=subprocess.PIPE)
            data, err = proc.communicate()
            retcode = proc.wait()
            if retcode:
                raise CompilerError(err)
        finally:
            if not self.options.keep_temporary_files:
                os.remove(fname)
            else:
                print >> sys.stderr, "Info: file '%s' not removed" % fname

    def try_create_xml(self, lines, xmlfile):
        """Create a temporary source file, 'compile' with gccxml to an
        xmlfile, and remove the source file again."""
        fname = self.create_source_file(lines)
        args = ["gccxml", fname]
        args.append("-fxml=%s" % xmlfile)
        if self.options.flags:
            args.extend(self.options.flags)

        if self.options.verbose:
            print >> sys.stderr, "running:", " ".join(args)
        proc = subprocess.Popen(args,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE,
                                stdin=subprocess.PIPE)
        data, err = proc.communicate()
        retcode = proc.wait()
        invalid_symbols = set()
        if retcode:
            invalid_symbols = self.parse_compiler_errors(err.splitlines())
            os.remove(fname)
        return invalid_symbols

    def parse_compiler_errors(self, lines):
        pat = re.compile(r"(.*\.cpp):(\d+):(.*)")
        invalid_symbols = set()
        for line in lines:
            match = pat.search(line)
            if match:
                fnm, lineno, errmsg = match.groups()
                if re.match(r"\d+:", errmsg):
                    errmsg = errmsg.split(":", 1)[1]
                src_line = linecache.getline(fnm, int(lineno)).rstrip()
                is_define = re.match(r"^  DEFINE\((.*)\);$", src_line)
                if is_define:
                    sym = is_define.group(1)
                    if sym not in invalid_symbols:
                        invalid_symbols.add(is_define.group(1))
        return invalid_symbols

    def get_defines(self, include_files):
        """'Compile' an include file with gccxml, and return a
        dictionary of preprocessor definitions.  Empty and compiler
        internal definitions are not included."""
        # compiler internal definitions
        lines = self.compile_and_dump()
        predefined = [line.split(None, 1)[0]
                      for line in lines]
        # all definitions
        code = ['#include "%s"' % fname for fname in include_files]
        lines = self.compile_and_dump(code)
        defined = [line.split(None, 1)
                   for line in lines]
        # remove empty and compiler internal definitions
        defined = [pair for pair in defined
                   if len(pair) == 2 and pair[0] not in predefined]

        return dict(defined)

    wordpat = re.compile("^[a-zA-Z_][a-zA-Z0-9_]*$")
    def is_excluded(self, name, value):
        INVALID_CHARS = "=/{}&;"
        if "(" in name:
            return "IS_FUNCTION"
        if name in self.excluded:
            return "excluded"
        if value[0] in INVALID_CHARS or value[-1] in INVALID_CHARS:
            return "cannot be a value"
        if self.wordpat.match(name) and self.wordpat.match(value):
            # aliases are handled later, when (and if!) the rhs is known
            return "IS_ALIAS"
        return False

    def filter_definitions(self, defines):
        """Return a dict of aliases, a dict of fucntion-like macros, and
        another dict of constants and literals"""
        result = {}
        aliases = {}
        functions = {}
        excluded = {}
        for name, value in defines.iteritems():
            why = self.is_excluded(name, value)
            if not why:
                result[name] = value
            elif why == "IS_ALIAS":
                aliases[name] = value
            elif why == "IS_FUNCTION":
                functions[name] = value
            else:
                excluded[name] = value
        return aliases, functions, excluded, result

    ################################################################

    def find_types(self, include_files, defines):
        for i in range(20):
            source = []
            for fname in include_files:
                source.append('#include "%s"' % fname)
            source.append("#define DECLARE(sym) template <typename T> T symbol_##sym(T) {}")
            source.append("#define DEFINE(sym) symbol_##sym(sym)")
            for name in defines:
                # create a function template for each value
                source.append("DECLARE(%s)" % name)
            source.append("int main() {")
            for name in defines:
                # instantiate a function template.
                # The return type of the function is the symbol's type.
                source.append("  DEFINE(%s);" % name)
            source.append("}")

            fd, fname = tempfile.mkstemp(".xml")
            os.close(fd)
            invalid_symbols = self.try_create_xml(source, fname)
            if not invalid_symbols:
                break
            if self.options.verbose:
                if i == 0:
                    print >> sys.stderr, "compiler errors caused by '-c' flag.\n" \
                          "Trying to resolve them in multiple passes."
                print >> sys.stderr, "pass %d:" % (i + 1)
            for n in invalid_symbols:
                del defines[n]
                if self.options.verbose:
                    print >> sys.stderr, "\t", n
        else:
            raise CompilerError()

        items = gccxmlparser.parse(fname)
        # make sure the temporary file is removed after using it
        if not self.options.keep_temporary_files:
            os.remove(fname)
        else:
            print >> sys.stderr, "Info: file '%s' not removed" % fname

        types = {}
        for i in items:
            name = getattr(i, "name", None)
            if name and name.startswith("symbol_"):
                name = name[len("symbol_"):]
                typ = i.returns
                try:
                    typ = self.c_type_name(i.returns)
                except TypeError, detail:
                    # XXX Warning?
                    ## print >> sys.stderr,  "skipped #define %s %s" % (name, defines[name]), detail
                    pass
                else:
                    types[name] = typ
        return types

    def create_final_xml(self, include_files, types, xmlfile=None):
        source = []
        for fname in include_files:
            source.append('#include "%s"' % fname)
        for name, value in types.iteritems():
            source.append("const %s cpp_sym_%s = (const %s) %s;" % (types[name], name, types[name], name))
        self.create_xml(source, xmlfile)

    def c_type_name(self, tp):
        "Return the C type name for this type."
        if isinstance(tp, typedesc.FundamentalType):
            return tp.name
        elif isinstance(tp, typedesc.PointerType):
            return "%s *" % self.c_type_name(tp.typ)
        elif isinstance(tp, typedesc.CvQualifiedType):
            return self.c_type_name(tp.typ)
        elif isinstance(tp, typedesc.Typedef):
            return self.c_type_name(tp.typ)
        elif isinstance(tp, typedesc.Structure):
            return tp.name
        raise TypeError, type(tp).__name__

    def dump_as_cdata(self, f, mapping, name):
            f.write('  <CPP_DUMP name="%s"><![CDATA[' % name)
            names = mapping.keys()
            names.sort()
            for n in names:
                v = mapping[n]
                f.write("%s %s\n" % (n, v))
            f.write("]]></CPP_DUMP>\n")

    ################################################################

    def parse(self, include_files):
        """Parse include files."""
        options = self.options

        types = {}
        functions = {}
        aliases = {}
        excluded = {}

        if options.cpp_symbols:
            if options.verbose:
                print >> sys.stderr, "compile for syntax check ..."
            # compile the input files to check for compilation errors,
            # before trying the fancy stuff with cpp_symbols.
            self.create_final_xml(include_files, types, None)

        if options.cpp_symbols:
            if options.verbose:
                print >> sys.stderr, "finding definitions ..."
            defines = self.get_defines(include_files)
            if options.verbose:
                print >> sys.stderr, "%d found" % len(defines)

                print >> sys.stderr, "filtering definitions ..."
            aliases, functions, excluded, defines = self.filter_definitions(defines)
            if options.verbose:
                print >> sys.stderr, "%d values, %d aliases" % (len(defines), len(aliases))

            if options.verbose:
                print >> sys.stderr, "finding definitions types ..."

            try:
                # invoke C++ template magic
                types = self.find_types(include_files, defines)
                if options.verbose:
                    print >> sys.stderr, "found %d types ..." % len(types)
            except CompilerError:
                print >> sys.stderr, "Could not determine the types of #define symbols."
                types = {}

        if options.verbose:
            print >> sys.stderr, "creating xml output file ..."
        self.create_final_xml(include_files, types, options.xmlfile)

        # Include additional preprecessor definitions into the XML file.

        if options.xmlfile:
            f = open(options.xmlfile, "r+")
            f.seek(-12, 2)
            data = f.read()
            if len(data) == 11:
                # text mode on windows is strange.  You read 12
                # characters, but get 11.
                assert data == "</GCC_XML>\n"
                f.seek(-12, 2)
            else:
                # linux, ...
                assert data == "\n</GCC_XML>\n"
                f.seek(-11, 2)
            f.flush()

            self.dump_as_cdata(f, functions, "functions")
            self.dump_as_cdata(f, aliases, "aliases")
            self.dump_as_cdata(f, excluded, "excluded")

            f.write("</GCC_XML>\n")
            f.close()
<commit_msg>Use the correct registry key to find gccxml.<commit_after>import sys, os, re, tempfile, linecache
import gccxmlparser, typedesc
import subprocess # the subprocess module is required

try:
    set
except NameError:
    from sets import Set as set

if sys.platform == "win32":

    def _locate_gccxml():
        import _winreg
        for subkey, valuename in [
            (r"Software\Kitware\gccxml 0.9.0", ""),
            (r"Software\Kitware\GCCXMLComplete 0.9.0", ""),
            (r"Software\gccxml", "loc"),
            (r"Software\Kitware\GCC_XML", "loc"),
            ]:
            for root in (_winreg.HKEY_CURRENT_USER, _winreg.HKEY_LOCAL_MACHINE):
                try:
                    hkey = _winreg.OpenKey(root, subkey, 0, _winreg.KEY_READ)
                except WindowsError, detail:
                    if detail.errno != 2:
                        raise
                else:
                    return _winreg.QueryValueEx(hkey, valuename)[0] + r"\bin"

    loc = _locate_gccxml()
    if loc:
        os.environ["PATH"] = loc

class CompilerError(Exception):
    pass

class IncludeParser(object):
    def __init__(self, options):
        """
        options must be an object having these attributes:
          verbose - integer
          flags - sequence of strings
          keep_temporary_files - true if temporary files should not be deleted
          cpp_symbols - whether to include preprocessor symbols in the XML file
          xml_file - pathname of output file (may be None)
        """
        self.options = options
        self.excluded = set()

    def create_source_file(self, lines, ext=".cpp"):
        "Create a temporary file, write lines to it, and return the filename"
        fd, fname = tempfile.mkstemp(ext, text=True)
        stream = os.fdopen(fd, "w")
        if lines:
            for line in lines:
                stream.write("%s\n" % line)
        stream.close()
        return fname

    def compile_and_dump(self, lines=None):
        """Create a temporary source file, dump preprocessor
        definitions, and remove the source file again."""
        fname = self.create_source_file(lines)
        try:
            args = ["gccxml", "--preprocess", "-dM", fname]
            if lines and self.options.flags:
                args.extend(self.options.flags)
            if self.options.verbose:
                print >> sys.stderr, "running:", " ".join(args)
            proc = subprocess.Popen(args,
                                    stdout=subprocess.PIPE,
                                    stdin=subprocess.PIPE)
            data, err = proc.communicate()
        finally:
            if not self.options.keep_temporary_files:
                os.remove(fname)
            else:
                print >> sys.stderr, "Info: file '%s' not removed" % fname
        return [line[len("#define "):]
                for line in data.splitlines()
                if line.startswith("#define ")]

    def create_xml(self, lines, xmlfile):
        """Create a temporary source file, 'compile' with gccxml to an
        xmlfile, and remove the source file again."""
        fname = self.create_source_file(lines)
        args = ["gccxml", fname]
        if xmlfile is not None:
            args.append("-fxml=%s" % xmlfile)
        if self.options.flags:
            args.extend(self.options.flags)
        try:
            if self.options.verbose:
                print >> sys.stderr, "running:", " ".join(args)
            proc = subprocess.Popen(args,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE,
                                    stdin=subprocess.PIPE)
            data, err = proc.communicate()
            retcode = proc.wait()
            if retcode:
                raise CompilerError(err)
        finally:
            if not self.options.keep_temporary_files:
                os.remove(fname)
            else:
                print >> sys.stderr, "Info: file '%s' not removed" % fname

    def try_create_xml(self, lines, xmlfile):
        """Create a temporary source file, 'compile' with gccxml to an
        xmlfile, and remove the source file again."""
        fname = self.create_source_file(lines)
        args = ["gccxml", fname]
        args.append("-fxml=%s" % xmlfile)
        if self.options.flags:
            args.extend(self.options.flags)

        if self.options.verbose:
            print >> sys.stderr, "running:", " ".join(args)
        proc = subprocess.Popen(args,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE,
                                stdin=subprocess.PIPE)
        data, err = proc.communicate()
        retcode = proc.wait()
        invalid_symbols = set()
        if retcode:
            invalid_symbols = self.parse_compiler_errors(err.splitlines())
            os.remove(fname)
        return invalid_symbols

    def parse_compiler_errors(self, lines):
        pat = re.compile(r"(.*\.cpp):(\d+):(.*)")
        invalid_symbols = set()
        for line in lines:
            match = pat.search(line)
            if match:
                fnm, lineno, errmsg = match.groups()
                if re.match(r"\d+:", errmsg):
                    errmsg = errmsg.split(":", 1)[1]
                src_line = linecache.getline(fnm, int(lineno)).rstrip()
                is_define = re.match(r"^  DEFINE\((.*)\);$", src_line)
                if is_define:
                    sym = is_define.group(1)
                    if sym not in invalid_symbols:
                        invalid_symbols.add(is_define.group(1))
        return invalid_symbols

    def get_defines(self, include_files):
        """'Compile' an include file with gccxml, and return a
        dictionary of preprocessor definitions.  Empty and compiler
        internal definitions are not included."""
        # compiler internal definitions
        lines = self.compile_and_dump()
        predefined = [line.split(None, 1)[0]
                      for line in lines]
        # all definitions
        code = ['#include "%s"' % fname for fname in include_files]
        lines = self.compile_and_dump(code)
        defined = [line.split(None, 1)
                   for line in lines]
        # remove empty and compiler internal definitions
        defined = [pair for pair in defined
                   if len(pair) == 2 and pair[0] not in predefined]

        return dict(defined)

    wordpat = re.compile("^[a-zA-Z_][a-zA-Z0-9_]*$")
    def is_excluded(self, name, value):
        INVALID_CHARS = "=/{}&;"
        if "(" in name:
            return "IS_FUNCTION"
        if name in self.excluded:
            return "excluded"
        if value[0] in INVALID_CHARS or value[-1] in INVALID_CHARS:
            return "cannot be a value"
        if self.wordpat.match(name) and self.wordpat.match(value):
            # aliases are handled later, when (and if!) the rhs is known
            return "IS_ALIAS"
        return False

    def filter_definitions(self, defines):
        """Return a dict of aliases, a dict of fucntion-like macros, and
        another dict of constants and literals"""
        result = {}
        aliases = {}
        functions = {}
        excluded = {}
        for name, value in defines.iteritems():
            why = self.is_excluded(name, value)
            if not why:
                result[name] = value
            elif why == "IS_ALIAS":
                aliases[name] = value
            elif why == "IS_FUNCTION":
                functions[name] = value
            else:
                excluded[name] = value
        return aliases, functions, excluded, result

    ################################################################

    def find_types(self, include_files, defines):
        for i in range(20):
            source = []
            for fname in include_files:
                source.append('#include "%s"' % fname)
            source.append("#define DECLARE(sym) template <typename T> T symbol_##sym(T) {}")
            source.append("#define DEFINE(sym) symbol_##sym(sym)")
            for name in defines:
                # create a function template for each value
                source.append("DECLARE(%s)" % name)
            source.append("int main() {")
            for name in defines:
                # instantiate a function template.
                # The return type of the function is the symbol's type.
                source.append("  DEFINE(%s);" % name)
            source.append("}")

            fd, fname = tempfile.mkstemp(".xml")
            os.close(fd)
            invalid_symbols = self.try_create_xml(source, fname)
            if not invalid_symbols:
                break
            if self.options.verbose:
                if i == 0:
                    print >> sys.stderr, "compiler errors caused by '-c' flag.\n" \
                          "Trying to resolve them in multiple passes."
                print >> sys.stderr, "pass %d:" % (i + 1)
            for n in invalid_symbols:
                del defines[n]
                if self.options.verbose:
                    print >> sys.stderr, "\t", n
        else:
            raise CompilerError()

        items = gccxmlparser.parse(fname)
        # make sure the temporary file is removed after using it
        if not self.options.keep_temporary_files:
            os.remove(fname)
        else:
            print >> sys.stderr, "Info: file '%s' not removed" % fname

        types = {}
        for i in items:
            name = getattr(i, "name", None)
            if name and name.startswith("symbol_"):
                name = name[len("symbol_"):]
                typ = i.returns
                try:
                    typ = self.c_type_name(i.returns)
                except TypeError, detail:
                    # XXX Warning?
                    ## print >> sys.stderr,  "skipped #define %s %s" % (name, defines[name]), detail
                    pass
                else:
                    types[name] = typ
        return types

    def create_final_xml(self, include_files, types, xmlfile=None):
        source = []
        for fname in include_files:
            source.append('#include "%s"' % fname)
        for name, value in types.iteritems():
            source.append("const %s cpp_sym_%s = (const %s) %s;" % (types[name], name, types[name], name))
        self.create_xml(source, xmlfile)

    def c_type_name(self, tp):
        "Return the C type name for this type."
        if isinstance(tp, typedesc.FundamentalType):
            return tp.name
        elif isinstance(tp, typedesc.PointerType):
            return "%s *" % self.c_type_name(tp.typ)
        elif isinstance(tp, typedesc.CvQualifiedType):
            return self.c_type_name(tp.typ)
        elif isinstance(tp, typedesc.Typedef):
            return self.c_type_name(tp.typ)
        elif isinstance(tp, typedesc.Structure):
            return tp.name
        raise TypeError, type(tp).__name__

    def dump_as_cdata(self, f, mapping, name):
            f.write('  <CPP_DUMP name="%s"><![CDATA[' % name)
            names = mapping.keys()
            names.sort()
            for n in names:
                v = mapping[n]
                f.write("%s %s\n" % (n, v))
            f.write("]]></CPP_DUMP>\n")

    ################################################################

    def parse(self, include_files):
        """Parse include files."""
        options = self.options

        types = {}
        functions = {}
        aliases = {}
        excluded = {}

        if options.cpp_symbols:
            if options.verbose:
                print >> sys.stderr, "compile for syntax check ..."
            # compile the input files to check for compilation errors,
            # before trying the fancy stuff with cpp_symbols.
            self.create_final_xml(include_files, types, None)

        if options.cpp_symbols:
            if options.verbose:
                print >> sys.stderr, "finding definitions ..."
            defines = self.get_defines(include_files)
            if options.verbose:
                print >> sys.stderr, "%d found" % len(defines)

                print >> sys.stderr, "filtering definitions ..."
            aliases, functions, excluded, defines = self.filter_definitions(defines)
            if options.verbose:
                print >> sys.stderr, "%d values, %d aliases" % (len(defines), len(aliases))

            if options.verbose:
                print >> sys.stderr, "finding definitions types ..."

            try:
                # invoke C++ template magic
                types = self.find_types(include_files, defines)
                if options.verbose:
                    print >> sys.stderr, "found %d types ..." % len(types)
            except CompilerError:
                print >> sys.stderr, "Could not determine the types of #define symbols."
                types = {}

        if options.verbose:
            print >> sys.stderr, "creating xml output file ..."
        self.create_final_xml(include_files, types, options.xmlfile)

        # Include additional preprecessor definitions into the XML file.

        if options.xmlfile:
            f = open(options.xmlfile, "r+")
            f.seek(-12, 2)
            data = f.read()
            if len(data) == 11:
                # text mode on windows is strange.  You read 12
                # characters, but get 11.
                assert data == "</GCC_XML>\n"
                f.seek(-12, 2)
            else:
                # linux, ...
                assert data == "\n</GCC_XML>\n"
                f.seek(-11, 2)
            f.flush()

            self.dump_as_cdata(f, functions, "functions")
            self.dump_as_cdata(f, aliases, "aliases")
            self.dump_as_cdata(f, excluded, "excluded")

            f.write("</GCC_XML>\n")
            f.close()
<|endoftext|>


<commit_before>import base64
import json
from operator import itemgetter
import psycopg2
import werkzeug
from werkzeug import url_encode

import openerp
from openerp import SUPERUSER_ID
from openerp import http
from openerp.exceptions import AccessError
from openerp.http import request

from openerp.addons.web.controllers.main import binary_content

class MailController(http.Controller):
    _cp_path = '/mail'

    def _redirect_to_messaging(self):
        messaging_action = request.env['mail.thread']._get_inbox_action_xml_id()
        url = '/web#%s' % url_encode({'action': messaging_action})
        return werkzeug.utils.redirect(url)

    @http.route('/mail/receive', type='json', auth='none')
    def receive(self, req):
        """ End-point to receive mail from an external SMTP server. """
        dbs = req.jsonrequest.get('databases')
        for db in dbs:
            message = dbs[db].decode('base64')
            try:
                registry = openerp.registry(db)
                with registry.cursor() as cr:
                    mail_thread = registry['mail.thread']
                    mail_thread.message_process(cr, SUPERUSER_ID, None, message)
            except psycopg2.Error:
                pass
        return True

    @http.route('/mail/read_followers', type='json', auth='user')
    def read_followers(self, follower_ids, res_model):
        followers = []
        is_editable = request.env.user.has_group('base.group_no_one')
        partner_id = request.env.user.partner_id
        follower_id = None
        for follower in request.env['mail.followers'].browse(follower_ids):
            is_uid = partner_id == follower.partner_id
            follower_id = follower.id if is_uid else follower_id
            followers.append({
                'id': follower.id,
                'name': follower.partner_id.name or follower.channel_id.name,
                'email': follower.partner_id.email if follower.partner_id else None,
                'res_model': 'res.partner' if follower.partner_id else 'mail.channel',
                'res_id': follower.partner_id.id or follower.channel_id.id,
                'is_editable': is_editable,
                'is_uid': is_uid,
            })
        return {
            'followers': followers,
            'subtypes': self.read_subscription_data(res_model, follower_id) if follower_id else None
        }

    @http.route('/mail/read_subscription_data', type='json', auth='user')
    def read_subscription_data(self, res_model, follower_id):
        """ Computes:
            - message_subtype_data: data about document subtypes: which are
                available, which are followed if any """
        followers = request.env['mail.followers'].browse(follower_id)

        # find current model subtypes, add them to a dictionary
        subtypes = request.env['mail.message.subtype'].search(['&', ('hidden', '=', False), '|', ('res_model', '=', res_model), ('res_model', '=', False)])
        subtypes_list = [{
            'name': subtype.name,
            'res_model': subtype.res_model,
            'sequence': subtype.sequence,
            'default': subtype.default,
            'internal': subtype.internal,
            'followed': subtype.id in followers.mapped('subtype_ids').ids,
            'parent_model': subtype.parent_id and subtype.parent_id.res_model or False,
            'id': subtype.id
        } for subtype in subtypes]
        subtypes_list = sorted(subtypes_list, key=itemgetter('parent_model', 'res_model', 'internal', 'sequence'))
        return subtypes_list

    @http.route('/mail/view', type='http', auth='none')
    def mail_action_view(self, model=None, res_id=None, message_id=None):
        """ Generic access point from notification emails. The heuristic to
        choose where to redirect the user is the following :

         - find a public URL
         - if none found
          - users with a read access are redirected to the document
          - users without read access are redirected to the Messaging
          - not logged users are redirected to the login page
        """
        uid = request.session.uid

        if message_id:
            try:
                message = request.env['mail.message'].sudo().browse(int(message_id)).exists()
            except:
                message = request.env['mail.message']
            if message:
                model, res_id = message.model, message.res_id
            else:
                # either a wrong message_id, either someone trying ids -> just go to messaging
                return self._redirect_to_messaging()
        elif res_id and isinstance(res_id, basestring):
            res_id = int(res_id)

        # no model / res_id, meaning no possible record -> redirect to login
        if not model or not res_id or model not in request.env:
            return self._redirect_to_messaging()

        # find the access action using sudo to have the details about the access link
        RecordModel = request.env[model]
        record_sudo = RecordModel.sudo().browse(res_id).exists()
        if not record_sudo:
            # record does not seem to exist -> redirect to login
            return self._redirect_to_messaging()
        record_action = record_sudo.get_access_action()

        # the record has an URL redirection: use it directly
        if record_action['type'] == 'ir.actions.act_url':
            return werkzeug.utils.redirect(record_action['url'])
        # other choice: act_window (no support of anything else currently)
        elif not record_action['type'] == 'ir.actions.act_window':
            return self._redirect_to_messaging()

        # the record has a window redirection: check access rights
        if not RecordModel.sudo(uid).check_access_rights('read', raise_exception=False):
            return self._redirect_to_messaging()
        try:
            RecordModel.sudo(uid).browse(res_id).exists().check_access_rule('read')
        except AccessError:
            return self._redirect_to_messaging()

        query = {}
        url_params = {
            'view_type': record_action['view_type'],
            'model': model,
            'id': res_id,
            'active_id': res_id,
            'view_id': record_sudo.get_formview_id(),
            'action': record_action.get('id'),
        }
        url = '/web?%s#%s' % (url_encode(query), url_encode(url_params))
        return werkzeug.utils.redirect(url)

    @http.route('/mail/follow', type='http', auth='user')
    def mail_action_follow(self, model, res_id):
        if model not in request.env:
            return self._redirect_to_messaging()
        Model = request.env[model]
        try:
            Model.browse(res_id).message_subscribe_users()
        except:
            return self._redirect_to_messaging()
        return werkzeug.utils.redirect('/mail/view?%s' % url_encode({'model': model, 'res_id': res_id}))

    @http.route('/mail/unfollow', type='http', auth='user')
    def mail_action_unfollow(self, model, res_id):
        if model not in request.env:
            return self._redirect_to_messaging()
        Model = request.env[model]
        try:
            Model.browse(res_id).sudo().message_unsubscribe_users([request.uid])
        except:
            return self._redirect_to_messaging()
        return werkzeug.utils.redirect('/mail/view?%s' % url_encode({'model': model, 'res_id': res_id}))

    @http.route('/mail/new', type='http', auth='user')
    def mail_action_new(self, model, res_id, **kwargs):
        if model not in request.env:
            return self._redirect_to_messaging()
        params = {'view_type': 'form', 'model': model}
        if kwargs.get('action_id'):
            params['action'] = kwargs['action_id']
        return werkzeug.utils.redirect('/web?#%s' % url_encode(params))

    @http.route('/mail/method', type='http', auth='user')
    def mail_action_method(self, model, res_id, method, **kwargs):
        # only public methods / check exists
        if method.strip().startswith('_') or model not in request.env:
            return self._redirect_to_messaging()
        Model = request.env[model]
        try:
            record = Model.browse(int(res_id)).exists()
            getattr(record, method)(**json.loads(kwargs.get('params', {})))
        except:
            return self._redirect_to_messaging()
        return werkzeug.utils.redirect('/mail/view?%s' % url_encode({'model': model, 'res_id': res_id}))

    @http.route('/mail/assign', type='http', auth='user')
    def mail_action_assign(self, model, res_id, **kwargs):
        if model not in request.env:
            return self._redirect_to_messaging()
        Model = request.env[model]
        try:
            Model.browse(int(res_id)).exists().write({'user_id': request.uid})
        except:
            return self._redirect_to_messaging()
        return werkzeug.utils.redirect('/mail/view?%s' % url_encode({'model': model, 'res_id': res_id}))

    @http.route('/mail/workflow', type='http', auth='user')
    def mail_action_workflow(self, model, res_id, signal, **kwargs):
        if model not in request.env:
            return self._redirect_to_messaging()
        Model = request.env[model]
        try:
            Model.browse(int(res_id)).exists().signal_workflow(signal)
        except:
            return self._redirect_to_messaging()
        return werkzeug.utils.redirect('/mail/view?%s' % url_encode({'model': model, 'res_id': res_id}))

    @http.route('/mail/<string:res_model>/<int:res_id>/avatar/<int:partner_id>', type='http', auth='public')
    def avatar(self, res_model, res_id, partner_id):
        headers = [[('Content-Type', 'image/png')]]
        content = 'R0lGODlhAQABAIABAP///wAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw=='  # default image is one white pixel
        if res_model in request.env:
            try:
                # if the current user has access to the document, get the partner avatar as sudo()
                request.env[res_model].browse(res_id).check_access_rule('read')
                if partner_id in request.env[res_model].browse(res_id).sudo().exists().message_ids.mapped('author_id').ids:
                    status, headers, _content = binary_content(model='res.partner', id=partner_id, field='image_medium', default_mimetype='image/png', env=request.env(user=openerp.SUPERUSER_ID))
                    # binary content return an empty string and not a placeholder if obj[field] is False
                    if _content != '':
                        content = _content
                    if status == 304:
                        return werkzeug.wrappers.Response(status=304)
            except AccessError:
                pass
        image_base64 = base64.b64decode(content)
        headers.append(('Content-Length', len(image_base64)))
        response = request.make_response(image_base64, headers)
        response.status = str(status)
        return response

    @http.route('/mail/needaction', type='json', auth='user')
    def needaction(self):
        return request.env['res.partner'].get_needaction_count()

    @http.route('/mail/client_action', type='json', auth='user')
    def mail_client_action(self):
        values = {
            'needaction_inbox_counter': request.env['res.partner'].get_needaction_count(),
            'starred_counter': request.env['res.partner'].get_starred_count(),
            'channel_slots': request.env['mail.channel'].channel_fetch_slot(),
            'commands': request.env['mail.channel'].get_mention_commands(),
            'mention_partner_suggestions': request.env['res.partner'].get_static_mention_suggestions(),
            'shortcodes': request.env['mail.shortcode'].sudo().search_read([], ['shortcode_type', 'source', 'substitution', 'description']),
            'menu_id': request.env['ir.model.data'].xmlid_to_res_id('mail.mail_channel_menu_root_chat'),
        }
        return values
<commit_msg>[FIX] mail: controller avatar<commit_after>import base64
import json
from operator import itemgetter
import psycopg2
import werkzeug
from werkzeug import url_encode

import openerp
from openerp import SUPERUSER_ID
from openerp import http
from openerp.exceptions import AccessError
from openerp.http import request

from openerp.addons.web.controllers.main import binary_content

class MailController(http.Controller):
    _cp_path = '/mail'

    def _redirect_to_messaging(self):
        messaging_action = request.env['mail.thread']._get_inbox_action_xml_id()
        url = '/web#%s' % url_encode({'action': messaging_action})
        return werkzeug.utils.redirect(url)

    @http.route('/mail/receive', type='json', auth='none')
    def receive(self, req):
        """ End-point to receive mail from an external SMTP server. """
        dbs = req.jsonrequest.get('databases')
        for db in dbs:
            message = dbs[db].decode('base64')
            try:
                registry = openerp.registry(db)
                with registry.cursor() as cr:
                    mail_thread = registry['mail.thread']
                    mail_thread.message_process(cr, SUPERUSER_ID, None, message)
            except psycopg2.Error:
                pass
        return True

    @http.route('/mail/read_followers', type='json', auth='user')
    def read_followers(self, follower_ids, res_model):
        followers = []
        is_editable = request.env.user.has_group('base.group_no_one')
        partner_id = request.env.user.partner_id
        follower_id = None
        for follower in request.env['mail.followers'].browse(follower_ids):
            is_uid = partner_id == follower.partner_id
            follower_id = follower.id if is_uid else follower_id
            followers.append({
                'id': follower.id,
                'name': follower.partner_id.name or follower.channel_id.name,
                'email': follower.partner_id.email if follower.partner_id else None,
                'res_model': 'res.partner' if follower.partner_id else 'mail.channel',
                'res_id': follower.partner_id.id or follower.channel_id.id,
                'is_editable': is_editable,
                'is_uid': is_uid,
            })
        return {
            'followers': followers,
            'subtypes': self.read_subscription_data(res_model, follower_id) if follower_id else None
        }

    @http.route('/mail/read_subscription_data', type='json', auth='user')
    def read_subscription_data(self, res_model, follower_id):
        """ Computes:
            - message_subtype_data: data about document subtypes: which are
                available, which are followed if any """
        followers = request.env['mail.followers'].browse(follower_id)

        # find current model subtypes, add them to a dictionary
        subtypes = request.env['mail.message.subtype'].search(['&', ('hidden', '=', False), '|', ('res_model', '=', res_model), ('res_model', '=', False)])
        subtypes_list = [{
            'name': subtype.name,
            'res_model': subtype.res_model,
            'sequence': subtype.sequence,
            'default': subtype.default,
            'internal': subtype.internal,
            'followed': subtype.id in followers.mapped('subtype_ids').ids,
            'parent_model': subtype.parent_id and subtype.parent_id.res_model or False,
            'id': subtype.id
        } for subtype in subtypes]
        subtypes_list = sorted(subtypes_list, key=itemgetter('parent_model', 'res_model', 'internal', 'sequence'))
        return subtypes_list

    @http.route('/mail/view', type='http', auth='none')
    def mail_action_view(self, model=None, res_id=None, message_id=None):
        """ Generic access point from notification emails. The heuristic to
        choose where to redirect the user is the following :

         - find a public URL
         - if none found
          - users with a read access are redirected to the document
          - users without read access are redirected to the Messaging
          - not logged users are redirected to the login page
        """
        uid = request.session.uid

        if message_id:
            try:
                message = request.env['mail.message'].sudo().browse(int(message_id)).exists()
            except:
                message = request.env['mail.message']
            if message:
                model, res_id = message.model, message.res_id
            else:
                # either a wrong message_id, either someone trying ids -> just go to messaging
                return self._redirect_to_messaging()
        elif res_id and isinstance(res_id, basestring):
            res_id = int(res_id)

        # no model / res_id, meaning no possible record -> redirect to login
        if not model or not res_id or model not in request.env:
            return self._redirect_to_messaging()

        # find the access action using sudo to have the details about the access link
        RecordModel = request.env[model]
        record_sudo = RecordModel.sudo().browse(res_id).exists()
        if not record_sudo:
            # record does not seem to exist -> redirect to login
            return self._redirect_to_messaging()
        record_action = record_sudo.get_access_action()

        # the record has an URL redirection: use it directly
        if record_action['type'] == 'ir.actions.act_url':
            return werkzeug.utils.redirect(record_action['url'])
        # other choice: act_window (no support of anything else currently)
        elif not record_action['type'] == 'ir.actions.act_window':
            return self._redirect_to_messaging()

        # the record has a window redirection: check access rights
        if not RecordModel.sudo(uid).check_access_rights('read', raise_exception=False):
            return self._redirect_to_messaging()
        try:
            RecordModel.sudo(uid).browse(res_id).exists().check_access_rule('read')
        except AccessError:
            return self._redirect_to_messaging()

        query = {}
        url_params = {
            'view_type': record_action['view_type'],
            'model': model,
            'id': res_id,
            'active_id': res_id,
            'view_id': record_sudo.get_formview_id(),
            'action': record_action.get('id'),
        }
        url = '/web?%s#%s' % (url_encode(query), url_encode(url_params))
        return werkzeug.utils.redirect(url)

    @http.route('/mail/follow', type='http', auth='user')
    def mail_action_follow(self, model, res_id):
        if model not in request.env:
            return self._redirect_to_messaging()
        Model = request.env[model]
        try:
            Model.browse(res_id).message_subscribe_users()
        except:
            return self._redirect_to_messaging()
        return werkzeug.utils.redirect('/mail/view?%s' % url_encode({'model': model, 'res_id': res_id}))

    @http.route('/mail/unfollow', type='http', auth='user')
    def mail_action_unfollow(self, model, res_id):
        if model not in request.env:
            return self._redirect_to_messaging()
        Model = request.env[model]
        try:
            Model.browse(res_id).sudo().message_unsubscribe_users([request.uid])
        except:
            return self._redirect_to_messaging()
        return werkzeug.utils.redirect('/mail/view?%s' % url_encode({'model': model, 'res_id': res_id}))

    @http.route('/mail/new', type='http', auth='user')
    def mail_action_new(self, model, res_id, **kwargs):
        if model not in request.env:
            return self._redirect_to_messaging()
        params = {'view_type': 'form', 'model': model}
        if kwargs.get('action_id'):
            params['action'] = kwargs['action_id']
        return werkzeug.utils.redirect('/web?#%s' % url_encode(params))

    @http.route('/mail/method', type='http', auth='user')
    def mail_action_method(self, model, res_id, method, **kwargs):
        # only public methods / check exists
        if method.strip().startswith('_') or model not in request.env:
            return self._redirect_to_messaging()
        Model = request.env[model]
        try:
            record = Model.browse(int(res_id)).exists()
            getattr(record, method)(**json.loads(kwargs.get('params', {})))
        except:
            return self._redirect_to_messaging()
        return werkzeug.utils.redirect('/mail/view?%s' % url_encode({'model': model, 'res_id': res_id}))

    @http.route('/mail/assign', type='http', auth='user')
    def mail_action_assign(self, model, res_id, **kwargs):
        if model not in request.env:
            return self._redirect_to_messaging()
        Model = request.env[model]
        try:
            Model.browse(int(res_id)).exists().write({'user_id': request.uid})
        except:
            return self._redirect_to_messaging()
        return werkzeug.utils.redirect('/mail/view?%s' % url_encode({'model': model, 'res_id': res_id}))

    @http.route('/mail/workflow', type='http', auth='user')
    def mail_action_workflow(self, model, res_id, signal, **kwargs):
        if model not in request.env:
            return self._redirect_to_messaging()
        Model = request.env[model]
        try:
            Model.browse(int(res_id)).exists().signal_workflow(signal)
        except:
            return self._redirect_to_messaging()
        return werkzeug.utils.redirect('/mail/view?%s' % url_encode({'model': model, 'res_id': res_id}))

    @http.route('/mail/<string:res_model>/<int:res_id>/avatar/<int:partner_id>', type='http', auth='public')
    def avatar(self, res_model, res_id, partner_id):
        headers = [('Content-Type', 'image/png')]
        status = 200
        content = 'R0lGODlhAQABAIABAP///wAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw=='  # default image is one white pixel
        if res_model in request.env:
            try:
                # if the current user has access to the document, get the partner avatar as sudo()
                request.env[res_model].browse(res_id).check_access_rule('read')
                if partner_id in request.env[res_model].browse(res_id).sudo().exists().message_ids.mapped('author_id').ids:
                    status, headers, _content = binary_content(model='res.partner', id=partner_id, field='image_medium', default_mimetype='image/png', env=request.env(user=openerp.SUPERUSER_ID))
                    # binary content return an empty string and not a placeholder if obj[field] is False
                    if _content != '':
                        content = _content
                    if status == 304:
                        return werkzeug.wrappers.Response(status=304)
            except AccessError:
                pass
        image_base64 = base64.b64decode(content)
        headers.append(('Content-Length', len(image_base64)))
        response = request.make_response(image_base64, headers)
        response.status = str(status)
        return response

    @http.route('/mail/needaction', type='json', auth='user')
    def needaction(self):
        return request.env['res.partner'].get_needaction_count()

    @http.route('/mail/client_action', type='json', auth='user')
    def mail_client_action(self):
        values = {
            'needaction_inbox_counter': request.env['res.partner'].get_needaction_count(),
            'starred_counter': request.env['res.partner'].get_starred_count(),
            'channel_slots': request.env['mail.channel'].channel_fetch_slot(),
            'commands': request.env['mail.channel'].get_mention_commands(),
            'mention_partner_suggestions': request.env['res.partner'].get_static_mention_suggestions(),
            'shortcodes': request.env['mail.shortcode'].sudo().search_read([], ['shortcode_type', 'source', 'substitution', 'description']),
            'menu_id': request.env['ir.model.data'].xmlid_to_res_id('mail.mail_channel_menu_root_chat'),
        }
        return values
<|endoftext|>


<commit_before>from __future__ import unicode_literals
from ..filters import Always, CLIFilter, Never
from ..utils import Callback

from collections import defaultdict

__all__ = (
    'Registry',
)

class _Binding(object):
    """
    (Immutable binding class.)
    """
    def __init__(self, keys, handler, filter=None):
        assert isinstance(keys, tuple)
        assert callable(handler)
        assert isinstance(filter, CLIFilter)

        self.keys = keys
        self._handler = handler
        self.filter = filter

    def call(self, event):
        return self._handler(event)

    def __repr__(self):
        return '%s(keys=%r, callable=%r)' % (
            self.__class__.__name__, self.keys, self._callable)

class Registry(object):
    """
    Key binding registry.

    ::

        r = Registry()

        @r.add_binding(Keys.ControlX, Keys.ControlC, filter=INSERT)
        def handler(event):
            # Handle ControlX-ControlC key sequence.
            pass
    """
    def __init__(self):
        self.key_bindings = []

        #: (tuple of keys) -> [list of bindings handling this sequence].
        self._keys_to_bindings = defaultdict(list)

        #: (tuple of keys) -> [list of bindings handling suffixes of this sequence].
        self._keys_to_bindings_suffixes = defaultdict(list)

        self.on_handler_called = Callback()

    def add_binding(self, *keys, **kwargs):
        """
        Decorator for annotating key bindings.
        """
        filter = kwargs.pop('filter', Always())

        assert not kwargs
        assert keys
        assert isinstance(filter, CLIFilter), 'Expected Filter instance, got %r' % filter

        def decorator(func):
            # When a filter is Never, it will always stay disabled, so in that case
            # don't bother putting it in the registry. It will slow down every key
            # press otherwise. (This happens for instance when a KeyBindingManager
            # is used, but some set of bindings are always disabled.)
            if not isinstance(filter, Never):
                binding = _Binding(keys, func, filter=filter)

                self.key_bindings.append(binding)
                self._keys_to_bindings[keys].append(binding)

                for i in range(1, len(keys)):
                    self._keys_to_bindings_suffixes[keys[:i]].append(binding)

            return func
        return decorator

    def get_bindings_for_keys(self, keys):
        """
        Return a list of key bindings that can handle this key.
        (This return also inactive bindings, so the `filter` still has to be
        called, for checking it.)

        :param keys: tuple of keys.
        """
        return self._keys_to_bindings[keys]

    def get_bindings_starting_with_keys(self, keys):
        """
        Return a list of key bindings that handle a key sequence starting with
        `keys`. (It does only return bindings for which the sequences are
        longer than `keys`. And like `get_bindings_for_keys`, it also includes
        inactive bindings.)

        :param keys: tuple of keys.
        """
        return self._keys_to_bindings_suffixes[keys]
<commit_msg>Fixed _Binding.__repr__.<commit_after>from __future__ import unicode_literals
from ..filters import Always, CLIFilter, Never
from ..utils import Callback

from collections import defaultdict

__all__ = (
    'Registry',
)

class _Binding(object):
    """
    (Immutable binding class.)
    """
    def __init__(self, keys, handler, filter=None):
        assert isinstance(keys, tuple)
        assert callable(handler)
        assert isinstance(filter, CLIFilter)

        self.keys = keys
        self._handler = handler
        self.filter = filter

    def call(self, event):
        return self._handler(event)

    def __repr__(self):
        return '%s(keys=%r, handler=%r)' % (
            self.__class__.__name__, self.keys, self._handler)

class Registry(object):
    """
    Key binding registry.

    ::

        r = Registry()

        @r.add_binding(Keys.ControlX, Keys.ControlC, filter=INSERT)
        def handler(event):
            # Handle ControlX-ControlC key sequence.
            pass
    """
    def __init__(self):
        self.key_bindings = []

        #: (tuple of keys) -> [list of bindings handling this sequence].
        self._keys_to_bindings = defaultdict(list)

        #: (tuple of keys) -> [list of bindings handling suffixes of this sequence].
        self._keys_to_bindings_suffixes = defaultdict(list)

        self.on_handler_called = Callback()

    def add_binding(self, *keys, **kwargs):
        """
        Decorator for annotating key bindings.
        """
        filter = kwargs.pop('filter', Always())

        assert not kwargs
        assert keys
        assert isinstance(filter, CLIFilter), 'Expected Filter instance, got %r' % filter

        def decorator(func):
            # When a filter is Never, it will always stay disabled, so in that case
            # don't bother putting it in the registry. It will slow down every key
            # press otherwise. (This happens for instance when a KeyBindingManager
            # is used, but some set of bindings are always disabled.)
            if not isinstance(filter, Never):
                binding = _Binding(keys, func, filter=filter)

                self.key_bindings.append(binding)
                self._keys_to_bindings[keys].append(binding)

                for i in range(1, len(keys)):
                    self._keys_to_bindings_suffixes[keys[:i]].append(binding)

            return func
        return decorator

    def get_bindings_for_keys(self, keys):
        """
        Return a list of key bindings that can handle this key.
        (This return also inactive bindings, so the `filter` still has to be
        called, for checking it.)

        :param keys: tuple of keys.
        """
        return self._keys_to_bindings[keys]

    def get_bindings_starting_with_keys(self, keys):
        """
        Return a list of key bindings that handle a key sequence starting with
        `keys`. (It does only return bindings for which the sequences are
        longer than `keys`. And like `get_bindings_for_keys`, it also includes
        inactive bindings.)

        :param keys: tuple of keys.
        """
        return self._keys_to_bindings_suffixes[keys]
<|endoftext|>


<commit_before>"""Plot to test logscale"""
import matplotlib.pyplot as plt
import numpy as np
import mpld3

def create_plot():
    fig = plt.figure()
    fig.subplots_adjust(hspace=0.4, wspace=0.4)

    ax1 = fig.add_subplot(2, 2, 1)
    ax2 = fig.add_subplot(2, 2, 2, sharey=ax1, xscale='log')
    ax3 = fig.add_subplot(2, 2, 3, sharex=ax1, yscale='log')
    ax4 = fig.add_subplot(2, 2, 4, sharex=ax2, sharey=ax3)

    x = np.linspace(1, 1e2)
    y = x ** 2

    for ax in [ax1, ax2, ax3, ax4]:
        ax.plot(x, y)

    return fig

def test_logscale():
    fig = create_plot()
    html = mpld3.fig_to_html(fig)
    plt.close(fig)

if __name__ == "__main__":
    mpld3.show(create_plot())
<commit_msg>Add TODO to broken test<commit_after>"""
Plot to test logscale

TODO (@vladh): `sharex` and `sharey` seem to cause the tick labels to go nuts. This needs to
be fixed.
"""
import matplotlib.pyplot as plt
import numpy as np
import mpld3

def create_plot():
    fig = plt.figure()
    fig.subplots_adjust(hspace=0.4, wspace=0.4)

    ax1 = fig.add_subplot(2, 2, 1)
    ax2 = fig.add_subplot(2, 2, 2, sharey=ax1, xscale='log')
    ax3 = fig.add_subplot(2, 2, 3, sharex=ax1, yscale='log')
    ax4 = fig.add_subplot(2, 2, 4, sharex=ax2, sharey=ax3)

    x = np.linspace(1, 1e2)
    y = x ** 2

    for ax in [ax1, ax2, ax3, ax4]:
        ax.plot(x, y)

    return fig

def test_logscale():
    fig = create_plot()
    html = mpld3.fig_to_html(fig)
    plt.close(fig)

if __name__ == "__main__":
    mpld3.show(create_plot())
<|endoftext|>