<commit_before>"""
This module contains C{L{OpenIDStore}} implementations that use
various SQL databases to back them.

Example of how to initialize a store database::

    python -c 'from openid.store import sqlstore; import pysqlite2.dbapi2; sqlstore.SQLiteStore(pysqlite2.dbapi2.connect("cstore.db")).createTables()'
"""
import re
import time

from openid.association import Association
from openid.store.interface import OpenIDStore
from openid.store import nonce

def _inTxn(func):
    def wrapped(self, *args, **kwargs):
        return self._callInTransaction(func, self, *args, **kwargs)

    if hasattr(func, '__name__'):
        try:
            wrapped.__name__ = func.__name__[4:]
        except TypeError:
            pass

    if hasattr(func, '__doc__'):
        wrapped.__doc__ = func.__doc__

    return wrapped

class SQLStore(OpenIDStore):
    """
    This is the parent class for the SQL stores, which contains the
    logic common to all of the SQL stores.

    The table names used are determined by the class variables
    C{L{settings_table}}, C{L{associations_table}}, and
    C{L{nonces_table}}.  To change the name of the tables used, pass
    new table names into the constructor.

    To create the tables with the proper schema, see the
    C{L{createTables}} method.

    This class shouldn't be used directly.  Use one of its subclasses
    instead, as those contain the code necessary to use a specific
    database.

    All methods other than C{L{__init__}} and C{L{createTables}}
    should be considered implementation details.

    @cvar settings_table: This is the default name of the table to
        keep this store's settings in.

    @cvar associations_table: This is the default name of the table to
        keep associations in

    @cvar nonces_table: This is the default name of the table to keep
        nonces in.

    @sort: __init__, createTables
    """

    settings_table = 'oid_settings'
    associations_table = 'oid_associations'
    nonces_table = 'oid_nonces'

    def __init__(self, conn, settings_table=None, associations_table=None,
                 nonces_table=None):
        """
        This creates a new SQLStore instance.  It requires an
        established database connection be given to it, and it allows
        overriding the default table names.

        @param conn: This must be an established connection to a
            database of the correct type for the SQLStore subclass
            you're using.

        @type conn: A python database API compatible connection
            object.

        @param settings_table: This is an optional parameter to
            specify the name of the table used for this store's
            settings.  The default value is specified in
            C{L{SQLStore.settings_table}}.

        @type settings_table: C{str}

        @param associations_table: This is an optional parameter to
            specify the name of the table used for storing
            associations.  The default value is specified in
            C{L{SQLStore.associations_table}}.

        @type associations_table: C{str}

        @param nonces_table: This is an optional parameter to specify
            the name of the table used for storing nonces.  The
            default value is specified in C{L{SQLStore.nonces_table}}.

        @type nonces_table: C{str}
        """
        self.conn = conn
        self.cur = None
        self._statement_cache = {}
        self._table_names = {
            'settings': settings_table or self.settings_table,
            'associations': associations_table or self.associations_table,
            'nonces': nonces_table or self.nonces_table,
            }
        self.max_nonce_age = 6 * 60 * 60 # Six hours, in seconds

    def blobDecode(self, blob):
        """Convert a blob as returned by the SQL engine into a str object.

        str -> str"""
        return blob

    def blobEncode(self, s):
        """Convert a str object into the necessary object for storing
        in the database as a blob."""
        return s

    def _getSQL(self, sql_name):
        try:
            return self._statement_cache[sql_name]
        except KeyError:
            sql = getattr(self, sql_name)
            sql %= self._table_names
            self._statement_cache[sql_name] = sql
            return sql

    def _execSQL(self, sql_name, *args):
        sql = self._getSQL(sql_name)
        self.cur.execute(sql, args)

    def __getattr__(self, attr):
        # if the attribute starts with db_, use a default
        # implementation that looks up the appropriate SQL statement
        # as an attribute of this object and executes it.
        if attr[:3] == 'db_':
            sql_name = attr[3:] + '_sql'
            def func(*args):
                return self._execSQL(sql_name, *args)
            setattr(self, attr, func)
            return func
        else:
            raise AttributeError('Attribute %r not found' % (attr,))

    def _callInTransaction(self, func, *args, **kwargs):
        """Execute the given function inside of a transaction, with an
        open cursor. If no exception is raised, the transaction is
        comitted, otherwise it is rolled back."""
        # No nesting of transactions
        self.conn.rollback()

        try:
            self.cur = self.conn.cursor()
            try:
                ret = func(*args, **kwargs)
            finally:
                self.cur.close()
                self.cur = None
        except:
            self.conn.rollback()
            raise
        else:
            self.conn.commit()

        return ret

    def txn_createTables(self):
        """
        This method creates the database tables necessary for this
        store to work.  It should not be called if the tables already
        exist.
        """
        self.db_create_nonce()
        self.db_create_assoc()
        self.db_create_settings()

    createTables = _inTxn(txn_createTables)

    def txn_storeAssociation(self, server_url, association):
        """Set the association for the server URL.

        Association -> NoneType
        """
        a = association
        self.db_set_assoc(
            server_url,
            a.handle,
            self.blobEncode(a.secret),
            a.issued,
            a.lifetime,
            a.assoc_type)

    storeAssociation = _inTxn(txn_storeAssociation)

    def txn_getAssociation(self, server_url, handle=None):
        """Get the most recent association that has been set for this
        server URL and handle.

        str -> NoneType or Association
        """
        if handle is not None:
            self.db_get_assoc(server_url, handle)
        else:
            self.db_get_assocs(server_url)

        rows = self.cur.fetchall()
        if len(rows) == 0:
            return None
        else:
            associations = []
            for values in rows:
                assoc = Association(*values)
                assoc.secret = self.blobDecode(assoc.secret)
                if assoc.getExpiresIn() == 0:
                    self.txn_removeAssociation(server_url, assoc.handle)
                else:
                    associations.append((assoc.issued, assoc))

            if associations:
                associations.sort()
                return associations[-1][1]
            else:
                return None

    getAssociation = _inTxn(txn_getAssociation)

    def txn_removeAssociation(self, server_url, handle):
        """Remove the association for the given server URL and handle,
        returning whether the association existed at all.

        (str, str) -> bool
        """
        self.db_remove_assoc(server_url, handle)
        return self.cur.rowcount > 0 # -1 is undefined

    removeAssociation = _inTxn(txn_removeAssociation)

    def txn_useNonce(self, server_url, timestamp, salt):
        """Return whether this nonce is present, and if it is, then
        remove it from the set.

        str -> bool"""
        if abs(timestamp - time.time()) > nonce.SKEW:
            return False

        try:
            self.db_add_nonce(server_url, timestamp, salt)
        except self.dbapi.IntegrityError:
            # The key uniqueness check failed
            return False
        else:
            # The nonce was successfully added
            return True

    useNonce = _inTxn(txn_useNonce)

    def txn_getExpired(self):
        """Get the server URLs for all associations that have expired"""
        self.db_get_expired(int(time.time()))
        rows = self.cur.fetchall()
        return [row[0] for row in rows]

    getExpired = _inTxn(txn_getExpired)

class SQLiteStore(SQLStore):
    """
    This is an SQLite-based specialization of C{L{SQLStore}}.

    To create an instance, see C{L{SQLStore.__init__}}.  To create the
    tables it will use, see C{L{SQLStore.createTables}}.

    All other methods are implementation details.
    """

    try:
        from pysqlite2 import dbapi2 as dbapi
    except ImportError:
        pass

    create_nonce_sql = """
    CREATE TABLE %(nonces)s (
        server_url VARCHAR,
        timestamp INTEGER,
        salt CHAR(40),
        UNIQUE(server_url, timestamp, salt)
    );
    """

    create_assoc_sql = """
    CREATE TABLE %(associations)s
    (
        server_url VARCHAR(2047),
        handle VARCHAR(255),
        secret BLOB(128),
        issued INTEGER,
        lifetime INTEGER,
        assoc_type VARCHAR(64),
        PRIMARY KEY (server_url, handle)
    );
    """

    create_settings_sql = """
    CREATE TABLE %(settings)s
    (
        setting VARCHAR(128) UNIQUE PRIMARY KEY,
        value BLOB(20)
    );
    """

    set_assoc_sql = ('INSERT OR REPLACE INTO %(associations)s '
                     'VALUES (?, ?, ?, ?, ?, ?);')
    get_assocs_sql = ('SELECT handle, secret, issued, lifetime, assoc_type '
                      'FROM %(associations)s WHERE server_url = ?;')
    get_assoc_sql = (
        'SELECT handle, secret, issued, lifetime, assoc_type '
        'FROM %(associations)s WHERE server_url = ? AND handle = ?;')

    get_expired_sql = ('SELECT server_url '
                       'FROM %(associations)s WHERE issued + lifetime < ?;')

    remove_assoc_sql = ('DELETE FROM %(associations)s '
                        'WHERE server_url = ? AND handle = ?;')

    add_nonce_sql = 'INSERT INTO %(nonces)s VALUES (?, ?, ?);'

    def blobDecode(self, buf):
        return str(buf)

    def blobEncode(self, s):
        return buffer(s)

    def useNonce(self, *args, **kwargs):
        # Older versions of the sqlite wrapper do not raise
        # IntegrityError as they should, so we have to detect the
        # message from the OperationalError.
        try:
            return super(SQLiteStore, self).useNonce(*args, **kwargs)
        except self.dbapi.OperationalError, why:
            if re.match('^columns .* are not unique$', why[0]):
                return False
            else:
                raise

class MySQLStore(SQLStore):
    """
    This is a MySQL-based specialization of C{L{SQLStore}}.

    Uses InnoDB tables for transaction support.

    To create an instance, see C{L{SQLStore.__init__}}.  To create the
    tables it will use, see C{L{SQLStore.createTables}}.

    All other methods are implementation details.
    """

    try:
        import MySQLdb as dbapi
    except ImportError:
        pass

    create_nonce_sql = """
    CREATE TABLE %(nonces)s (
        server_url BLOB,
        timestamp INTEGER,
        salt CHAR(40),
        PRIMARY KEY (server_url(255), timestamp, salt)
    )
    TYPE=InnoDB;
    """

    create_assoc_sql = """
    CREATE TABLE %(associations)s
    (
        server_url BLOB,
        handle VARCHAR(255),
        secret BLOB,
        issued INTEGER,
        lifetime INTEGER,
        assoc_type VARCHAR(64),
        PRIMARY KEY (server_url(255), handle)
    )
    TYPE=InnoDB;
    """

    create_settings_sql = """
    CREATE TABLE %(settings)s
    (
        setting VARCHAR(128) UNIQUE PRIMARY KEY,
        value BLOB
    )
    TYPE=InnoDB;
    """

    set_assoc_sql = ('REPLACE INTO %(associations)s '
                     'VALUES (%%s, %%s, %%s, %%s, %%s, %%s);')
    get_assocs_sql = ('SELECT handle, secret, issued, lifetime, assoc_type'
                      ' FROM %(associations)s WHERE server_url = %%s;')
    get_expired_sql = ('SELECT server_url '
                       'FROM %(associations)s WHERE issued + lifetime < %%s;')

    get_assoc_sql = (
        'SELECT handle, secret, issued, lifetime, assoc_type'
        ' FROM %(associations)s WHERE server_url = %%s AND handle = %%s;')
    remove_assoc_sql = ('DELETE FROM %(associations)s '
                        'WHERE server_url = %%s AND handle = %%s;')

    add_nonce_sql = 'INSERT INTO %(nonces)s VALUES (%%s, %%s, %%s);'

    def blobDecode(self, blob):
        return blob.tostring()

class PostgreSQLStore(SQLStore):
    """
    This is a PostgreSQL-based specialization of C{L{SQLStore}}.

    To create an instance, see C{L{SQLStore.__init__}}.  To create the
    tables it will use, see C{L{SQLStore.createTables}}.

    All other methods are implementation details.
    """

    try:
        import psycopg as dbapi
    except ImportError:
        pass

    create_nonce_sql = """
    CREATE TABLE %(nonces)s (
        server_url VARCHAR(2047),
        timestamp INTEGER,
        salt CHAR(40),
        PRIMARY KEY (server_url, timestamp, salt)
    );
    """

    create_assoc_sql = """
    CREATE TABLE %(associations)s
    (
        server_url VARCHAR(2047),
        handle VARCHAR(255),
        secret BYTEA,
        issued INTEGER,
        lifetime INTEGER,
        assoc_type VARCHAR(64),
        PRIMARY KEY (server_url, handle),
        CONSTRAINT secret_length_constraint CHECK (LENGTH(secret) <= 128)
    );
    """

    create_settings_sql = """
    CREATE TABLE %(settings)s
    (
        setting VARCHAR(128) UNIQUE PRIMARY KEY,
        value BYTEA,
        CONSTRAINT value_length_constraint CHECK (LENGTH(value) <= 20)
    );
    """

    def db_set_assoc(self, server_url, handle, secret, issued, lifetime, assoc_type):
        """
        Set an association.  This is implemented as a method because
        REPLACE INTO is not supported by PostgreSQL (and is not
        standard SQL).
        """
        result = self.db_get_assoc(server_url, handle)
        rows = self.cur.fetchall()
        if len(rows):
            # Update the table since this associations already exists.
            return self.db_update_assoc(secret, issued, lifetime, assoc_type,
                                        server_url, handle)
        else:
            # Insert a new record because this association wasn't
            # found.
            return self.db_new_assoc(server_url, handle, secret, issued,
                                     lifetime, assoc_type)

    new_assoc_sql = ('INSERT INTO %(associations)s '
                     'VALUES (%%s, %%s, %%s, %%s, %%s, %%s);')
    update_assoc_sql = ('UPDATE %(associations)s SET '
                        'secret = %%s, issued = %%s, '
                        'lifetime = %%s, assoc_type = %%s '
                        'WHERE server_url = %%s AND handle = %%s;')
    get_assocs_sql = ('SELECT handle, secret, issued, lifetime, assoc_type'
                      ' FROM %(associations)s WHERE server_url = %%s;')
    get_expired_sql = ('SELECT server_url '
                       'FROM %(associations)s WHERE issued + lifetime < %%s;')

    get_assoc_sql = (
        'SELECT handle, secret, issued, lifetime, assoc_type'
        ' FROM %(associations)s WHERE server_url = %%s AND handle = %%s;')
    remove_assoc_sql = ('DELETE FROM %(associations)s '
                        'WHERE server_url = %%s AND handle = %%s;')

    add_nonce_sql = 'INSERT INTO %(nonces)s VALUES (%%s, %%s, %%s);'

    def blobEncode(self, blob):
        import psycopg
        return psycopg.Binary(blob)
<commit_msg>[project @ store.sqlstore.SQLStore.cleanupNonces: added]<commit_after>"""
This module contains C{L{OpenIDStore}} implementations that use
various SQL databases to back them.

Example of how to initialize a store database::

    python -c 'from openid.store import sqlstore; import pysqlite2.dbapi2; sqlstore.SQLiteStore(pysqlite2.dbapi2.connect("cstore.db")).createTables()'
"""
import re
import time

from openid.association import Association
from openid.store.interface import OpenIDStore
from openid.store import nonce

def _inTxn(func):
    def wrapped(self, *args, **kwargs):
        return self._callInTransaction(func, self, *args, **kwargs)

    if hasattr(func, '__name__'):
        try:
            wrapped.__name__ = func.__name__[4:]
        except TypeError:
            pass

    if hasattr(func, '__doc__'):
        wrapped.__doc__ = func.__doc__

    return wrapped

class SQLStore(OpenIDStore):
    """
    This is the parent class for the SQL stores, which contains the
    logic common to all of the SQL stores.

    The table names used are determined by the class variables
    C{L{settings_table}}, C{L{associations_table}}, and
    C{L{nonces_table}}.  To change the name of the tables used, pass
    new table names into the constructor.

    To create the tables with the proper schema, see the
    C{L{createTables}} method.

    This class shouldn't be used directly.  Use one of its subclasses
    instead, as those contain the code necessary to use a specific
    database.

    All methods other than C{L{__init__}} and C{L{createTables}}
    should be considered implementation details.

    @cvar settings_table: This is the default name of the table to
        keep this store's settings in.

    @cvar associations_table: This is the default name of the table to
        keep associations in

    @cvar nonces_table: This is the default name of the table to keep
        nonces in.

    @sort: __init__, createTables
    """

    settings_table = 'oid_settings'
    associations_table = 'oid_associations'
    nonces_table = 'oid_nonces'

    def __init__(self, conn, settings_table=None, associations_table=None,
                 nonces_table=None):
        """
        This creates a new SQLStore instance.  It requires an
        established database connection be given to it, and it allows
        overriding the default table names.

        @param conn: This must be an established connection to a
            database of the correct type for the SQLStore subclass
            you're using.

        @type conn: A python database API compatible connection
            object.

        @param settings_table: This is an optional parameter to
            specify the name of the table used for this store's
            settings.  The default value is specified in
            C{L{SQLStore.settings_table}}.

        @type settings_table: C{str}

        @param associations_table: This is an optional parameter to
            specify the name of the table used for storing
            associations.  The default value is specified in
            C{L{SQLStore.associations_table}}.

        @type associations_table: C{str}

        @param nonces_table: This is an optional parameter to specify
            the name of the table used for storing nonces.  The
            default value is specified in C{L{SQLStore.nonces_table}}.

        @type nonces_table: C{str}
        """
        self.conn = conn
        self.cur = None
        self._statement_cache = {}
        self._table_names = {
            'settings': settings_table or self.settings_table,
            'associations': associations_table or self.associations_table,
            'nonces': nonces_table or self.nonces_table,
            }
        self.max_nonce_age = 6 * 60 * 60 # Six hours, in seconds

    def blobDecode(self, blob):
        """Convert a blob as returned by the SQL engine into a str object.

        str -> str"""
        return blob

    def blobEncode(self, s):
        """Convert a str object into the necessary object for storing
        in the database as a blob."""
        return s

    def _getSQL(self, sql_name):
        try:
            return self._statement_cache[sql_name]
        except KeyError:
            sql = getattr(self, sql_name)
            sql %= self._table_names
            self._statement_cache[sql_name] = sql
            return sql

    def _execSQL(self, sql_name, *args):
        sql = self._getSQL(sql_name)
        self.cur.execute(sql, args)

    def __getattr__(self, attr):
        # if the attribute starts with db_, use a default
        # implementation that looks up the appropriate SQL statement
        # as an attribute of this object and executes it.
        if attr[:3] == 'db_':
            sql_name = attr[3:] + '_sql'
            def func(*args):
                return self._execSQL(sql_name, *args)
            setattr(self, attr, func)
            return func
        else:
            raise AttributeError('Attribute %r not found' % (attr,))

    def _callInTransaction(self, func, *args, **kwargs):
        """Execute the given function inside of a transaction, with an
        open cursor. If no exception is raised, the transaction is
        comitted, otherwise it is rolled back."""
        # No nesting of transactions
        self.conn.rollback()

        try:
            self.cur = self.conn.cursor()
            try:
                ret = func(*args, **kwargs)
            finally:
                self.cur.close()
                self.cur = None
        except:
            self.conn.rollback()
            raise
        else:
            self.conn.commit()

        return ret

    def txn_createTables(self):
        """
        This method creates the database tables necessary for this
        store to work.  It should not be called if the tables already
        exist.
        """
        self.db_create_nonce()
        self.db_create_assoc()
        self.db_create_settings()

    createTables = _inTxn(txn_createTables)

    def txn_storeAssociation(self, server_url, association):
        """Set the association for the server URL.

        Association -> NoneType
        """
        a = association
        self.db_set_assoc(
            server_url,
            a.handle,
            self.blobEncode(a.secret),
            a.issued,
            a.lifetime,
            a.assoc_type)

    storeAssociation = _inTxn(txn_storeAssociation)

    def txn_getAssociation(self, server_url, handle=None):
        """Get the most recent association that has been set for this
        server URL and handle.

        str -> NoneType or Association
        """
        if handle is not None:
            self.db_get_assoc(server_url, handle)
        else:
            self.db_get_assocs(server_url)

        rows = self.cur.fetchall()
        if len(rows) == 0:
            return None
        else:
            associations = []
            for values in rows:
                assoc = Association(*values)
                assoc.secret = self.blobDecode(assoc.secret)
                if assoc.getExpiresIn() == 0:
                    self.txn_removeAssociation(server_url, assoc.handle)
                else:
                    associations.append((assoc.issued, assoc))

            if associations:
                associations.sort()
                return associations[-1][1]
            else:
                return None

    getAssociation = _inTxn(txn_getAssociation)

    def txn_removeAssociation(self, server_url, handle):
        """Remove the association for the given server URL and handle,
        returning whether the association existed at all.

        (str, str) -> bool
        """
        self.db_remove_assoc(server_url, handle)
        return self.cur.rowcount > 0 # -1 is undefined

    removeAssociation = _inTxn(txn_removeAssociation)

    def txn_useNonce(self, server_url, timestamp, salt):
        """Return whether this nonce is present, and if it is, then
        remove it from the set.

        str -> bool"""
        if abs(timestamp - time.time()) > nonce.SKEW:
            return False

        try:
            self.db_add_nonce(server_url, timestamp, salt)
        except self.dbapi.IntegrityError:
            # The key uniqueness check failed
            return False
        else:
            # The nonce was successfully added
            return True

    useNonce = _inTxn(txn_useNonce)

    def txn_getExpired(self):
        """Get the server URLs for all associations that have expired"""
        self.db_get_expired(int(time.time()))
        rows = self.cur.fetchall()
        return [row[0] for row in rows]

    getExpired = _inTxn(txn_getExpired)

    def txn_cleanupNonces(self):
        self.db_clean_nonce(int(time.time()) - nonce.SKEW)
        return self.cur.rowcount

    cleanupNonces = _inTxn(txn_cleanupNonces)

class SQLiteStore(SQLStore):
    """
    This is an SQLite-based specialization of C{L{SQLStore}}.

    To create an instance, see C{L{SQLStore.__init__}}.  To create the
    tables it will use, see C{L{SQLStore.createTables}}.

    All other methods are implementation details.
    """

    try:
        from pysqlite2 import dbapi2 as dbapi
    except ImportError:
        pass

    create_nonce_sql = """
    CREATE TABLE %(nonces)s (
        server_url VARCHAR,
        timestamp INTEGER,
        salt CHAR(40),
        UNIQUE(server_url, timestamp, salt)
    );
    """

    create_assoc_sql = """
    CREATE TABLE %(associations)s
    (
        server_url VARCHAR(2047),
        handle VARCHAR(255),
        secret BLOB(128),
        issued INTEGER,
        lifetime INTEGER,
        assoc_type VARCHAR(64),
        PRIMARY KEY (server_url, handle)
    );
    """

    create_settings_sql = """
    CREATE TABLE %(settings)s
    (
        setting VARCHAR(128) UNIQUE PRIMARY KEY,
        value BLOB(20)
    );
    """

    set_assoc_sql = ('INSERT OR REPLACE INTO %(associations)s '
                     'VALUES (?, ?, ?, ?, ?, ?);')
    get_assocs_sql = ('SELECT handle, secret, issued, lifetime, assoc_type '
                      'FROM %(associations)s WHERE server_url = ?;')
    get_assoc_sql = (
        'SELECT handle, secret, issued, lifetime, assoc_type '
        'FROM %(associations)s WHERE server_url = ? AND handle = ?;')

    get_expired_sql = ('SELECT server_url '
                       'FROM %(associations)s WHERE issued + lifetime < ?;')

    remove_assoc_sql = ('DELETE FROM %(associations)s '
                        'WHERE server_url = ? AND handle = ?;')

    add_nonce_sql = 'INSERT INTO %(nonces)s VALUES (?, ?, ?);'

    clean_nonce_sql = 'DELETE FROM %(nonces)s WHERE timestamp < ?;'

    def blobDecode(self, buf):
        return str(buf)

    def blobEncode(self, s):
        return buffer(s)

    def useNonce(self, *args, **kwargs):
        # Older versions of the sqlite wrapper do not raise
        # IntegrityError as they should, so we have to detect the
        # message from the OperationalError.
        try:
            return super(SQLiteStore, self).useNonce(*args, **kwargs)
        except self.dbapi.OperationalError, why:
            if re.match('^columns .* are not unique$', why[0]):
                return False
            else:
                raise

class MySQLStore(SQLStore):
    """
    This is a MySQL-based specialization of C{L{SQLStore}}.

    Uses InnoDB tables for transaction support.

    To create an instance, see C{L{SQLStore.__init__}}.  To create the
    tables it will use, see C{L{SQLStore.createTables}}.

    All other methods are implementation details.
    """

    try:
        import MySQLdb as dbapi
    except ImportError:
        pass

    create_nonce_sql = """
    CREATE TABLE %(nonces)s (
        server_url BLOB,
        timestamp INTEGER,
        salt CHAR(40),
        PRIMARY KEY (server_url(255), timestamp, salt)
    )
    TYPE=InnoDB;
    """

    create_assoc_sql = """
    CREATE TABLE %(associations)s
    (
        server_url BLOB,
        handle VARCHAR(255),
        secret BLOB,
        issued INTEGER,
        lifetime INTEGER,
        assoc_type VARCHAR(64),
        PRIMARY KEY (server_url(255), handle)
    )
    TYPE=InnoDB;
    """

    create_settings_sql = """
    CREATE TABLE %(settings)s
    (
        setting VARCHAR(128) UNIQUE PRIMARY KEY,
        value BLOB
    )
    TYPE=InnoDB;
    """

    set_assoc_sql = ('REPLACE INTO %(associations)s '
                     'VALUES (%%s, %%s, %%s, %%s, %%s, %%s);')
    get_assocs_sql = ('SELECT handle, secret, issued, lifetime, assoc_type'
                      ' FROM %(associations)s WHERE server_url = %%s;')
    get_expired_sql = ('SELECT server_url '
                       'FROM %(associations)s WHERE issued + lifetime < %%s;')

    get_assoc_sql = (
        'SELECT handle, secret, issued, lifetime, assoc_type'
        ' FROM %(associations)s WHERE server_url = %%s AND handle = %%s;')
    remove_assoc_sql = ('DELETE FROM %(associations)s '
                        'WHERE server_url = %%s AND handle = %%s;')

    add_nonce_sql = 'INSERT INTO %(nonces)s VALUES (%%s, %%s, %%s);'

    clean_nonce_sql = 'DELETE FROM %(nonces)s WHERE timestamp < %%s;'

    def blobDecode(self, blob):
        return blob.tostring()

class PostgreSQLStore(SQLStore):
    """
    This is a PostgreSQL-based specialization of C{L{SQLStore}}.

    To create an instance, see C{L{SQLStore.__init__}}.  To create the
    tables it will use, see C{L{SQLStore.createTables}}.

    All other methods are implementation details.
    """

    try:
        import psycopg as dbapi
    except ImportError:
        pass

    create_nonce_sql = """
    CREATE TABLE %(nonces)s (
        server_url VARCHAR(2047),
        timestamp INTEGER,
        salt CHAR(40),
        PRIMARY KEY (server_url, timestamp, salt)
    );
    """

    create_assoc_sql = """
    CREATE TABLE %(associations)s
    (
        server_url VARCHAR(2047),
        handle VARCHAR(255),
        secret BYTEA,
        issued INTEGER,
        lifetime INTEGER,
        assoc_type VARCHAR(64),
        PRIMARY KEY (server_url, handle),
        CONSTRAINT secret_length_constraint CHECK (LENGTH(secret) <= 128)
    );
    """

    create_settings_sql = """
    CREATE TABLE %(settings)s
    (
        setting VARCHAR(128) UNIQUE PRIMARY KEY,
        value BYTEA,
        CONSTRAINT value_length_constraint CHECK (LENGTH(value) <= 20)
    );
    """

    def db_set_assoc(self, server_url, handle, secret, issued, lifetime, assoc_type):
        """
        Set an association.  This is implemented as a method because
        REPLACE INTO is not supported by PostgreSQL (and is not
        standard SQL).
        """
        result = self.db_get_assoc(server_url, handle)
        rows = self.cur.fetchall()
        if len(rows):
            # Update the table since this associations already exists.
            return self.db_update_assoc(secret, issued, lifetime, assoc_type,
                                        server_url, handle)
        else:
            # Insert a new record because this association wasn't
            # found.
            return self.db_new_assoc(server_url, handle, secret, issued,
                                     lifetime, assoc_type)

    new_assoc_sql = ('INSERT INTO %(associations)s '
                     'VALUES (%%s, %%s, %%s, %%s, %%s, %%s);')
    update_assoc_sql = ('UPDATE %(associations)s SET '
                        'secret = %%s, issued = %%s, '
                        'lifetime = %%s, assoc_type = %%s '
                        'WHERE server_url = %%s AND handle = %%s;')
    get_assocs_sql = ('SELECT handle, secret, issued, lifetime, assoc_type'
                      ' FROM %(associations)s WHERE server_url = %%s;')
    get_expired_sql = ('SELECT server_url '
                       'FROM %(associations)s WHERE issued + lifetime < %%s;')

    get_assoc_sql = (
        'SELECT handle, secret, issued, lifetime, assoc_type'
        ' FROM %(associations)s WHERE server_url = %%s AND handle = %%s;')
    remove_assoc_sql = ('DELETE FROM %(associations)s '
                        'WHERE server_url = %%s AND handle = %%s;')

    add_nonce_sql = 'INSERT INTO %(nonces)s VALUES (%%s, %%s, %%s);'

    clean_nonce_sql = 'DELETE FROM %(nonces)s WHERE timestamp < %%s;'

    def blobEncode(self, blob):
        import psycopg
        return psycopg.Binary(blob)
<|endoftext|>


<commit_before>from compas.geometry import Point
from compas.geometry import Vector
from compas.geometry import Line
from compas.geometry import Plane
from compas.geometry import Frame
from compas.geometry import Circle
from compas.geometry import Ellipse
from compas.geometry import Polyline
from compas.geometry import Polygon

from Rhino.Geometry import Point3d
from Rhino.Geometry import Vector3d
from Rhino.Geometry import Line as RhinoLine
from Rhino.Geometry import Plane as RhinoPlane
from Rhino.Geometry import Circle as RhinoCircle
from Rhino.Geometry import Ellipse as RhinoEllipse
from Rhino.Geometry import Polyline as RhinoPolyline

def point_to_compas(point):
    """Convert a Rhino point to a COMPAS point.

    Parameters
    ----------
    point : :class:`Rhino.Geometry.Point3d`

    Returns
    -------
    :class:`compas.geometry.Point`
    """
    return Point(point.X, point.Y, point.Z)

def point_to_rhino(point):
    """Convert a COMPAS point to a Rhino point.

    Parameters
    ----------
    point : :class:`compas.geometry.Point`

    Returns
    -------
    :class:`Rhino.Geometry.Point3d`
    """
    return Point3d(point.x, point.y, point.z)

def vector_to_compas(vector):
    """Convert a Rhino vector to a COMPAS vector.

    Parameters
    ----------
    vector : :class:`Rhino.Geometry.Vector3d`

    Returns
    -------
    :class:`compas.geometry.Vector`
    """
    return Vector(vector.X, vector.Y, vector.Z)

def vector_to_rhino(vector):
    """Convert a COMPAS vector to a Rhino vector.

    Parameters
    ----------
    vector : :class:`compas.geometry.Vector`

    Returns
    -------
    :class:`Rhino.Geometry.Vector3d`
    """
    return Vector3d(vector.x, vector.y, vector.z)

def line_to_compas(line):
    """Convert a Rhino line to a COMPAS line.

    Parameters
    ----------
    line : :class:`Rhino.Geometry.Line`

    Returns
    -------
    :class:`compas.geometry.Line`
    """
    return Line(point_to_compas(line.From),
                point_to_compas(line.To))

def line_to_rhino(line):
    """Convert a COMPAS line to a Rhino line.

    Parameters
    ----------
    line : :class:`compas.geometry.Line`

    Returns
    -------
    :class:`Rhino.Geometry.Line`
    """
    return RhinoLine(point_to_rhino(line.start),
                     point_to_rhino(line.end))

def plane_to_compas(plane):
    """Convert a Rhino plane to a COMPAS plane.

    Parameters
    ----------
    plane : :class:`Rhino.Geometry.Plane`

    Returns
    -------
    :class:`compas.geometry.Plane`
    """
    return Plane(point_to_compas(plane.Origin),
                 vector_to_compas(plane.Normal))

def plane_to_rhino(plane):
    """Convert a COMPAS plane to a Rhino plane.

    Parameters
    ----------
    plane : :class:`compas.geometry.Plane`

    Returns
    -------
    :class:`Rhino.Geometry.Plane`
    """
    return RhinoPlane(point_to_rhino(plane.point),
                      vector_to_rhino(plane.normal))

def plane_to_compas_frame(plane):
    """Convert a Rhino plane to a COMPAS frame.

    Parameters
    ----------
    plane : :class:`Rhino.Geometry.Plane`

    Returns
    -------
    :class:`compas.geometry.Frame`
    """
    return Frame(point_to_compas(plane.Origin),
                 vector_to_compas(plane.XAxis),
                 vector_to_compas(plane.YAxis))

def frame_to_rhino(frame):
    """Convert a COMPAS frame to a Rhino plane.

    Parameters
    ----------
    frame : :class:`compas.geometry.Frame`

    Returns
    -------
    :class:`Rhino.Geometry.Plane`
    """
    return RhinoPlane(point_to_rhino(frame.point),
                      vector_to_rhino(frame.xaxis),
                      vector_to_rhino(frame.yaxis))

def circle_to_compas(circle):
    """Convert a Rhino circle to a COMPAS circle.

    Parameters
    ----------
    circle : :class:`Rhino.Geometry.Circle`

    Returns
    -------
    :class:`compas.geometry.Circle`
    """
    return Circle(plane_to_compas(circle.Plane), circle.Radius)

def circle_to_rhino(circle):
    """Convert a COMPAS circle to a Rhino circle.

    Parameters
    ----------
    circle : :class:`compas.geometry.Circle`

    Returns
    -------
    :class:`Rhino.Geometry.Circle`
    """
    return RhinoCircle(plane_to_rhino(circle.plane), circle.radius)

def ellipse_to_compas(ellipse):
    """Convert a Rhino ellipse to a COMPAS ellipse.

    Parameters
    ----------
    ellipse : :class:`Rhino.Geometry.Ellipse`

    Returns
    -------
    :class:`compas.geometry.Ellipse`
    """
    return Ellipse(plane_to_compas(ellipse.Plane), ellipse.major, ellipse.minor)

def ellipse_to_rhino(ellipse):
    """Convert a COMPAS ellipse to a Rhino ellipse.

    Parameters
    ----------
    ellipse : :class:`compas.geometry.Ellipse`

    Returns
    -------
    :class:`Rhino.Geometry.Ellipse`
    """
    return RhinoEllipse(plane_to_rhino(ellipse.plane), ellipse.major, ellipse.minor)

def polyline_to_compas(polyline):
    """Convert a Rhino polyline to a COMPAS polyline.

    Parameters
    ----------
    polyline : :class:`Rhino.Geometry.Polyline`

    Returns
    -------
    :class:`compas.geometry.Polyline`
    """
    return Polyline([point_to_compas(point) for point in polyline])

def polyline_to_rhino(polyline):
    """Convert a COMPAS polyline to a Rhino polyline.

    Parameters
    ----------
    polyline : :class:`compas.geometry.Polyline`

    Returns
    -------
    :class:`Rhino.Geometry.Ellipse`
    """
    return RhinoPolyline([point_to_rhino(point) for point in polyline])

def polygon_to_compas(polygon):
    """Convert a Rhino polygon to a COMPAS polygon.

    Parameters
    ----------
    polygon : :class:`Rhino.Geometry.Polygon`

    Returns
    -------
    :class:`compas.geometry.Polygon`
    """
    return Polygon([point_to_compas(point) for point in polygon])

def polygon_to_rhino(polygon):
    """Convert a COMPAS polygon to a Rhino polygon.

    Parameters
    ----------
    polygon : :class:`compas.geometry.Polygon`

    Returns
    -------
    :class:`Rhino.Geometry.Polygon`
    """
    raise NotImplementedError
<commit_msg>compatibility with native representations<commit_after>from compas.geometry import Point
from compas.geometry import Vector
from compas.geometry import Line
from compas.geometry import Plane
from compas.geometry import Frame
from compas.geometry import Circle
from compas.geometry import Ellipse
from compas.geometry import Polyline
from compas.geometry import Polygon

from Rhino.Geometry import Point3d
from Rhino.Geometry import Vector3d
from Rhino.Geometry import Line as RhinoLine
from Rhino.Geometry import Plane as RhinoPlane
from Rhino.Geometry import Circle as RhinoCircle
from Rhino.Geometry import Ellipse as RhinoEllipse
from Rhino.Geometry import Polyline as RhinoPolyline

def point_to_compas(point):
    """Convert a Rhino point to a COMPAS point.

    Parameters
    ----------
    point : :class:`Rhino.Geometry.Point3d`

    Returns
    -------
    :class:`compas.geometry.Point`
    """
    return Point(point.X, point.Y, point.Z)

def point_to_rhino(point):
    """Convert a COMPAS point to a Rhino point.

    Parameters
    ----------
    point : :class:`compas.geometry.Point`

    Returns
    -------
    :class:`Rhino.Geometry.Point3d`
    """
    return Point3d(point[0], point[1], point[2])

def vector_to_compas(vector):
    """Convert a Rhino vector to a COMPAS vector.

    Parameters
    ----------
    vector : :class:`Rhino.Geometry.Vector3d`

    Returns
    -------
    :class:`compas.geometry.Vector`
    """
    return Vector(vector.X, vector.Y, vector.Z)

def vector_to_rhino(vector):
    """Convert a COMPAS vector to a Rhino vector.

    Parameters
    ----------
    vector : :class:`compas.geometry.Vector`

    Returns
    -------
    :class:`Rhino.Geometry.Vector3d`
    """
    return Vector3d(vector[0], vector[1], vector[2])

def line_to_compas(line):
    """Convert a Rhino line to a COMPAS line.

    Parameters
    ----------
    line : :class:`Rhino.Geometry.Line`

    Returns
    -------
    :class:`compas.geometry.Line`
    """
    return Line(point_to_compas(line.From),
                point_to_compas(line.To))

def line_to_rhino(line):
    """Convert a COMPAS line to a Rhino line.

    Parameters
    ----------
    line : :class:`compas.geometry.Line`

    Returns
    -------
    :class:`Rhino.Geometry.Line`
    """
    return RhinoLine(point_to_rhino(line[0]),
                     point_to_rhino(line[1]))

def plane_to_compas(plane):
    """Convert a Rhino plane to a COMPAS plane.

    Parameters
    ----------
    plane : :class:`Rhino.Geometry.Plane`

    Returns
    -------
    :class:`compas.geometry.Plane`
    """
    return Plane(point_to_compas(plane.Origin),
                 vector_to_compas(plane.Normal))

def plane_to_rhino(plane):
    """Convert a COMPAS plane to a Rhino plane.

    Parameters
    ----------
    plane : :class:`compas.geometry.Plane`

    Returns
    -------
    :class:`Rhino.Geometry.Plane`
    """
    return RhinoPlane(point_to_rhino(plane[0]),
                      vector_to_rhino(plane[1]))

def plane_to_compas_frame(plane):
    """Convert a Rhino plane to a COMPAS frame.

    Parameters
    ----------
    plane : :class:`Rhino.Geometry.Plane`

    Returns
    -------
    :class:`compas.geometry.Frame`
    """
    return Frame(point_to_compas(plane.Origin),
                 vector_to_compas(plane.XAxis),
                 vector_to_compas(plane.YAxis))

def frame_to_rhino(frame):
    """Convert a COMPAS frame to a Rhino plane.

    Parameters
    ----------
    frame : :class:`compas.geometry.Frame`

    Returns
    -------
    :class:`Rhino.Geometry.Plane`
    """
    return RhinoPlane(point_to_rhino(frame[0]),
                      vector_to_rhino(frame[1]),
                      vector_to_rhino(frame[2]))

def circle_to_compas(circle):
    """Convert a Rhino circle to a COMPAS circle.

    Parameters
    ----------
    circle : :class:`Rhino.Geometry.Circle`

    Returns
    -------
    :class:`compas.geometry.Circle`
    """
    return Circle(plane_to_compas(circle.Plane), circle.Radius)

def circle_to_rhino(circle):
    """Convert a COMPAS circle to a Rhino circle.

    Parameters
    ----------
    circle : :class:`compas.geometry.Circle`

    Returns
    -------
    :class:`Rhino.Geometry.Circle`
    """
    return RhinoCircle(plane_to_rhino(circle[0]), circle[1])

def ellipse_to_compas(ellipse):
    """Convert a Rhino ellipse to a COMPAS ellipse.

    Parameters
    ----------
    ellipse : :class:`Rhino.Geometry.Ellipse`

    Returns
    -------
    :class:`compas.geometry.Ellipse`
    """
    return Ellipse(plane_to_compas(ellipse.Plane), ellipse.Major, ellipse.Minor)

def ellipse_to_rhino(ellipse):
    """Convert a COMPAS ellipse to a Rhino ellipse.

    Parameters
    ----------
    ellipse : :class:`compas.geometry.Ellipse`

    Returns
    -------
    :class:`Rhino.Geometry.Ellipse`
    """
    return RhinoEllipse(plane_to_rhino(ellipse[0]), ellipse[1], ellipse[2])

def polyline_to_compas(polyline):
    """Convert a Rhino polyline to a COMPAS polyline.

    Parameters
    ----------
    polyline : :class:`Rhino.Geometry.Polyline`

    Returns
    -------
    :class:`compas.geometry.Polyline`
    """
    return Polyline([point_to_compas(point) for point in polyline])

def polyline_to_rhino(polyline):
    """Convert a COMPAS polyline to a Rhino polyline.

    Parameters
    ----------
    polyline : :class:`compas.geometry.Polyline`

    Returns
    -------
    :class:`Rhino.Geometry.Ellipse`
    """
    return RhinoPolyline([point_to_rhino(point) for point in polyline])

def polygon_to_compas(polygon):
    """Convert a Rhino polygon to a COMPAS polygon.

    Parameters
    ----------
    polygon : :class:`Rhino.Geometry.Polygon`

    Returns
    -------
    :class:`compas.geometry.Polygon`
    """
    return Polygon([point_to_compas(point) for point in polygon])

def polygon_to_rhino(polygon):
    """Convert a COMPAS polygon to a Rhino polygon.

    Parameters
    ----------
    polygon : :class:`compas.geometry.Polygon`

    Returns
    -------
    :class:`Rhino.Geometry.Polygon`
    """
    raise NotImplementedError
<|endoftext|>


<commit_before>from time import time
import numpy as np

from sklearn.externals.joblib import Parallel, delayed, cpu_count
from sklearn.utils import gen_even_slices, shuffle

from .ssvm import BaseSSVM
from ..utils import find_constraint

class SubgradientSSVM(BaseSSVM):
    """Structured SVM solver using subgradient descent.

    Implements a margin rescaled with l1 slack penalty.
    By default, a constant learning rate is used.
    It is also possible to use the adaptive learning rate found by AdaGrad.

    This class implements online subgradient descent. If n_jobs != 1,
    small batches of size n_jobs are used to exploit parallel inference.
    If inference is fast, use n_jobs=1.

    Parameters
    ----------
    model : StructuredModel
        Object containing model structure. Has to implement
        `loss`, `inference` and `loss_augmented_inference`.

    max_iter : int, default=100
        Maximum number of passes over dataset to find constraints and perform
        updates.

    C : float, default=1.
        Regularization parameter.

    verbose : int, default=0
        Verbosity.

    learning_rate : float or 'auto', default='auto'
        Learning rate used in subgradient descent. If 'auto', the pegasos schedule is used,
        which starts with learning_rate = n_samples * C.

    momentum : float, default=0.9
        Momentum used in subgradient descent.

    n_jobs : int, default=1
        Number of parallel jobs for inference. -1 means as many as cpus.

    batch_size : int, default=None
        Ignored if n_jobs > 1. If n_jobs=1, inference will be done in mini
        batches of size batch_size. If n_jobs=-1, batch learning will be
        performed, that is the whole dataset will be used to compute each
        subgradient.

    show_loss_every : int, default=0
        Controlls how often the hamming loss is computed (for monitoring
        purposes). Zero means never, otherwise it will be computed very
        show_loss_every'th epoch.

    decay_exponent : float, default=0
        Exponent for decaying learning rate. Effective learning rate is
        ``learning_rate / (t0 + t)** decay_exponent``. Zero means no decay.

    decay_t0 : float, default=10
        Offset for decaying learning rate. Effective learning rate is
        ``learning_rate / (t0 + t)** decay_exponent``. Zero means no decay.

    break_on_no_constraints : bool, default=True
        Break when there are no new constraints found.

    logger : logger object.

    averaging : string, default='linear'
        Whether and how to average weights. Possible options are 'linear', 'squared' and None.
        The string reflects the weighting of the averaging:

            - linear: w_avg ~ w_1 + 2 * w_2 + ... + t * w_t

            - squared: w_avg ~ w_1 + 4 * w_2 + ... + t**2 * w_t

        Uniform averaging is not implemented as it is worth than linear
        weighted averaging or no averaging.

    shuffle : bool, default=False
        Whether to shuffle the dataset in each iteration.

    Attributes
    ----------
    w : nd-array, shape=(model.size_psi,)
        The learned weights of the SVM.

    ``loss_curve_`` : list of float
        List of loss values if show_loss_every > 0.

    ``objective_curve_`` : list of float
       Primal objective after each pass through the dataset.

    ``timestamps_`` : list of int
       Total training time stored before each iteration.
    """
    def __init__(self, model, max_iter=100, C=1.0, verbose=0, momentum=0.0,
                 learning_rate='auto', n_jobs=1,
                 show_loss_every=0, decay_exponent=1,
                 break_on_no_constraints=True, logger=None, batch_size=None,
                 decay_t0=10, averaging=None, shuffle=False):
        BaseSSVM.__init__(self, model, max_iter, C, verbose=verbose,
                          n_jobs=n_jobs, show_loss_every=show_loss_every,
                          logger=logger)
        self.averaging = averaging
        self.break_on_no_constraints = break_on_no_constraints
        self.momentum = momentum
        self.learning_rate = learning_rate
        self.t = 0
        self.decay_exponent = decay_exponent
        self.decay_t0 = decay_t0
        self.batch_size = batch_size
        self.shuffle = shuffle

    def _solve_subgradient(self, dpsi, n_samples, w):
        """Do a single subgradient step."""
        grad = (dpsi - w / (self.C * n_samples))

        self.grad_old = ((1 - self.momentum) * grad
                         + self.momentum * self.grad_old)
        if self.decay_exponent == 0:
            effective_lr = self.learning_rate_
        else:
            effective_lr = (self.learning_rate_
                            / (self.t + self.decay_t0)
                            ** self.decay_exponent)
        w += effective_lr * self.grad_old

        if self.averaging == 'linear':
            rho = 2. / (self.t + 2.)
            self.w = (1. - rho) * self.w + rho * w
        elif self.averaging == 'squared':
            rho = 6. * (self.t + 1) / ((self.t + 2) * (2 * self.t + 3))
            self.w = (1. - rho) * self.w + rho * w
        else:
            self.w = w
        self.t += 1.
        return w

    def fit(self, X, Y, constraints=None, warm_start=False, initialize=True):
        """Learn parameters using subgradient descent.

        Parameters
        ----------
        X : iterable
            Traing instances. Contains the structured input objects.
            No requirement on the particular form of entries of X is made.

        Y : iterable
            Training labels. Contains the strctured labels for inputs in X.
            Needs to have the same length as X.

        constraints : None
            Discarded. Only for API compatibility currently.

        warm_start : boolean, default=False
            Whether to restart a previous fit.

        initialize : boolean, default=True
            Whether to initialize the model for the data.
            Leave this true except if you really know what you are doing.
        """
        if initialize:
            self.model.initialize(X, Y)
        print("Training primal subgradient structural SVM")
        self.grad_old = np.zeros(self.model.size_psi)
        self.w = getattr(self, "w", np.zeros(self.model.size_psi))
        w = self.w.copy()
        if not warm_start:
            self.primal_objective_curve_ = []
            self.timestamps_ = [time()]
            if self.learning_rate == "auto":
                self.learning_rate_ = self.C * len(X)
            else:
                self.learning_rate_ = self.learning_rate
        else:
            self.timestamps_ = (np.array(self.timestamps_) - time()).tolist()
        try:
            # catch ctrl+c to stop training
            for iteration in xrange(self.max_iter):
                if self.shuffle:
                    X, Y = shuffle(X, Y)
                if self.n_jobs == 1:
                    objective, positive_slacks, w = self._sequential_learning(X, Y, w)
                else:
                    objective, positive_slacks, w = self._parallel_learning(X, Y, w)

                # some statistics
                objective = objective * self.C + np.sum(w ** 2) / 2.

                if positive_slacks == 0:
                    print("No additional constraints")
                    if self.break_on_no_constraints:
                        break
                if self.verbose > 0:
                    print(self)
                    print("iteration %d" % iteration)
                    print("positive slacks: %d,"
                          "objective: %f" %
                          (positive_slacks, objective))
                self.timestamps_.append(time() - self.timestamps_[0])
                self.objective_curve_.append(objective)

                if self.verbose > 2:
                    print(self.w)

                self._compute_training_loss(X, Y, iteration)
                if self.logger is not None:
                    self.logger(self, iteration)

        except KeyboardInterrupt:
            pass

        if self.verbose:
            print("Computing final objective")

        self.timestamps_.append(time() - self.timestamps_[0])
        self.objective_curve_.append(self._objective(X, Y))
        if self.logger is not None:
            self.logger(self, 'final')
        if self.verbose:
            if self.objective_curve_:
                print("final objective: %f" % self.objective_curve_[-1])
            if self.verbose and self.n_jobs == 1:
                print("calls to inference: %d" % self.model.inference_calls)

        return self

    def _parallel_learning(self, X, Y, w):
        n_samples = len(X)
        objective, positive_slacks = 0, 0
        verbose = max(0, self.verbose - 3)
        if self.batch_size is not None:
            raise ValueError("If n_jobs != 1, batch_size needs to"
                             "be None")
        # generate batches of size n_jobs
        # to speed up inference
        if self.n_jobs == -1:
            n_jobs = cpu_count()
        else:
            n_jobs = self.n_jobs

        n_batches = int(np.ceil(float(len(X)) / n_jobs))
        slices = gen_even_slices(n_samples, n_batches)
        for batch in slices:
            X_b = X[batch]
            Y_b = Y[batch]
            candidate_constraints = Parallel(
                n_jobs=self.n_jobs,
                verbose=verbose)(delayed(find_constraint)(
                    self.model, x, y, w)
                    for x, y in zip(X_b, Y_b))
            dpsi = np.zeros(self.model.size_psi)
            for x, y, constraint in zip(X_b, Y_b,
                                        candidate_constraints):
                y_hat, delta_psi, slack, loss = constraint
                if slack > 0:
                    objective += slack
                    dpsi += delta_psi
                    positive_slacks += 1
            w = self._solve_subgradient(dpsi, n_samples, w)
        return objective, positive_slacks, w

    def _sequential_learning(self, X, Y, w):
        n_samples = len(X)
        objective, positive_slacks = 0, 0
        if self.batch_size in [None, 1]:
            # online learning
            for x, y in zip(X, Y):
                y_hat, delta_psi, slack, loss = \
                    find_constraint(self.model, x, y, w)
                objective += slack
                if slack > 0:
                    positive_slacks += 1
                self._solve_subgradient(delta_psi, n_samples, w)
        else:
            # mini batch learning
            if self.batch_size == -1:
                slices = [slice(0, len(X)), None]
            else:
                n_batches = int(np.ceil(float(len(X)) / self.batch_size))
                slices = gen_even_slices(n_samples, n_batches)
            for batch in slices:
                X_b = X[batch]
                Y_b = Y[batch]
                Y_hat = self.model.batch_loss_augmented_inference(
                    X_b, Y_b, w, relaxed=True)
                delta_psi = (self.model.batch_psi(X_b, Y_b)
                             - self.model.batch_psi(X_b, Y_hat))
                loss = np.sum(self.model.batch_loss(Y_b, Y_hat))

                violation = np.maximum(0, loss - np.dot(w, delta_psi))
                objective += violation
                positive_slacks += self.batch_size
                self._solve_subgradient(delta_psi / len(X_b), n_samples, w)
        return objective, positive_slacks, w
<commit_msg>fix backporting problem<commit_after>from time import time
import numpy as np

from sklearn.externals.joblib import Parallel, delayed, cpu_count
from sklearn.utils import gen_even_slices, shuffle

from .ssvm import BaseSSVM
from ..utils import find_constraint

class SubgradientSSVM(BaseSSVM):
    """Structured SVM solver using subgradient descent.

    Implements a margin rescaled with l1 slack penalty.
    By default, a constant learning rate is used.
    It is also possible to use the adaptive learning rate found by AdaGrad.

    This class implements online subgradient descent. If n_jobs != 1,
    small batches of size n_jobs are used to exploit parallel inference.
    If inference is fast, use n_jobs=1.

    Parameters
    ----------
    model : StructuredModel
        Object containing model structure. Has to implement
        `loss`, `inference` and `loss_augmented_inference`.

    max_iter : int, default=100
        Maximum number of passes over dataset to find constraints and perform
        updates.

    C : float, default=1.
        Regularization parameter.

    verbose : int, default=0
        Verbosity.

    learning_rate : float or 'auto', default='auto'
        Learning rate used in subgradient descent. If 'auto', the pegasos schedule is used,
        which starts with learning_rate = n_samples * C.

    momentum : float, default=0.9
        Momentum used in subgradient descent.

    n_jobs : int, default=1
        Number of parallel jobs for inference. -1 means as many as cpus.

    batch_size : int, default=None
        Ignored if n_jobs > 1. If n_jobs=1, inference will be done in mini
        batches of size batch_size. If n_jobs=-1, batch learning will be
        performed, that is the whole dataset will be used to compute each
        subgradient.

    show_loss_every : int, default=0
        Controlls how often the hamming loss is computed (for monitoring
        purposes). Zero means never, otherwise it will be computed very
        show_loss_every'th epoch.

    decay_exponent : float, default=0
        Exponent for decaying learning rate. Effective learning rate is
        ``learning_rate / (t0 + t)** decay_exponent``. Zero means no decay.

    decay_t0 : float, default=10
        Offset for decaying learning rate. Effective learning rate is
        ``learning_rate / (t0 + t)** decay_exponent``. Zero means no decay.

    break_on_no_constraints : bool, default=True
        Break when there are no new constraints found.

    logger : logger object.

    averaging : string, default='linear'
        Whether and how to average weights. Possible options are 'linear', 'squared' and None.
        The string reflects the weighting of the averaging:

            - linear: w_avg ~ w_1 + 2 * w_2 + ... + t * w_t

            - squared: w_avg ~ w_1 + 4 * w_2 + ... + t**2 * w_t

        Uniform averaging is not implemented as it is worth than linear
        weighted averaging or no averaging.

    shuffle : bool, default=False
        Whether to shuffle the dataset in each iteration.

    Attributes
    ----------
    w : nd-array, shape=(model.size_psi,)
        The learned weights of the SVM.

    ``loss_curve_`` : list of float
        List of loss values if show_loss_every > 0.

    ``objective_curve_`` : list of float
       Primal objective after each pass through the dataset.

    ``timestamps_`` : list of int
       Total training time stored before each iteration.
    """
    def __init__(self, model, max_iter=100, C=1.0, verbose=0, momentum=0.0,
                 learning_rate='auto', n_jobs=1,
                 show_loss_every=0, decay_exponent=1,
                 break_on_no_constraints=True, logger=None, batch_size=None,
                 decay_t0=10, averaging=None, shuffle=False):
        BaseSSVM.__init__(self, model, max_iter, C, verbose=verbose,
                          n_jobs=n_jobs, show_loss_every=show_loss_every,
                          logger=logger)
        self.averaging = averaging
        self.break_on_no_constraints = break_on_no_constraints
        self.momentum = momentum
        self.learning_rate = learning_rate
        self.t = 0
        self.decay_exponent = decay_exponent
        self.decay_t0 = decay_t0
        self.batch_size = batch_size
        self.shuffle = shuffle

    def _solve_subgradient(self, dpsi, n_samples, w):
        """Do a single subgradient step."""
        grad = (dpsi - w / (self.C * n_samples))

        self.grad_old = ((1 - self.momentum) * grad
                         + self.momentum * self.grad_old)
        if self.decay_exponent == 0:
            effective_lr = self.learning_rate_
        else:
            effective_lr = (self.learning_rate_
                            / (self.t + self.decay_t0)
                            ** self.decay_exponent)
        w += effective_lr * self.grad_old

        if self.averaging == 'linear':
            rho = 2. / (self.t + 2.)
            self.w = (1. - rho) * self.w + rho * w
        elif self.averaging == 'squared':
            rho = 6. * (self.t + 1) / ((self.t + 2) * (2 * self.t + 3))
            self.w = (1. - rho) * self.w + rho * w
        else:
            self.w = w
        self.t += 1.
        return w

    def fit(self, X, Y, constraints=None, warm_start=False, initialize=True):
        """Learn parameters using subgradient descent.

        Parameters
        ----------
        X : iterable
            Traing instances. Contains the structured input objects.
            No requirement on the particular form of entries of X is made.

        Y : iterable
            Training labels. Contains the strctured labels for inputs in X.
            Needs to have the same length as X.

        constraints : None
            Discarded. Only for API compatibility currently.

        warm_start : boolean, default=False
            Whether to restart a previous fit.

        initialize : boolean, default=True
            Whether to initialize the model for the data.
            Leave this true except if you really know what you are doing.
        """
        if initialize:
            self.model.initialize(X, Y)
        print("Training primal subgradient structural SVM")
        self.grad_old = np.zeros(self.model.size_psi)
        self.w = getattr(self, "w", np.zeros(self.model.size_psi))
        w = self.w.copy()
        if not warm_start:
            self.objective_curve_ = []
            self.timestamps_ = [time()]
            if self.learning_rate == "auto":
                self.learning_rate_ = self.C * len(X)
            else:
                self.learning_rate_ = self.learning_rate
        else:
            self.timestamps_ = (np.array(self.timestamps_) - time()).tolist()
        try:
            # catch ctrl+c to stop training
            for iteration in xrange(self.max_iter):
                if self.shuffle:
                    X, Y = shuffle(X, Y)
                if self.n_jobs == 1:
                    objective, positive_slacks, w = self._sequential_learning(X, Y, w)
                else:
                    objective, positive_slacks, w = self._parallel_learning(X, Y, w)

                # some statistics
                objective = objective * self.C + np.sum(w ** 2) / 2.

                if positive_slacks == 0:
                    print("No additional constraints")
                    if self.break_on_no_constraints:
                        break
                if self.verbose > 0:
                    print(self)
                    print("iteration %d" % iteration)
                    print("positive slacks: %d,"
                          "objective: %f" %
                          (positive_slacks, objective))
                self.timestamps_.append(time() - self.timestamps_[0])
                self.objective_curve_.append(objective)

                if self.verbose > 2:
                    print(self.w)

                self._compute_training_loss(X, Y, iteration)
                if self.logger is not None:
                    self.logger(self, iteration)

        except KeyboardInterrupt:
            pass

        if self.verbose:
            print("Computing final objective")

        self.timestamps_.append(time() - self.timestamps_[0])
        self.objective_curve_.append(self._objective(X, Y))
        if self.logger is not None:
            self.logger(self, 'final')
        if self.verbose:
            if self.objective_curve_:
                print("final objective: %f" % self.objective_curve_[-1])
            if self.verbose and self.n_jobs == 1:
                print("calls to inference: %d" % self.model.inference_calls)

        return self

    def _parallel_learning(self, X, Y, w):
        n_samples = len(X)
        objective, positive_slacks = 0, 0
        verbose = max(0, self.verbose - 3)
        if self.batch_size is not None:
            raise ValueError("If n_jobs != 1, batch_size needs to"
                             "be None")
        # generate batches of size n_jobs
        # to speed up inference
        if self.n_jobs == -1:
            n_jobs = cpu_count()
        else:
            n_jobs = self.n_jobs

        n_batches = int(np.ceil(float(len(X)) / n_jobs))
        slices = gen_even_slices(n_samples, n_batches)
        for batch in slices:
            X_b = X[batch]
            Y_b = Y[batch]
            candidate_constraints = Parallel(
                n_jobs=self.n_jobs,
                verbose=verbose)(delayed(find_constraint)(
                    self.model, x, y, w)
                    for x, y in zip(X_b, Y_b))
            dpsi = np.zeros(self.model.size_psi)
            for x, y, constraint in zip(X_b, Y_b,
                                        candidate_constraints):
                y_hat, delta_psi, slack, loss = constraint
                if slack > 0:
                    objective += slack
                    dpsi += delta_psi
                    positive_slacks += 1
            w = self._solve_subgradient(dpsi, n_samples, w)
        return objective, positive_slacks, w

    def _sequential_learning(self, X, Y, w):
        n_samples = len(X)
        objective, positive_slacks = 0, 0
        if self.batch_size in [None, 1]:
            # online learning
            for x, y in zip(X, Y):
                y_hat, delta_psi, slack, loss = \
                    find_constraint(self.model, x, y, w)
                objective += slack
                if slack > 0:
                    positive_slacks += 1
                self._solve_subgradient(delta_psi, n_samples, w)
        else:
            # mini batch learning
            if self.batch_size == -1:
                slices = [slice(0, len(X)), None]
            else:
                n_batches = int(np.ceil(float(len(X)) / self.batch_size))
                slices = gen_even_slices(n_samples, n_batches)
            for batch in slices:
                X_b = X[batch]
                Y_b = Y[batch]
                Y_hat = self.model.batch_loss_augmented_inference(
                    X_b, Y_b, w, relaxed=True)
                delta_psi = (self.model.batch_psi(X_b, Y_b)
                             - self.model.batch_psi(X_b, Y_hat))
                loss = np.sum(self.model.batch_loss(Y_b, Y_hat))

                violation = np.maximum(0, loss - np.dot(w, delta_psi))
                objective += violation
                positive_slacks += self.batch_size
                self._solve_subgradient(delta_psi / len(X_b), n_samples, w)
        return objective, positive_slacks, w
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals
from rest_framework import viewsets, serializers, permissions
from rest_framework import filters
from notification.models import TaskHistory
from logical.models import Database, DatabaseHistory

RELEVANCE_DICT = {
        0: "CRITICAL",
        1: "ERROR",
        2: "WARNING",
        3: "INFO",
        4: "DEBUG"
    }

class TaskSerializer(serializers.ModelSerializer):
    database = serializers.SerializerMethodField('get_database')
    rollback = serializers.SerializerMethodField('had_rollback')
    relevance = serializers.SerializerMethodField('get_relevance')

    class Meta:
        model = TaskHistory
        fields = (
            'id',
            'task_id',
            'task_status',
            'db_id',
            'object_class',
            'object_id',
            'updated_at',
            'user',
            'created_at',
            'task_name',
            'database',
            'rollback',
            'relevance',
        )

    def get_relevance(self, task):
        return RELEVANCE_DICT[task.relevance]

    def had_rollback(self, task):
        if not task.details:
            return False

        if task.task_name == 'notification.tasks.destroy_database':
            return False

        # TODO: see if do details.lower() is expensive
        return 'rollback step' in task.details.lower()

    def get_database(self, task):
        def make_dict_from_model(instance):
            engine = instance.databaseinfra.engine
            return {
                'name': instance.name,
                'environment': instance.environment.name,
                'engine': '{} {}'.format(
                    engine.engine_type.name,
                    engine.version
                )
            }

        def make_dict_from_history(instance):
            return {
                'name': instance.name,
                'environment': instance.environment,
                'engine': instance.engine
            }

        if task.object_class == Database._meta.db_table:
            try:
                database = (
                    Database.objects
                    .select_related(
                        'environment',
                        'databaseinfra',
                        'databaseinfra__engine',
                        'databaseinfra__engine__enginetype'
                    ).get(id=task.object_id)
                )
            except Database.DoesNotExist:
                try:
                    database_history = DatabaseHistory.objects.get(database_id=task.object_id)
                except DatabaseHistory.DoesNotExist:
                    return None
                else:
                    return make_dict_from_history(database_history)
            else:
                return make_dict_from_model(database)

        return None

class TaskAPI(viewsets.ReadOnlyModelViewSet):

    """
    Task API
    """

    chg_tasks_names = [
        'notification.tasks.destroy_database',
        'notification.tasks.create_database',
        'notification.tasks.destroy_database',
        'notification.tasks.resize_database',
        'notification.tasks.clone_database',
        'notification.tasks.database_disk_resize',
        'notification.tasks.add_instances_to_database',
        'notification.tasks.remove_readonly_instance',
        'database_disk_resize',
        'backup.tasks.restore_snapshot',
        'notification.tasks.upgrade_mongodb_24_to_30',
        'notification.tasks.upgrade_database',
        'notification.tasks.reinstall_vm_database'
    ]
    model = TaskHistory
    serializer_class = TaskSerializer
    permission_classes = (permissions.IsAuthenticatedOrReadOnly,)
    filter_backends = (filters.OrderingFilter,)
    filter_fields = (
        'task_id',
        'task_status',
        'object_class',
        'object_id',
        'updated_at',
        'created_at',
        'user',
        'relevance'
    )
    ordering_fields = ('created_at', 'updated_at', 'id')
    ordering = ('-created_at',)
    datetime_fields = ('created_at', 'updated_at')

    def get_queryset(self):
        params = self.request.GET.dict()
        filter_params = {}
        for k, v in params.iteritems():
            if k == 'exclude_system_tasks':
                filter_params['task_name__in'] = self.chg_tasks_names
            elif k.split('__')[0] in self.filter_fields:
                filter_params[k] = v
        return self.model.objects.filter(**filter_params)
<commit_msg>Deleting unnecessary code<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals
from rest_framework import viewsets, serializers, permissions
from rest_framework import filters
from notification.models import TaskHistory
from logical.models import Database, DatabaseHistory

# RELEVANCE_DICT = {
#         0: "CRITICAL",
#         1: "ERROR",
#         2: "WARNING",
#         3: "INFO",
#         4: "DEBUG"
#     }

class TaskSerializer(serializers.ModelSerializer):
    database = serializers.SerializerMethodField('get_database')
    rollback = serializers.SerializerMethodField('had_rollback')
    relevance = serializers.SerializerMethodField('get_relevance')

    class Meta:
        model = TaskHistory
        fields = (
            'id',
            'task_id',
            'task_status',
            'db_id',
            'object_class',
            'object_id',
            'updated_at',
            'user',
            'created_at',
            'task_name',
            'database',
            'rollback',
            'relevance',
        )

    def get_relevance(self, task):
        return TaskHistory.RELEVANCE_CHOICES[task.relevance][1]
        # return RELEVANCE_DICT[task.relevance]

    def had_rollback(self, task):
        if not task.details:
            return False

        if task.task_name == 'notification.tasks.destroy_database':
            return False

        # TODO: see if do details.lower() is expensive
        return 'rollback step' in task.details.lower()

    def get_database(self, task):
        def make_dict_from_model(instance):
            engine = instance.databaseinfra.engine
            return {
                'name': instance.name,
                'environment': instance.environment.name,
                'engine': '{} {}'.format(
                    engine.engine_type.name,
                    engine.version
                )
            }

        def make_dict_from_history(instance):
            return {
                'name': instance.name,
                'environment': instance.environment,
                'engine': instance.engine
            }

        if task.object_class == Database._meta.db_table:
            try:
                database = (
                    Database.objects
                    .select_related(
                        'environment',
                        'databaseinfra',
                        'databaseinfra__engine',
                        'databaseinfra__engine__enginetype'
                    ).get(id=task.object_id)
                )
            except Database.DoesNotExist:
                try:
                    database_history = DatabaseHistory.objects.get(database_id=task.object_id)
                except DatabaseHistory.DoesNotExist:
                    return None
                else:
                    return make_dict_from_history(database_history)
            else:
                return make_dict_from_model(database)

        return None

class TaskAPI(viewsets.ReadOnlyModelViewSet):

    """
    Task API
    """

    chg_tasks_names = [
        'notification.tasks.destroy_database',
        'notification.tasks.create_database',
        'notification.tasks.destroy_database',
        'notification.tasks.resize_database',
        'notification.tasks.clone_database',
        'notification.tasks.database_disk_resize',
        'notification.tasks.add_instances_to_database',
        'notification.tasks.remove_readonly_instance',
        'database_disk_resize',
        'backup.tasks.restore_snapshot',
        'notification.tasks.upgrade_mongodb_24_to_30',
        'notification.tasks.upgrade_database',
        'notification.tasks.reinstall_vm_database'
    ]
    model = TaskHistory
    serializer_class = TaskSerializer
    permission_classes = (permissions.IsAuthenticatedOrReadOnly,)
    filter_backends = (filters.OrderingFilter,)
    filter_fields = (
        'task_id',
        'task_status',
        'object_class',
        'object_id',
        'updated_at',
        'created_at',
        'user',
        'relevance'
    )
    ordering_fields = ('created_at', 'updated_at', 'id')
    ordering = ('-created_at',)
    datetime_fields = ('created_at', 'updated_at')

    def get_queryset(self):
        params = self.request.GET.dict()
        filter_params = {}
        for k, v in params.iteritems():
            if k == 'exclude_system_tasks':
                filter_params['task_name__in'] = self.chg_tasks_names
            elif k.split('__')[0] in self.filter_fields:
                filter_params[k] = v
        return self.model.objects.filter(**filter_params)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2010 British Broadcasting Corporation and Kamaelia Contributors(1)
#
# (1) Kamaelia Contributors are listed in the AUTHORS file and at
#     http://www.kamaelia.org/AUTHORS - please extend this file,
#     not this notice.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# -------------------------------------------------------------------------
#

import Axon
import zlib
from Axon.Ipc import WaitComplete, producerFinished, shutdownMicroprocess
from Kamaelia.UI.PygameDisplay import PygameDisplay
import pygame

try:
    import Image
except ImportError:
    print "WARNING: Python Imaging Library Not available, defaulting to bmp only mode"

class Canvas(Axon.Component.component):
    """\
    Canvas component - pygame surface that accepts drawing instructions
    """

    Inboxes =  { "inbox"   : "Receives drawing instructions",
                 "control" : "",
                 "fromDisplay"  : "For receiving replies from PygameDisplay service",
                 "eventsIn" : "For receiving PygameDisplay events",
               }
    Outboxes = { "outbox" : "Issues drawing instructions",
                 "signal" : "",
                 "toDisplay" : "For sending requests to PygameDisplay service",
                 "eventsOut" : "Events forwarded out of here",
                 "surfacechanged" : "If the surface gets changed from last load/save a 'dirty' message is emitted here",
               }

    def __init__(self, position=(0,0), size=(1024,768), bgcolour=(255,255,255)):
        """x.__init__(...) initializes x; see x.__class__.__doc__ for signature"""
        super(Canvas,self).__init__()
        self.position = position
        self.size = size
        self.antialias = False
        self.bgcolour = bgcolour
        if self.antialias == True:
            self.pygame_draw_line = pygame.draw.aaline
        else:
            self.pygame_draw_line = pygame.draw.line
        self.dirty_sent = False

    def waitBox(self,boxname):
        waiting = True
        while waiting:
            if self.dataReady(boxname):
                return
            else:
                yield 1

    def requestDisplay(self, **argd):
        displayservice = PygameDisplay.getDisplayService()
        self.link((self,"toDisplay"), displayservice)
        argd["transparency"] = self.bgcolour
        self.send(argd, "toDisplay")
        for _ in self.waitBox("fromDisplay"):
            yield 1
        self.surface = self.recv("fromDisplay")

    def finished(self):
        while self.dataReady("control"):
            msg = self.recv("control")
            if isinstance(msg, producerFinished) or isinstance(msg, shutdownMicroprocess):
                self.send(msg, "signal")
                return True
        return False

    def main(self):
        """Main loop"""
        yield 1
        yield 1
        yield 1
        yield 1
        yield WaitComplete(
              self.requestDisplay( DISPLAYREQUEST=True,
                                   callback = (self,"fromDisplay"),
                                   events = (self, "eventsIn"),
                                   size = self.size,
                                   position = self.position,
                                 )
              )

        self.surface.fill( (self.bgcolour) )
        self.send({"REDRAW":True, "surface":self.surface}, "toDisplay")

        self.send( {"ADDLISTENEVENT" : pygame.MOUSEBUTTONDOWN, "surface" : self.surface},
                   "toDisplay" )
        self.send( {"ADDLISTENEVENT" : pygame.MOUSEMOTION, "surface" : self.surface},
                   "toDisplay" )
        self.send( {"ADDLISTENEVENT" : pygame.MOUSEBUTTONUP, "surface" : self.surface},
                   "toDisplay" )

        while not self.finished():

            self.redrawNeeded = False
            while self.dataReady("inbox"):
                msgs = self.recv("inbox")
#                \
#print repr(msgs)
                for msg in msgs:
                    cmd = msg[0]
                    args = msg[1:]
                    # parse commands here
                    self.handleCommand(cmd, *args)
                yield 1

            if self.redrawNeeded:
                self.send({"REDRAW":True, "surface":self.surface}, "toDisplay")
                if not self.clean:
                    if not self.dirty_sent:
                        self.send("dirty", "surfacechanged")
                        self.dirty_sent = True

            # pass on events received from pygame display
            while self.dataReady("eventsIn"):
                self.send( self.recv("eventsIn"), "eventsOut" )

            self.pause()
            yield 1

    def handleCommand(self, cmd, *args):
        #
        # Could really take a dispatch pattern
        # Would then be pluggable.
        #
        cmd = cmd.upper()
        if   cmd=="CLEAR":
            self.clear(args)
            self.clean = True
            self.dirty_sent = False
        elif cmd=="LINE":
             self.line(args)
        elif cmd=="CIRCLE":
            self.circle(args)
            self.clean = False
        elif cmd=="LOAD":
            self.load(args)
            self.clean = True
            self.dirty_sent = False
        elif cmd=="SAVE":
            self.save(args)
            self.clean = True
            self.dirty_sent = False
        elif cmd=="GETIMG":
            self.getimg(args)
            self.clean = False
        elif cmd=="SETIMG":
            self.setimg(args)
            self.clean = False
        elif cmd=="WRITE":
            self.write(args)
            self.clean = False

    def line(self, args):
        (r,g,b,sx,sy,ex,ey) = [int(v) for v in args[0:7]]
        self.pygame_draw_line(self.surface, (r,g,b), (sx,sy), (ex,ey))
#        pygame.draw.aaline(self.surface, (r,g,b), (sx,sy), (ex,ey))
        self.redrawNeeded = True
        if not((sy <0) or (ey <0)):
            self.clean = False

    def clear(self, args):
        if len(args) == 3:
            self.surface.fill( [int(a) for a in args[0:3]] )
        else:
            self.surface.fill( (self.bgcolour) )
        self.redrawNeeded = True
        self.dirty_sent = True
        self.clean = True

    def circle(self, args):
        (r,g,b,x,y,radius) = [int(v) for v in args[0:6]]
        pygame.draw.circle(self.surface, (r,g,b), (x,y), radius, 0)
        self.redrawNeeded = True

    def load(self, args):
            filename = args[0]
#            print "ARGS", args
            try:
                loadedimage = pygame.image.load(filename)
            except:
                pass
            else:
                self.surface.blit(loadedimage, (0,0))
            self.redrawNeeded = True
            if not ( (len(args) >1) and args[1] == "nopropogate" ):
                self.getimg(())
            self.clean = True

    def save(self, args):
        filename = args[0]
        try:
            imagestring = pygame.image.tostring(self.surface,"RGB")
            pilImage = Image.fromstring("RGB", self.surface.get_size(), imagestring)
            pilImage.save(filename)
        except NameError:
            pygame.image.save(self.surface, filename)
        self.clean = True

    def getimg(self, args):
            imagestring = pygame.image.tostring(self.surface,"RGB")
            imagestring = zlib.compress(imagestring)
            w,h = self.surface.get_size()
            self.send( [["SETIMG",imagestring,str(w),str(h),"RGB"]], "outbox" )
#            print "GETIMG"

    def setimg(self, args):
            w,h = int(args[1]), int(args[2])
            imagestring = zlib.decompress(args[0])
            recvsurface = pygame.image.frombuffer(imagestring, (w,h), args[3])
            self.surface.blit(recvsurface, (0,0))
            self.redrawNeeded = True

    def write(self, args):
            x,y,size,r,g,b = [int(a) for a in args[0:6]]
            text = args[6]
            font = pygame.font.Font(None,size)
            textimg = font.render(text, self.antialias, (r,g,b))
            self.surface.blit(textimg, (x,y))
            self.redrawNeeded = True

<commit_msg>Arghh.... <commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2010 British Broadcasting Corporation and Kamaelia Contributors(1)
#
# (1) Kamaelia Contributors are listed in the AUTHORS file and at
#     http://www.kamaelia.org/AUTHORS - please extend this file,
#     not this notice.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# -------------------------------------------------------------------------
#

import Axon
import zlib
from Axon.Ipc import WaitComplete, producerFinished, shutdownMicroprocess
from Kamaelia.UI.PygameDisplay import PygameDisplay
import pygame

try:
    import Image
except ImportError:
    print "WARNING: Python Imaging Library Not available, defaulting to bmp only mode"

class Canvas(Axon.Component.component):
    """\
    Canvas component - pygame surface that accepts drawing instructions
    """

    Inboxes =  { "inbox"   : "Receives drawing instructions",
                 "control" : "",
                 "fromDisplay"  : "For receiving replies from PygameDisplay service",
                 "eventsIn" : "For receiving PygameDisplay events",
               }
    Outboxes = { "outbox" : "Issues drawing instructions",
                 "signal" : "",
                 "toDisplay" : "For sending requests to PygameDisplay service",
                 "eventsOut" : "Events forwarded out of here",
                 "surfacechanged" : "If the surface gets changed from last load/save a 'dirty' message is emitted here",
               }

    def __init__(self, position=(0,0), size=(1024,768), bgcolour=(255,255,255)):
        """x.__init__(...) initializes x; see x.__class__.__doc__ for signature"""
        super(Canvas,self).__init__()
        self.position = position
        self.size = size
        self.antialias = False
        self.bgcolour = bgcolour
        if self.antialias == True:
            self.pygame_draw_line = pygame.draw.aaline
        else:
            self.pygame_draw_line = pygame.draw.line
        self.dirty_sent = False

    def waitBox(self,boxname):
        waiting = True
        while waiting:
            if self.dataReady(boxname):
                return
            else:
                yield 1

    def requestDisplay(self, **argd):
        displayservice = PygameDisplay.getDisplayService()
        self.link((self,"toDisplay"), displayservice)
        argd["transparency"] = self.bgcolour
        self.send(argd, "toDisplay")
        for _ in self.waitBox("fromDisplay"):
            yield 1
        self.surface = self.recv("fromDisplay")

    def finished(self):
        while self.dataReady("control"):
            msg = self.recv("control")
            if isinstance(msg, producerFinished) or isinstance(msg, shutdownMicroprocess):
                self.send(msg, "signal")
                return True
        return False

    def main(self):
        """Main loop"""
        yield 1 #FIXME: AWFUL CODE VOMIT
        yield 1 #FIXME: AWFUL CODE VOMIT
        yield 1 #FIXME: AWFUL CODE VOMIT
        yield 1 #FIXME: AWFUL CODE VOMIT
        yield WaitComplete(
              self.requestDisplay( DISPLAYREQUEST=True,
                                   callback = (self,"fromDisplay"),
                                   events = (self, "eventsIn"),
                                   size = self.size,
                                   position = self.position,
                                 )
              )

        self.surface.fill( (self.bgcolour) )
        self.send({"REDRAW":True, "surface":self.surface}, "toDisplay")

        self.send( {"ADDLISTENEVENT" : pygame.MOUSEBUTTONDOWN, "surface" : self.surface},
                   "toDisplay" )
        self.send( {"ADDLISTENEVENT" : pygame.MOUSEMOTION, "surface" : self.surface},
                   "toDisplay" )
        self.send( {"ADDLISTENEVENT" : pygame.MOUSEBUTTONUP, "surface" : self.surface},
                   "toDisplay" )

        while not self.finished():

            self.redrawNeeded = False
            while self.dataReady("inbox"):
                msgs = self.recv("inbox")
#                \
#print repr(msgs)
                for msg in msgs:
                    cmd = msg[0]
                    args = msg[1:]
                    # parse commands here
                    self.handleCommand(cmd, *args)
                yield 1

            if self.redrawNeeded:
                self.send({"REDRAW":True, "surface":self.surface}, "toDisplay")
                if not self.clean:
                    if not self.dirty_sent:
                        self.send("dirty", "surfacechanged")
                        self.dirty_sent = True

            # pass on events received from pygame display
            while self.dataReady("eventsIn"):
                self.send( self.recv("eventsIn"), "eventsOut" )

            self.pause()
            yield 1

    def handleCommand(self, cmd, *args):
        #
        # Could really take a dispatch pattern
        # Would then be pluggable.
        #
        cmd = cmd.upper()
        if   cmd=="CLEAR":
            self.clear(args)
            self.clean = True
            self.dirty_sent = False
        elif cmd=="LINE":
             self.line(args)
        elif cmd=="CIRCLE":
            self.circle(args)
            self.clean = False
        elif cmd=="LOAD":
            self.load(args)
            self.clean = True
            self.dirty_sent = False
        elif cmd=="SAVE":
            self.save(args)
            self.clean = True
            self.dirty_sent = False
        elif cmd=="GETIMG":
            self.getimg(args)
            self.clean = False
        elif cmd=="SETIMG":
            self.setimg(args)
            self.clean = False
        elif cmd=="WRITE":
            self.write(args)
            self.clean = False

    def line(self, args):
        (r,g,b,sx,sy,ex,ey) = [int(v) for v in args[0:7]]
        self.pygame_draw_line(self.surface, (r,g,b), (sx,sy), (ex,ey))
#        pygame.draw.aaline(self.surface, (r,g,b), (sx,sy), (ex,ey))
        self.redrawNeeded = True
        if not((sy <0) or (ey <0)):
            self.clean = False

    def clear(self, args):
        if len(args) == 3:
            self.surface.fill( [int(a) for a in args[0:3]] )
        else:
            self.surface.fill( (self.bgcolour) )
        self.redrawNeeded = True
        self.dirty_sent = True
        self.clean = True

    def circle(self, args):
        (r,g,b,x,y,radius) = [int(v) for v in args[0:6]]
        pygame.draw.circle(self.surface, (r,g,b), (x,y), radius, 0)
        self.redrawNeeded = True

    def load(self, args):
            filename = args[0]
#            print "ARGS", args
            try:
                loadedimage = pygame.image.load(filename)
            except:
                pass
            else:
                self.surface.blit(loadedimage, (0,0))
            self.redrawNeeded = True
            if not ( (len(args) >1) and args[1] == "nopropogate" ):
                self.getimg(())
            self.clean = True

    def save(self, args):
        filename = args[0]
        try:
            imagestring = pygame.image.tostring(self.surface,"RGB")
            pilImage = Image.fromstring("RGB", self.surface.get_size(), imagestring)
            pilImage.save(filename)
        except NameError:
            pygame.image.save(self.surface, filename)
        self.clean = True

    def getimg(self, args):
            imagestring = pygame.image.tostring(self.surface,"RGB")
            imagestring = zlib.compress(imagestring)
            w,h = self.surface.get_size()
            self.send( [["SETIMG",imagestring,str(w),str(h),"RGB"]], "outbox" )
#            print "GETIMG"

    def setimg(self, args):
            w,h = int(args[1]), int(args[2])
            imagestring = zlib.decompress(args[0])
            recvsurface = pygame.image.frombuffer(imagestring, (w,h), args[3])
            self.surface.blit(recvsurface, (0,0))
            self.redrawNeeded = True

    def write(self, args):
            x,y,size,r,g,b = [int(a) for a in args[0:6]]
            text = args[6]
            font = pygame.font.Font(None,size)
            textimg = font.render(text, self.antialias, (r,g,b))
            self.surface.blit(textimg, (x,y))
            self.redrawNeeded = True

<|endoftext|>