<commit_before>#!/usr/bin/env python

"""
$Id$

This file is part of the sqlmap project, http://sqlmap.sourceforge.net.

Copyright (c) 2007-2009 Bernardo Damele A. G. <bernardo.damele@gmail.com>
Copyright (c) 2006 Daniele Bellucci <daniele.bellucci@gmail.com>

sqlmap is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free
Software Foundation version 2 of the License.

sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
details.

You should have received a copy of the GNU General Public License along
with sqlmap; if not, write to the Free Software Foundation, Inc., 51
Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
"""

import difflib
import os
import re
import shutil
import sys
import tempfile
import time
import urlparse
import zipfile

from distutils.dir_util import mkpath
from xml.dom.minidom import Document

from subprocess import PIPE
from subprocess import Popen as execute

from lib.core.common import dataToStdout
from lib.core.common import pollProcess
from lib.core.common import readInput
from lib.core.data import conf
from lib.core.data import logger
from lib.core.data import paths
from lib.core.exception import sqlmapConnectionException
from lib.core.exception import sqlmapFilePathException
from lib.core.settings import MSSQL_VERSIONS_URL
from lib.core.settings import SQLMAP_VERSION_URL
from lib.core.settings import SQLMAP_SOURCE_URL
from lib.core.settings import VERSION
from lib.request.connect import Connect as Request

def __updateMSSQLXML():
    infoMsg = "updating Microsoft SQL Server XML versions file"
    logger.info(infoMsg)

    try:
        mssqlVersionsHtmlString, _ = Request.getPage(url=MSSQL_VERSIONS_URL, direct=True)
    except sqlmapConnectionException, _:
        __mssqlPath     = urlparse.urlsplit(MSSQL_VERSIONS_URL)
        __mssqlHostname = __mssqlPath[1]

        warnMsg  = "sqlmap was unable to connect to %s," % __mssqlHostname
        warnMsg += " check your Internet connection and retry"
        logger.warn(warnMsg)

        return

    releases      = re.findall("class=\"BCC_DV_01DarkBlueTitle\">SQL Server ([\d\.]+) Builds", mssqlVersionsHtmlString, re.I | re.M)
    releasesCount = len(releases)

    # Create the minidom document
    doc = Document()

    # Create the <root> base element
    root = doc.createElement("root")
    doc.appendChild(root)

    for index in range(0, releasesCount):
        release = releases[index]

        # Skip Microsoft SQL Server 6.5 because the HTML
        # table is in another format
        if release == "6.5":
            continue

        # Create the <signatures> base element
        signatures = doc.createElement("signatures")
        signatures.setAttribute("release", release)
        root.appendChild(signatures)

        startIdx = mssqlVersionsHtmlString.index("SQL Server %s Builds" % releases[index])

        if index == releasesCount - 1:
            stopIdx  = len(mssqlVersionsHtmlString)
        else:
            stopIdx  = mssqlVersionsHtmlString.index("SQL Server %s Builds" % releases[index + 1])

        mssqlVersionsReleaseString = mssqlVersionsHtmlString[startIdx:stopIdx]
        servicepackVersion = re.findall("	[7\.0|2000|2005|2008]*(.*?)	<td.*?([\d\.]+)	[\r]*\n", mssqlVersionsReleaseString, re.I | re.M)

        for servicePack, version in servicepackVersion:
            if servicePack.startswith(" "):
                servicePack = servicePack[1:]
            if "/" in servicePack:
                servicePack = servicePack[:servicePack.index("/")]
            if "(" in servicePack:
                servicePack = servicePack[:servicePack.index("(")]
            if "-" in servicePack:
                servicePack = servicePack[:servicePack.index("-")]
            if "*" in servicePack:
                servicePack = servicePack[:servicePack.index("*")]
            if servicePack.startswith("+"):
                servicePack = "0%s" % servicePack

            servicePack = servicePack.replace("\t", " ")
            servicePack = servicePack.replace("  ", " ")
            servicePack = servicePack.replace("No SP", "0")
            servicePack = servicePack.replace("RTM", "0")
            servicePack = servicePack.replace("SP", "")
            servicePack = servicePack.replace("Service Pack", "")
            servicePack = servicePack.replace("<a href=\"http:", "")

            if servicePack.endswith(" "):
                servicePack = servicePack[:-1]

            if servicePack and version:
                # Create the main <card> element
                signature = doc.createElement("signature")
                signatures.appendChild(signature)

                # Create a <version> element
                versionElement = doc.createElement("version")
                signature.appendChild(versionElement)

                # Give the <version> elemenet some text
                versionText = doc.createTextNode(version)
                versionElement.appendChild(versionText)

                # Create a <servicepack> element
                servicepackElement = doc.createElement("servicepack")
                signature.appendChild(servicepackElement)

                # Give the <servicepack> elemenet some text
                servicepackText = doc.createTextNode(servicePack)
                servicepackElement.appendChild(servicepackText)

    # Get the XML old file content to a local variable
    mssqlXml = open(paths.MSSQL_XML, "r")
    oldMssqlXml = mssqlXml.read()
    oldMssqlXmlSignatures = oldMssqlXml.count("<signature>")
    oldMssqlXmlList = oldMssqlXml.splitlines(1)
    mssqlXml.close()

    # Backup the XML old file
    shutil.copy(paths.MSSQL_XML, "%s.bak" % paths.MSSQL_XML)

    # Save our newly created XML to the signatures file
    mssqlXml = open(paths.MSSQL_XML, "w")
    doc.writexml(writer=mssqlXml, addindent="    ", newl="\n")
    mssqlXml.close()

    # Get the XML new file content to a local variable
    mssqlXml = open(paths.MSSQL_XML, "r")
    newMssqlXml = mssqlXml.read()
    newMssqlXmlSignatures = newMssqlXml.count("<signature>")
    newMssqlXmlList = newMssqlXml.splitlines(1)
    mssqlXml.close()

    # If the new XML versions file differs from the old one it probably
    # means that we have got new Microsoft SQL Server versions
    if oldMssqlXmlSignatures != newMssqlXmlSignatures:
        infoMsg  = "Microsoft SQL Server XML versions file updated successfully. "

        if oldMssqlXmlSignatures < newMssqlXmlSignatures:
            infoMsg += "%d " % (newMssqlXmlSignatures - oldMssqlXmlSignatures)
            infoMsg += "new signatures added since the last update"

        # NOTE: This should never happen, in this rare case it might
        # be that the Microsoft SQL Server versions database
        # (MSSQL_VERSIONS_URL) changed its structure
        else:
            infoMsg += "%d " % (oldMssqlXmlSignatures - newMssqlXmlSignatures)
            infoMsg += "signatures removed since the last update"

        logger.info(infoMsg)

        message = "Do you want to see the differences? [Y/n] "
        test = readInput(message, default="Y")

        if not test or test[0] in ("y", "Y"):
            infoMsg = "Differences:"
            logger.info(infoMsg)

            # Compare the old XML file with the new one
            diff = difflib.unified_diff(oldMssqlXmlList, newMssqlXmlList, "%s.bak" % paths.MSSQL_XML, paths.MSSQL_XML)
            sys.stdout.writelines(diff)
    else:
        infoMsg  = "no new Microsoft SQL Server versions since the "
        infoMsg += "last update"
        logger.info(infoMsg)

def __updateSqlmap():
    rootDir = paths.SQLMAP_ROOT_PATH

    infoMsg = "updating sqlmap to latest development version from the "
    infoMsg += "subversion repository"
    logger.info(infoMsg)

    try:
        import pysvn

        debugMsg = "sqlmap will update itself using installed python-svn "
        debugMsg += "third-party library, http://pysvn.tigris.org/"
        logger.debug(debugMsg)

        def notify(event_dict):
            action = str(event_dict['action'])

            if action.find('_update') != -1:
                return

            index = action.find('_')
            prefix = action[index + 1].upper() if index != -1 else action.capitalize()

            if action.find('_completed') == -1:
                print "%s    %s" % (prefix, event_dict['path'])
            else:
                revision = str(event_dict['revision'])
                index = revision.find('number ')

                if index != -1:
                    revision = revision[index+7:].strip('>')

                logger.info('updated to the latest revision %s' % revision)

        client = pysvn.Client()
        client.callback_notify = notify
        client.update(rootDir)
    except ImportError, _:
        debugMsg = "sqlmap will try to update itself using 'svn' command"
        logger.debug(debugMsg)

        process = execute("svn update %s" % rootDir, shell=True, stdout=PIPE, stderr=PIPE)

        dataToStdout("\r[%s] [INFO] update in progress " % time.strftime("%X"))
        pollProcess(process)
        svnStdout, svnStderr = process.communicate()

        if svnStderr:
            errMsg = svnStderr.strip()
            logger.error(errMsg)
        elif svnStdout:
            revision = re.search("revision\s+([\d]+)", svnStdout, re.I)
            if revision:
                logger.info('updated to the latest revision %s' % revision.group(1))

def update():
    if not conf.updateAll:
        return

    __updateSqlmap()
    __updateMSSQLXML()
<commit_msg>More tweaking on --update<commit_after>#!/usr/bin/env python

"""
$Id$

This file is part of the sqlmap project, http://sqlmap.sourceforge.net.

Copyright (c) 2007-2009 Bernardo Damele A. G. <bernardo.damele@gmail.com>
Copyright (c) 2006 Daniele Bellucci <daniele.bellucci@gmail.com>

sqlmap is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free
Software Foundation version 2 of the License.

sqlmap is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
details.

You should have received a copy of the GNU General Public License along
with sqlmap; if not, write to the Free Software Foundation, Inc., 51
Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
"""

import difflib
import os
import re
import shutil
import sys
import tempfile
import time
import urlparse
import zipfile

from distutils.dir_util import mkpath
from xml.dom.minidom import Document

from subprocess import PIPE
from subprocess import Popen as execute

from lib.core.common import dataToStdout
from lib.core.common import pollProcess
from lib.core.common import readInput
from lib.core.data import conf
from lib.core.data import logger
from lib.core.data import paths
from lib.core.exception import sqlmapConnectionException
from lib.core.exception import sqlmapFilePathException
from lib.core.settings import MSSQL_VERSIONS_URL
from lib.core.settings import SQLMAP_VERSION_URL
from lib.core.settings import SQLMAP_SOURCE_URL
from lib.core.settings import VERSION
from lib.request.connect import Connect as Request

def __updateMSSQLXML():
    infoMsg = "updating Microsoft SQL Server XML versions file"
    logger.info(infoMsg)

    try:
        mssqlVersionsHtmlString, _ = Request.getPage(url=MSSQL_VERSIONS_URL, direct=True)
    except sqlmapConnectionException, _:
        __mssqlPath     = urlparse.urlsplit(MSSQL_VERSIONS_URL)
        __mssqlHostname = __mssqlPath[1]

        warnMsg  = "sqlmap was unable to connect to %s," % __mssqlHostname
        warnMsg += " check your Internet connection and retry"
        logger.warn(warnMsg)

        return

    releases      = re.findall("class=\"BCC_DV_01DarkBlueTitle\">SQL Server ([\d\.]+) Builds", mssqlVersionsHtmlString, re.I | re.M)
    releasesCount = len(releases)

    # Create the minidom document
    doc = Document()

    # Create the <root> base element
    root = doc.createElement("root")
    doc.appendChild(root)

    for index in range(0, releasesCount):
        release = releases[index]

        # Skip Microsoft SQL Server 6.5 because the HTML
        # table is in another format
        if release == "6.5":
            continue

        # Create the <signatures> base element
        signatures = doc.createElement("signatures")
        signatures.setAttribute("release", release)
        root.appendChild(signatures)

        startIdx = mssqlVersionsHtmlString.index("SQL Server %s Builds" % releases[index])

        if index == releasesCount - 1:
            stopIdx  = len(mssqlVersionsHtmlString)
        else:
            stopIdx  = mssqlVersionsHtmlString.index("SQL Server %s Builds" % releases[index + 1])

        mssqlVersionsReleaseString = mssqlVersionsHtmlString[startIdx:stopIdx]
        servicepackVersion = re.findall("	[7\.0|2000|2005|2008]*(.*?)	<td.*?([\d\.]+)	[\r]*\n", mssqlVersionsReleaseString, re.I | re.M)

        for servicePack, version in servicepackVersion:
            if servicePack.startswith(" "):
                servicePack = servicePack[1:]
            if "/" in servicePack:
                servicePack = servicePack[:servicePack.index("/")]
            if "(" in servicePack:
                servicePack = servicePack[:servicePack.index("(")]
            if "-" in servicePack:
                servicePack = servicePack[:servicePack.index("-")]
            if "*" in servicePack:
                servicePack = servicePack[:servicePack.index("*")]
            if servicePack.startswith("+"):
                servicePack = "0%s" % servicePack

            servicePack = servicePack.replace("\t", " ")
            servicePack = servicePack.replace("  ", " ")
            servicePack = servicePack.replace("No SP", "0")
            servicePack = servicePack.replace("RTM", "0")
            servicePack = servicePack.replace("SP", "")
            servicePack = servicePack.replace("Service Pack", "")
            servicePack = servicePack.replace("<a href=\"http:", "")

            if servicePack.endswith(" "):
                servicePack = servicePack[:-1]

            if servicePack and version:
                # Create the main <card> element
                signature = doc.createElement("signature")
                signatures.appendChild(signature)

                # Create a <version> element
                versionElement = doc.createElement("version")
                signature.appendChild(versionElement)

                # Give the <version> elemenet some text
                versionText = doc.createTextNode(version)
                versionElement.appendChild(versionText)

                # Create a <servicepack> element
                servicepackElement = doc.createElement("servicepack")
                signature.appendChild(servicepackElement)

                # Give the <servicepack> elemenet some text
                servicepackText = doc.createTextNode(servicePack)
                servicepackElement.appendChild(servicepackText)

    # Get the XML old file content to a local variable
    mssqlXml = open(paths.MSSQL_XML, "r")
    oldMssqlXml = mssqlXml.read()
    oldMssqlXmlSignatures = oldMssqlXml.count("<signature>")
    oldMssqlXmlList = oldMssqlXml.splitlines(1)
    mssqlXml.close()

    # Backup the XML old file
    shutil.copy(paths.MSSQL_XML, "%s.bak" % paths.MSSQL_XML)

    # Save our newly created XML to the signatures file
    mssqlXml = open(paths.MSSQL_XML, "w")
    doc.writexml(writer=mssqlXml, addindent="    ", newl="\n")
    mssqlXml.close()

    # Get the XML new file content to a local variable
    mssqlXml = open(paths.MSSQL_XML, "r")
    newMssqlXml = mssqlXml.read()
    newMssqlXmlSignatures = newMssqlXml.count("<signature>")
    newMssqlXmlList = newMssqlXml.splitlines(1)
    mssqlXml.close()

    # If the new XML versions file differs from the old one it probably
    # means that we have got new Microsoft SQL Server versions
    if oldMssqlXmlSignatures != newMssqlXmlSignatures:
        infoMsg  = "Microsoft SQL Server XML versions file updated successfully. "

        if oldMssqlXmlSignatures < newMssqlXmlSignatures:
            infoMsg += "%d " % (newMssqlXmlSignatures - oldMssqlXmlSignatures)
            infoMsg += "new signatures added since the last update"

        # NOTE: This should never happen, in this rare case it might
        # be that the Microsoft SQL Server versions database
        # (MSSQL_VERSIONS_URL) changed its structure
        else:
            infoMsg += "%d " % (oldMssqlXmlSignatures - newMssqlXmlSignatures)
            infoMsg += "signatures removed since the last update"

        logger.info(infoMsg)

        message = "Do you want to see the differences? [Y/n] "
        test = readInput(message, default="Y")

        if not test or test[0] in ("y", "Y"):
            infoMsg = "Differences:"
            logger.info(infoMsg)

            # Compare the old XML file with the new one
            diff = difflib.unified_diff(oldMssqlXmlList, newMssqlXmlList, "%s.bak" % paths.MSSQL_XML, paths.MSSQL_XML)
            sys.stdout.writelines(diff)
    else:
        infoMsg  = "no new Microsoft SQL Server versions since the "
        infoMsg += "last update"
        logger.info(infoMsg)

def __updateSqlmap():
    rootDir = paths.SQLMAP_ROOT_PATH

    infoMsg = "updating sqlmap to latest development version from the "
    infoMsg += "subversion repository"
    logger.info(infoMsg)

    try:
        import pysvn

        debugMsg = "sqlmap will update itself using installed python-svn "
        debugMsg += "third-party library, http://pysvn.tigris.org/"
        logger.debug(debugMsg)

        def notify(event_dict):
            action = str(event_dict['action'])
            index = action.find('_')
            prefix = action[index + 1].upper() if index != -1 else action.capitalize()

            if action.find('_update') != -1:
                return

            if action.find('_completed') == -1:
                print "%s\t%s" % (prefix, event_dict['path'])
            else:
                revision = str(event_dict['revision'])
                index = revision.find('number ')

                if index != -1:
                    revision = revision[index+7:].strip('>')

                logger.info('updated to the latest revision %s' % revision)

        client = pysvn.Client()
        client.callback_notify = notify
        client.update(rootDir)
    except ImportError, _:
        debugMsg = "sqlmap will try to update itself using 'svn' command"
        logger.debug(debugMsg)

        process = execute("svn update %s" % rootDir, shell=True, stdout=PIPE, stderr=PIPE)

        dataToStdout("\r[%s] [INFO] update in progress " % time.strftime("%X"))
        pollProcess(process)
        svnStdout, svnStderr = process.communicate()

        if svnStderr:
            errMsg = svnStderr.strip()
            logger.error(errMsg)
        elif svnStdout:
            revision = re.search("revision\s+([\d]+)", svnStdout, re.I)
            if revision:
                logger.info('updated to the latest revision %s' % revision.group(1))

def update():
    if not conf.updateAll:
        return

    __updateSqlmap()
    __updateMSSQLXML()
<|endoftext|>


<commit_before>"""Jinja environment filters for Pynuts."""

from flask import escape
from flask.ext.wtf import QuerySelectField, QuerySelectMultipleField

def data(field):
    """Return data according to a specific field."""
    if isinstance(field, QuerySelectMultipleField):
        if field.data:
            return escape(
                u', '.join(field.get_label(data) for data in field.data))
    elif isinstance(field, QuerySelectField):
        if field.data:
            return escape(field.get_label(field.data))
    return escape(field.data)
<commit_msg>Add a read filter for boolean fields<commit_after># -*- coding: utf-8 -*-

"""Jinja environment filters for Pynuts."""

from flask import escape
from flask.ext.wtf import (
    QuerySelectField, QuerySelectMultipleField, BooleanField)

def data(field):
    """Return data according to a specific field."""
    if isinstance(field, QuerySelectMultipleField):
        if field.data:
            return escape(
                u', '.join(field.get_label(data) for data in field.data))
    elif isinstance(field, QuerySelectField):
        if field.data:
            return escape(field.get_label(field.data))
    elif isinstance(field, BooleanField):
        return u'✓' if field.data else u'✕'
    return escape(field.data)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""Events framework with publisher, subscriber and repository."""

__author__ = 'Dave Foster <dfoster@asascience.com>, Michael Meisinger'
__license__ = 'Apache 2.0'

import functools
import sys
import traceback
from gevent import event as gevent_event

from pyon.core import bootstrap
from pyon.core.exception import BadRequest, IonException, StreamException
from pyon.datastore.datastore import DataStore
from pyon.ion.identifier import create_unique_event_id
from pyon.net.endpoint import Publisher, Subscriber
from pyon.util.async import spawn
from pyon.util.containers import get_ion_ts, is_valid_ts
from pyon.util.log import log

from interface.objects import Event

# @TODO: configurable
EVENTS_XP = "pyon.events"
EVENTS_XP_TYPE = "topic"

#The event will be ignored if older than this time period
VALID_EVENT_TIME_PERIOD = 365 * 24 * 60 * 60 * 1000   # one year

def get_events_exchange_point():
    return "%s.%s" % (bootstrap.get_sys_name(), EVENTS_XP)

class EventError(IonException):
    status_code = 500

class EventPublisher(Publisher):

    def __init__(self, event_type=None, xp=None, **kwargs):
        """
        Constructs a publisher of events for a specific type.

        @param  event_type  The name of the event type object
        @param  xp          Exchange (AMQP) name, can be none, will use events default.
        """

        self.event_type = event_type

        if bootstrap.container_instance and getattr(bootstrap.container_instance, 'event_repository', None):
            self.event_repo = bootstrap.container_instance.event_repository
        else:
            self.event_repo = None

        # generate an exchange name to publish events to
        xp = xp or get_events_exchange_point()
        name = (xp, None)

        Publisher.__init__(self, to_name=name, **kwargs)

    def _topic(self, event_object):
        """
        Builds the topic that this event should be published to.
        """
        assert event_object
        base_types = event_object.base_types or []
        base_str = ".".join(reversed(base_types))
        sub_type = event_object.sub_type or "_"
        origin_type = event_object.origin_type or "_"
        routing_key = "%s.%s.%s.%s.%s" % (base_str, event_object._get_type(), sub_type, origin_type, event_object.origin)
        return routing_key

    def publish_event_object(self, event_object):
        """
        Publishes an event of given type for the given origin. Event_type defaults to an
        event_type set when initializing the EventPublisher. Other kwargs fill out the fields
        of the event. This operation will fail with an exception.
        @param event_object     the event object to be published
        @retval event_object    the event object which was published
        """
        assert event_object

        topic = self._topic(event_object)
        to_name = (self._send_name.exchange, topic)
        log.trace("Publishing event message to %s", to_name)

        current_time = int(get_ion_ts())

        #Ensure valid created timestamp if supplied
        if event_object.ts_created:

            if not is_valid_ts(event_object.ts_created):
                raise BadRequest("The ts_created value is not a valid timestamp: '%s'" % (event_object.ts_created))

            #Reject events that are older than specified time
            if int(event_object.ts_created) > ( current_time + VALID_EVENT_TIME_PERIOD ):
                raise BadRequest("This ts_created value is too far in the future:'%s'" % (event_object.ts_created))

            #Reject events that are older than specified time
            if int(event_object.ts_created) < (current_time - VALID_EVENT_TIME_PERIOD) :
                raise BadRequest("This ts_created value is too old:'%s'" % (event_object.ts_created))

        else:
            event_object.ts_created = str(current_time)

        #Validate this object
        #TODO - enable this once the resource agent issue sending a dict is figured out
        #event_object._validate()

        #Ensure the event object has a unique id
        if '_id' in event_object:
            raise BadRequest("The event object cannot contain a _id field '%s'" % (event_object))

        #Generate a unique ID for this event
        event_object._id = create_unique_event_id()

        try:
            self.publish(event_object, to_name=to_name)
        except Exception as ex:
            log.exception("Failed to publish event (%s): '%s'" % (ex.message, event_object))
            raise

        return event_object

    def publish_event(self, origin=None, event_type=None, **kwargs):
        """
        Publishes an event of given type for the given origin. Event_type defaults to an
        event_type set when initializing the EventPublisher. Other kwargs fill out the fields
        of the event. This operation will fail with an exception.
        @param origin     the origin field value
        @param event_type the event type (defaults to the EventPublisher's event_type if set)
        @param kwargs     additional event fields
        @retval event_object    the event object which was published
        """

        event_type = event_type or self.event_type
        assert event_type

        event_object = bootstrap.IonObject(event_type, origin=origin, **kwargs)
        event_object.base_types = event_object._get_extends()
        ret_val = self.publish_event_object(event_object)
        return ret_val

class BaseEventSubscriberMixin(object):
    """
    A mixin class for Event subscribers to facilitate inheritance.

    EventSubscribers must come in both standard and process level versions, which
    rely on common base code. It is difficult to multiple inherit due to both of
    them sharing a base class, so this mixin is preferred.
    """

    @staticmethod
    def _topic(event_type, origin, sub_type=None, origin_type=None):
        """
        Builds the topic that this event should be published to.
        If either side of the event_id.origin pair are missing, will subscribe to anything.
        """
        if event_type == "Event":
            event_type = "Event.#"
        elif event_type:
            event_type = "#.%s.#" % event_type
        else:
            event_type = "#"

        sub_type = sub_type or "*.#"
        origin_type = origin_type or "*"
        origin      = origin or "*"

        return "%s.%s.%s.%s" % (event_type, sub_type, origin_type, origin)

    def __init__(self, xp_name=None, event_type=None, origin=None, queue_name=None,
                 sub_type=None, origin_type=None, pattern=None):
        self.event_type = event_type
        self.sub_type = sub_type
        self.origin_type = origin_type
        self.origin = origin

        xp_name = xp_name or get_events_exchange_point()
        if pattern:
            binding = pattern
        else:
            binding = self._topic(event_type, origin, sub_type, origin_type)
        self.binding = binding

        # TODO: Provide a case where we can have multiple bindings (e.g. different event_types)

        # prefix the queue_name, if specified, with the sysname
        # this is because queue names transcend xp boundaries (see R1 OOIION-477)
        if queue_name is not None:
            if not queue_name.startswith(bootstrap.get_sys_name()):
                queue_name = "%s.%s" % (bootstrap.get_sys_name(), queue_name)
                log.warn("queue_name specified, prepending sys_name to it: %s", queue_name)

        # set this name to be picked up by inherited folks
        self._ev_recv_name = (xp_name, queue_name)

class EventSubscriber(Subscriber, BaseEventSubscriberMixin):

    ALL_EVENTS = "#"

    def __init__(self, xp_name=None, event_type=None, origin=None, queue_name=None, callback=None,
                 sub_type=None, origin_type=None, pattern=None, auto_delete=None, *args, **kwargs):
        """
        Initializer.

        If the queue_name is specified here, the sysname is prefixed automatically to it. This is because
        named queues are not namespaces to their exchanges, so two different systems on the same broker
        can cross-pollute messages if a named queue is used.

        Note: an EventSubscriber needs to be closed to free broker resources
        """
        self._cbthread = None
        self._auto_delete = auto_delete

        BaseEventSubscriberMixin.__init__(self, xp_name=xp_name, event_type=event_type, origin=origin,
                                          queue_name=queue_name, sub_type=sub_type, origin_type=origin_type, pattern=pattern)

        log.debug("EventPublisher events pattern %s", self.binding)

        Subscriber.__init__(self, from_name=self._ev_recv_name, binding=self.binding, callback=callback, **kwargs)

    def start(self):
        """
        Pass in a subscriber here, this will make it listen in a background greenlet.
        """
        assert not self._cbthread, "start called twice on EventSubscriber"
        gl = spawn(self.listen)
        self._cbthread = gl
        if not self._ready_event.wait(timeout=5):
            log.warning('EventSubscriber start timed out.')
        log.info("EventSubscriber started. Event pattern=%s" % self.binding)
        return gl

    def stop(self):
        self.close()
        self._cbthread.join(timeout=5)
        self._cbthread.kill()
        self._cbthread = None
        log.info("EventSubscriber stopped. Event pattern=%s" % self.binding)

    def __str__(self):
        return "EventSubscriber at %s:\n\trecv_name: %s\n\tcb: %s" % (hex(id(self)), str(self._recv_name), str(self._callback))

    def _create_channel(self, **kwargs):
        """
        Override to set the channel's queue_auto_delete property.
        """
        ch = Subscriber._create_channel(self, **kwargs)
        if self._auto_delete is not None:
            ch.queue_auto_delete = self._auto_delete

        return ch

class EventRepository(object):
    """
    Class that uses a data store to provide a persistent repository for ION events.
    """

    def __init__(self, datastore_manager=None, container=None):
        self.container = container or bootstrap.container_instance

        # Get an instance of datastore configured as directory.
        # May be persistent or mock, forced clean, with indexes
        datastore_manager = datastore_manager or self.container.datastore_manager
        self.event_store = datastore_manager.get_datastore("events", DataStore.DS_PROFILE.EVENTS)

    def start(self):
        pass

    def stop(self):
        self.close()

    def close(self):
        """
        Pass-through method to close the underlying datastore.
        """
        self.event_store.close()

    def put_event(self, event):
        log.trace("Store event persistently %s", event)
        if not isinstance(event, Event):
            raise BadRequest("event must be type Event, not %s" % type(event))
        event_id = event.__dict__.pop("_id", None)
        return self.event_store.create(event, event_id)

    def put_events(self, events):
        log.debug("Store %s events persistently", len(events))
        if type(events) is not list:
            raise BadRequest("events must be type list, not %s" % type(events))
        if not all([isinstance(event, Event) for event in events]):
            raise BadRequest("events must all be type Event")

        if events:
            return self.event_store.create_mult(events, allow_ids=True)
        else:
            return None

    def get_event(self, event_id):
        log.trace("Retrieving persistent event for id=%s", event_id)
        event_obj = self.event_store.read(event_id)
        return event_obj

    def find_events(self, event_type=None, origin=None, start_ts=None, end_ts=None, **kwargs):
        log.trace("Retrieving persistent event for event_type=%s, origin=%s, start_ts=%s, end_ts=%s, descending=%s, limit=%s",
                  event_type,origin,start_ts,end_ts,kwargs.get("descending", None),kwargs.get("limit",None))
        events = None

        design_name = "event"
        view_name = None
        start_key = []
        end_key = []
        if origin and event_type:
            view_name = "by_origintype"
            start_key = [origin, event_type]
            end_key = [origin, event_type]
        elif origin:
            view_name = "by_origin"
            start_key = [origin]
            end_key = [origin]
        elif event_type:
            view_name = "by_type"
            start_key = [event_type]
            end_key = [event_type]
        elif start_ts or end_ts:
            view_name = "by_time"
            start_key = []
            end_key = []
        else:
            view_name = "by_time"
            if kwargs.get("limit", 0) < 1:
                kwargs["limit"] = 100
                log.warn("Querying all events, no limit given. Set limit to 100")

        if start_ts:
            start_key.append(start_ts)
        if end_ts:
            end_key.append(end_ts)

        events = self.event_store.find_by_view(design_name, view_name, start_key=start_key, end_key=end_key,
                                               id_only=False, **kwargs)

        return events

class EventGate(EventSubscriber):
    def __init__(self, *args, **kwargs):
        EventSubscriber.__init__(self, *args, callback=self.trigger_cb, **kwargs)

    def trigger_cb(self, event):
        self.stop()
        self.gate.set()

    def await(self, timeout=None):
        self.gate = gevent_event.Event()
        self.start()
        return self.gate.wait(timeout)

    def check_or_await(self):
        pass

def handle_stream_exception(iorigin="stream_exception"):
    """
    decorator for stream exceptions
    """
    def real_decorator(fn):
        @functools.wraps(fn)
        def wrapped(*args, **kwargs):
            try:
                fn(*args, **kwargs)
            except StreamException as e:
                info = "".join(traceback.format_tb(sys.exc_info()[2]))
                pub = EventPublisher(event_type="ExceptionEvent")
                pub.publish_event(origin=iorigin, description="stream exception event", exception_type=str(type(e)), message=info)
        return wrapped
    return real_decorator

<commit_msg>Enable event validation but only log validation errors<commit_after>#!/usr/bin/env python

"""Events framework with publisher, subscriber and repository."""

__author__ = 'Dave Foster <dfoster@asascience.com>, Michael Meisinger'
__license__ = 'Apache 2.0'

import functools
import sys
import traceback
from gevent import event as gevent_event

from pyon.core import bootstrap
from pyon.core.exception import BadRequest, IonException, StreamException
from pyon.datastore.datastore import DataStore
from pyon.ion.identifier import create_unique_event_id
from pyon.net.endpoint import Publisher, Subscriber
from pyon.util.async import spawn
from pyon.util.containers import get_ion_ts, is_valid_ts
from pyon.util.log import log

from interface.objects import Event

# @TODO: configurable
EVENTS_XP = "pyon.events"
EVENTS_XP_TYPE = "topic"

#The event will be ignored if older than this time period
VALID_EVENT_TIME_PERIOD = 365 * 24 * 60 * 60 * 1000   # one year

def get_events_exchange_point():
    return "%s.%s" % (bootstrap.get_sys_name(), EVENTS_XP)

class EventError(IonException):
    status_code = 500

class EventPublisher(Publisher):

    def __init__(self, event_type=None, xp=None, **kwargs):
        """
        Constructs a publisher of events for a specific type.

        @param  event_type  The name of the event type object
        @param  xp          Exchange (AMQP) name, can be none, will use events default.
        """

        self.event_type = event_type

        if bootstrap.container_instance and getattr(bootstrap.container_instance, 'event_repository', None):
            self.event_repo = bootstrap.container_instance.event_repository
        else:
            self.event_repo = None

        # generate an exchange name to publish events to
        xp = xp or get_events_exchange_point()
        name = (xp, None)

        Publisher.__init__(self, to_name=name, **kwargs)

    def _topic(self, event_object):
        """
        Builds the topic that this event should be published to.
        """
        assert event_object
        base_types = event_object.base_types or []
        base_str = ".".join(reversed(base_types))
        sub_type = event_object.sub_type or "_"
        origin_type = event_object.origin_type or "_"
        routing_key = "%s.%s.%s.%s.%s" % (base_str, event_object._get_type(), sub_type, origin_type, event_object.origin)
        return routing_key

    def publish_event_object(self, event_object):
        """
        Publishes an event of given type for the given origin. Event_type defaults to an
        event_type set when initializing the EventPublisher. Other kwargs fill out the fields
        of the event. This operation will fail with an exception.
        @param event_object     the event object to be published
        @retval event_object    the event object which was published
        """
        assert event_object

        topic = self._topic(event_object)
        to_name = (self._send_name.exchange, topic)
        log.trace("Publishing event message to %s", to_name)

        current_time = int(get_ion_ts())

        #Ensure valid created timestamp if supplied
        if event_object.ts_created:

            if not is_valid_ts(event_object.ts_created):
                raise BadRequest("The ts_created value is not a valid timestamp: '%s'" % (event_object.ts_created))

            #Reject events that are older than specified time
            if int(event_object.ts_created) > ( current_time + VALID_EVENT_TIME_PERIOD ):
                raise BadRequest("This ts_created value is too far in the future:'%s'" % (event_object.ts_created))

            #Reject events that are older than specified time
            if int(event_object.ts_created) < (current_time - VALID_EVENT_TIME_PERIOD) :
                raise BadRequest("This ts_created value is too old:'%s'" % (event_object.ts_created))

        else:
            event_object.ts_created = str(current_time)

        #Validate this object - ideally the validator should pass on problems, but for now just log
        #any errors and keep going, since seeing invalid situations are better than skipping validation.
        try:
            event_object._validate()
        except Exception, e:
            log.exception(e)

        #Ensure the event object has a unique id
        if '_id' in event_object:
            raise BadRequest("The event object cannot contain a _id field '%s'" % (event_object))

        #Generate a unique ID for this event
        event_object._id = create_unique_event_id()

        try:
            self.publish(event_object, to_name=to_name)
        except Exception as ex:
            log.exception("Failed to publish event (%s): '%s'" % (ex.message, event_object))
            raise

        return event_object

    def publish_event(self, origin=None, event_type=None, **kwargs):
        """
        Publishes an event of given type for the given origin. Event_type defaults to an
        event_type set when initializing the EventPublisher. Other kwargs fill out the fields
        of the event. This operation will fail with an exception.
        @param origin     the origin field value
        @param event_type the event type (defaults to the EventPublisher's event_type if set)
        @param kwargs     additional event fields
        @retval event_object    the event object which was published
        """

        event_type = event_type or self.event_type
        assert event_type

        event_object = bootstrap.IonObject(event_type, origin=origin, **kwargs)
        event_object.base_types = event_object._get_extends()
        ret_val = self.publish_event_object(event_object)
        return ret_val

class BaseEventSubscriberMixin(object):
    """
    A mixin class for Event subscribers to facilitate inheritance.

    EventSubscribers must come in both standard and process level versions, which
    rely on common base code. It is difficult to multiple inherit due to both of
    them sharing a base class, so this mixin is preferred.
    """

    @staticmethod
    def _topic(event_type, origin, sub_type=None, origin_type=None):
        """
        Builds the topic that this event should be published to.
        If either side of the event_id.origin pair are missing, will subscribe to anything.
        """
        if event_type == "Event":
            event_type = "Event.#"
        elif event_type:
            event_type = "#.%s.#" % event_type
        else:
            event_type = "#"

        sub_type = sub_type or "*.#"
        origin_type = origin_type or "*"
        origin      = origin or "*"

        return "%s.%s.%s.%s" % (event_type, sub_type, origin_type, origin)

    def __init__(self, xp_name=None, event_type=None, origin=None, queue_name=None,
                 sub_type=None, origin_type=None, pattern=None):
        self.event_type = event_type
        self.sub_type = sub_type
        self.origin_type = origin_type
        self.origin = origin

        xp_name = xp_name or get_events_exchange_point()
        if pattern:
            binding = pattern
        else:
            binding = self._topic(event_type, origin, sub_type, origin_type)
        self.binding = binding

        # TODO: Provide a case where we can have multiple bindings (e.g. different event_types)

        # prefix the queue_name, if specified, with the sysname
        # this is because queue names transcend xp boundaries (see R1 OOIION-477)
        if queue_name is not None:
            if not queue_name.startswith(bootstrap.get_sys_name()):
                queue_name = "%s.%s" % (bootstrap.get_sys_name(), queue_name)
                log.warn("queue_name specified, prepending sys_name to it: %s", queue_name)

        # set this name to be picked up by inherited folks
        self._ev_recv_name = (xp_name, queue_name)

class EventSubscriber(Subscriber, BaseEventSubscriberMixin):

    ALL_EVENTS = "#"

    def __init__(self, xp_name=None, event_type=None, origin=None, queue_name=None, callback=None,
                 sub_type=None, origin_type=None, pattern=None, auto_delete=None, *args, **kwargs):
        """
        Initializer.

        If the queue_name is specified here, the sysname is prefixed automatically to it. This is because
        named queues are not namespaces to their exchanges, so two different systems on the same broker
        can cross-pollute messages if a named queue is used.

        Note: an EventSubscriber needs to be closed to free broker resources
        """
        self._cbthread = None
        self._auto_delete = auto_delete

        BaseEventSubscriberMixin.__init__(self, xp_name=xp_name, event_type=event_type, origin=origin,
                                          queue_name=queue_name, sub_type=sub_type, origin_type=origin_type, pattern=pattern)

        log.debug("EventPublisher events pattern %s", self.binding)

        Subscriber.__init__(self, from_name=self._ev_recv_name, binding=self.binding, callback=callback, **kwargs)

    def start(self):
        """
        Pass in a subscriber here, this will make it listen in a background greenlet.
        """
        assert not self._cbthread, "start called twice on EventSubscriber"
        gl = spawn(self.listen)
        self._cbthread = gl
        if not self._ready_event.wait(timeout=5):
            log.warning('EventSubscriber start timed out.')
        log.info("EventSubscriber started. Event pattern=%s" % self.binding)
        return gl

    def stop(self):
        self.close()
        self._cbthread.join(timeout=5)
        self._cbthread.kill()
        self._cbthread = None
        log.info("EventSubscriber stopped. Event pattern=%s" % self.binding)

    def __str__(self):
        return "EventSubscriber at %s:\n\trecv_name: %s\n\tcb: %s" % (hex(id(self)), str(self._recv_name), str(self._callback))

    def _create_channel(self, **kwargs):
        """
        Override to set the channel's queue_auto_delete property.
        """
        ch = Subscriber._create_channel(self, **kwargs)
        if self._auto_delete is not None:
            ch.queue_auto_delete = self._auto_delete

        return ch

class EventRepository(object):
    """
    Class that uses a data store to provide a persistent repository for ION events.
    """

    def __init__(self, datastore_manager=None, container=None):
        self.container = container or bootstrap.container_instance

        # Get an instance of datastore configured as directory.
        # May be persistent or mock, forced clean, with indexes
        datastore_manager = datastore_manager or self.container.datastore_manager
        self.event_store = datastore_manager.get_datastore("events", DataStore.DS_PROFILE.EVENTS)

    def start(self):
        pass

    def stop(self):
        self.close()

    def close(self):
        """
        Pass-through method to close the underlying datastore.
        """
        self.event_store.close()

    def put_event(self, event):
        log.trace("Store event persistently %s", event)
        if not isinstance(event, Event):
            raise BadRequest("event must be type Event, not %s" % type(event))
        event_id = event.__dict__.pop("_id", None)
        return self.event_store.create(event, event_id)

    def put_events(self, events):
        log.debug("Store %s events persistently", len(events))
        if type(events) is not list:
            raise BadRequest("events must be type list, not %s" % type(events))
        if not all([isinstance(event, Event) for event in events]):
            raise BadRequest("events must all be type Event")

        if events:
            return self.event_store.create_mult(events, allow_ids=True)
        else:
            return None

    def get_event(self, event_id):
        log.trace("Retrieving persistent event for id=%s", event_id)
        event_obj = self.event_store.read(event_id)
        return event_obj

    def find_events(self, event_type=None, origin=None, start_ts=None, end_ts=None, **kwargs):
        log.trace("Retrieving persistent event for event_type=%s, origin=%s, start_ts=%s, end_ts=%s, descending=%s, limit=%s",
                  event_type,origin,start_ts,end_ts,kwargs.get("descending", None),kwargs.get("limit",None))
        events = None

        design_name = "event"
        view_name = None
        start_key = []
        end_key = []
        if origin and event_type:
            view_name = "by_origintype"
            start_key = [origin, event_type]
            end_key = [origin, event_type]
        elif origin:
            view_name = "by_origin"
            start_key = [origin]
            end_key = [origin]
        elif event_type:
            view_name = "by_type"
            start_key = [event_type]
            end_key = [event_type]
        elif start_ts or end_ts:
            view_name = "by_time"
            start_key = []
            end_key = []
        else:
            view_name = "by_time"
            if kwargs.get("limit", 0) < 1:
                kwargs["limit"] = 100
                log.warn("Querying all events, no limit given. Set limit to 100")

        if start_ts:
            start_key.append(start_ts)
        if end_ts:
            end_key.append(end_ts)

        events = self.event_store.find_by_view(design_name, view_name, start_key=start_key, end_key=end_key,
                                               id_only=False, **kwargs)

        return events

class EventGate(EventSubscriber):
    def __init__(self, *args, **kwargs):
        EventSubscriber.__init__(self, *args, callback=self.trigger_cb, **kwargs)

    def trigger_cb(self, event):
        self.stop()
        self.gate.set()

    def await(self, timeout=None):
        self.gate = gevent_event.Event()
        self.start()
        return self.gate.wait(timeout)

    def check_or_await(self):
        pass

def handle_stream_exception(iorigin="stream_exception"):
    """
    decorator for stream exceptions
    """
    def real_decorator(fn):
        @functools.wraps(fn)
        def wrapped(*args, **kwargs):
            try:
                fn(*args, **kwargs)
            except StreamException as e:
                info = "".join(traceback.format_tb(sys.exc_info()[2]))
                pub = EventPublisher(event_type="ExceptionEvent")
                pub.publish_event(origin=iorigin, description="stream exception event", exception_type=str(type(e)), message=info)
        return wrapped
    return real_decorator

<|endoftext|>


<commit_before>__version__ = '0.1b.7dev'<commit_msg>pypet v.0.1b.7RC<commit_after>__version__ = '0.1b.7RC'<|endoftext|>


<commit_before>import base64
import sys
import os
import fnmatch
import cPickle
import tarfile
import tempfile
import compressor
from ppd import PPD

def read_file_in_syspath(filename):
    """Reads the file in filename in each sys.path.

    If we couldn't find, throws the last IOError caught.

    """
    last_exception = None
    for path in sys.path:
        try:
            return open(path + "/" + filename).read()
        except IOError as ex:
            last_exception = ex
            continue
    raise last_exception

def find_files(directory, pattern):
    """Yields each file that matches pattern in directory."""
    abs_directory = os.path.abspath(directory)
    for root, dirnames, filenames in os.walk(abs_directory):
        for filename in fnmatch.filter(filenames, pattern):
            yield os.path.join(root, filename)

def compress(directory):
    """Compresses and indexes all *.ppd files in directory returning as a string.

    The directory is walked recursively, inserting all ppds found in a tar file.
    For each, it parses and saves its name, description (in the format CUPS needs)
    and path into a dictionary, used as an index.
    Then, it compresses the tar file, adds into the dictionary as key ARCHIVE and
    string of all ppds concatenated and returns the pickle dump of the dictionary.

    """
    tmp = tempfile.NamedTemporaryFile()
    ppds = tarfile.open(fileobj = tmp.file, mode = 'w')
    ppds_index = {}

    for ppd_path in find_files(directory, "*.ppd"):
        try:
            ppd_file = open(ppd_path).read()
            ppds.add(ppd_path)

            # Removes the first slash in the path, so we can find the file in
            # the tar archive.
            ppd_path = ppd_path[1:]

            a_ppd = PPD(ppd_file)
            ppds_index[a_ppd.name] = (ppd_path, str(a_ppd))
        except:
            raise

    ppds_index['ARCHIVE'] = compressor.compress_file(tmp.name)
    ppds_pickle = compressor.compress(cPickle.dumps(ppds_index))
    ppds.close()

    return ppds_pickle

def create_archive(ppds_directory):
    """Returns a string with the decompressor, its dependencies and the archive.

    It reads the template at pyppd/pyppd-ppdfile.in, inserts the dependencies
    and the archive encoded in base64, and returns as a string.

    """
    ppds_compressed = base64.b64encode(compress(ppds_directory))

    template = read_file_in_syspath("pyppd/pyppd-ppdfile.in")
    compressor_py = read_file_in_syspath("pyppd/compressor.py")

    template = template.replace("@compressor@", compressor_py)
    template = template.replace("@ppds_compressed_b64@", ppds_compressed)

    return template
<commit_msg>Reorganizing methods, the most important at the top<commit_after>import base64
import sys
import os
import fnmatch
import cPickle
import tarfile
import tempfile
import compressor
from ppd import PPD

def create_archive(ppds_directory):
    """Returns a string with the decompressor, its dependencies and the archive.

    It reads the template at pyppd/pyppd-ppdfile.in, inserts the dependencies
    and the archive encoded in base64, and returns as a string.

    """
    ppds_compressed = base64.b64encode(compress(ppds_directory))

    template = read_file_in_syspath("pyppd/pyppd-ppdfile.in")
    compressor_py = read_file_in_syspath("pyppd/compressor.py")

    template = template.replace("@compressor@", compressor_py)
    template = template.replace("@ppds_compressed_b64@", ppds_compressed)

    return template

def compress(directory):
    """Compresses and indexes all *.ppd files in directory returning as a string.

    The directory is walked recursively, inserting all ppds found in a tar file.
    For each, it parses and saves its name, description (in the format CUPS needs)
    and path into a dictionary, used as an index.
    Then, it compresses the tar file, adds into the dictionary as key ARCHIVE and
    string of all ppds concatenated and returns the pickle dump of the dictionary.

    """
    tmp = tempfile.NamedTemporaryFile()
    ppds = tarfile.open(fileobj = tmp.file, mode = 'w')
    ppds_index = {}

    for ppd_path in find_files(directory, "*.ppd"):
        try:
            ppd_file = open(ppd_path).read()
            ppds.add(ppd_path)

            # Removes the first slash in the path, so we can find the file in
            # the tar archive.
            ppd_path = ppd_path[1:]

            a_ppd = PPD(ppd_file)
            ppds_index[a_ppd.name] = (ppd_path, str(a_ppd))
        except:
            raise

    ppds_index['ARCHIVE'] = compressor.compress_file(tmp.name)
    ppds_pickle = compressor.compress(cPickle.dumps(ppds_index))
    ppds.close()

    return ppds_pickle

def read_file_in_syspath(filename):
    """Reads the file in filename in each sys.path.

    If we couldn't find, throws the last IOError caught.

    """
    last_exception = None
    for path in sys.path:
        try:
            return open(path + "/" + filename).read()
        except IOError as ex:
            last_exception = ex
            continue
    raise last_exception

def find_files(directory, pattern):
    """Yields each file that matches pattern in directory."""
    abs_directory = os.path.abspath(directory)
    for root, dirnames, filenames in os.walk(abs_directory):
        for filename in fnmatch.filter(filenames, pattern):
            yield os.path.join(root, filename)
<|endoftext|>


<commit_before>import pysb.core
import pysb.bng
import numpy
from scipy.integrate import ode
from scipy.weave import inline
import scipy.weave.build_tools
import distutils.errors
import sympy
import re
import itertools

# some sane default options for a few well-known integrators
default_integrator_options = {
    'vode': {
        'method': 'bdf',
        'with_jacobian': True,
        # Set nsteps as high as possible to give our users flexibility in
        # choosing their time step. (Let's be safe and assume vode was compiled
        # with 32-bit ints. What would actually happen if it was and we passed
        # 2**64-1 though?)
        'nsteps': 2**31 - 1,
        },
    'cvode': {
        'method': 'bdf',
        'iteration': 'newton',
        },
    }

class Solver(object):
    """An interface for numeric integration of models.

    Parameters
    ----------
    model : pysb.Model
        Model to integrate.
    tspan : vector-like
        Time values over which to integrate. The first and last values define
        the time range, and the returned trajectories will be sampled at every
        value.
    integrator : string, optional
        Name of the integrator to use, taken from the list of integrators known
        to :py:class:`scipy.integrate.ode`.
    integrator_params
        Additional parameters for the integrator.

    Attributes
    ----------
    model : pysb.Model
        Model passed to the constructor
    tspan : vector-like
        Time values passed to the constructor.
    y : numpy.ndarray
        Species trajectories. Dimensionality is ``(len(tspan),
        len(model.species))``.
    yobs : numpy.ndarray with record-style data-type
        Observable trajectories. Length is is ``len(tspan)`` and record names
        follow ``model.observables`` names.
    yobs_view : numpy.ndarray
        An array view (sharing the same data buffer) on ``yobs``. Dimensionality
        is ``(len(tspan), len(model.observables))``.
    integrator : scipy.integrate.ode
        Integrator object.

    Notes
    -----
    The expensive step of generating the code for the right-hand side of the
    model's ODEs is performed during initialization. If you need to integrate
    the same model repeatedly with different parameters then you should build a
    single Solver object and then call its ``run`` method as needed.

    """

    @staticmethod
    def _test_inline():
        """Detect whether scipy.weave.inline is functional."""
        if not hasattr(Solver, '_use_inline'):
            Solver._use_inline = False
            try:
                inline('int i;', force=1)
                Solver._use_inline = True
            except (scipy.weave.build_tools.CompileError,
                    distutils.errors.CompileError, ImportError):
                pass

    def __init__(self, model, tspan, integrator='vode', **integrator_options):

        pysb.bng.generate_equations(model)

        code_eqs = '\n'.join(['ydot[%d] = %s;' % (i, sympy.ccode(model.odes[i])) for i in range(len(model.odes))])
        code_eqs = re.sub(r's(\d+)', lambda m: 'y[%s]' % (int(m.group(1))), code_eqs)
        for e in model.expressions:
            code_eqs = re.sub(r'\b(%s)\b' % e.name,
                              sympy.ccode(e.expand_expr()), code_eqs)
        for i, p in enumerate(model.parameters):
            code_eqs = re.sub(r'\b(%s)\b' % p.name, 'p[%d]' % i, code_eqs)

        Solver._test_inline()
        # If we can't use weave.inline to run the C code, compile it as Python code instead for use with
        # exec. Note: C code with array indexing, basic math operations, and pow() just happens to also
        # be valid Python.  If the equations ever have more complex things in them, this might fail.
        if not Solver._use_inline:
            code_eqs_py = compile(code_eqs, '<%s odes>' % model.name, 'exec')
        else:
            for arr_name in ('ydot', 'y', 'p'):
                macro = arr_name.upper() + '1'
                code_eqs = re.sub(r'\b%s\[(\d+)\]' % arr_name,
                                  '%s(\\1)' % macro, code_eqs)

        def rhs(t, y, p):
            ydot = self.ydot
            # note that the evaluated code sets ydot as a side effect
            if Solver._use_inline:
                inline(code_eqs, ['ydot', 't', 'y', 'p']);
            else:
                exec code_eqs_py in locals()
            return ydot

        # build integrator options list from our defaults and any kwargs passed to this function
        options = {}
        try:
            options.update(default_integrator_options[integrator])
        except KeyError as e:
            pass
        options.update(integrator_options)

        self.model = model
        self.tspan = tspan
        self.y = numpy.ndarray((len(tspan), len(model.species)))
        self.ydot = numpy.ndarray(len(model.species))
        if len(model.observables):
            self.yobs = numpy.ndarray(len(tspan), zip(model.observables.keys(),
                                                      itertools.repeat(float)))
        else:
            self.yobs = numpy.ndarray((len(tspan), 0))
        exprs = model.expressions_dynamic()
        if len(exprs):
            self.yexpr = numpy.ndarray(len(tspan), zip(exprs.keys(),
                                                       itertools.repeat(float)))
        else:
            self.yexpr = numpy.ndarray((len(tspan), 0))
        self.yobs_view = self.yobs.view(float).reshape(len(self.yobs), -1)
        self.integrator = ode(rhs).set_integrator(integrator, **options)

    def run(self, param_values=None, y0=None):
        """Perform an integration.

        Returns nothing; access the Solver object's ``y``, ``yobs``, or
        ``yobs_view`` attributes to retrieve the results.

        Parameters
        ----------
        param_values : vector-like, optional
            Values to use for every parameter in the model. Ordering is
            determined by the order of model.parameters. If not specified,
            parameter values will be taken directly from model.parameters.
        y0 : vector-like, optional
            Values to use for the initial condition of all species. Ordering is
            determined by the order of model.species. If not specified, initial
            conditions will be taken from model.initial_conditions (with initial
            condition parameter values taken from `param_values` if specified).

        """

        if param_values is not None:
            # accept vector of parameter values as an argument
            if len(param_values) != len(self.model.parameters):
                raise Exception("param_values must be the same length as model.parameters")
            if not isinstance(param_values, numpy.ndarray):
                param_values = numpy.array(param_values)
        else:
            # create parameter vector from the values in the model
            param_values = numpy.array([p.value for p in self.model.parameters])

        subs = dict((p, param_values[i])
                    for i, p in enumerate(self.model.parameters))
        if y0 is not None:
            # accept vector of species amounts as an argument
            if len(y0) != self.y.shape[1]:
                raise Exception("y0 must be the same length as model.species")
            if not isinstance(y0, numpy.ndarray):
                y0 = numpy.array(y0)
        else:
            y0 = numpy.zeros((self.y.shape[1],))
            for cp, value_obj in self.model.initial_conditions:
                if value_obj in self.model.parameters:
                    pi = self.model.parameters.index(value_obj)
                    value = param_values[pi]
                elif value_obj in self.model.expressions:
                    value = value_obj.expand_expr().evalf(subs=subs)
                else:
                    raise ValueError("Unexpected initial condition value type")
                si = self.model.get_species_index(cp)
                if si is None:
                    raise Exception("Species not found in model: %s" % repr(cp))
                y0[si] = value

        # perform the actual integration
        self.integrator.set_initial_value(y0, self.tspan[0])
        self.integrator.set_f_params(param_values)
        self.y[0] = y0
        i = 1
        while self.integrator.successful() and self.integrator.t < self.tspan[-1]:
            self.y[i] = self.integrator.integrate(self.tspan[i])
            i += 1
        if self.integrator.t < self.tspan[-1]:
            self.y[i:, :] = 'nan'

        for i, obs in enumerate(self.model.observables):
            self.yobs_view[:, i] = \
                (self.y[:, obs.species] * obs.coefficients).sum(1)
        obs_names = self.model.observables.keys()
        obs_dict = dict((k, self.yobs[k]) for k in obs_names)
        for expr in self.model.expressions_dynamic():
            expr_subs = expr.expand_expr().subs(subs)
            func = sympy.lambdify(obs_names, expr_subs)
            self.yexpr[expr.name] = func(**obs_dict)

def odesolve(model, tspan, param_values=None, y0=None, integrator='vode',
             **integrator_options):
    """Integrate a model's ODEs over a given timespan.

    This is a simple function-based interface to integrating (a.k.a. solving or
    simulating) a model. If you need to integrate a model repeatedly with
    different parameter values or initial conditions (as in parameter
    estimation), using the Solver class directly will provide much better
    performance.

    Parameters
    ----------
    model : pysb.Model
        Model to integrate.
    tspan : vector-like
        Time values over which to integrate. The first and last values define
        the time range, and the returned trajectories will be sampled at every
        value.
    param_values : vector-like, optional
        Values to use for every parameter in the model. Ordering is determined
        by the order of model.parameters. If not specified, parameter values
        will be taken directly from model.parameters.
    y0 : vector-like, optional
        Values to use for the initial condition of all species. Ordering is
        determined by the order of model.species. If not specified, initial
        conditions will be taken from model.initial_conditions (with initial
        condition parameter values taken from `param_values` if specified).
    integrator : string, optional
        Name of the integrator to use, taken from the list of integrators known
        to :py:class:`scipy.integrate.ode`.
    integrator_params
        Additional parameters for the integrator.

    Returns
    -------
    yfull : record array
        The trajectories calculated by the integration. The first dimension is
        time and its length is identical to that of `tspan`. The second
        dimension is species/observables and its length is the sum of the
        lengths of model.species and model.observables. The dtype of the array
        specifies field names: '__s0', '__s1', etc. for the species and
        observable names for the observables. See Notes below for further
        explanation and caveats.

    Notes
    -----
    This function was the first implementation of integration support and
    accordingly it has a few warts:

    * It performs expensive code generation every time it is called.

    * The returned array, with its record-style data-type, allows convenient
      selection of individual columns by their field names, but does not permit
      slice ranges or indexing by integers for columns. If you only need access
      to your model's observables this is usually not a problem, but sometimes
      it's more convenient to have a "regular" array. See Examples below for
      code to do this.

    The actual integration code has since been moved to the Solver class and
    split up such that the code generation is only performed on
    initialization. The model may then be integrated repeatedly with different
    parameter values or initial conditions with much better
    performance. Additionally, Solver makes the species trajectories available
    as a simple array and only uses the record array for the observables where
    it makes sense.

    This function now simply serves as a wrapper for creating a Solver object,
    calling its ``run`` method, and building the record array to return.

    Examples
    --------
    Simulate a model and display the results for an observable:

    >>> from pysb.examples.robertson import model
    >>> from numpy import linspace
    >>> numpy.set_printoptions(precision=4)
    >>> yfull = odesolve(model, linspace(0, 40, 10))
    >>> print yfull['A_total']   #doctest: +NORMALIZE_WHITESPACE
    [ 1.      0.899   0.8506  0.8179  0.793   0.7728  0.7557  0.7408  0.7277
    0.7158]

    Obtain a view on a returned record array which uses an atomic data-type and
    integer indexing (note that the view's data buffer is shared with the
    original array so there is no extra memory cost):

    >>> print yfull.shape
    (10,)
    >>> print yfull.dtype   #doctest: +NORMALIZE_WHITESPACE
    [('__s0', '<f8'), ('__s1', '<f8'), ('__s2', '<f8'), ('A_total', '<f8'),
    ('B_total', '<f8'), ('C_total', '<f8')]
    >>> print yfull[0:4, 1:3]
    Traceback (most recent call last):
      ...
    IndexError: too many indices
    >>> yarray = yfull.view(float).reshape(len(yfull), -1)
    >>> print yarray.shape
    (10, 6)
    >>> print yarray.dtype
    float64
    >>> print yarray[0:4, 1:3]
    [[  0.0000e+00   0.0000e+00]
     [  2.1672e-05   1.0093e-01]
     [  1.6980e-05   1.4943e-01]
     [  1.4502e-05   1.8209e-01]]

    """

    solver = Solver(model, tspan, integrator, **integrator_options)
    solver.run(param_values, y0)

    species_names = ['__s%d' % i for i in range(solver.y.shape[1])]
    yfull_dtype = zip(species_names, itertools.repeat(float))
    if len(solver.yobs.dtype):
        yfull_dtype += solver.yobs.dtype.descr
    yfull = numpy.ndarray(len(solver.y), yfull_dtype)

    yfull_view = yfull.view(float).reshape(len(yfull), -1)
    yfull_view[:, :solver.y.shape[1]] = solver.y
    yfull_view[:, solver.y.shape[1]:] = solver.yobs_view

    return yfull
<commit_msg>keep noisy distutils output from breaking doctests<commit_after>import pysb.core
import pysb.bng
import numpy
from scipy.integrate import ode
from scipy.weave import inline
import scipy.weave.build_tools
import distutils.errors
import sympy
import re
import itertools

# some sane default options for a few well-known integrators
default_integrator_options = {
    'vode': {
        'method': 'bdf',
        'with_jacobian': True,
        # Set nsteps as high as possible to give our users flexibility in
        # choosing their time step. (Let's be safe and assume vode was compiled
        # with 32-bit ints. What would actually happen if it was and we passed
        # 2**64-1 though?)
        'nsteps': 2**31 - 1,
        },
    'cvode': {
        'method': 'bdf',
        'iteration': 'newton',
        },
    }

class Solver(object):
    """An interface for numeric integration of models.

    Parameters
    ----------
    model : pysb.Model
        Model to integrate.
    tspan : vector-like
        Time values over which to integrate. The first and last values define
        the time range, and the returned trajectories will be sampled at every
        value.
    integrator : string, optional
        Name of the integrator to use, taken from the list of integrators known
        to :py:class:`scipy.integrate.ode`.
    integrator_params
        Additional parameters for the integrator.

    Attributes
    ----------
    model : pysb.Model
        Model passed to the constructor
    tspan : vector-like
        Time values passed to the constructor.
    y : numpy.ndarray
        Species trajectories. Dimensionality is ``(len(tspan),
        len(model.species))``.
    yobs : numpy.ndarray with record-style data-type
        Observable trajectories. Length is is ``len(tspan)`` and record names
        follow ``model.observables`` names.
    yobs_view : numpy.ndarray
        An array view (sharing the same data buffer) on ``yobs``. Dimensionality
        is ``(len(tspan), len(model.observables))``.
    integrator : scipy.integrate.ode
        Integrator object.

    Notes
    -----
    The expensive step of generating the code for the right-hand side of the
    model's ODEs is performed during initialization. If you need to integrate
    the same model repeatedly with different parameters then you should build a
    single Solver object and then call its ``run`` method as needed.

    """

    @staticmethod
    def _test_inline():
        """Detect whether scipy.weave.inline is functional."""
        if not hasattr(Solver, '_use_inline'):
            Solver._use_inline = False
            try:
                inline('int i;', force=1)
                Solver._use_inline = True
            except (scipy.weave.build_tools.CompileError,
                    distutils.errors.CompileError, ImportError):
                pass

    def __init__(self, model, tspan, integrator='vode', **integrator_options):

        pysb.bng.generate_equations(model)

        code_eqs = '\n'.join(['ydot[%d] = %s;' % (i, sympy.ccode(model.odes[i])) for i in range(len(model.odes))])
        code_eqs = re.sub(r's(\d+)', lambda m: 'y[%s]' % (int(m.group(1))), code_eqs)
        for e in model.expressions:
            code_eqs = re.sub(r'\b(%s)\b' % e.name,
                              sympy.ccode(e.expand_expr()), code_eqs)
        for i, p in enumerate(model.parameters):
            code_eqs = re.sub(r'\b(%s)\b' % p.name, 'p[%d]' % i, code_eqs)

        Solver._test_inline()
        # If we can't use weave.inline to run the C code, compile it as Python code instead for use with
        # exec. Note: C code with array indexing, basic math operations, and pow() just happens to also
        # be valid Python.  If the equations ever have more complex things in them, this might fail.
        if not Solver._use_inline:
            code_eqs_py = compile(code_eqs, '<%s odes>' % model.name, 'exec')
        else:
            for arr_name in ('ydot', 'y', 'p'):
                macro = arr_name.upper() + '1'
                code_eqs = re.sub(r'\b%s\[(\d+)\]' % arr_name,
                                  '%s(\\1)' % macro, code_eqs)

        def rhs(t, y, p):
            ydot = self.ydot
            # note that the evaluated code sets ydot as a side effect
            if Solver._use_inline:
                inline(code_eqs, ['ydot', 't', 'y', 'p']);
            else:
                exec code_eqs_py in locals()
            return ydot

        # build integrator options list from our defaults and any kwargs passed to this function
        options = {}
        try:
            options.update(default_integrator_options[integrator])
        except KeyError as e:
            pass
        options.update(integrator_options)

        self.model = model
        self.tspan = tspan
        self.y = numpy.ndarray((len(tspan), len(model.species)))
        self.ydot = numpy.ndarray(len(model.species))
        if len(model.observables):
            self.yobs = numpy.ndarray(len(tspan), zip(model.observables.keys(),
                                                      itertools.repeat(float)))
        else:
            self.yobs = numpy.ndarray((len(tspan), 0))
        exprs = model.expressions_dynamic()
        if len(exprs):
            self.yexpr = numpy.ndarray(len(tspan), zip(exprs.keys(),
                                                       itertools.repeat(float)))
        else:
            self.yexpr = numpy.ndarray((len(tspan), 0))
        self.yobs_view = self.yobs.view(float).reshape(len(self.yobs), -1)
        self.integrator = ode(rhs).set_integrator(integrator, **options)

    def run(self, param_values=None, y0=None):
        """Perform an integration.

        Returns nothing; access the Solver object's ``y``, ``yobs``, or
        ``yobs_view`` attributes to retrieve the results.

        Parameters
        ----------
        param_values : vector-like, optional
            Values to use for every parameter in the model. Ordering is
            determined by the order of model.parameters. If not specified,
            parameter values will be taken directly from model.parameters.
        y0 : vector-like, optional
            Values to use for the initial condition of all species. Ordering is
            determined by the order of model.species. If not specified, initial
            conditions will be taken from model.initial_conditions (with initial
            condition parameter values taken from `param_values` if specified).

        """

        if param_values is not None:
            # accept vector of parameter values as an argument
            if len(param_values) != len(self.model.parameters):
                raise Exception("param_values must be the same length as model.parameters")
            if not isinstance(param_values, numpy.ndarray):
                param_values = numpy.array(param_values)
        else:
            # create parameter vector from the values in the model
            param_values = numpy.array([p.value for p in self.model.parameters])

        subs = dict((p, param_values[i])
                    for i, p in enumerate(self.model.parameters))
        if y0 is not None:
            # accept vector of species amounts as an argument
            if len(y0) != self.y.shape[1]:
                raise Exception("y0 must be the same length as model.species")
            if not isinstance(y0, numpy.ndarray):
                y0 = numpy.array(y0)
        else:
            y0 = numpy.zeros((self.y.shape[1],))
            for cp, value_obj in self.model.initial_conditions:
                if value_obj in self.model.parameters:
                    pi = self.model.parameters.index(value_obj)
                    value = param_values[pi]
                elif value_obj in self.model.expressions:
                    value = value_obj.expand_expr().evalf(subs=subs)
                else:
                    raise ValueError("Unexpected initial condition value type")
                si = self.model.get_species_index(cp)
                if si is None:
                    raise Exception("Species not found in model: %s" % repr(cp))
                y0[si] = value

        # perform the actual integration
        self.integrator.set_initial_value(y0, self.tspan[0])
        self.integrator.set_f_params(param_values)
        self.y[0] = y0
        i = 1
        while self.integrator.successful() and self.integrator.t < self.tspan[-1]:
            self.y[i] = self.integrator.integrate(self.tspan[i])
            i += 1
        if self.integrator.t < self.tspan[-1]:
            self.y[i:, :] = 'nan'

        for i, obs in enumerate(self.model.observables):
            self.yobs_view[:, i] = \
                (self.y[:, obs.species] * obs.coefficients).sum(1)
        obs_names = self.model.observables.keys()
        obs_dict = dict((k, self.yobs[k]) for k in obs_names)
        for expr in self.model.expressions_dynamic():
            expr_subs = expr.expand_expr().subs(subs)
            func = sympy.lambdify(obs_names, expr_subs)
            self.yexpr[expr.name] = func(**obs_dict)

def odesolve(model, tspan, param_values=None, y0=None, integrator='vode',
             **integrator_options):
    """Integrate a model's ODEs over a given timespan.

    This is a simple function-based interface to integrating (a.k.a. solving or
    simulating) a model. If you need to integrate a model repeatedly with
    different parameter values or initial conditions (as in parameter
    estimation), using the Solver class directly will provide much better
    performance.

    Parameters
    ----------
    model : pysb.Model
        Model to integrate.
    tspan : vector-like
        Time values over which to integrate. The first and last values define
        the time range, and the returned trajectories will be sampled at every
        value.
    param_values : vector-like, optional
        Values to use for every parameter in the model. Ordering is determined
        by the order of model.parameters. If not specified, parameter values
        will be taken directly from model.parameters.
    y0 : vector-like, optional
        Values to use for the initial condition of all species. Ordering is
        determined by the order of model.species. If not specified, initial
        conditions will be taken from model.initial_conditions (with initial
        condition parameter values taken from `param_values` if specified).
    integrator : string, optional
        Name of the integrator to use, taken from the list of integrators known
        to :py:class:`scipy.integrate.ode`.
    integrator_params
        Additional parameters for the integrator.

    Returns
    -------
    yfull : record array
        The trajectories calculated by the integration. The first dimension is
        time and its length is identical to that of `tspan`. The second
        dimension is species/observables and its length is the sum of the
        lengths of model.species and model.observables. The dtype of the array
        specifies field names: '__s0', '__s1', etc. for the species and
        observable names for the observables. See Notes below for further
        explanation and caveats.

    Notes
    -----
    This function was the first implementation of integration support and
    accordingly it has a few warts:

    * It performs expensive code generation every time it is called.

    * The returned array, with its record-style data-type, allows convenient
      selection of individual columns by their field names, but does not permit
      slice ranges or indexing by integers for columns. If you only need access
      to your model's observables this is usually not a problem, but sometimes
      it's more convenient to have a "regular" array. See Examples below for
      code to do this.

    The actual integration code has since been moved to the Solver class and
    split up such that the code generation is only performed on
    initialization. The model may then be integrated repeatedly with different
    parameter values or initial conditions with much better
    performance. Additionally, Solver makes the species trajectories available
    as a simple array and only uses the record array for the observables where
    it makes sense.

    This function now simply serves as a wrapper for creating a Solver object,
    calling its ``run`` method, and building the record array to return.

    Examples
    --------
    Simulate a model and display the results for an observable:

    >>> from pysb.examples.robertson import model
    >>> from numpy import linspace
    >>> numpy.set_printoptions(precision=4)
    >>> yfull = odesolve(model, linspace(0, 40, 10))
    >>> print yfull['A_total']   #doctest: +NORMALIZE_WHITESPACE
    [ 1.      0.899   0.8506  0.8179  0.793   0.7728  0.7557  0.7408  0.7277
    0.7158]

    Obtain a view on a returned record array which uses an atomic data-type and
    integer indexing (note that the view's data buffer is shared with the
    original array so there is no extra memory cost):

    >>> print yfull.shape
    (10,)
    >>> print yfull.dtype   #doctest: +NORMALIZE_WHITESPACE
    [('__s0', '<f8'), ('__s1', '<f8'), ('__s2', '<f8'), ('A_total', '<f8'),
    ('B_total', '<f8'), ('C_total', '<f8')]
    >>> print yfull[0:4, 1:3]
    Traceback (most recent call last):
      ...
    IndexError: too many indices
    >>> yarray = yfull.view(float).reshape(len(yfull), -1)
    >>> print yarray.shape
    (10, 6)
    >>> print yarray.dtype
    float64
    >>> print yarray[0:4, 1:3]
    [[  0.0000e+00   0.0000e+00]
     [  2.1672e-05   1.0093e-01]
     [  1.6980e-05   1.4943e-01]
     [  1.4502e-05   1.8209e-01]]

    """

    solver = Solver(model, tspan, integrator, **integrator_options)
    solver.run(param_values, y0)

    species_names = ['__s%d' % i for i in range(solver.y.shape[1])]
    yfull_dtype = zip(species_names, itertools.repeat(float))
    if len(solver.yobs.dtype):
        yfull_dtype += solver.yobs.dtype.descr
    yfull = numpy.ndarray(len(solver.y), yfull_dtype)

    yfull_view = yfull.view(float).reshape(len(yfull), -1)
    yfull_view[:, :solver.y.shape[1]] = solver.y
    yfull_view[:, solver.y.shape[1]:] = solver.yobs_view

    return yfull

def setup_module(module):
    """Doctest fixture for nose."""
    # Distutils' temp directory creation code has a more-or-less unsuppressable
    # print to stdout which will break the doctest which triggers it (via
    # scipy.weave.inline). So here we run an end-to-end test of the inlining
    # system to get that print out of the way at a point where it won't matter.
    # As a bonus, the test harness is suppressing writes to stdout at this time
    # anyway so the message is just swallowed silently.
    Solver._test_inline()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
#   cubefile.py
#   Copyright 2011 William Trevor Olson <trevor@heytrevor.com>
#
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import numpy as np
from .labels import parse_file_label
from .specialpixels import SPECIAL_PIXELS

class CubeFile(object):
    """ A Isis Cube file reader.

    Attributes:
        filename: The filename if given, otherwise none.
        label: The parsed label header in dictionary form.
        data: A numpy array representing the image data.
        multiplier: A multiplicative factor by which to scale pixel DN.
        base: An additive factor by which to offset pixel DN.
    """

    PIXEL_TYPES = {
        'UnsignedByte': np.dtype('uint8'),
        'SignedByte': np.dtype('int8'),
        'UnsignedWord': np.dtype('uint16'),
        'SignedWord': np.dtype('int16'),
        'UnsignedInteger': np.dtype('uint32'),
        'SignedInteger': np.dtype('int32'),
        'Real': np.dtype('float32'),
        'Double': np.dtype('float64')
    }

    BYTE_ORDERS = {
        'NoByteOrder': '=',
        'Lsb': '<', # little-endian
        'Msb': '>' # big-endian
    }

    SPECIAL_PIXELS = SPECIAL_PIXELS

    @staticmethod
    def open(stream, filename=None):
        """ Open an Isis Cube file.

        Arguments:
            stream: The file name or file object to read as an isis file.
            filename: If stream is a file object, an optional filename to attach
                to the object.

        Returns:
            A new isis cube object from the specified file.
        """
        if isinstance(stream, basestring):
            with open(stream) as f:
                return CubeFile()._open(f, stream)

        return CubeFile()._open(stream, filename)

    def apply_scaling(self, copy=True):
        """ Scale pixel values to there true DN.

        Arguments:
            copy: Whether to apply the scalling to a copy of the pixel data
                and leave the orginial unaffected.

        Returns:
            A scalled version of the pixel data.
        """
        if copy:
            return self.multiplier * self.data + self.base

        if self.multiplier != 1:
            self.data *= self.multiplier

        if self.base != 0:
            self.data += self.base

        return self.data

    def specials_mask(self):
        """ Create a pixel map for special pixels.

        Returns:
            An array where the value is False if the pixel is special and True
            otherwise.
        """
        mask  = self.data >= self.specials['Min']
        mask &= self.data <= self.specials['Max']
        return mask

    def get_image_array(self):
        """ Create an array for use in making an image.

        Creates a linear stretch of the image and scales it to between 0 and
        255. Null, Lis and Lrs pixels are set to 0. His and Hrs pixels are set
        to 255.

        Usage:
            from pysis import CubeFile
            from PIL import image

            # Read in the image and create the image data
            im = CubeFile.open('test.cub')
            data = im.get_image_array()

            # Save the first band to a new file
            Image.fromarray(data[0]).save('test.png')

        Returns:
            A uint8 array of pixel values.
        """
        specials_mask = self.specials_mask()
        data          = self.data.copy()

        data[specials_mask] -= data[specials_mask].min()
        data[specials_mask] *= 255 / data[specials_mask].max()

        data[data == self.specials['His']] = 255
        data[data == self.specials['Hrs']] = 255

        return data.astype(np.uint8)

    def _open(self, stream, filename):
        self.filename = filename
        self.label    = parse_file_label(stream)

        (self.bands, self.lines, self.samples)   = self._get_dims()
        (self.dtype, self.specials, self.base, self.multiplier) = self._get_pixel_info()

        start = self._get_data_start()
        stream.seek(start)

        self.format = self.label['IsisCube']['Core']['Format']
        if self.format == 'BandSequential':
            self.data = self._read_bs_data(stream)

        elif self.format == 'Tile':
            self.data = self._read_tile_data(stream)

        else:
            raise Excption('Unkown Isis Cube format (%s)' % self.format)

        return self

    def _get_dims(self):
        dims = self.label['IsisCube']['Core']['Dimensions']
        return dims['Bands'], dims['Lines'], dims['Samples']

    def _get_pixel_info(self):
        pixels_group = self.label['IsisCube']['Core']['Pixels']

        byte_order   = self.BYTE_ORDERS[pixels_group['ByteOrder']]
        pixel_type   = self.PIXEL_TYPES[pixels_group['Type']]
        specials     = self.SPECIAL_PIXELS[pixels_group['Type']]
        dtype        = pixel_type.newbyteorder(byte_order)

        return dtype, specials, pixels_group['Base'], pixels_group['Multiplier']

    def _get_data_start(self):
        return self.label['IsisCube']['Core']['StartByte'] - 1

    def _read_bs_data(self, stream):
        size = self.bands * self.lines * self.samples
        data = np.fromfile(stream, self.dtype, size)
        return data.reshape((self.bands, self.lines, self.samples))

    def _read_tile_data(self, stream):
        tile_lines   = self.label['IsisCube']['Core']['TileLines']
        tile_samples = self.label['IsisCube']['Core']['TileSamples']
        tile_size    = tile_samples * tile_lines

        samples = xrange(0, self.samples, tile_samples)
        lines   = xrange(0, self.lines, tile_lines)

        data = np.empty((self.bands, self.lines, self.samples),
                        dtype=self.dtype)

        for band in data:
            for line in lines:
                for sample in samples:
                    sample_end = sample + tile_samples
                    line_end   = line   + tile_lines
                    chunk      = band[line:line_end, sample:sample_end]

                    tile = np.fromfile(stream, self.dtype, tile_size)
                    tile = tile.reshape((tile_samples, tile_lines))

                    chunk_lines, chunk_samples = chunk.shape
                    chunk[:] = tile[:chunk_lines, :chunk_samples]

        return data
<commit_msg>Broke label parsing into individual properties.<commit_after># -*- coding: utf-8 -*-
#
#   cubefile.py
#   Copyright 2011 William Trevor Olson <trevor@heytrevor.com>
#
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import numpy as np
from .labels import parse_file_label
from .specialpixels import SPECIAL_PIXELS

class CubeFile(object):
    """ A Isis Cube file reader.

    Attributes:
        filename: The filename if given, otherwise none.
        label: The parsed label header in dictionary form.
        data: A numpy array representing the image data.
        multiplier: A multiplicative factor by which to scale pixel DN.
        base: An additive factor by which to offset pixel DN.
    """

    PIXEL_TYPES = {
        'UnsignedByte': np.dtype('uint8'),
        'SignedByte': np.dtype('int8'),
        'UnsignedWord': np.dtype('uint16'),
        'SignedWord': np.dtype('int16'),
        'UnsignedInteger': np.dtype('uint32'),
        'SignedInteger': np.dtype('int32'),
        'Real': np.dtype('float32'),
        'Double': np.dtype('float64')
    }

    BYTE_ORDERS = {
        'NoByteOrder': '=',
        'Lsb': '<', # little-endian
        'Msb': '>' # big-endian
    }

    SPECIAL_PIXELS = SPECIAL_PIXELS

    @staticmethod
    def open(stream, filename=None):
        """ Open an Isis Cube file.

        Arguments:
            stream: The file name or file object to read as an isis file.
            filename: If stream is a file object, an optional filename to attach
                to the object.

        Returns:
            A new isis cube object from the specified file.
        """
        return CubeFile(stream, filename)

    def apply_scaling(self, copy=True):
        """ Scale pixel values to there true DN.

        Arguments:
            copy: Whether to apply the scalling to a copy of the pixel data
                and leave the orginial unaffected.

        Returns:
            A scalled version of the pixel data.
        """
        if copy:
            return self.multiplier * self.data + self.base

        if self.multiplier != 1:
            self.data *= self.multiplier

        if self.base != 0:
            self.data += self.base

        return self.data

    def specials_mask(self):
        """ Create a pixel map for special pixels.

        Returns:
            An array where the value is False if the pixel is special and True
            otherwise.
        """
        mask  = self.data >= self.specials['Min']
        mask &= self.data <= self.specials['Max']
        return mask

    def get_image_array(self):
        """ Create an array for use in making an image.

        Creates a linear stretch of the image and scales it to between 0 and
        255. Null, Lis and Lrs pixels are set to 0. His and Hrs pixels are set
        to 255.

        Usage:
            from pysis import CubeFile
            from PIL import image

            # Read in the image and create the image data
            im = CubeFile.open('test.cub')
            data = im.get_image_array()

            # Save the first band to a new file
            Image.fromarray(data[0]).save('test.png')

        Returns:
            A uint8 array of pixel values.
        """
        specials_mask = self.specials_mask()
        data          = self.data.copy()

        data[specials_mask] -= data[specials_mask].min()
        data[specials_mask] *= 255 / data[specials_mask].max()

        data[data == self.specials['His']] = 255
        data[data == self.specials['Hrs']] = 255

        return data.astype(np.uint8)

    @property
    def bands(self):
        return self.label['IsisCube']['Core']['Dimensions']['Bands']

    @property
    def lines(self):
        return self.label['IsisCube']['Core']['Dimensions']['Lines']

    @property
    def samples(self):
        return self.label['IsisCube']['Core']['Dimensions']['Samples']

    @property
    def tile_lines(self):
        if self.format != 'Tile':
            return None

        return self.label['IsisCube']['Core']['TileLines']

    @property
    def tile_samples(self):
        if self.format != 'Tile':
            return None

        return self.label['IsisCube']['Core']['TileSamples']

    @property
    def format(self):
        return self.label['IsisCube']['Core']['Format']

    @property
    def dtype(self):
        pixels_group = self.label['IsisCube']['Core']['Pixels']
        byte_order   = self.BYTE_ORDERS[pixels_group['ByteOrder']]
        pixel_type   = self.PIXEL_TYPES[pixels_group['Type']]
        return pixel_type.newbyteorder(byte_order)

    @property
    def specials(self):
        pixel_type = self.label['IsisCube']['Core']['Pixels']['Type']
        return self.SPECIAL_PIXELS[pixel_type]

    @property
    def base(self):
        pixel_type = self.label['IsisCube']['Core']['Pixels']['Base']
        return self.SPECIAL_PIXELS[pixel_type]

    @property
    def multiplier(self):
        pixel_type = self.label['IsisCube']['Core']['Pixels']['Multiplier']
        return self.SPECIAL_PIXELS[pixel_type]

    @property
    def start_byte(self):
        return self.label['IsisCube']['Core']['StartByte'] - 1

    @property
    def shape(self):
        return (self.bands, self.lines, self.samples)

    @property
    def size(self):
        return self.bands * self.lines * self.samples

    def __init__(self, stream, filename=None):
        """ Create an Isis Cube file.

        Arguments:
            stream: The file name or file object to read as an isis file.
            filename: If stream is a file object, an optional filename to attach
                to the object.
        """
        if isinstance(stream, basestring):
            self.filename = stream
            with open(self.filename) as f:
                self._open(f)

        else:
            self.filename = filename
            self._open(stream)

    def _open(self, stream):
        self.label = parse_file_label(stream)
        stream.seek(self.start_byte)

        if self.format == 'BandSequential':
            self.data = self._get_bs_data(stream)

        elif self.format == 'Tile':
            self.data = self._get_tile_data(stream)

        else:
            raise Exception('Unkown Isis Cube format (%s)' % self.format)

    def _get_bs_data(self, stream):
        data = np.fromfile(stream, self.dtype, self.size)
        return data.reshape(self.shape)

    def _get_tile_data(self, stream):
        tile_lines   = self.tile_lines
        tile_samples = self.tile_samples
        tile_size    = tile_lines * tile_samples

        lines     = xrange(0, self.lines, self.tile_lines)
        samples   = xrange(0, self.samples, self.tile_samples)

        dtype = self.dtype
        data  = np.empty(self.shape, dtype=dtype)

        for band in data:
            for line in lines:
                for sample in samples:
                    sample_end = sample + tile_samples
                    line_end   = line   + tile_lines
                    chunk      = band[line:line_end, sample:sample_end]

                    tile = np.fromfile(stream, dtype, tile_size)
                    tile = tile.reshape((tile_samples, tile_lines))

                    chunk_lines, chunk_samples = chunk.shape
                    chunk[:] = tile[:chunk_lines, :chunk_samples]

        return data
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Author: Teemu Henrikki Pitkänen <teemu.pitkanen@helsinki.fi>
# University of Helsinki / Helsinki Institute for Information Technology 2016
#

import mrptlib

class MRPTIndex(object):
    """
    Wraps the extension module written in C++ and ensures that the arguments are given as lists, not ndarrays.
    """
    def __init__(self, X, n0, n_trees):
        self.index = mrptlib.MrptIndex(X.tolist(), n0, n_trees)

    def ann(self, q, k, n_extra_branches=0, votes_required=0):
        if votes_required == 0:
            return self.index.old_ann(q.tolist(), k)
        return self.index.ann(q.tolist(), k, votes_required, n_extra_branches)
<commit_msg>Removed cppmrpt.py duplicate<commit_after><|endoftext|>