<commit_before>
import sys
import urllib2
import urllib
import json
import operator
import pprint
import time
from StringIO import StringIO

import numpy as np
import smap.util as util
import tscache
from smap.core import SmapException

from twisted.internet import reactor
from twisted.internet.protocol import ReconnectingClientFactory
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.protocols.basic import LineReceiver
from twisted.web.client import Agent
from twisted.web.http_headers import Headers
from twisted.python import log

try:
    from twisted.web.client import FileBodyProducer
except ImportError:
    from smap.contrib.client import FileBodyProducer

try:
    from smap.iface.http.httpcurl import get
except ImportError:
    print >>sys.stderr, """Warning -- can't find httpcurl
  --> Falling back on urllib2
  --> Install pycURL for parallel data downloads"""
    from smap.iface.http.httputils import get

class SmapClient:
    """Blocking client class for the archiver API.
    """
    def __init__(self, base='http://new.openbms.org/backend',
                 key=None, private=False, timeout=5.0):
        self.base = base
        self.timeout = timeout
        self.key = key
        self.private = private

    def _build_qdict(self):
        """return a query dict to be passed with all requests"""
        rv = {}
        if self.key:
            rv['key'] = self.key
        if self.private:
            rv['private'] = ''
        return rv

    def query(self, q):
        """Send a query using the ARD query language to the server, and return
the result.

:param str q: the query
:return: the parsed JSON object returned by the server
"""
        try:
            fp = urllib2.urlopen(self.base + '/api/query?' +
                                 urllib.urlencode(self._build_qdict()),
                                 data=q,
                                 timeout=self.timeout)
            rv = json.load(fp)
        except urllib2.HTTPError:
            log.err("Bad request running query: ""%s"" " % q)
            raise SmapException()
        fp.close()
        return rv

    def tags(self, qbody, tags='*', nest=False):
        """Look up tags associated with a specific query body"""
        tags = self.query('select %s where %s' % (tags, qbody))
        if not nest:
            return map(lambda t: dict(util.buildkv('', t)), tags)
        else:
            return tags

    def _data_uuid(self, uuid, start, end, cache):
        """Construct a list of urls we need to load for a single uuid"""
        if cache:
            cache = tscache.TimeseriesCache(uuid)
            cached_data = cache.read(0, start, end)
            cache.close()
        else:
            cached_data = []

        cached_data = [((0, start), None)] + \
            cached_data + \
            [((end, 0), None)]
        load_list = []
        for idx in range(0, len(cached_data) - 1):
            fetch_start, fetch_end = cached_data[idx][0][1], cached_data[idx+1][0][0]
            load_list.append([(fetch_start, fetch_end)])

        return load_list, map(operator.itemgetter(1), cached_data[1:-1])

    def data_uuid(self, uuids, start, end, cache=True):
        """Load a time range of data for a list of uuids

Attempts to use cached data and load missing data in parallel.

:param list uuids: a list of stringified UUIDs
:param int start: the timestamp of the first record in seconds, inclusive
:param int end: the timestamp of the last record, exclusive
:return: a list of data vectors.  Each element is
  :py:class:`numpy.array` of data in the same order as the input list of
  uuids
        """
        qdict = self._build_qdict()
        qdict['limit'] = -1
        data, urls = {}, []
        start, end = start * 1000, end * 1000
        now = int((time.time() - 300) * 1000)

        # construct a list of all holes in the cache
        for u in uuids:
            data[u] = self._data_uuid(u, start, end, cache)

            # these are the regions of missing data
            for region in data[u][0]:
                qdict['starttime'] = str(region[0][0])
                qdict['endtime'] = str(region[0][1])
                dlurl = str(self.base + '/api/data/uuid/' + u + '?' +
                            urllib.urlencode(qdict))
                if qdict['starttime'] != qdict['endtime']:
                    region.append(dlurl)
                    urls.append(dlurl)
                else:
                    region.append(None)

        # load all of the missing chunks in parallel
        newdata = dict(get(urls))

        # insert all the new data and return the result
        rv = []
        for u in uuids:
            loaddata = []
            for range, url in data[u][0]:
                if url != None and len(newdata[url][0]['Readings']) > 0:
                    assert newdata[url][0]['uuid'] == u
                    loaddata.append(np.array(newdata[url][0]['Readings']))
                    print "downloaded", len(loaddata[-1])

                    if cache and range[0] < now:
                        c = tscache.TimeseriesCache(u)
                        c.insert(0, range[0], range[1] if range[1] < now else now,
                                 loaddata[-1][np.nonzero(loaddata[-1][:,0] < now)])
                        c.close()
                else:
                    v = np.array([])
                    v.shape = (0, 2)
                    loaddata.append(v)

            def interleave(x, y):
                lst = [None] * (len(x) * 2 - 1)
                for idx in xrange(0, len(x)):
                    lst[idx * 2] = x[idx]
                for idx in xrange(0, len(y)):
                    lst[idx * 2 + 1] = y[idx]
                return lst
            rv.append(np.vstack(interleave(loaddata, data[u][1])))

        return rv

    def _data(self, selector, where, limit, streamlimit):
        qbody = "select data %s limit %i streamlimit %i where %s" % \
            (selector, limit, streamlimit, where)
        return self.query(qbody)

    def data(self, qbody, start, end):
        """Load data for streams matching a particular query

        Uses the local time-series cache in the .cache directory.
        """
        uids = self.query('select distinct uuid where %s' % qbody)
        data = self.data_uuid(uids, start, end)
        return uids, data

    def prev(self, qbody, ref, limit=1, streamlimit=10):
        """Load data before a reference time.

        :param str qbody: a selector identifying the streams to query
        :param int ref: reference timestamp
        :param int limit: the maximum number of points to retrieve per stream
        :param int streamlimit: the maximum number of streams to query
        """
        return self._data("before %i" % (ref * 1000), where, limit, streamlimit)

    def next(self, where, ref, limit=1, streamlimit=10):
        """Load data after a reference time.

        See `prev` for args.
        """
        return self._data("after %i" % (ref * 1000), where, limit, streamlimit)

    def latest(self, qbody, limit=1, streamlimit=10):
        """Load the last data in a time-series.

        See `prev` for args.
        """
        return self._data("before 4294967295000", where, limit, streamlimit)

class RepublishClient:
    """Listener for streaming data from a sMAP source or archiver's /republish feed
    """
    def __init__(self, url, datacb, reconnect=True, restrict=None):
        """
:param str url: url of the source
:param datacb: callable to be called with each new sMAP object
:param bool reconnect: weather to reconnect if the socket connection is dropped.
:param str restrict: "where" clause restricting data to be delivered.
    This is only evaluated once, when connecting.
        """
        self.url = url
        self.datacb = datacb
        self.agent = Agent(reactor)
        self.reconnect = reconnect
        self.failcount = 0
        self.restrict = restrict

    class DataReceiver(LineReceiver):
        """Make our own LineReceiver to read back the streaming data
        from the server.  Use the right delimiter and make sure we can
        handle big objects."""
        MAX_LENGTH = 1e7
        delimiter = '\n\n'

        def __init__(self, client):
            self.client = client

        def lineReceived(self, line):
            self.failcount = 0
            try:
                obj = util.json_decode(line)
                self.client.datacb(obj)
            except:
                log.err()
                print line

        def connectionLost(self, reason):
            self.client._failed()
            self.client._reconnect()

    def _failed(self):
        if self.failcount < 5:
            self.failcount += 1

    def __request(self, response):
        receiver = RepublishClient.DataReceiver(self)
        receiver.setLineMode()
        response.deliverBody(receiver)

    def _reconnect(self):
        """Exponential backup on the reconnect policy"""
        if self.reconnect:
            print "connection failed, reconnecting in", (self.failcount ** 2)
            reactor.callLater(self.failcount ** 2, self.connect)

    def _connect_failed(self, reason):
        self._failed()
        self._reconnect()

    def connect(self):
        """Subscribe and start receiving data
        """
        if not self.restrict:
            d = self.agent.request('GET',
                                   self.url + '/republish',
                                   Headers(),
                                   None)
        else:
            d = self.agent.request('POST',
                                   self.url + '/republish',
                                   Headers(),
                                   FileBodyProducer(StringIO(str(self.restrict))))
        d.addCallback(self.__request)
        d.addErrback(self._connect_failed)

# if __name__ == '__main__':
#     def cb(line):
#         print line
#         pass

#     c = RepublishClient('http://smote.cs.berkeley.edu:8079', cb,
#                         restrict="Metadata/SourceName ~ '^410'")
#     c.connect()
#     reactor.run()

if __name__ == '__main__':
    import time
    # c = SmapClient('http://new.openbms.org/backend')
    # c = SmapClient('http://local.cs.berkeley.edu:8079')
    c = SmapClient('http://localhost:8079')
#     print c.tags('Metadata/SourceName = "410 Labjacks"')
#     print c.latest('Metadata/SourceName = "Cory Hall Dent Meters"', limit=1)
    #c._data_uuid('018eba5e-51b6-5a8d-920f-b5a831546610',
    #             int(time.time()) - 3600 * 24, int(time.time()))
    data = c.data_uuid(['018eba5e-51b6-5a8d-920f-b5a831546610'],
                       int(time.time()) - 3600 * 300, int(time.time()))

    # print data
    print len(np.unique(data[0][:,0])), len(data[0][:,0])
<commit_msg>fix latest<commit_after>
import sys
import urllib2
import urllib
import json
import operator
import pprint
import time
from StringIO import StringIO

import numpy as np
import smap.util as util
import tscache
from smap.core import SmapException

from twisted.internet import reactor
from twisted.internet.protocol import ReconnectingClientFactory
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.protocols.basic import LineReceiver
from twisted.web.client import Agent
from twisted.web.http_headers import Headers
from twisted.python import log

try:
    from twisted.web.client import FileBodyProducer
except ImportError:
    from smap.contrib.client import FileBodyProducer

try:
    from smap.iface.http.httpcurl import get
except ImportError:
    print >>sys.stderr, """Warning -- can't find httpcurl
  --> Falling back on urllib2
  --> Install pycURL for parallel data downloads"""
    from smap.iface.http.httputils import get

class SmapClient:
    """Blocking client class for the archiver API.
    """
    def __init__(self, base='http://new.openbms.org/backend',
                 key=None, private=False, timeout=5.0):
        self.base = base
        self.timeout = timeout
        self.key = key
        self.private = private

    def _build_qdict(self):
        """return a query dict to be passed with all requests"""
        rv = {}
        if self.key:
            rv['key'] = self.key
        if self.private:
            rv['private'] = ''
        return rv

    def query(self, q):
        """Send a query using the ARD query language to the server, and return
the result.

:param str q: the query
:return: the parsed JSON object returned by the server
"""
        try:
            fp = urllib2.urlopen(self.base + '/api/query?' +
                                 urllib.urlencode(self._build_qdict()),
                                 data=q,
                                 timeout=self.timeout)
            rv = json.load(fp)
        except urllib2.HTTPError:
            log.err("Bad request running query: ""%s"" " % q)
            raise SmapException()
        fp.close()
        return rv

    def tags(self, qbody, tags='*', nest=False):
        """Look up tags associated with a specific query body"""
        tags = self.query('select %s where %s' % (tags, qbody))
        if not nest:
            return map(lambda t: dict(util.buildkv('', t)), tags)
        else:
            return tags

    def _data_uuid(self, uuid, start, end, cache):
        """Construct a list of urls we need to load for a single uuid"""
        if cache:
            cache = tscache.TimeseriesCache(uuid)
            cached_data = cache.read(0, start, end)
            cache.close()
        else:
            cached_data = []

        cached_data = [((0, start), None)] + \
            cached_data + \
            [((end, 0), None)]
        load_list = []
        for idx in range(0, len(cached_data) - 1):
            fetch_start, fetch_end = cached_data[idx][0][1], cached_data[idx+1][0][0]
            load_list.append([(fetch_start, fetch_end)])

        return load_list, map(operator.itemgetter(1), cached_data[1:-1])

    def data_uuid(self, uuids, start, end, cache=True):
        """Load a time range of data for a list of uuids

Attempts to use cached data and load missing data in parallel.

:param list uuids: a list of stringified UUIDs
:param int start: the timestamp of the first record in seconds, inclusive
:param int end: the timestamp of the last record, exclusive
:return: a list of data vectors.  Each element is
  :py:class:`numpy.array` of data in the same order as the input list of
  uuids
        """
        qdict = self._build_qdict()
        qdict['limit'] = -1
        data, urls = {}, []
        start, end = start * 1000, end * 1000
        now = int((time.time() - 300) * 1000)

        # construct a list of all holes in the cache
        for u in uuids:
            data[u] = self._data_uuid(u, start, end, cache)

            # these are the regions of missing data
            for region in data[u][0]:
                qdict['starttime'] = str(region[0][0])
                qdict['endtime'] = str(region[0][1])
                dlurl = str(self.base + '/api/data/uuid/' + u + '?' +
                            urllib.urlencode(qdict))
                if qdict['starttime'] != qdict['endtime']:
                    region.append(dlurl)
                    urls.append(dlurl)
                else:
                    region.append(None)

        # load all of the missing chunks in parallel
        newdata = dict(get(urls))

        # insert all the new data and return the result
        rv = []
        for u in uuids:
            loaddata = []
            for range, url in data[u][0]:
                if url != None and len(newdata[url][0]['Readings']) > 0:
                    assert newdata[url][0]['uuid'] == u
                    loaddata.append(np.array(newdata[url][0]['Readings']))
                    print "downloaded", len(loaddata[-1])

                    if cache and range[0] < now:
                        c = tscache.TimeseriesCache(u)
                        c.insert(0, range[0], range[1] if range[1] < now else now,
                                 loaddata[-1][np.nonzero(loaddata[-1][:,0] < now)])
                        c.close()
                else:
                    v = np.array([])
                    v.shape = (0, 2)
                    loaddata.append(v)

            def interleave(x, y):
                lst = [None] * (len(x) * 2 - 1)
                for idx in xrange(0, len(x)):
                    lst[idx * 2] = x[idx]
                for idx in xrange(0, len(y)):
                    lst[idx * 2 + 1] = y[idx]
                return lst
            rv.append(np.vstack(interleave(loaddata, data[u][1])))

        return rv

    def _data(self, selector, where, limit, streamlimit):
        qbody = "select data %s limit %i streamlimit %i where %s" % \
            (selector, limit, streamlimit, where)
        return self.query(qbody)

    def data(self, qbody, start, end):
        """Load data for streams matching a particular query

        Uses the local time-series cache in the .cache directory.
        """
        uids = self.query('select distinct uuid where %s' % qbody)
        data = self.data_uuid(uids, start, end)
        return uids, data

    def prev(self, qbody, ref, limit=1, streamlimit=10):
        """Load data before a reference time.

        :param str qbody: a selector identifying the streams to query
        :param int ref: reference timestamp
        :param int limit: the maximum number of points to retrieve per stream
        :param int streamlimit: the maximum number of streams to query
        """
        return self._data("before %i" % (ref * 1000), where, limit, streamlimit)

    def next(self, where, ref, limit=1, streamlimit=10):
        """Load data after a reference time.

        See `prev` for args.
        """
        return self._data("after %i" % (ref * 1000), where, limit, streamlimit)

    def latest(self, where, limit=1, streamlimit=10):
        """Load the last data in a time-series.

        See `prev` for args.
        """
        return self._data("before 4294967295000", where, limit, streamlimit)

class RepublishClient:
    """Listener for streaming data from a sMAP source or archiver's /republish feed
    """
    def __init__(self, url, datacb, reconnect=True, restrict=None):
        """
:param str url: url of the source
:param datacb: callable to be called with each new sMAP object
:param bool reconnect: weather to reconnect if the socket connection is dropped.
:param str restrict: "where" clause restricting data to be delivered.
    This is only evaluated once, when connecting.
        """
        self.url = url
        self.datacb = datacb
        self.agent = Agent(reactor)
        self.reconnect = reconnect
        self.failcount = 0
        self.restrict = restrict

    class DataReceiver(LineReceiver):
        """Make our own LineReceiver to read back the streaming data
        from the server.  Use the right delimiter and make sure we can
        handle big objects."""
        MAX_LENGTH = 1e7
        delimiter = '\n\n'

        def __init__(self, client):
            self.client = client

        def lineReceived(self, line):
            self.failcount = 0
            try:
                obj = util.json_decode(line)
                self.client.datacb(obj)
            except:
                log.err()
                print line

        def connectionLost(self, reason):
            self.client._failed()
            self.client._reconnect()

    def _failed(self):
        if self.failcount < 5:
            self.failcount += 1

    def __request(self, response):
        receiver = RepublishClient.DataReceiver(self)
        receiver.setLineMode()
        response.deliverBody(receiver)

    def _reconnect(self):
        """Exponential backup on the reconnect policy"""
        if self.reconnect:
            print "connection failed, reconnecting in", (self.failcount ** 2)
            reactor.callLater(self.failcount ** 2, self.connect)

    def _connect_failed(self, reason):
        self._failed()
        self._reconnect()

    def connect(self):
        """Subscribe and start receiving data
        """
        if not self.restrict:
            d = self.agent.request('GET',
                                   self.url + '/republish',
                                   Headers(),
                                   None)
        else:
            d = self.agent.request('POST',
                                   self.url + '/republish',
                                   Headers(),
                                   FileBodyProducer(StringIO(str(self.restrict))))
        d.addCallback(self.__request)
        d.addErrback(self._connect_failed)

# if __name__ == '__main__':
#     def cb(line):
#         print line
#         pass

#     c = RepublishClient('http://smote.cs.berkeley.edu:8079', cb,
#                         restrict="Metadata/SourceName ~ '^410'")
#     c.connect()
#     reactor.run()

if __name__ == '__main__':
    import time
    # c = SmapClient('http://new.openbms.org/backend')
    # c = SmapClient('http://local.cs.berkeley.edu:8079')
    c = SmapClient('http://localhost:8079')
#     print c.tags('Metadata/SourceName = "410 Labjacks"')
#     print c.latest('Metadata/SourceName = "Cory Hall Dent Meters"', limit=1)
    #c._data_uuid('018eba5e-51b6-5a8d-920f-b5a831546610',
    #             int(time.time()) - 3600 * 24, int(time.time()))
    data = c.data_uuid(['018eba5e-51b6-5a8d-920f-b5a831546610'],
                       int(time.time()) - 3600 * 300, int(time.time()))

    # print data
    print len(np.unique(data[0][:,0])), len(data[0][:,0])
<|endoftext|>


<commit_before>#
# Copyright (C) 2012 Satoru SATOH <satoru.satoh@gmail.com>
# SPDX-License-Identifier: MIT
#
# pylint: disable=unused-import,import-error,invalid-name
r"""Public APIs of anyconfig module.

.. versionchanged:: 0.10.2

   - Re-structured APIs and split into sub modules
   - Added type hints

.. versionchanged:: 0.9.9

   - Removed the API 'find_loader'
   - Added new APIs :func:`find` and :func:`findall` to :func:`find parsers`
     (loaders and dumpers) to suppport to find multiple parsers, and replace
     the API 'find_loader'
   - Added new APIs :func:`list_by_cid`, :func:`list_by_type` and
     :func:`list_by_extension` to list parsers by various viewpoints.

.. versionadded:: 0.9.8

   - Added new API load_plugins to [re-]load plugins

.. versionadded:: 0.9.5

   - Added pathlib support. Now all of load and dump APIs can process
     pathlib.Path object basically.
   - 'ignore_missing' keyword option for load APIs are now marked as deprecated
     and will be removed soon.
   - Allow to load data other than mapping objects for some backends such as
     JSON and YAML.

.. versionadded:: 0.8.3

   - Added ac_dict keyword option to pass dict factory (any callable like
     function or class) to make dict-like object in backend parsers.
   - Added ac_query keyword option to query data with JMESPath expression.
   - Added experimental query api to query data with JMESPath expression.
   - Removed ac_namedtuple keyword option.
   - Export :func:`merge`.
   - Stop exporting :func:`to_container` which was deprecated and removed.

.. versionadded:: 0.8.2

   - Added new API, version to provide version information.

.. versionadded:: 0.8.0

   - Removed set_loglevel API as it does not help much.
   - Added :func:`open` API to open files with appropriate open mode.
   - Added custom exception classes, :class:`UnknownProcessorTypeError` and
     :class:`UnknownFileTypeError` to express specific errors.
   - Change behavior of the API :func:`find_loader` and others to make them
     fail firt and raise exceptions (ValueError, UnknownProcessorTypeError or
     UnknownFileTypeError) as much as possible if wrong parser type for uknown
     file type was given.

.. versionadded:: 0.5.0

   - Most keyword arguments passed to APIs are now position independent.
   - Added ac_namedtuple parameter to \*load and \*dump APIs.

.. versionchanged:: 0.3

   - Replaced 'forced_type' optional argument of some public APIs with
     'ac_parser' to allow skip of config parser search by passing parser object
     previously found and instantiated.

     Also removed some optional arguments, 'ignore_missing', 'merge' and
     'marker', from definitions of some public APIs as these may not be changed
     from default in common use cases.

.. versionchanged:: 0.2

   - Now APIs :func:`find_loader`, :func:`single_load`, :func:`multi_load`,
     :func:`load` and :func:`dump` can process a file/file-like object or a
     list of file/file-like objects instead of a file path or a list of file
     paths.

.. versionadded:: 0.2

   - Export factory method (create) of anyconfig.mergeabledict.MergeableDict
"""
from .datatypes import MaybeDataT
from ._dump import (
    dump, dumps
)
from ._load import (
    single_load, multi_load, load, loads
)
from ._open import open  # pylint: disable=redefined-builtin

# Export some more APIs originally from other sub modules.
from ..backend import ParserT
from ..common import (
    IOInfo, InDataT,
    UnknownFileTypeError, UnknownParserTypeError,
    UnknownProcessorTypeError, ValidationError
)
from ..dicts import (
    MS_REPLACE, MS_NO_REPLACE, MS_DICTS, MS_DICTS_AND_LISTS, MERGE_STRATEGIES,
    merge, get, set_,
)
from ..parsers import (
    load_plugins, list_types, list_by_cid, list_by_type, list_by_extension,
    findall, find, MaybeParserT
)
from ..query import try_query
from ..schema import (
    validate, is_valid, gen_schema
)

__version__ = '0.10.1'

def version():
    """
    :return: A tuple of version info, (major, minor, release), e.g. (0, 8, 2)
    """
    return __version__.split('.')

__all__ = [
    'MaybeDataT',
    'dump', 'dumps',
    'single_load', 'multi_load', 'load', 'loads',
    'open', 'version',

    # anyconfig.backend
    'ParserT',

    # anyconfig.common
    'IOInfo', 'InDataT',
    'UnknownFileTypeError', 'UnknownParserTypeError',
    'UnknownProcessorTypeError', 'ValidationError',

    # anyconfig.dicsts
    'MS_REPLACE', 'MS_NO_REPLACE', 'MS_DICTS', 'MS_DICTS_AND_LISTS',
    'MERGE_STRATEGIES',
    'merge', 'get', 'set_',

    # anyconfig.parsers
    'load_plugins', 'list_types', 'list_by_cid', 'list_by_type',
    'list_by_extension', 'findall', 'find',
    'MaybeParserT',

    # anyconfig.query
    'try_query',

    # anyconfig.validate
    'validate', 'is_valid', 'gen_schema'
]

# vim:sw=4:ts=4:et:
<commit_msg>0.11.0 rc2<commit_after>#
# Copyright (C) 2012 Satoru SATOH <satoru.satoh@gmail.com>
# SPDX-License-Identifier: MIT
#
# pylint: disable=unused-import,import-error,invalid-name
r"""Public APIs of anyconfig module.

.. versionchanged:: 0.10.2

   - Re-structured APIs and split into sub modules
   - Added type hints

.. versionchanged:: 0.9.9

   - Removed the API 'find_loader'
   - Added new APIs :func:`find` and :func:`findall` to :func:`find parsers`
     (loaders and dumpers) to suppport to find multiple parsers, and replace
     the API 'find_loader'
   - Added new APIs :func:`list_by_cid`, :func:`list_by_type` and
     :func:`list_by_extension` to list parsers by various viewpoints.

.. versionadded:: 0.9.8

   - Added new API load_plugins to [re-]load plugins

.. versionadded:: 0.9.5

   - Added pathlib support. Now all of load and dump APIs can process
     pathlib.Path object basically.
   - 'ignore_missing' keyword option for load APIs are now marked as deprecated
     and will be removed soon.
   - Allow to load data other than mapping objects for some backends such as
     JSON and YAML.

.. versionadded:: 0.8.3

   - Added ac_dict keyword option to pass dict factory (any callable like
     function or class) to make dict-like object in backend parsers.
   - Added ac_query keyword option to query data with JMESPath expression.
   - Added experimental query api to query data with JMESPath expression.
   - Removed ac_namedtuple keyword option.
   - Export :func:`merge`.
   - Stop exporting :func:`to_container` which was deprecated and removed.

.. versionadded:: 0.8.2

   - Added new API, version to provide version information.

.. versionadded:: 0.8.0

   - Removed set_loglevel API as it does not help much.
   - Added :func:`open` API to open files with appropriate open mode.
   - Added custom exception classes, :class:`UnknownProcessorTypeError` and
     :class:`UnknownFileTypeError` to express specific errors.
   - Change behavior of the API :func:`find_loader` and others to make them
     fail firt and raise exceptions (ValueError, UnknownProcessorTypeError or
     UnknownFileTypeError) as much as possible if wrong parser type for uknown
     file type was given.

.. versionadded:: 0.5.0

   - Most keyword arguments passed to APIs are now position independent.
   - Added ac_namedtuple parameter to \*load and \*dump APIs.

.. versionchanged:: 0.3

   - Replaced 'forced_type' optional argument of some public APIs with
     'ac_parser' to allow skip of config parser search by passing parser object
     previously found and instantiated.

     Also removed some optional arguments, 'ignore_missing', 'merge' and
     'marker', from definitions of some public APIs as these may not be changed
     from default in common use cases.

.. versionchanged:: 0.2

   - Now APIs :func:`find_loader`, :func:`single_load`, :func:`multi_load`,
     :func:`load` and :func:`dump` can process a file/file-like object or a
     list of file/file-like objects instead of a file path or a list of file
     paths.

.. versionadded:: 0.2

   - Export factory method (create) of anyconfig.mergeabledict.MergeableDict
"""
from .datatypes import MaybeDataT
from ._dump import (
    dump, dumps
)
from ._load import (
    single_load, multi_load, load, loads
)
from ._open import open  # pylint: disable=redefined-builtin

# Export some more APIs originally from other sub modules.
from ..backend import ParserT
from ..common import (
    IOInfo, InDataT,
    UnknownFileTypeError, UnknownParserTypeError,
    UnknownProcessorTypeError, ValidationError
)
from ..dicts import (
    MS_REPLACE, MS_NO_REPLACE, MS_DICTS, MS_DICTS_AND_LISTS, MERGE_STRATEGIES,
    merge, get, set_,
)
from ..parsers import (
    load_plugins, list_types, list_by_cid, list_by_type, list_by_extension,
    findall, find, MaybeParserT
)
from ..query import try_query
from ..schema import (
    validate, is_valid, gen_schema
)

__version__ = '0.11.0'

def version():
    """
    :return: A tuple of version info, (major, minor, release), e.g. (0, 8, 2)
    """
    return __version__.split('.')

__all__ = [
    'MaybeDataT',
    'dump', 'dumps',
    'single_load', 'multi_load', 'load', 'loads',
    'open', 'version',

    # anyconfig.backend
    'ParserT',

    # anyconfig.common
    'IOInfo', 'InDataT',
    'UnknownFileTypeError', 'UnknownParserTypeError',
    'UnknownProcessorTypeError', 'ValidationError',

    # anyconfig.dicsts
    'MS_REPLACE', 'MS_NO_REPLACE', 'MS_DICTS', 'MS_DICTS_AND_LISTS',
    'MERGE_STRATEGIES',
    'merge', 'get', 'set_',

    # anyconfig.parsers
    'load_plugins', 'list_types', 'list_by_cid', 'list_by_type',
    'list_by_extension', 'findall', 'find',
    'MaybeParserT',

    # anyconfig.query
    'try_query',

    # anyconfig.validate
    'validate', 'is_valid', 'gen_schema'
]

# vim:sw=4:ts=4:et:
<|endoftext|>


<commit_before>import shutil
import tempfile
from numpy import array, array_equal
from thunder.clustering import KMeans
from test_utils import PySparkTestCase

class ClusteringTestCase(PySparkTestCase):
    def setUp(self):
        super(ClusteringTestCase, self).setUp()
        self.outputdir = tempfile.mkdtemp()

    def tearDown(self):
        super(ClusteringTestCase, self).tearDown()
        shutil.rmtree(self.outputdir)

class TestKMeans(ClusteringTestCase):
    def test_kmeans(self):
        """ With k=1 always get one cluster centered on the mean"""

        data_local = [
            array([1.0, 2.0, 6.0]),
            array([1.0, 3.0, 0.0]),
            array([1.0, 4.0, 6.0])]

        data = self.sc.parallelize(zip(range(1, 4), data_local))

        model = KMeans(k=1, maxiter=20, tol=0.001).train(data)
        labels = model.predict(data)
        assert array_equal(model.centers[0], array([1.0, 3.0, 4.0]))
        assert array_equal(labels.map(lambda (_, v): v[0]).collect(), array([0, 0, 0]))

<commit_msg>Fixed bugs, added more tests<commit_after>import shutil
import tempfile
from numpy import array, array_equal
from thunder.clustering.kmeans import KMeans, KMeansModel
from thunder.utils import DataSets
from test_utils import PySparkTestCase

class ClusteringTestCase(PySparkTestCase):
    def setUp(self):
        super(ClusteringTestCase, self).setUp()
        self.outputdir = tempfile.mkdtemp()

    def tearDown(self):
        super(ClusteringTestCase, self).tearDown()
        shutil.rmtree(self.outputdir)

class TestKMeans(ClusteringTestCase):
    def test_kmeans_k1(self):
        """ With k=1 always get one cluster centered on the mean"""

        data_local = [
            array([1.0, 2.0, 6.0]),
            array([1.0, 3.0, 0.0]),
            array([1.0, 4.0, 6.0])]

        data = self.sc.parallelize(zip(range(1, 4), data_local))

        model = KMeans(k=1, maxiter=20, tol=0.001).train(data)
        labels = model.predict(data)
        assert array_equal(model.centers[0], array([1.0, 3.0, 4.0]))
        assert array_equal(labels.map(lambda (_, v): v).collect(), array([0, 0, 0]))

    def test_kmeans_k2(self):
        """ Test k=2 also with more points"""

        data, centerstrue = DataSets.make(self.sc, "kmeans",
                                          k=2, nrecords=50, npartitions=5, seed=42, returnparams=True)
        centerstrue = KMeansModel(centerstrue)

        model = KMeans(k=2, maxiter=20, tol=0.001, init="sample").train(data)

        labels = array(model.predict(data).values().collect())
        labelstrue = array(centerstrue.predict(data).values().collect())
        print(labels)
        print(labelstrue)

        assert(array_equal(labels, labelstrue) or array_equal(labels, 1 - labelstrue))

<|endoftext|>


<commit_before>from numpy import arctan2, sqrt, pi, abs, dstack, clip, transpose, inf, \
    random, zeros, ones, asarray, corrcoef, allclose, maximum, add, multiply, \
    nan_to_num

class Colorize(object):
    """
    Class for turning numerical data into colors.

    Supports a set of custom conversions (rgb, hsv, polar, and indexed)
    as well as conversions to standard matplotlib colormaps through
    either a passed colormap or a string specification.

    If vmax and vmin are not specified, numerical data will be automatically
    scaled by its maximum and minimum values.

    Supports two-dimensional and three-dimensional data.

    Attributes
    ----------
    cmap : string, optional, default = rainbow
        The colormap to convert to, can be one of a special set of conversions
        (rgb, hsv, hv, polar, angle, indexed), a matplotlib colormap object,
        or a string specification of a matplotlib colormap

    scale : float, optional, default = 1
        How to scale amplitude during color conversion, controls brighthness

    colors : list, optional, default = None
        List of colors for 'indexed' option

    vmin : scalar, optional, default = None
        Numerical value to set to 0 during normalization, values below will be clipped

    vmax : scalar, optional, default = None
        Numerical value to set to 1 during normalization, values above will be clipped.
    """

    def __init__(self, cmap='rainbow', scale=1, colors=None, vmin=None, vmax=None):
        self.cmap = cmap
        self.scale = scale
        self.colors = colors
        if self.cmap == 'index' and self.colors is None:
            raise Exception("Must specify colors for indexed conversion")
        self.vmin = vmin
        self.vmax = vmax

    @staticmethod
    def show(img, cmap='gray', bar=False, nans=True):
        """
        Streamlined display of images using matplotlib.

        Parameters
        ----------
        img : ndarray, 2D or 3D
            The image to display

        bar : boolean, optional, default = False
            Whether to append a colorbar

        cmap : str or Colormap, optional, default = 'gray'
            A colormap to use, for non RGB images

        nans : boolean, optional, deafult = True
            Whether to replace NaNs, if True, will replace with 0s

        """
        from matplotlib.pyplot import imshow, axis, colorbar

        if nans is True:
            img = nan_to_num(asarray(img))

        if img.ndim == 3:
            if bar:
                raise Exception("Cannot show meaningful colorbar for RGB images")
            if img.shape[2] != 3:
                raise Exception("Size of third dimension must be 3 for RGB images, got %g" % img.shape[2])
            mn = img.min()
            mx = img.max()
            if mn < 0.0 or mx > 1.0:
                raise Exception("Values must be between 0.0 and 1.0 for RGB images, got range (%g, %g)" % (mn, mx))
            imshow(img, interpolation='none')
        else:
            imshow(img, cmap=cmap, interpolation='none')

        if bar is True:
            colorbar()

        axis('off')

    def transform(self, img, mask=None, background=None, mixing=1.0):
        """
        Colorize numerical image data.

        Input can either be a single array or a list of arrays.
        Depending on the colorization option, each array must either be
        2 or 3 dimensional, see parameters for details.

        Parameters
        ----------
        img : array
            The image(s) to colorize. For rgb, hsv, polar, and indexed conversions,
            must be of shape (c, x, y, z) or (c, x, y), where c is the dimension
            containing the information for colorizing. For colormap conversions,
            must be of shape (x, y, z) or (x, y).

        mask : array
            An additional image to mask the luminance channel of the first one.
            Must be of shape (x, y, z) or (x, y), and must match dimensions of images.
            Must be strictly positive (and will be clipped below at 0).

        background : array
            An additional image to display as a grayscale background.
            Must be of shape (x, y, z) or (x, y), and must match dimensions of images.

        mixing : scalar
            If adding a background image, mixing controls the relative scale.
            Values larger than 1.0 will emphasize the background more.

        Returns
        -------
        Arrays with RGB values, with shape (x, y, z, 3) or (x, y, 3)
        """

        from matplotlib.cm import get_cmap
        from matplotlib.colors import ListedColormap, LinearSegmentedColormap, hsv_to_rgb, Normalize

        img = asarray(img)
        dims = img.shape
        self._checkDims(dims)

        if self.cmap not in ['polar', 'angle']:

            if self.cmap in ['rgb', 'hv', 'hsv', 'indexed']:
                img = copy(img)
                for i, im in enumerate(img):
                    norm = Normalize(vmin=self.vmin, vmax=self.vmax, clip=True)
                    img[i] = norm(im)

            if isinstance(self.cmap, ListedColormap) or isinstance(self.cmap, str):
                norm = Normalize(vmin=self.vmin, vmax=self.vmax, clip=True)
                img = norm(copy(img))

        if mask is not None:
            mask = self._prepareMask(mask)
            self._checkMixedDims(mask.shape, dims)

        if background is not None:
            background = self._prepareBackground(background, mixing)
            self._checkMixedDims(background.shape, dims)

        if self.cmap == 'rgb':
            if img.ndim == 4:
                out = transpose(img, [1, 2, 3, 0])
            if img.ndim == 3:
                out = transpose(img, [1, 2, 0])

        elif self.cmap == 'hv':
            saturation = ones((dims[1], dims[2])) * 0.8
            if img.ndim == 4:
                out = zeros((dims[1], dims[2], dims[3], 3))
                for i in range(0, dims[3]):
                    out[:, :, i, :] = hsv_to_rgb(dstack((img[0][:, :, i], saturation, img[2][:, :, i])))
            if img.ndim == 3:
                out = hsv_to_rgb(dstack((img[0], saturation, img[2])))

        elif self.cmap == 'hsv':
            if img.ndim == 4:
                out = zeros((dims[1], dims[2], dims[3], 3))
                for i in range(0, dims[3]):
                    out[:, :, i, :] = hsv_to_rgb(dstack((img[0][:, :, i], img[1][:, :, i], img[2][:, :, i])))
            if img.ndim == 3:
                out = hsv_to_rgb(dstack((img[0], img[1], img[2])))

        elif self.cmap == 'angle':
            theta = ((arctan2(-img[0], -img[1]) + pi/2) % (pi*2)) / (2 * pi)
            saturation = ones((dims[1], dims[2])) * 0.8
            rho = ones((dims[1], dims[2]))
            if img.ndim == 4:
                out = zeros((dims[1], dims[2], dims[3], 3))
                for i in range(0, dims[3]):
                    out[:, :, i, :] = hsv_to_rgb(dstack((theta[:, :, i], saturation, rho)))
            if img.ndim == 3:
                out = hsv_to_rgb(dstack((theta, saturation, rho)))

        elif self.cmap == 'polar':
            theta = ((arctan2(-img[0], -img[1]) + pi/2) % (pi*2)) / (2 * pi)
            rho = sqrt(img[0]**2 + img[1]**2)
            saturation = ones((dims[1], dims[2]))
            if img.ndim == 4:
                out = zeros((dims[1], dims[2], dims[3], 3))
                for i in range(0, dims[3]):
                    out[:, :, i, :] = hsv_to_rgb(dstack((theta[:, :, i], saturation, self.scale * rho[:, :, i])))
            if img.ndim == 3:
                out = hsv_to_rgb(dstack((theta, saturation, self.scale * rho)))

        elif self.cmap == 'indexed':
            if img.ndim == 4:
                out = zeros((dims[1], dims[2], dims[3], 3))
            if img.ndim == 3:
                out = zeros((dims[1], dims[2], 3))
            for ix, clr in enumerate(self.colors):
                cmap = LinearSegmentedColormap.from_list('blend', [[0, 0, 0], clr])
                tmp = cmap(img[ix])
                if img.ndim == 4:
                    tmp = tmp[:, :, :, 0:3]
                if img.ndim == 3:
                    tmp = tmp[:, :, 0:3]
                out = maximum(out, clip(tmp, 0, 1))

        elif isinstance(self.cmap, ListedColormap):
            if img.ndim == 3:
                out = self.cmap(img)
                out = out[:, :, :, 0:3]
            if img.ndim == 2:
                out = self.cmap(img)
                out = out[:, :, 0:3]

        elif isinstance(self.cmap, str):
            func = lambda x: get_cmap(self.cmap, 256)(x)
            out = func(img)
            if img.ndim == 3:
                out = out[:, :, :, 0:3]
            if img.ndim == 2:
                out = out[:, :, 0:3]

        else:
            raise Exception('Colorization method not understood')

        out = clip(out * self.scale, 0, 1)

        if mask is not None:
            out = self.blend(out, mask, multiply)

        if background is not None:
            out = self.blend(out, background, add)

        return clip(out, 0, 1)

    @staticmethod
    def blend(img, mask, op=add):

        if mask.ndim == 3:
            for i in range(0, 3):
                img[:, :, :, i] = op(img[:, :, :, i], mask)
        else:
            for i in range(0, 3):
                img[:, :, i] = op(img[:, :, i], mask)

        return img

    def _checkDims(self, dims):

        from matplotlib.colors import ListedColormap

        if self.cmap in ['rgb', 'hsv', 'hv', 'polar', 'angle', 'indexed']:
            if self.cmap in ['rgb', 'hsv']:
                if dims[0] != 3:
                    raise Exception('First dimension must be 3 for %s conversion' % self.cmap)
            if self.cmap in ['polar', 'angle', 'hv']:
                if dims[0] != 2:
                    raise Exception('First dimension must be 2 for %s conversion' % self.cmap)
            if self.cmap in ['indexed']:
                if dims[0] != len(self.colors):
                    raise Exception('First dimension must be %g for %s conversion with list %s'
                                    % (len(self.colors), self.cmap, self.colors))

        elif isinstance(self.cmap, ListedColormap) or isinstance(self.cmap, str):
            if len(dims) not in [2, 3]:
                raise Exception('Number of dimensions must be 2 or 3 for %s conversion' % self.cmap)

    def _checkMixedDims(self, dims1, dims2):

        from matplotlib.colors import ListedColormap

        if self.cmap in ['rgb', 'hsv', 'hv', 'polar', 'angle', 'indexed']:
            if not allclose(dims1, dims2[1:]):
                raise Exception

        elif isinstance(self.cmap, ListedColormap) or isinstance(self.cmap, str):
            if not allclose(dims1, dims2):
                raise Exception

    @staticmethod
    def _prepareMask(mask):

        mask = asarray(mask)
        mask = clip(mask, 0, inf)

        return mask / mask.max()

    @staticmethod
    def _prepareBackground(background, mixing):

        from matplotlib.colors import Normalize

        background = asarray(background)
        background = Normalize()(background)

        return background * mixing

    @classmethod
    def optimize(cls, mat, asCmap=False):
        """
        Optimal colors based on array data similarity.

        Given an (n, m) data array with n m-dimensional data points,
        tries to find an optimal set of n colors such that the similarity
        between colors in 3-dimensional space is well-matched to the similarity
        between the data points in m-dimensional space.

        Parameters
        ----------
        mat : array-like
            Array of data points to use for estimating similarity.

        asCmap : boolean, optional, default = False
            Whether to return a matplotlib colormap, if False will
            return a list of colors.
        """

        mat = asarray(mat)

        if mat.ndim < 2:
            raise Exception('Input array must be two-dimensional')

        nclrs = mat.shape[0]

        from scipy.spatial.distance import pdist, squareform
        from scipy.optimize import minimize

        distMat = squareform(pdist(mat, metric='cosine')).flatten()

        optFunc = lambda x: 1 - corrcoef(distMat, squareform(pdist(x.reshape(nclrs, 3), 'cosine')).flatten())[0, 1]
        init = random.rand(nclrs*3)
        bounds = [(0, 1) for _ in range(0, nclrs * 3)]
        res = minimize(optFunc, init, bounds=bounds, method='L-BFGS-B')
        newClrs = res.x.reshape(nclrs, 3).tolist()

        from matplotlib.colors import ListedColormap

        if asCmap:
            newClrs = ListedColormap(newClrs, name='from_list')

        return newClrs<commit_msg>Add clim option<commit_after>from numpy import arctan2, sqrt, pi, abs, dstack, clip, transpose, inf, \
    random, zeros, ones, asarray, corrcoef, allclose, maximum, add, multiply, \
    nan_to_num, copy

class Colorize(object):
    """
    Class for turning numerical data into colors.

    Supports a set of custom conversions (rgb, hsv, polar, and indexed)
    as well as conversions to standard matplotlib colormaps through
    either a passed colormap or a string specification.

    If vmax and vmin are not specified, numerical data will be automatically
    scaled by its maximum and minimum values.

    Supports two-dimensional and three-dimensional data.

    Attributes
    ----------
    cmap : string, optional, default = rainbow
        The colormap to convert to, can be one of a special set of conversions
        (rgb, hsv, hv, polar, angle, indexed), a matplotlib colormap object,
        or a string specification of a matplotlib colormap

    scale : float, optional, default = 1
        How to scale amplitude during color conversion, controls brighthness

    colors : list, optional, default = None
        List of colors for 'indexed' option

    vmin : scalar, optional, default = None
        Numerical value to set to 0 during normalization, values below will be clipped

    vmax : scalar, optional, default = None
        Numerical value to set to 1 during normalization, values above will be clipped.
    """

    def __init__(self, cmap='rainbow', scale=1, colors=None, vmin=None, vmax=None):
        self.cmap = cmap
        self.scale = scale
        self.colors = colors
        if self.cmap == 'index' and self.colors is None:
            raise Exception("Must specify colors for indexed conversion")
        self.vmin = vmin
        self.vmax = vmax

    @staticmethod
    def show(img, cmap='gray', bar=False, nans=True, clim=None):
        """
        Streamlined display of images using matplotlib.

        Parameters
        ----------
        img : ndarray, 2D or 3D
            The image to display

        bar : boolean, optional, default = False
            Whether to append a colorbar

        cmap : str or Colormap, optional, default = 'gray'
            A colormap to use, for non RGB images

        nans : boolean, optional, deafult = True
            Whether to replace NaNs, if True, will replace with 0s

        clim : tuple, optional, default = None
            Limits for scaling image

        """
        from matplotlib.pyplot import imshow, axis, colorbar

        if nans is True:
            img = nan_to_num(asarray(img))

        if img.ndim == 3:
            if bar:
                raise Exception("Cannot show meaningful colorbar for RGB images")
            if img.shape[2] != 3:
                raise Exception("Size of third dimension must be 3 for RGB images, got %g" % img.shape[2])
            mn = img.min()
            mx = img.max()
            if mn < 0.0 or mx > 1.0:
                raise Exception("Values must be between 0.0 and 1.0 for RGB images, got range (%g, %g)" % (mn, mx))
            imshow(img, interpolation='none', clim=clim)
        else:
            imshow(img, cmap=cmap, interpolation='none', clim=clim)

        if bar is True:
            colorbar()

        axis('off')

    def transform(self, img, mask=None, background=None, mixing=1.0):
        """
        Colorize numerical image data.

        Input can either be a single array or a list of arrays.
        Depending on the colorization option, each array must either be
        2 or 3 dimensional, see parameters for details.

        Parameters
        ----------
        img : array
            The image(s) to colorize. For rgb, hsv, polar, and indexed conversions,
            must be of shape (c, x, y, z) or (c, x, y), where c is the dimension
            containing the information for colorizing. For colormap conversions,
            must be of shape (x, y, z) or (x, y).

        mask : array
            An additional image to mask the luminance channel of the first one.
            Must be of shape (x, y, z) or (x, y), and must match dimensions of images.
            Must be strictly positive (and will be clipped below at 0).

        background : array
            An additional image to display as a grayscale background.
            Must be of shape (x, y, z) or (x, y), and must match dimensions of images.

        mixing : scalar
            If adding a background image, mixing controls the relative scale.
            Values larger than 1.0 will emphasize the background more.

        Returns
        -------
        Arrays with RGB values, with shape (x, y, z, 3) or (x, y, 3)
        """

        from matplotlib.cm import get_cmap
        from matplotlib.colors import ListedColormap, LinearSegmentedColormap, hsv_to_rgb, Normalize

        img = asarray(img)
        dims = img.shape
        self._checkDims(dims)

        if self.cmap not in ['polar', 'angle']:

            if self.cmap in ['rgb', 'hv', 'hsv', 'indexed']:
                img = copy(img)
                for i, im in enumerate(img):
                    norm = Normalize(vmin=self.vmin, vmax=self.vmax, clip=True)
                    img[i] = norm(im)

            if isinstance(self.cmap, ListedColormap) or isinstance(self.cmap, str):
                norm = Normalize(vmin=self.vmin, vmax=self.vmax, clip=True)
                img = norm(copy(img))

        if mask is not None:
            mask = self._prepareMask(mask)
            self._checkMixedDims(mask.shape, dims)

        if background is not None:
            background = self._prepareBackground(background, mixing)
            self._checkMixedDims(background.shape, dims)

        if self.cmap == 'rgb':
            if img.ndim == 4:
                out = transpose(img, [1, 2, 3, 0])
            if img.ndim == 3:
                out = transpose(img, [1, 2, 0])

        elif self.cmap == 'hv':
            saturation = ones((dims[1], dims[2])) * 0.8
            if img.ndim == 4:
                out = zeros((dims[1], dims[2], dims[3], 3))
                for i in range(0, dims[3]):
                    out[:, :, i, :] = hsv_to_rgb(dstack((img[0][:, :, i], saturation, img[2][:, :, i])))
            if img.ndim == 3:
                out = hsv_to_rgb(dstack((img[0], saturation, img[2])))

        elif self.cmap == 'hsv':
            if img.ndim == 4:
                out = zeros((dims[1], dims[2], dims[3], 3))
                for i in range(0, dims[3]):
                    out[:, :, i, :] = hsv_to_rgb(dstack((img[0][:, :, i], img[1][:, :, i], img[2][:, :, i])))
            if img.ndim == 3:
                out = hsv_to_rgb(dstack((img[0], img[1], img[2])))

        elif self.cmap == 'angle':
            theta = ((arctan2(-img[0], -img[1]) + pi/2) % (pi*2)) / (2 * pi)
            saturation = ones((dims[1], dims[2])) * 0.8
            rho = ones((dims[1], dims[2]))
            if img.ndim == 4:
                out = zeros((dims[1], dims[2], dims[3], 3))
                for i in range(0, dims[3]):
                    out[:, :, i, :] = hsv_to_rgb(dstack((theta[:, :, i], saturation, rho)))
            if img.ndim == 3:
                out = hsv_to_rgb(dstack((theta, saturation, rho)))

        elif self.cmap == 'polar':
            theta = ((arctan2(-img[0], -img[1]) + pi/2) % (pi*2)) / (2 * pi)
            rho = sqrt(img[0]**2 + img[1]**2)
            saturation = ones((dims[1], dims[2]))
            if img.ndim == 4:
                out = zeros((dims[1], dims[2], dims[3], 3))
                for i in range(0, dims[3]):
                    out[:, :, i, :] = hsv_to_rgb(dstack((theta[:, :, i], saturation, self.scale * rho[:, :, i])))
            if img.ndim == 3:
                out = hsv_to_rgb(dstack((theta, saturation, self.scale * rho)))

        elif self.cmap == 'indexed':
            if img.ndim == 4:
                out = zeros((dims[1], dims[2], dims[3], 3))
            if img.ndim == 3:
                out = zeros((dims[1], dims[2], 3))
            for ix, clr in enumerate(self.colors):
                cmap = LinearSegmentedColormap.from_list('blend', [[0, 0, 0], clr])
                tmp = cmap(img[ix])
                if img.ndim == 4:
                    tmp = tmp[:, :, :, 0:3]
                if img.ndim == 3:
                    tmp = tmp[:, :, 0:3]
                out = maximum(out, clip(tmp, 0, 1))

        elif isinstance(self.cmap, ListedColormap):
            if img.ndim == 3:
                out = self.cmap(img)
                out = out[:, :, :, 0:3]
            if img.ndim == 2:
                out = self.cmap(img)
                out = out[:, :, 0:3]

        elif isinstance(self.cmap, str):
            func = lambda x: get_cmap(self.cmap, 256)(x)
            out = func(img)
            if img.ndim == 3:
                out = out[:, :, :, 0:3]
            if img.ndim == 2:
                out = out[:, :, 0:3]

        else:
            raise Exception('Colorization method not understood')

        out = clip(out * self.scale, 0, 1)

        if mask is not None:
            out = self.blend(out, mask, multiply)

        if background is not None:
            out = self.blend(out, background, add)

        return clip(out, 0, 1)

    @staticmethod
    def blend(img, mask, op=add):

        if mask.ndim == 3:
            for i in range(0, 3):
                img[:, :, :, i] = op(img[:, :, :, i], mask)
        else:
            for i in range(0, 3):
                img[:, :, i] = op(img[:, :, i], mask)

        return img

    def _checkDims(self, dims):

        from matplotlib.colors import ListedColormap

        if self.cmap in ['rgb', 'hsv', 'hv', 'polar', 'angle', 'indexed']:
            if self.cmap in ['rgb', 'hsv']:
                if dims[0] != 3:
                    raise Exception('First dimension must be 3 for %s conversion' % self.cmap)
            if self.cmap in ['polar', 'angle', 'hv']:
                if dims[0] != 2:
                    raise Exception('First dimension must be 2 for %s conversion' % self.cmap)
            if self.cmap in ['indexed']:
                if dims[0] != len(self.colors):
                    raise Exception('First dimension must be %g for %s conversion with list %s'
                                    % (len(self.colors), self.cmap, self.colors))

        elif isinstance(self.cmap, ListedColormap) or isinstance(self.cmap, str):
            if len(dims) not in [2, 3]:
                raise Exception('Number of dimensions must be 2 or 3 for %s conversion' % self.cmap)

    def _checkMixedDims(self, dims1, dims2):

        from matplotlib.colors import ListedColormap

        if self.cmap in ['rgb', 'hsv', 'hv', 'polar', 'angle', 'indexed']:
            if not allclose(dims1, dims2[1:]):
                raise Exception

        elif isinstance(self.cmap, ListedColormap) or isinstance(self.cmap, str):
            if not allclose(dims1, dims2):
                raise Exception

    @staticmethod
    def _prepareMask(mask):

        mask = asarray(mask)
        mask = clip(mask, 0, inf)

        return mask / mask.max()

    @staticmethod
    def _prepareBackground(background, mixing):

        from matplotlib.colors import Normalize

        background = asarray(background)
        background = Normalize()(background)

        return background * mixing

    @classmethod
    def optimize(cls, mat, asCmap=False):
        """
        Optimal colors based on array data similarity.

        Given an (n, m) data array with n m-dimensional data points,
        tries to find an optimal set of n colors such that the similarity
        between colors in 3-dimensional space is well-matched to the similarity
        between the data points in m-dimensional space.

        Parameters
        ----------
        mat : array-like
            Array of data points to use for estimating similarity.

        asCmap : boolean, optional, default = False
            Whether to return a matplotlib colormap, if False will
            return a list of colors.
        """

        mat = asarray(mat)

        if mat.ndim < 2:
            raise Exception('Input array must be two-dimensional')

        nclrs = mat.shape[0]

        from scipy.spatial.distance import pdist, squareform
        from scipy.optimize import minimize

        distMat = squareform(pdist(mat, metric='cosine')).flatten()

        optFunc = lambda x: 1 - corrcoef(distMat, squareform(pdist(x.reshape(nclrs, 3), 'cosine')).flatten())[0, 1]
        init = random.rand(nclrs*3)
        bounds = [(0, 1) for _ in range(0, nclrs * 3)]
        res = minimize(optFunc, init, bounds=bounds, method='L-BFGS-B')
        newClrs = res.x.reshape(nclrs, 3).tolist()

        from matplotlib.colors import ListedColormap

        if asCmap:
            newClrs = ListedColormap(newClrs, name='from_list')

        return newClrs<|endoftext|>


<commit_before>#Recursion
def countup(n):
	if n >= 11:
		pass
	else:
		print n
		countup(n+1)

def countupfrom(start, stop):
	if start >= stop+1:
		pass
	else:
		print start
		countupfrom(start+1, stop)

def countdownfrom(start, stop):
	if start <= stop-1:
		pass
	else:
		print start
		countdownfrom(start-1, stop)

def adder(number):
	total = 0
	total += number
	count = (raw_input("Running total: " + str(total) + """:
Next number: """))
	if count == '':
		print "The sum is " + str(total)
	else:
		total += float(count)
		adder(total)

def biggest(lastnumber):
	number = raw_input("Next: ")
	if number == '':
		print str(lastnumber)
	elif float(number) > lastnumber:
		biggest(float(number))
	else:
		biggest(lastnumber)

def smallest(lastnumber):
	number = raw_input("Next: ")
	if number == '':
		print str(lastnumber)
	elif float(number) < lastnumber:
		smallest(float(number))
	else:
		smallest(lastnumber)

def pow(x, n):
	x

def main():
#	randnum = int(raw_input("Write a random number and it will count up to 10: "))
#	countup(randnum)
#	countupfrom(1,10)
#	countdownfrom(10,1)
#	adder(0)
	biggest(-float("inf"))
	smallest(float('inf'))

main()

<commit_msg>Delete conditionalexecution2.py<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Created by René Meusel
This file is part of the CernVM File System auxiliary tools.
"""

import datetime
import collections
import hashlib
import os

from _common import _split_md5, DatabaseObject
from dirent  import DirectoryEntry, Chunk

class CatalogIterator:
    """ Iterates through all directory entries of a Catalog """

    def __init__(self, catalog):
        self.catalog = catalog
        self.backlog = collections.deque()
        root_path = ""
        if not self.catalog.is_root():
            root_path = self.catalog.root_prefix
        self._push((root_path, self.catalog.find_directory_entry(root_path)))

    def __iter__(self):
        return self

    def next(self):
        if not self._has_more():
            raise StopIteration()
        return self._recursion_step()

    def _has_more(self):
        return len(self.backlog) > 0

    def _push(self, path):
        self.backlog.append(path)

    def _pop(self):
        return self.backlog.popleft()

    def _recursion_step(self):
        path, dirent = self._pop()
        if dirent.is_directory():
            new_dirents = self.catalog.list_directory_split_md5(dirent.md5path_1, \
                                                                dirent.md5path_2)
            for new_dirent in new_dirents:
                self._push((path + "/" + new_dirent.name, new_dirent))
        return path, dirent

class CatalogReference:
    """ Wraps a catalog reference to nested catalogs as found in Catalogs """

    def __init__(self, root_path, clg_hash, clg_size = 0):
        self.root_path = root_path
        self.hash      = clg_hash
        self.size      = clg_size

    def __str__(self):
        return "<CatalogReference for " + self.root_path + " - " + self.hash + ">"

    def __repr__(self):
        return "<CatalogReference for " + self.root_path + ">"

    def retrieve_from(self, source_repository):
        return source_repository.retrieve_catalog(self.hash)

class CatalogStatistics:
    """ Provides a convenience data wrapper around catalog statistics """

    def __init__(self, catalog):
        self.catalog = catalog
        if catalog.schema >= 2.1:
            self._read_statistics(catalog)

    def __str__(self):
        return "<CatalogStatistics for " + self.catalog.root_prefix + ">"

    def __repr__(self):
        return self.__str__()

    def num_entries(self):
        return self._get_stat('regular') + \
               self._get_stat('dir')     + \
               self._get_stat('symlink')

    def num_subtree_entries(self):
        return self._get_stat('all_regular') + \
               self._get_stat('all_dir')     + \
               self._get_stat('all_symlink')

    def num_chunked_files(self):
        return self._get_stat('chunked')

    def num_subtree_chunked_files(self):
        return self._get_stat('all_chunked')

    def num_file_chunks(self):
        return self._get_stat('chunks')

    def num_subtree_file_chunks(self):
        return self._get_stat('all_chunks')

    def data_size(self):
        return self._get_stat('file_size')

    def subtree_data_size(self):
        return self._get_stat('all_file_size')

    def _read_statistics(self, catalog):
        stats = catalog.run_sql("SELECT * FROM statistics ORDER BY counter;")
        for stat, value in stats:
            if stat.startswith('self_'):
                setattr(self, stat[5:], value)
            elif stat.startswith('subtree_'):
                setattr(self, "all_" + stat[8:], value + getattr(self, stat[8:]))

    def _get_stat(self, stat):
        if not hasattr(self, stat):
            raise Exception("Statistic '" + stat + "' not provided.")
        return getattr(self, stat)

class Catalog(DatabaseObject):
    """ Wraps the basic functionality of CernVM-FS Catalogs """

    def __init__(self, catalog_file, catalog_hash = ""):
        DatabaseObject.__init__(self, catalog_file)
        self.hash = catalog_hash
        self._read_properties()
        self._guess_root_prefix_if_needed()
        self._guess_last_modified_if_needed()
        self._check_validity()

    def __str__(self):
        return "<Catalog " + self.root_prefix + ">"

    def __repr__(self):
        return self.__str__()

    def __iter__(self):
        return CatalogIterator(self)

    def has_nested(self):
        return self.nested_count() > 0

    def nested_count(self):
        """ Returns the number of nested catalogs in this catalog """
        num_catalogs = self.run_sql("SELECT count(*) FROM nested_catalogs;")
        return num_catalogs[0][0]

    def list_nested(self):
        """ List CatalogReferences to all contained nested catalogs """
        new_version = (self.schema <= 1.2 and self.schema_revision > 0)
        if new_version:
            sql_query = "SELECT path, sha1, size FROM nested_catalogs;"
        else:
            sql_query = "SELECT path, sha1 FROM nested_catalogs;"
        catalogs = self.run_sql(sql_query)
        if new_version:
            return [ CatalogReference(clg[0], clg[1], clg[2]) for clg in catalogs ]
        else:
            return [ CatalogReference(clg[0], clg[1]) for clg in catalogs ]

    def get_statistics(self):
        """ returns the embedded catalog statistics (if available) """
        return CatalogStatistics(self)

    def find_nested_for_path(self, needle_path):
        """ Find the best matching nested CatalogReference for a given path """
        nested_catalogs  = self.list_nested()
        best_match       = None
        best_match_score = 0
        real_needle_path = self._canonicalize_path(needle_path)
        for nested_catalog in nested_catalogs:
            if real_needle_path.startswith(nested_catalog.root_path) and \
                len(nested_catalog.root_path) > best_match_score:
                    best_match_score = len(nested_catalog.root_path)
                    best_match       = nested_catalog
        return best_match

    def list_directory(self, path):
        """ Create a directory listing of the given directory path """
        real_path = self._canonicalize_path(path)
        parent_1, parent_2 = _split_md5(hashlib.md5(real_path).digest())
        return self.list_directory_split_md5(parent_1, parent_2)

    def list_directory_split_md5(self, parent_1, parent_2):
        """ Create a directory listing of DirectoryEntry items based on MD5 path """
        res = self.run_sql("SELECT " + DirectoryEntry._catalog_db_fields() + " \
                            FROM catalog                                       \
                            WHERE parent_1 = " + str(parent_1) + " AND         \
                                  parent_2 = " + str(parent_2) + "             \
                            ORDER BY name ASC;")
        return [ self._make_directory_entry(result) for result in res ]

    def find_directory_entry(self, path):
        """ Finds the DirectoryEntry for a given path """
        real_path = self._canonicalize_path(path)
        md5path = hashlib.md5(real_path)
        return self.find_directory_entry_md5(md5path)

    def find_directory_entry_md5(self, md5path):
        """ Finds the DirectoryEntry for a given MD5 hashed path """
        lo, hi = _split_md5(md5path.digest())
        return self.find_directory_entry_split_md5(lo, hi)

    def find_directory_entry_split_md5(self, md5path_1, md5path_2):
        """ Finds the DirectoryEntry for the given split MD5 hashed path """
        res = self.run_sql("SELECT " + DirectoryEntry._catalog_db_fields() + " \
                            FROM catalog                                       \
                            WHERE md5path_1 = " + str(md5path_1) + " AND       \
                                  md5path_2 = " + str(md5path_2) + "           \
                            LIMIT 1;")
        return self._make_directory_entry(res[0]) if len(res) == 1 else None

    def is_root(self):
        """ Checks if this is the root catalog (based on the root prefix) """
        return self.root_prefix == "/"

    def has_predecessor(self):
        return hasattr(self, "previous_revision")

    def get_predecessor(self):
        if not self.has_predecessor():
            return None
        return CatalogReference(self.root_prefix, self.previous_revision)

    def _read_properties(self):
        self.read_properties_table(lambda prop_key, prop_value:
            self._read_property(prop_key, prop_value))
        if not hasattr(self, 'schema_revision'):
            self.schema_revision = 0

    def _read_property(self, prop_key, prop_value):
        """ Detect catalog properties and store them as public class members """
        if prop_key == "revision":
            self.revision          = prop_value
        if prop_key == "schema":
            self.schema            = float(prop_value)
        if prop_key == "schema_revision":
            self.schema_revision   = float(prop_value)
        if prop_key == "last_modified":
            self.last_modified     = datetime.datetime.fromtimestamp(int(prop_value))
        if prop_key == "previous_revision":
            self.previous_revision = prop_value
        if prop_key == "root_prefix":
            self.root_prefix       = prop_value

    def _make_directory_entry(self, result_set):
        dirent = DirectoryEntry(result_set)
        self._read_chunks(dirent)
        return dirent

    def _read_chunks(self, dirent):
        """ Finds and adds the file chunk of a DirectoryEntry """
        if self.schema < 2.4:
            return
        res = self.run_sql("SELECT " + Chunk._catalog_db_fields() + "           \
                            FROM chunks                                         \
                            WHERE md5path_1 = " + str(dirent.md5path_1) + " AND \
                                  md5path_2 = " + str(dirent.md5path_2) + "     \
                            ORDER BY offset ASC;")
        dirent._add_chunks(res)

    def _guess_root_prefix_if_needed(self):
        """ Root catalogs don't have a root prefix property (fixed here) """
        if not hasattr(self, 'root_prefix'):
            self.root_prefix = "/"

    def _guess_last_modified_if_needed(self):
        """ Catalog w/o a last_modified field, we set it to 0 """
        if not hasattr(self, 'last_modified'):
            self.last_modified = datetime.datetime.min

    def _canonicalize_path(self, path):
        if not path:
            return ""
        return os.path.realpath(path)

    def _check_validity(self):
        """ Check that all crucial properties have been found in the database """
        if not hasattr(self, 'revision'):
          raise Exception("Catalog lacks a revision entry")
        if not hasattr(self, 'schema'):
          raise Exception("Catalog lacks a schema entry")
        if not hasattr(self, 'root_prefix'):
          raise Exception("Catalog lacks a root prefix entry")
        if not hasattr(self, 'last_modified'):
          raise Exception("Catalog lacks a last modification entry")

<commit_msg>add python CatalogStatistics::get_all_fields()<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Created by René Meusel
This file is part of the CernVM File System auxiliary tools.
"""

import datetime
import collections
import hashlib
import os

from _common import _split_md5, DatabaseObject
from dirent  import DirectoryEntry, Chunk

class CatalogIterator:
    """ Iterates through all directory entries of a Catalog """

    def __init__(self, catalog):
        self.catalog = catalog
        self.backlog = collections.deque()
        root_path = ""
        if not self.catalog.is_root():
            root_path = self.catalog.root_prefix
        self._push((root_path, self.catalog.find_directory_entry(root_path)))

    def __iter__(self):
        return self

    def next(self):
        if not self._has_more():
            raise StopIteration()
        return self._recursion_step()

    def _has_more(self):
        return len(self.backlog) > 0

    def _push(self, path):
        self.backlog.append(path)

    def _pop(self):
        return self.backlog.popleft()

    def _recursion_step(self):
        path, dirent = self._pop()
        if dirent.is_directory():
            new_dirents = self.catalog.list_directory_split_md5(dirent.md5path_1, \
                                                                dirent.md5path_2)
            for new_dirent in new_dirents:
                self._push((path + "/" + new_dirent.name, new_dirent))
        return path, dirent

class CatalogReference:
    """ Wraps a catalog reference to nested catalogs as found in Catalogs """

    def __init__(self, root_path, clg_hash, clg_size = 0):
        self.root_path = root_path
        self.hash      = clg_hash
        self.size      = clg_size

    def __str__(self):
        return "<CatalogReference for " + self.root_path + " - " + self.hash + ">"

    def __repr__(self):
        return "<CatalogReference for " + self.root_path + ">"

    def retrieve_from(self, source_repository):
        return source_repository.retrieve_catalog(self.hash)

class CatalogStatistics:
    """ Provides a convenience data wrapper around catalog statistics """

    def __init__(self, catalog):
        self.catalog = catalog
        if catalog.schema >= 2.1:
            self._read_statistics(catalog)

    def __str__(self):
        return "<CatalogStatistics for " + self.catalog.root_prefix + ">"

    def __repr__(self):
        return self.__str__()

    def num_entries(self):
        return self._get_stat('regular') + \
               self._get_stat('dir')     + \
               self._get_stat('symlink')

    def num_subtree_entries(self):
        return self._get_stat('all_regular') + \
               self._get_stat('all_dir')     + \
               self._get_stat('all_symlink')

    def num_chunked_files(self):
        return self._get_stat('chunked')

    def num_subtree_chunked_files(self):
        return self._get_stat('all_chunked')

    def num_file_chunks(self):
        return self._get_stat('chunks')

    def num_subtree_file_chunks(self):
        return self._get_stat('all_chunks')

    def data_size(self):
        return self._get_stat('file_size')

    def subtree_data_size(self):
        return self._get_stat('all_file_size')

    def get_all_fields(self):
        return self._get_stat('all_regular') , self._get_stat('all_dir') ,          \
               self._get_stat('all_symlink') , self._get_stat('all_file_size') ,    \
               self._get_stat('all_chunked') , self._get_stat('all_chunked_size') , \
               self._get_stat('all_chunks')  , self._get_stat('all_nested')

    def _read_statistics(self, catalog):
        stats = catalog.run_sql("SELECT * FROM statistics ORDER BY counter;")
        for stat, value in stats:
            if stat.startswith('self_'):
                setattr(self, stat[5:], value)
            elif stat.startswith('subtree_'):
                setattr(self, "all_" + stat[8:], value + getattr(self, stat[8:]))

    def _get_stat(self, stat):
        if not hasattr(self, stat):
            raise Exception("Statistic '" + stat + "' not provided.")
        return getattr(self, stat)

class Catalog(DatabaseObject):
    """ Wraps the basic functionality of CernVM-FS Catalogs """

    def __init__(self, catalog_file, catalog_hash = ""):
        DatabaseObject.__init__(self, catalog_file)
        self.hash = catalog_hash
        self._read_properties()
        self._guess_root_prefix_if_needed()
        self._guess_last_modified_if_needed()
        self._check_validity()

    def __str__(self):
        return "<Catalog " + self.root_prefix + ">"

    def __repr__(self):
        return self.__str__()

    def __iter__(self):
        return CatalogIterator(self)

    def has_nested(self):
        return self.nested_count() > 0

    def nested_count(self):
        """ Returns the number of nested catalogs in this catalog """
        num_catalogs = self.run_sql("SELECT count(*) FROM nested_catalogs;")
        return num_catalogs[0][0]

    def list_nested(self):
        """ List CatalogReferences to all contained nested catalogs """
        new_version = (self.schema <= 1.2 and self.schema_revision > 0)
        if new_version:
            sql_query = "SELECT path, sha1, size FROM nested_catalogs;"
        else:
            sql_query = "SELECT path, sha1 FROM nested_catalogs;"
        catalogs = self.run_sql(sql_query)
        if new_version:
            return [ CatalogReference(clg[0], clg[1], clg[2]) for clg in catalogs ]
        else:
            return [ CatalogReference(clg[0], clg[1]) for clg in catalogs ]

    def get_statistics(self):
        """ returns the embedded catalog statistics (if available) """
        return CatalogStatistics(self)

    def find_nested_for_path(self, needle_path):
        """ Find the best matching nested CatalogReference for a given path """
        nested_catalogs  = self.list_nested()
        best_match       = None
        best_match_score = 0
        real_needle_path = self._canonicalize_path(needle_path)
        for nested_catalog in nested_catalogs:
            if real_needle_path.startswith(nested_catalog.root_path) and \
                len(nested_catalog.root_path) > best_match_score:
                    best_match_score = len(nested_catalog.root_path)
                    best_match       = nested_catalog
        return best_match

    def list_directory(self, path):
        """ Create a directory listing of the given directory path """
        real_path = self._canonicalize_path(path)
        parent_1, parent_2 = _split_md5(hashlib.md5(real_path).digest())
        return self.list_directory_split_md5(parent_1, parent_2)

    def list_directory_split_md5(self, parent_1, parent_2):
        """ Create a directory listing of DirectoryEntry items based on MD5 path """
        res = self.run_sql("SELECT " + DirectoryEntry._catalog_db_fields() + " \
                            FROM catalog                                       \
                            WHERE parent_1 = " + str(parent_1) + " AND         \
                                  parent_2 = " + str(parent_2) + "             \
                            ORDER BY name ASC;")
        return [ self._make_directory_entry(result) for result in res ]

    def find_directory_entry(self, path):
        """ Finds the DirectoryEntry for a given path """
        real_path = self._canonicalize_path(path)
        md5path = hashlib.md5(real_path)
        return self.find_directory_entry_md5(md5path)

    def find_directory_entry_md5(self, md5path):
        """ Finds the DirectoryEntry for a given MD5 hashed path """
        lo, hi = _split_md5(md5path.digest())
        return self.find_directory_entry_split_md5(lo, hi)

    def find_directory_entry_split_md5(self, md5path_1, md5path_2):
        """ Finds the DirectoryEntry for the given split MD5 hashed path """
        res = self.run_sql("SELECT " + DirectoryEntry._catalog_db_fields() + " \
                            FROM catalog                                       \
                            WHERE md5path_1 = " + str(md5path_1) + " AND       \
                                  md5path_2 = " + str(md5path_2) + "           \
                            LIMIT 1;")
        return self._make_directory_entry(res[0]) if len(res) == 1 else None

    def is_root(self):
        """ Checks if this is the root catalog (based on the root prefix) """
        return self.root_prefix == "/"

    def has_predecessor(self):
        return hasattr(self, "previous_revision")

    def get_predecessor(self):
        if not self.has_predecessor():
            return None
        return CatalogReference(self.root_prefix, self.previous_revision)

    def _read_properties(self):
        self.read_properties_table(lambda prop_key, prop_value:
            self._read_property(prop_key, prop_value))
        if not hasattr(self, 'schema_revision'):
            self.schema_revision = 0

    def _read_property(self, prop_key, prop_value):
        """ Detect catalog properties and store them as public class members """
        if prop_key == "revision":
            self.revision          = prop_value
        if prop_key == "schema":
            self.schema            = float(prop_value)
        if prop_key == "schema_revision":
            self.schema_revision   = float(prop_value)
        if prop_key == "last_modified":
            self.last_modified     = datetime.datetime.fromtimestamp(int(prop_value))
        if prop_key == "previous_revision":
            self.previous_revision = prop_value
        if prop_key == "root_prefix":
            self.root_prefix       = prop_value

    def _make_directory_entry(self, result_set):
        dirent = DirectoryEntry(result_set)
        self._read_chunks(dirent)
        return dirent

    def _read_chunks(self, dirent):
        """ Finds and adds the file chunk of a DirectoryEntry """
        if self.schema < 2.4:
            return
        res = self.run_sql("SELECT " + Chunk._catalog_db_fields() + "           \
                            FROM chunks                                         \
                            WHERE md5path_1 = " + str(dirent.md5path_1) + " AND \
                                  md5path_2 = " + str(dirent.md5path_2) + "     \
                            ORDER BY offset ASC;")
        dirent._add_chunks(res)

    def _guess_root_prefix_if_needed(self):
        """ Root catalogs don't have a root prefix property (fixed here) """
        if not hasattr(self, 'root_prefix'):
            self.root_prefix = "/"

    def _guess_last_modified_if_needed(self):
        """ Catalog w/o a last_modified field, we set it to 0 """
        if not hasattr(self, 'last_modified'):
            self.last_modified = datetime.datetime.min

    def _canonicalize_path(self, path):
        if not path:
            return ""
        return os.path.realpath(path)

    def _check_validity(self):
        """ Check that all crucial properties have been found in the database """
        if not hasattr(self, 'revision'):
          raise Exception("Catalog lacks a revision entry")
        if not hasattr(self, 'schema'):
          raise Exception("Catalog lacks a schema entry")
        if not hasattr(self, 'root_prefix'):
          raise Exception("Catalog lacks a root prefix entry")
        if not hasattr(self, 'last_modified'):
          raise Exception("Catalog lacks a last modification entry")

<|endoftext|>


<commit_before># Django settings for production instance of pyvideo.ru
import os
from .settings import *

DEBUG = False

SECRET_KEY = os.environ.get('SECRET_KEY')

import dj_database_url
DATABASES['default'] = dj_database_url.config()

# Honor the 'X-Forwarded-Proto' header for request.is_secure()
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')

HAYSTACK_CONNECTIONS = {
    'default': {
        'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
        'URL': os.environ.get('BONSAI_URL'),
        'INDEX_NAME': 'pyvideo',
    },
}<commit_msg>SITE_URL for browserid<commit_after># Django settings for production instance of pyvideo.ru
import os
from .settings import *

DEBUG = False

SECRET_KEY = os.environ.get('SECRET_KEY')

import dj_database_url
DATABASES['default'] = dj_database_url.config()

# Honor the 'X-Forwarded-Proto' header for request.is_secure()
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')

HAYSTACK_CONNECTIONS = {
    'default': {
        'ENGINE': 'haystack.backends.elasticsearch_backend.ElasticsearchSearchEngine',
        'URL': os.environ.get('BONSAI_URL'),
        'INDEX_NAME': 'pyvideo',
    },
}

SITE_URL = 'http://pyvideo.herokuapp.com'<|endoftext|>


<commit_before>import unittest

import os.path
import os
import sys
sys.path.append(".")

from pywinauto.timings import Timings
Timings.Fast()

excludes = ['test_sendkeys']

def run_tests():
    testfolder = os.path.abspath(os.path.split(__file__)[0])

    sys.path.append(testfolder)

    for root, dirs, files in os.walk(testfolder):
        test_modules = [
            file.replace('.py', '') for file in files if
                file.startswith('test_') and
                file.endswith('.py')]

        test_modules = [mod for mod in test_modules if mod.lower() not in excludes]
        for mod in test_modules:

            #globals().update(__import__(mod, globals(), locals()).__dict__)
            # import it
            imported_mod = __import__(mod, globals(), locals())
            #print imported_mod.__dict__
            globals().update(imported_mod.__dict__)

    #runner = unittest.TextTestRunner(verbosity = 2)
    unittest.main()#testRunner = runner)

if __name__ == '__main__':
    run_tests()<commit_msg> * Removed the 'fast' timing from testall - tests are more likely to fail because of this!<commit_after>import unittest

import os.path
import os
import sys
sys.path.append(".")

#from pywinauto.timings import Timings
#Timings.Fast()

excludes = ['test_sendkeys']

def run_tests():
    testfolder = os.path.abspath(os.path.split(__file__)[0])

    sys.path.append(testfolder)

    for root, dirs, files in os.walk(testfolder):
        test_modules = [
            file.replace('.py', '') for file in files if
                file.startswith('test_') and
                file.endswith('.py')]

        test_modules = [mod for mod in test_modules if mod.lower() not in excludes]
        for mod in test_modules:

            #globals().update(__import__(mod, globals(), locals()).__dict__)
            # import it
            imported_mod = __import__(mod, globals(), locals())
            #print imported_mod.__dict__
            globals().update(imported_mod.__dict__)

    #runner = unittest.TextTestRunner(verbosity = 2)
    unittest.main()#testRunner = runner)

if __name__ == '__main__':
    run_tests()<|endoftext|>


<commit_before># -----------------------------------------------------------------------------
# Copyright (c) 2014--, The Qiita Development Team.
#
# Distributed under the terms of the BSD 3-clause License.
#
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------

from unittest import TestCase, main

from qiita_core.util import qiita_test_checker
from qiita_db.ontology import Ontology
from qiita_db.util import convert_from_id, convert_to_id

@qiita_test_checker()
class TestOntology(TestCase):
    def setUp(self):
        self.ontology = Ontology(999999999)

    def testConvertToID(self):
        self.assertEqual(convert_to_id('ENA_test', 'ontology'), 999999999)

    def testConvertFromID(self):
        self.assertEqual(convert_from_id(999999999, 'ontology'), 'ENA_test')

    def testShortNameProperty(self):
        self.assertEqual(self.ontology.shortname, 'ENA_test')

    def testTerms(self):
        obs = self.ontology.terms
        self.assertEqual(obs, [
            'Whole Genome Sequencing',
            'Metagenomics',
            'Transcriptome Analysis',
            'Resequencing',
            'Epigenetics',
            'Synthetic Genomics',
            'Forensic or Paleo-genomics',
            'Gene Regulation Study',
            'Cancer Genomics',
            'Population Genomics',
            'RNASeq',
            'Exome Sequencing',
            'Pooled Clone Sequencing',
            'Other'])

    def test_user_defined_terms(self):
        obs = self.ontology.user_defined_terms
        self.assertEqual(obs, [])

    def test_term_type(self):
        obs = self.ontology.term_type('RNASeq')
        self.assertEqual('ontology', obs)

        obs = self.ontology.term_type('Sasquatch')
        self.assertEqual('not_ontology', obs)

        self.ontology.add_user_defined_term('Test Term')
        obs = self.ontology.term_type('Test Term')
        self.assertEqual('user_defined', obs)

    def test_add_user_defined_term(self):
        self.assertFalse('Test Term' in self.ontology.user_defined_terms)
        pre = len(self.ontology.user_defined_terms)
        self.ontology.add_user_defined_term('Test Term')
        post = len(self.ontology.user_defined_terms)
        self.assertTrue('Test Term' in self.ontology.user_defined_terms)
        self.assertEqual(post-pre, 1)

    def testContains(self):
        self.assertTrue('Metagenomics' in self.ontology)
        self.assertFalse('NotATerm' in self.ontology)

if __name__ == '__main__':
    main()
<commit_msg>Fix failing tests<commit_after># -----------------------------------------------------------------------------
# Copyright (c) 2014--, The Qiita Development Team.
#
# Distributed under the terms of the BSD 3-clause License.
#
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------

from unittest import TestCase, main

from qiita_core.util import qiita_test_checker
from qiita_db.ontology import Ontology
from qiita_db.util import convert_from_id, convert_to_id

@qiita_test_checker()
class TestOntology(TestCase):
    def setUp(self):
        self.ontology = Ontology(999999999)

    def testConvertToID(self):
        self.assertEqual(convert_to_id('ENA', 'ontology'), 999999999)

    def testConvertFromID(self):
        self.assertEqual(convert_from_id(999999999, 'ontology'), 'ENA')

    def testShortNameProperty(self):
        self.assertEqual(self.ontology.shortname, 'ENA')

    def testTerms(self):
        obs = self.ontology.terms
        self.assertEqual(obs, [
            'Whole Genome Sequencing',
            'Metagenomics',
            'Transcriptome Analysis',
            'Resequencing',
            'Epigenetics',
            'Synthetic Genomics',
            'Forensic or Paleo-genomics',
            'Gene Regulation Study',
            'Cancer Genomics',
            'Population Genomics',
            'RNASeq',
            'Exome Sequencing',
            'Pooled Clone Sequencing',
            'Other'])

    def test_user_defined_terms(self):
        obs = self.ontology.user_defined_terms
        self.assertEqual(obs, [])

    def test_term_type(self):
        obs = self.ontology.term_type('RNASeq')
        self.assertEqual('ontology', obs)

        obs = self.ontology.term_type('Sasquatch')
        self.assertEqual('not_ontology', obs)

        self.ontology.add_user_defined_term('Test Term')
        obs = self.ontology.term_type('Test Term')
        self.assertEqual('user_defined', obs)

    def test_add_user_defined_term(self):
        self.assertFalse('Test Term' in self.ontology.user_defined_terms)
        pre = len(self.ontology.user_defined_terms)
        self.ontology.add_user_defined_term('Test Term')
        post = len(self.ontology.user_defined_terms)
        self.assertTrue('Test Term' in self.ontology.user_defined_terms)
        self.assertEqual(post-pre, 1)

    def testContains(self):
        self.assertTrue('Metagenomics' in self.ontology)
        self.assertFalse('NotATerm' in self.ontology)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
""" override sale order """

import pprint
import logging

from openerp import models, fields, api, exceptions, tools
_logger = logging.getLogger(__name__)

class Order(models.Model):
    """ override sale order """

    _inherit = 'sale.order'

    incipit = fields.Text()
    sale_offer_markup = fields.Float(related='project_id.sale_offer_markup', readonly=True)
    real_project_id = fields.Many2one('project.project', string="Project",
                                      related="project_id.project_id", readonly=True)
    unrelated_task_ids = fields.One2many('project.task', string="Related Tasks",
                                         compute="compute_unrelated_task_ids")
    task_to_invoice_ids = fields.One2many('project.task', 'sale_order_id', string="Billing plan")

    @api.multi
    def write(self, values):
        """ annullo eventuali task di fatturazione collegati
            all'annullamento dell'ordine
            annullo anche task non chiusi che contengono il nome ordine
            (fatturazione, gest produzione etc..) """

        if 'unrelated_task_ids' in values and len(self) == 1:
            for unreltask in values['unrelated_task_ids']:
                for task in self.unrelated_task_ids:
                    if task.id == unreltask[1]:
                        if unreltask[2] and 'direct_sale_line_id' in unreltask[2]:
                            task.direct_sale_line_id = unreltask[2]['direct_sale_line_id']
                        else:
                            task.direct_sale_line_id = False

        for order in self:
            if 'state' in values and values['state'] == 'cancel':

                order.task_to_invoice_ids.filtered(lambda record:
                                                   record.stage_id not in [7]).write(
                                                       {'stage_id': 8}
                                                    )

                order.unrelated_task_ids.filtered(lambda record:
                                                  record.stage_id not in [7] and order.name in record.name).write(
                                                      {'stage_id': 8}
                                                  )

        res = super(Order, self).write(values)
        return res

    @api.one
    def compute_unrelated_task_ids(self):
        """ compute_unrelated_task_ids """

        if self.real_project_id:
            unrelated_recordset = self.env["project.task"].sudo().search(
                [
                    ("direct_sale_line_id", "=", False),
                    '|',
                    ("project_id", "=", self.real_project_id.id),
                    ("project_ref_id", "=", self.real_project_id.id)
                ]
            )
            self.unrelated_task_ids = unrelated_recordset

    @api.multi
    def calculate_project_costs(self):
        """ calculate_project_costs """

        for line in self.order_line:
            line_cost = 0

            for task in line.sudo().tasks_ids:
                task.compute_price()
                line_cost = task.cost + line_cost

            if not line.sudo().tasks_ids:
                line_cost = line.purchase_price

            update = {
                "purchase_price": line_cost,
                "price_unit": line_cost * self.sale_offer_markup if not line.fixed_price else line.price_unit
            }

            line.write(update)
<commit_msg>show total<commit_after># -*- coding: utf-8 -*-
""" override sale order """

import pprint
import logging

from openerp import models, fields, api, exceptions, tools
_logger = logging.getLogger(__name__)

class Order(models.Model):
    """ override sale order """

    _inherit = 'sale.order'

    incipit = fields.Text()
    sale_offer_markup = fields.Float(related='project_id.sale_offer_markup', readonly=True)
    real_project_id = fields.Many2one('project.project', string="Project",
                                      related="project_id.project_id", readonly=True)
    unrelated_task_ids = fields.One2many('project.task', string="Related Tasks",
                                         compute="compute_unrelated_task_ids")
    task_to_invoice_ids = fields.One2many('project.task', 'sale_order_id', string="Billing plan")
    show_total = fields.Boolean(default=True)

    @api.multi
    def write(self, values):
        """ annullo eventuali task di fatturazione collegati
            all'annullamento dell'ordine
            annullo anche task non chiusi che contengono il nome ordine
            (fatturazione, gest produzione etc..) """

        if 'unrelated_task_ids' in values and len(self) == 1:
            for unreltask in values['unrelated_task_ids']:
                for task in self.unrelated_task_ids:
                    if task.id == unreltask[1]:
                        if unreltask[2] and 'direct_sale_line_id' in unreltask[2]:
                            task.direct_sale_line_id = unreltask[2]['direct_sale_line_id']
                        else:
                            task.direct_sale_line_id = False

        for order in self:
            if 'state' in values and values['state'] == 'cancel':

                order.task_to_invoice_ids.filtered(lambda record:
                                                   record.stage_id not in [7]).write(
                                                       {'stage_id': 8}
                                                    )

                order.unrelated_task_ids.filtered(lambda record:
                                                  record.stage_id not in [7] and order.name in record.name).write(
                                                      {'stage_id': 8}
                                                  )

        res = super(Order, self).write(values)
        return res

    @api.one
    def compute_unrelated_task_ids(self):
        """ compute_unrelated_task_ids """

        if self.real_project_id:
            unrelated_recordset = self.env["project.task"].sudo().search(
                [
                    ("direct_sale_line_id", "=", False),
                    '|',
                    ("project_id", "=", self.real_project_id.id),
                    ("project_ref_id", "=", self.real_project_id.id)
                ]
            )
            self.unrelated_task_ids = unrelated_recordset

    @api.multi
    def calculate_project_costs(self):
        """ calculate_project_costs """

        for line in self.order_line:
            line_cost = 0

            for task in line.sudo().tasks_ids:
                task.compute_price()
                line_cost = task.cost + line_cost

            if not line.sudo().tasks_ids:
                line_cost = line.purchase_price

            update = {
                "purchase_price": line_cost,
                "price_unit": line_cost * self.sale_offer_markup if not line.fixed_price else line.price_unit
            }

            line.write(update)
<|endoftext|>