<commit_before>PROJECT='dcagent'
VERSION='0.0.1'
DEBUG = 1    #0/1
DEFS = []
VERBOSE = 'on'    #on/off
EXTRA_C_FLAGS = '-Wno-unused-parameter'
EXTRA_CXX_FLAGS = '--std=c++11'
LIBS = [
        {
            'name':'dcbase',
            'subdir':'base',
            'linklibs' : [],
            'includes':[],
            'src_dirs':['base'],
        },
        {
            'name':'dcnode',
            'subdir':'dcnode',
            'includes':['/usr/local/include','/usr/local/include/libmongoc-1.0','3rd'],
            'src_dirs':['dcnode/proto','3rd/pbjson'],
            'genobj': {
                'out':'${CMAKE_CURRENT_SOURCE_DIR}/proto/dcnode.pb.cc',
                'dep':'${CMAKE_CURRENT_SOURCE_DIR}/proto/dcnode.proto',
                'cmd':'protoc ${CMAKE_CURRENT_SOURCE_DIR}/proto/dcnode.proto -I${CMAKE_CURRENT_SOURCE_DIR}/proto --cpp_out=${CMAKE_CURRENT_SOURCE_DIR}/proto'
            }
        },
        {
            'name':'dcagent',
            'subdir':'dcagent',
            'includes':['/usr/local/include','/usr/local/include/libmongoc-1.0','3rd'],
            'src_dirs':['dcagent/proto'],
            'genobj': {
                'out':'${CMAKE_CURRENT_SOURCE_DIR}/proto/dcagent.pb.cc',
                'dep':'${CMAKE_CURRENT_SOURCE_DIR}/proto/dcagent.proto',
                'cmd':'protoc ${CMAKE_CURRENT_SOURCE_DIR}/proto/dcagent.proto -I${CMAKE_CURRENT_SOURCE_DIR}/proto --cpp_out=${CMAKE_CURRENT_SOURCE_DIR}/proto'
            }
        },
        {
            'name':'mongoproxyapi',
            'subdir':'app/mongoproxy/api',
            'includes':['/usr/local/include/libmongoc-1.0','3rd'],
            'linkpaths':['/usr/local/lib'],
            'src_dirs': ['app/mongoproxy/proto'],
        },
        {
            'name': 'dcrepoter',
            'subdir': 'reporter',
            'includes': ['base','dcnode','dcagent','/usr/local/include/libmongoc-1.0','3rd'],
        },
        {
            'name': 'pbjson',
            'subdir': '3rd/pbjson/',
            'includes': ['3rd'],
        },
        {
            'name': 'dcutil-redis',
            'subdir': 'utility/redis',
            'includes': ['3rd'],
        },
        {
            'name': 'dcrpc',
            'subdir': 'dcrpc',
            'includes': [],
            'src_dirs': ['./dcrpc/client/','./dcrpc/server/','./dcrpc/share/','./dcrpc/share/dcrpc.pb.cc'],
            'genobj': {
                'out':'${CMAKE_CURRENT_SOURCE_DIR}/share/dcrpc.pb.cc',
                'dep':'${CMAKE_CURRENT_SOURCE_DIR}/proto/dcrpc.proto',
                'cmd':'protoc ${CMAKE_CURRENT_SOURCE_DIR}/proto/dcrpc.proto -I${CMAKE_CURRENT_SOURCE_DIR}/proto --cpp_out=${CMAKE_CURRENT_SOURCE_DIR}/share'
            }
        },
        {
            'name':'dcutil-mysql',
            'subdir':'utility/mysql',
            'linklibs' : [],
            'includes':['/usr/local/include'],
        },
        {
            'name':'dcutil-script',
            'subdir':'utility/script',
            'linklibs' : [],
            'includes':['/usr/local/include'],
        },
        {
            'name':'dcutil-mongo',
            'subdir':'utility/mongo',
            'linklibs' : [],
            'includes':['/usr/local/include/libbson-1.0','3rd'],
        },
        {
            'name':'dcutil-drs',
            'subdir':'utility/drs',
            'linklibs' : [],
            'includes':['/usr/local/include','3rd'],
        },
        {
            'name':'dcutil-crypt',
            'subdir':'utility/crypt',
            'linklibs' : [],
            'includes':[],
            'src_dirs':[],
            'extra_srcs': [],
        },

]
EXES = [
        {
            'name':'dctest',
            'subdir':'app/test',
            'includes':['/usr/local/include','/usr/local/include/libmongoc-1.0'],
            'linkpaths':['/usr/local/lib'],
            'linklibs' : [
                'dcnode',
                'dcagent',
                'dcutil-drs',
                'dcutil-mysql',
                'dcutil-mongo',
                'dcutil-script',
                'dcbase',
                'pbjson',
                'python2.7',
                'libprotobuf.a',
                'mysqlclient',
                'mongoc-1.0',
                'bson-1.0',
            ],
            'genobj': {
                'out':'${CMAKE_CURRENT_SOURCE_DIR}/test_conf.pb.cc',
                'dep':'${CMAKE_CURRENT_SOURCE_DIR}/test_conf.proto',
                'cmd':'protoc ${CMAKE_CURRENT_SOURCE_DIR}/test_conf.proto -I${CMAKE_CURRENT_SOURCE_DIR} --cpp_out=${CMAKE_CURRENT_SOURCE_DIR}'
            }
        },
        {
            'name':'mongoproxy',
            'subdir':'app/mongoproxy',
            'includes':['/usr/local/include','/usr/local/include/libmongoc-1.0'],
            'linkpaths':['/usr/local/lib'],
            'src_dirs': ['app/mongoproxy/proto'],
            'linklibs' : [
                'mongoproxyapi',
                'dcnode',
                'dcagent',
                'dcutil-mongo',
                'dcutil-drs',
                'dcbase',
                'pbjson',
                'python2.7',
                'libprotobuf.a',
                'mongoc-1.0',
                'bson-1.0'
            ]
        },
        {
            'name':'mongoproxy_testapi',
            'subdir':'app/mongoproxy/testapi',
            'includes':['/usr/local/include'],
            'linkpaths':['/usr/local/lib'],
            'src_dirs': [],
            'linklibs' : [
                'mongoproxyapi',
                'dcnode',
                'dcagent',
                'dcutil-mongo',
                'dcutil-drs',
                'dcbase',
                'pbjson',
                'python2.7',
                'libprotobuf.a',
                'mongoc-1.0',
                'bson-1.0'
            ]
        },
        {
            'name':'reporter',
            'subdir':'app/reporter',
            'includes':['/usr/local/include','/usr/local/include/libmongoc-1.0'],
            'linklibs' : [
                'dcrepoter',
                'dcagent',
                'dcnode',
                'dcutil-script',
                'dcutil-drs',
                'dcbase',
                'pbjson',
                'python2.7',
                'protobuf',
            ]
        },
        {
            'name':'collector',
            'subdir':'app/collector',
            'includes':['/usr/local/include','/usr/local/include/libmongoc-1.0'],
            'linklibs' : [
                'dcrepoter',
                'dcagent',
                'dcnode',
                'dcutil-script',
                'dcutil-drs',
                'dcbase',
                'pbjson',
                'python2.7',
                'protobuf',
            ]
        },
        {
            'name':'pingpong',
            'subdir':'app/pingpong',
            'includes':['/usr/local/include','/usr/local/include/libmongoc-1.0'],
            'linklibs' : [
                'dcnode',
                'dcutil-drs',
                'dcbase',
                'pbjson',
                'protobuf',
            ]
        },
]
<commit_msg>add mongoproxy makelist<commit_after>PROJECT='dcagent'
VERSION='0.0.1'
DEBUG = 1    #0/1
DEFS = []
VERBOSE = 'on'    #on/off
EXTRA_C_FLAGS = '-Wno-unused-parameter'
EXTRA_CXX_FLAGS = '--std=c++11'
LIBS = [
        {
            'name':'dcbase',
            'subdir':'base',
            'linklibs' : [],
            'includes':[],
            'src_dirs':['base'],
        },
        {
            'name':'dcnode',
            'subdir':'dcnode',
            'includes':['/usr/local/include','/usr/local/include/libmongoc-1.0','3rd'],
            'src_dirs':['dcnode/proto','3rd/pbjson'],
            'genobj': {
                'out':'${CMAKE_CURRENT_SOURCE_DIR}/proto/dcnode.pb.cc',
                'dep':'${CMAKE_CURRENT_SOURCE_DIR}/proto/dcnode.proto',
                'cmd':'protoc ${CMAKE_CURRENT_SOURCE_DIR}/proto/dcnode.proto -I${CMAKE_CURRENT_SOURCE_DIR}/proto --cpp_out=${CMAKE_CURRENT_SOURCE_DIR}/proto'
            }
        },
        {
            'name':'dcagent',
            'subdir':'dcagent',
            'includes':['/usr/local/include','/usr/local/include/libmongoc-1.0','3rd'],
            'src_dirs':['dcagent/proto'],
            'genobj': {
                'out':'${CMAKE_CURRENT_SOURCE_DIR}/proto/dcagent.pb.cc',
                'dep':'${CMAKE_CURRENT_SOURCE_DIR}/proto/dcagent.proto',
                'cmd':'protoc ${CMAKE_CURRENT_SOURCE_DIR}/proto/dcagent.proto -I${CMAKE_CURRENT_SOURCE_DIR}/proto --cpp_out=${CMAKE_CURRENT_SOURCE_DIR}/proto'
            }
        },
        {
            'name':'mongoproxyapi',
            'subdir':'app/mongoproxy/api',
            'includes':['/usr/local/include/libmongoc-1.0','3rd'],
            'linkpaths':['/usr/local/lib'],
            'src_dirs': ['app/mongoproxy/proto'],
            'genobj': {
                'out':'${CMAKE_CURRENT_SOURCE_DIR}/../proto/mongo.pb.cc',
                'dep':'${CMAKE_CURRENT_SOURCE_DIR}/../proto/mongo.proto',
                'cmd':'protoc ${CMAKE_CURRENT_SOURCE_DIR}/proto/mongo.proto -I${CMAKE_CURRENT_SOURCE_DIR}/../proto --cpp_out=${CMAKE_CURRENT_SOURCE_DIR}/../proto'
            }
        },
        {
            'name': 'dcrepoter',
            'subdir': 'reporter',
            'includes': ['base','dcnode','dcagent','/usr/local/include/libmongoc-1.0','3rd'],
        },
        {
            'name': 'pbjson',
            'subdir': '3rd/pbjson/',
            'includes': ['3rd'],
        },
        {
            'name': 'dcutil-redis',
            'subdir': 'utility/redis',
            'includes': ['3rd'],
        },
        {
            'name': 'dcrpc',
            'subdir': 'dcrpc',
            'includes': [],
            'src_dirs': ['./dcrpc/client/','./dcrpc/server/','./dcrpc/share/','./dcrpc/share/dcrpc.pb.cc'],
            'genobj': {
                'out':'${CMAKE_CURRENT_SOURCE_DIR}/share/dcrpc.pb.cc',
                'dep':'${CMAKE_CURRENT_SOURCE_DIR}/proto/dcrpc.proto',
                'cmd':'protoc ${CMAKE_CURRENT_SOURCE_DIR}/proto/dcrpc.proto -I${CMAKE_CURRENT_SOURCE_DIR}/proto --cpp_out=${CMAKE_CURRENT_SOURCE_DIR}/share'
            }
        },
        {
            'name':'dcutil-mysql',
            'subdir':'utility/mysql',
            'linklibs' : [],
            'includes':['/usr/local/include'],
        },
        {
            'name':'dcutil-script',
            'subdir':'utility/script',
            'linklibs' : [],
            'includes':['/usr/local/include'],
        },
        {
            'name':'dcutil-mongo',
            'subdir':'utility/mongo',
            'linklibs' : [],
            'includes':['/usr/local/include/libbson-1.0','3rd'],
        },
        {
            'name':'dcutil-drs',
            'subdir':'utility/drs',
            'linklibs' : [],
            'includes':['/usr/local/include','3rd'],
        },
        {
            'name':'dcutil-crypt',
            'subdir':'utility/crypt',
            'linklibs' : [],
            'includes':[],
            'src_dirs':[],
            'extra_srcs': [],
        },

]
EXES = [
        {
            'name':'dctest',
            'subdir':'app/test',
            'includes':['/usr/local/include','/usr/local/include/libmongoc-1.0'],
            'linkpaths':['/usr/local/lib'],
            'linklibs' : [
                'dcnode',
                'dcagent',
                'dcutil-drs',
                'dcutil-mysql',
                'dcutil-mongo',
                'dcutil-script',
                'dcbase',
                'pbjson',
                'python2.7',
                'libprotobuf.a',
                'mysqlclient',
                'mongoc-1.0',
                'bson-1.0',
            ],
            'genobj': {
                'out':'${CMAKE_CURRENT_SOURCE_DIR}/test_conf.pb.cc',
                'dep':'${CMAKE_CURRENT_SOURCE_DIR}/test_conf.proto',
                'cmd':'protoc ${CMAKE_CURRENT_SOURCE_DIR}/test_conf.proto -I${CMAKE_CURRENT_SOURCE_DIR} --cpp_out=${CMAKE_CURRENT_SOURCE_DIR}'
            }
        },
        {
            'name':'mongoproxy',
            'subdir':'app/mongoproxy',
            'includes':['/usr/local/include','/usr/local/include/libmongoc-1.0'],
            'linkpaths':['/usr/local/lib'],
            'src_dirs': ['app/mongoproxy/proto'],
            'linklibs' : [
                'mongoproxyapi',
                'dcnode',
                'dcagent',
                'dcutil-mongo',
                'dcutil-drs',
                'dcbase',
                'pbjson',
                'python2.7',
                'libprotobuf.a',
                'mongoc-1.0',
                'bson-1.0'
            ]
        },
        {
            'name':'mongoproxy_testapi',
            'subdir':'app/mongoproxy/testapi',
            'includes':['/usr/local/include'],
            'linkpaths':['/usr/local/lib'],
            'src_dirs': [],
            'linklibs' : [
                'mongoproxyapi',
                'dcnode',
                'dcagent',
                'dcutil-mongo',
                'dcutil-drs',
                'dcbase',
                'pbjson',
                'python2.7',
                'libprotobuf.a',
                'mongoc-1.0',
                'bson-1.0'
            ]
        },
        {
            'name':'reporter',
            'subdir':'app/reporter',
            'includes':['/usr/local/include','/usr/local/include/libmongoc-1.0'],
            'linklibs' : [
                'dcrepoter',
                'dcagent',
                'dcnode',
                'dcutil-script',
                'dcutil-drs',
                'dcbase',
                'pbjson',
                'python2.7',
                'protobuf',
            ]
        },
        {
            'name':'collector',
            'subdir':'app/collector',
            'includes':['/usr/local/include','/usr/local/include/libmongoc-1.0'],
            'linklibs' : [
                'dcrepoter',
                'dcagent',
                'dcnode',
                'dcutil-script',
                'dcutil-drs',
                'dcbase',
                'pbjson',
                'python2.7',
                'protobuf',
            ]
        },
        {
            'name':'pingpong',
            'subdir':'app/pingpong',
            'includes':['/usr/local/include','/usr/local/include/libmongoc-1.0'],
            'linklibs' : [
                'dcnode',
                'dcutil-drs',
                'dcbase',
                'pbjson',
                'protobuf',
            ]
        },
]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import logging
import os
import resource
import sys

import cv2
from joblib import Parallel, parallel_backend, delayed

logger = logging.getLogger(__name__)

abspath = os.path.abspath(os.path.dirname(__file__))
SENSOR = os.path.join(abspath, 'data', 'sensor_data.json')
BOW_PATH = os.path.join(abspath, 'data', 'bow')

# Handle different OpenCV versions
OPENCV3 = int(cv2.__version__.split('.')[0]) >= 3

if hasattr(cv2, 'flann_Index'):
    flann_Index = cv2.flann_Index
elif hasattr(cv2, 'flann') and hasattr(cv2.flann, 'Index'):
    flann_Index = cv2.flann.Index
else:
    logger.warning('Unable to find flann Index')
    flann_Index = None

# Parallel processes
def parallel_map(func, args, num_proc, max_batch_size=1):
    """Run function for all arguments using multiple processes."""
    num_proc = min(num_proc, len(args))
    if num_proc <= 1:
        return list(map(func, args))
    else:
        with parallel_backend('loky', n_jobs=num_proc):
            batch_size = max(1, int(len(args)/(num_proc*2)))
            batch_size = min(batch_size, max_batch_size) if max_batch_size else batch_size
            return Parallel(batch_size=batch_size)(delayed(func)(arg) for arg in args)

# Memory usage
if sys.platform == 'darwin':
    rusage_unit = 1
else:
    rusage_unit = 1024

def memory_available():
    """Available memory in MB.

    Only works on linux and returns None otherwise.
    """
    lines = os.popen('free -t -m').readlines()
    if not lines:
        return None
    available_mem = int(lines[1].split()[6])
    return available_mem

def processes_that_fit_in_memory(desired, per_process):
    """Amount of parallel BoW process that fit in memory."""
    available_mem = memory_available()
    if available_mem is not None:
        fittable = max(1, int(available_mem / per_process))
        return min(desired, fittable)
    else:
        return desired

def current_memory_usage():
    return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * rusage_unit
<commit_msg>feat: de-activate and restore opencv inner threading<commit_after># -*- coding: utf-8 -*-
import logging
import os
import resource
import sys

import cv2
from joblib import Parallel, parallel_backend, delayed

logger = logging.getLogger(__name__)

abspath = os.path.abspath(os.path.dirname(__file__))
SENSOR = os.path.join(abspath, 'data', 'sensor_data.json')
BOW_PATH = os.path.join(abspath, 'data', 'bow')

# Handle different OpenCV versions
OPENCV3 = int(cv2.__version__.split('.')[0]) >= 3

if hasattr(cv2, 'flann_Index'):
    flann_Index = cv2.flann_Index
elif hasattr(cv2, 'flann') and hasattr(cv2.flann, 'Index'):
    flann_Index = cv2.flann.Index
else:
    logger.warning('Unable to find flann Index')
    flann_Index = None

# Parallel processes
def parallel_map(func, args, num_proc, max_batch_size=1):
    """Run function for all arguments using multiple processes."""
    num_proc = min(num_proc, len(args))
    if num_proc <= 1:
        return list(map(func, args))
    else:
        # De-activate/Restore any inner OpenCV threading
        threads_used = cv2.getNumThreads()
        cv2.setNumThreads(0)
        with parallel_backend('loky', n_jobs=num_proc):
            batch_size = max(1, int(len(args)/(num_proc*2)))
            batch_size = min(batch_size, max_batch_size) if max_batch_size else batch_size
            res = Parallel(batch_size=batch_size)(delayed(func)(arg) for arg in args)
            cv2.setNumThreads(threads_used)
            return res

# Memory usage
if sys.platform == 'darwin':
    rusage_unit = 1
else:
    rusage_unit = 1024

def memory_available():
    """Available memory in MB.

    Only works on linux and returns None otherwise.
    """
    lines = os.popen('free -t -m').readlines()
    if not lines:
        return None
    available_mem = int(lines[1].split()[6])
    return available_mem

def processes_that_fit_in_memory(desired, per_process):
    """Amount of parallel BoW process that fit in memory."""
    available_mem = memory_available()
    if available_mem is not None:
        fittable = max(1, int(available_mem / per_process))
        return min(desired, fittable)
    else:
        return desired

def current_memory_usage():
    return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * rusage_unit
<|endoftext|>


<commit_before># Copyright Hybrid Logic Ltd.  See LICENSE file for details.

"""
Rackspace provisioner.
"""

from ._libcloud import monkeypatch, LibcloudProvisioner
from ._install import (
    provision,
    task_open_control_firewall,
    task_upgrade_kernel,
)
from ._ssh import run_remotely

from ._effect import sequence
from effect import Func, Effect

def provision_rackspace(node, package_source, distribution, variants):
    """
    Provision flocker on this node.

    :param LibcloudNode node: Node to provision.
    :param PackageSource package_source: See func:`task_install_flocker`
    :param bytes distribution: See func:`task_install_flocker`
    :param set variants: The set of variant configurations to use when
        provisioning
    """
    commands = []
    if distribution in ('centos-7',):
        commands.append(run_remotely(
            username='root',
            address=node.address,
            commands=sequence([
                task_upgrade_kernel('centos-7'),
                Effect(Func(node.reboot)),
            ]),
        ))

    commands.append(run_remotely(
        username='root',
        address=node.address,
        commands=sequence([
            provision(
                package_source=package_source,
                distribution=node.distribution,
                variants=variants,
            ),
            # https://clusterhq.atlassian.net/browse/FLOC-1550
            # This should be part of ._install.configure_cluster
            task_open_control_firewall(node.distribution),
        ]),
    ))

    return sequence(commands)

IMAGE_NAMES = {
    'fedora-20': u'Fedora 20 (Heisenbug) (PVHVM)',
    'centos-7': u'CentOS 7 (PVHVM)',
    'ubuntu-14.04': u'Ubuntu 14.04 LTS (Trusty Tahr) (PVHVM)'
}

def rackspace_provisioner(username, key, region, keyname):
    """
    Create a LibCloudProvisioner for provisioning nodes on rackspace.

    :param bytes username: The user to connect to rackspace with.
    :param bytes key: The API key associated with the user.
    :param bytes region: The rackspace region in which to launch the instance.
    :param bytes keyname: The name of an existing ssh public key configured in
       rackspace. The provision step assumes the corresponding private key is
       available from an agent.
    """
    # Import these here, so that this can be imported without
    # installng libcloud.
    from libcloud.compute.providers import get_driver, Provider
    monkeypatch()
    driver = get_driver(Provider.RACKSPACE)(
        key=username,
        secret=key,
        region=region)

    provisioner = LibcloudProvisioner(
        driver=driver,
        keyname=keyname,
        image_names=IMAGE_NAMES,
        create_node_arguments=lambda **kwargs: {
            "ex_config_drive": "true",
        },
        provision=provision_rackspace,
        default_size="performance1-8",
    )

    return provisioner
<commit_msg>Use specified distribution, rather than doing manual value propogation.<commit_after># Copyright Hybrid Logic Ltd.  See LICENSE file for details.

"""
Rackspace provisioner.
"""

from ._libcloud import monkeypatch, LibcloudProvisioner
from ._install import (
    provision,
    task_open_control_firewall,
    task_upgrade_kernel,
)
from ._ssh import run_remotely

from ._effect import sequence
from effect import Func, Effect

def provision_rackspace(node, package_source, distribution, variants):
    """
    Provision flocker on this node.

    :param LibcloudNode node: Node to provision.
    :param PackageSource package_source: See func:`task_install_flocker`
    :param bytes distribution: See func:`task_install_flocker`
    :param set variants: The set of variant configurations to use when
        provisioning
    """
    commands = []
    if distribution in ('centos-7',):
        commands.append(run_remotely(
            username='root',
            address=node.address,
            commands=sequence([
                task_upgrade_kernel(node.distribution),
                Effect(Func(node.reboot)),
            ]),
        ))

    commands.append(run_remotely(
        username='root',
        address=node.address,
        commands=sequence([
            provision(
                package_source=package_source,
                distribution=node.distribution,
                variants=variants,
            ),
            # https://clusterhq.atlassian.net/browse/FLOC-1550
            # This should be part of ._install.configure_cluster
            task_open_control_firewall(node.distribution),
        ]),
    ))

    return sequence(commands)

IMAGE_NAMES = {
    'fedora-20': u'Fedora 20 (Heisenbug) (PVHVM)',
    'centos-7': u'CentOS 7 (PVHVM)',
    'ubuntu-14.04': u'Ubuntu 14.04 LTS (Trusty Tahr) (PVHVM)'
}

def rackspace_provisioner(username, key, region, keyname):
    """
    Create a LibCloudProvisioner for provisioning nodes on rackspace.

    :param bytes username: The user to connect to rackspace with.
    :param bytes key: The API key associated with the user.
    :param bytes region: The rackspace region in which to launch the instance.
    :param bytes keyname: The name of an existing ssh public key configured in
       rackspace. The provision step assumes the corresponding private key is
       available from an agent.
    """
    # Import these here, so that this can be imported without
    # installng libcloud.
    from libcloud.compute.providers import get_driver, Provider
    monkeypatch()
    driver = get_driver(Provider.RACKSPACE)(
        key=username,
        secret=key,
        region=region)

    provisioner = LibcloudProvisioner(
        driver=driver,
        keyname=keyname,
        image_names=IMAGE_NAMES,
        create_node_arguments=lambda **kwargs: {
            "ex_config_drive": "true",
        },
        provision=provision_rackspace,
        default_size="performance1-8",
    )

    return provisioner
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from osv import fields,osv
from osv.orm import except_orm
import tools

class ir_attachment(osv.osv):
    def check(self, cr, uid, ids, mode, context=None):
        if not ids:
            return
        ima = self.pool.get('ir.model.access')
        if isinstance(ids, (int, long)):
            ids = [ids]
        cr.execute('select distinct res_model from ir_attachment where id IN %s', (tuple(ids),))
        for obj in cr.fetchall():
            if obj[0]:
                ima.check(cr, uid, obj[0], mode, context=context)

    def search(self, cr, uid, args, offset=0, limit=None, order=None,
            context=None, count=False):
        ids = super(ir_attachment, self).search(cr, uid, args, offset=offset,
                                                limit=limit, order=order,
                                                context=context, count=False)
        if not ids:
            if count:
                return 0
            return []
        models = super(ir_attachment,self).read(cr, uid, ids, ['id', 'res_model'])
        cache = {}
        ima = self.pool.get('ir.model.access')
        for m in models:
            if m['res_model']:
                if m['res_model'] not in cache:
                    cache[m['res_model']] = ima.check(cr, uid, m['res_model'], 'read',
                                                      raise_exception=False, context=context)
                if not cache[m['res_model']]:
                    ids.remove(m['id'])

        if count:
            return len(ids)
        return ids

    def read(self, cr, uid, ids, fields_to_read=None, context=None, load='_classic_read'):
        self.check(cr, uid, ids, 'read', context=context)
        return super(ir_attachment, self).read(cr, uid, ids, fields_to_read, context, load)

    def write(self, cr, uid, ids, vals, context=None):
        self.check(cr, uid, ids, 'write', context=context)
        return super(ir_attachment, self).write(cr, uid, ids, vals, context)

    def copy(self, cr, uid, id, default=None, context=None):
        self.check(cr, uid, [id], 'write', context=context)
        return super(ir_attachment, self).copy(cr, uid, id, default, context)

    def unlink(self, cr, uid, ids, context=None):
        self.check(cr, uid, ids, 'unlink', context=context)
        return super(ir_attachment, self).unlink(cr, uid, ids, context)

    def create(self, cr, uid, values, context=None):
        if 'res_model' in values and values['res_model'] != '':
            self.pool.get('ir.model.access').check(cr, uid, values['res_model'], 'create', context=context)
        return super(ir_attachment, self).create(cr, uid, values, context)

    def action_get(self, cr, uid, context=None):
        dataobj = self.pool.get('ir.model.data')
        data_id = dataobj._get_id(cr, 1, 'base', 'action_attachment')
        res_id = dataobj.browse(cr, uid, data_id, context).res_id
        return self.pool.get('ir.actions.act_window').read(cr, uid, res_id, [], context)

    def _name_get_resname(self, cr, uid, ids, object,method, context):
        data = {}
        for attachment in self.browse(cr, uid, ids, context=context):
            model_object = attachment.res_model
            res_id = attachment.res_id
            if model_object and res_id:
                model_pool = self.pool.get(model_object)
                res = model_pool.name_get(cr,uid,[res_id],context)
                data[attachment.id] = res[0][1]
            else:
                 data[attachment.id] = False
        return data

    _name = 'ir.attachment'
    _columns = {
        'name': fields.char('Attachment Name',size=256, required=True),
        'datas': fields.binary('Data'),
        'datas_fname': fields.char('Filename',size=256),
        'description': fields.text('Description'),
        'res_name': fields.function(_name_get_resname, type='char',
                string='Resource Name', method=True),
        'res_model': fields.char('Resource Object',size=64, readonly=True,
                help="The database object this attachment will be attached to"),
        'res_id': fields.integer('Resource ID', readonly=True,
                help="The record id this is attached to"),
        'url': fields.char('Url', size=512, oldname="link"),
        'type': fields.selection(
                [ ('url','URL'), ('binary','Binary'), ],
                'Type', help="Binary File or external URL", required=True),

        'create_date': fields.datetime('Date Created', readonly=True),
        'create_uid':  fields.many2one('res.users', 'Creator', readonly=True),
    }

    _defaults = {
        'type': 'binary',
    }

ir_attachment()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<commit_msg>ir.attachment: handle missing name from res_model<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from osv import fields,osv
from osv.orm import except_orm
import tools

class ir_attachment(osv.osv):
    def check(self, cr, uid, ids, mode, context=None):
        if not ids:
            return
        ima = self.pool.get('ir.model.access')
        if isinstance(ids, (int, long)):
            ids = [ids]
        cr.execute('select distinct res_model from ir_attachment where id IN %s', (tuple(ids),))
        for obj in cr.fetchall():
            if obj[0]:
                ima.check(cr, uid, obj[0], mode, context=context)

    def search(self, cr, uid, args, offset=0, limit=None, order=None,
            context=None, count=False):
        ids = super(ir_attachment, self).search(cr, uid, args, offset=offset,
                                                limit=limit, order=order,
                                                context=context, count=False)
        if not ids:
            if count:
                return 0
            return []
        models = super(ir_attachment,self).read(cr, uid, ids, ['id', 'res_model'])
        cache = {}
        ima = self.pool.get('ir.model.access')
        for m in models:
            if m['res_model']:
                if m['res_model'] not in cache:
                    cache[m['res_model']] = ima.check(cr, uid, m['res_model'], 'read',
                                                      raise_exception=False, context=context)
                if not cache[m['res_model']]:
                    ids.remove(m['id'])

        if count:
            return len(ids)
        return ids

    def read(self, cr, uid, ids, fields_to_read=None, context=None, load='_classic_read'):
        self.check(cr, uid, ids, 'read', context=context)
        return super(ir_attachment, self).read(cr, uid, ids, fields_to_read, context, load)

    def write(self, cr, uid, ids, vals, context=None):
        self.check(cr, uid, ids, 'write', context=context)
        return super(ir_attachment, self).write(cr, uid, ids, vals, context)

    def copy(self, cr, uid, id, default=None, context=None):
        self.check(cr, uid, [id], 'write', context=context)
        return super(ir_attachment, self).copy(cr, uid, id, default, context)

    def unlink(self, cr, uid, ids, context=None):
        self.check(cr, uid, ids, 'unlink', context=context)
        return super(ir_attachment, self).unlink(cr, uid, ids, context)

    def create(self, cr, uid, values, context=None):
        if 'res_model' in values and values['res_model'] != '':
            self.pool.get('ir.model.access').check(cr, uid, values['res_model'], 'create', context=context)
        return super(ir_attachment, self).create(cr, uid, values, context)

    def action_get(self, cr, uid, context=None):
        dataobj = self.pool.get('ir.model.data')
        data_id = dataobj._get_id(cr, 1, 'base', 'action_attachment')
        res_id = dataobj.browse(cr, uid, data_id, context).res_id
        return self.pool.get('ir.actions.act_window').read(cr, uid, res_id, [], context)

    def _name_get_resname(self, cr, uid, ids, object,method, context):
        data = {}
        for attachment in self.browse(cr, uid, ids, context=context):
            model_object = attachment.res_model
            res_id = attachment.res_id
            if model_object and res_id:
                model_pool = self.pool.get(model_object)
                res = model_pool.name_get(cr,uid,[res_id],context)
                data[attachment.id] = (res and res[0][1]) or False
            else:
                 data[attachment.id] = False
        return data

    _name = 'ir.attachment'
    _columns = {
        'name': fields.char('Attachment Name',size=256, required=True),
        'datas': fields.binary('Data'),
        'datas_fname': fields.char('Filename',size=256),
        'description': fields.text('Description'),
        'res_name': fields.function(_name_get_resname, type='char',
                string='Resource Name', method=True),
        'res_model': fields.char('Resource Object',size=64, readonly=True,
                help="The database object this attachment will be attached to"),
        'res_id': fields.integer('Resource ID', readonly=True,
                help="The record id this is attached to"),
        'url': fields.char('Url', size=512, oldname="link"),
        'type': fields.selection(
                [ ('url','URL'), ('binary','Binary'), ],
                'Type', help="Binary File or external URL", required=True),

        'create_date': fields.datetime('Date Created', readonly=True),
        'create_uid':  fields.many2one('res.users', 'Creator', readonly=True),
    }

    _defaults = {
        'type': 'binary',
    }

ir_attachment()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<|endoftext|>


<commit_before># Copyright (c) Charl P. Botha, TU Delft.
# All rights reserved.
# See COPYRIGHT for details.

import itk
import re

def get_img_type_and_dim(itk_img):

    try:
        t = itk_img.this
    except AttributeError, e:
        g = None
    else:
        # g will be e.g. ('float', '3') or ('unsigned_char', '2')
        # note that we use the NON-greedy version so it doesn't break
        # on vectors
        # example strings:
        # Image.F3: _f04b4f1b_p_itk__SmartPointerTitk__ImageTfloat_3u_t_t
        # Image.VF33: _600e411b_p_itk__SmartPointerTitk__ImageTitk__VectorTfloat_3u_t_3u_t_t'
        mo = re.search('.*itk__ImageT(.*?)_([0-9]+)u*_t',
                      itk_img.this)

    if not mo:
        raise TypeError, 'This method requires an ITK Image as input.'
    else:
        g = mo.groups()

    # see if it's a vector
    if g[0].startswith('itk__VectorT'):
        vectorString = 'V'
        # it's a vector, so let's remove the 'itk__VectorT' bit
        g = list(g)
        g[0] = g[0][len('itk__VectorT'):]
        g = tuple(g)

    else:
        vectorString = ''

    # this could also be ('float', '3', 'V'), or ('unsigned_char', '2', '')
    return g + (vectorString,)

def get_img_type_and_dim_shortstring(itk_img):

    tdv = get_img_type_and_dim(itk_img)

    # this turns 'unsigned_char' into 'UC' and 'float' into 'F'
    itkTypeC = ''.join([i.upper()[0] for i in tdv[0].split('_')])

    # the 'this' signature contains 'short_int', but the ITK shortstring for
    # this is SS
    if itkTypeC == 'SI':
        itkTypeC = 'SS'

    if len(tdv[2]) > 0:
        # this will be for instance VF33 or VF22
        shortstring = '%s%s%s%s' % (tdv[2], itkTypeC, tdv[1], tdv[1])

    else:
        # and this F3 or UC2
        shortstring = '%s%s' % (itkTypeC, tdv[1])

    return shortstring

def setupITKObjectProgress(dvModule, obj, nameOfObject, progressText,
                           objEvals=None):

    # objEvals is on optional TUPLE of obj attributes that will be called
    # at each progress callback and filled into progressText via the Python
    # % operator.  In other words, all string attributes in objEvals will be
    # eval'ed on the object instance and these values will be filled into
    # progressText, which has to contain the necessary number of format tokens

    # first we have to find the attribute of dvModule that binds
    # to obj.  We _don't_ want to have a binding to obj hanging around
    # in our callable, because this means that the ITK object can never
    # be destroyed.  Instead we look up the binding everytime the callable
    # is invoked by making use of getattr on the devideModule binding.

    # find attribute string of obj in dvModule
    di = dvModule.__dict__.items()
    objAttrString = None
    for key, value in di:
        if value == obj:
            objAttrString = key
            break

    if not objAttrString:
        raise Exception, 'Could not determine attribute string for ' \
              'object %s.' % (obj.__class__.__name__)

    mm = dvModule._module_manager

    # sanity check objEvals
    if type(objEvals) != type(()) and objEvals != None:
        raise TypeError, 'objEvals should be a tuple or None.'

    def commandCallable():
        # setup for and get values of all requested objEvals
        values = []

        if type(objEvals) == type(()):
            for objEval in objEvals:
                values.append(
                    eval('dvModule.%s.%s' % (objAttrString, objEval)))

        values = tuple(values)
        print values

        # do the actual callback
        mm.generic_progress_callback(getattr(dvModule, objAttrString),
            nameOfObject, getattr(dvModule, objAttrString).GetProgress(),
            progressText % values)

        # get rid of all bindings
        del values

    pc = itk.PyCommand.New()
    pc.SetCommandCallable(commandCallable)
    res = obj.AddObserver(itk.ProgressEvent(), pc.GetPointer())

    # we DON'T have to store a binding to the PyCommand; during AddObserver,
    # the ITK object makes its own copy.  The ITK object will also take care
    # of destroying the observer when it (the ITK object) is destroyed
    #obj.progressPyCommand = pc
<commit_msg>Fixed possible bug in type-checking code.<commit_after># Copyright (c) Charl P. Botha, TU Delft.
# All rights reserved.
# See COPYRIGHT for details.

import itk
import re

def get_img_type_and_dim(itk_img):
    """Returns itk image type as a tuple with ('type', 'dim', 'v').
    """

    try:
        t = itk_img.this
    except AttributeError, e:
        raise TypeError, 'This method requires an ITK image as input.'
    else:
        # g will be e.g. ('float', '3') or ('unsigned_char', '2')
        # note that we use the NON-greedy version so it doesn't break
        # on vectors
        # example strings:
        # Image.F3: _f04b4f1b_p_itk__SmartPointerTitk__ImageTfloat_3u_t_t
        # Image.VF33: _600e411b_p_itk__SmartPointerTitk__ImageTitk__VectorTfloat_3u_t_3u_t_t'
        mo = re.search('.*itk__ImageT(.*?)_([0-9]+)u*_t',
                      itk_img.this)

        if not mo:
            raise TypeError, 'This method requires an ITK Image as input.'
        else:
            g = mo.groups()

    # see if it's a vector
    if g[0].startswith('itk__VectorT'):
        vectorString = 'V'
        # it's a vector, so let's remove the 'itk__VectorT' bit
        g = list(g)
        g[0] = g[0][len('itk__VectorT'):]
        g = tuple(g)

    else:
        vectorString = ''

    # this could also be ('float', '3', 'V'), or ('unsigned_char', '2', '')
    return g + (vectorString,)

def get_img_type_and_dim_shortstring(itk_img):

    tdv = get_img_type_and_dim(itk_img)

    # this turns 'unsigned_char' into 'UC' and 'float' into 'F'
    itkTypeC = ''.join([i.upper()[0] for i in tdv[0].split('_')])

    # the 'this' signature contains 'short_int', but the ITK shortstring for
    # this is SS
    if itkTypeC == 'SI':
        itkTypeC = 'SS'

    if len(tdv[2]) > 0:
        # this will be for instance VF33 or VF22
        shortstring = '%s%s%s%s' % (tdv[2], itkTypeC, tdv[1], tdv[1])

    else:
        # and this F3 or UC2
        shortstring = '%s%s' % (itkTypeC, tdv[1])

    return shortstring

def setupITKObjectProgress(dvModule, obj, nameOfObject, progressText,
                           objEvals=None):

    # objEvals is on optional TUPLE of obj attributes that will be called
    # at each progress callback and filled into progressText via the Python
    # % operator.  In other words, all string attributes in objEvals will be
    # eval'ed on the object instance and these values will be filled into
    # progressText, which has to contain the necessary number of format tokens

    # first we have to find the attribute of dvModule that binds
    # to obj.  We _don't_ want to have a binding to obj hanging around
    # in our callable, because this means that the ITK object can never
    # be destroyed.  Instead we look up the binding everytime the callable
    # is invoked by making use of getattr on the devideModule binding.

    # find attribute string of obj in dvModule
    di = dvModule.__dict__.items()
    objAttrString = None
    for key, value in di:
        if value == obj:
            objAttrString = key
            break

    if not objAttrString:
        raise Exception, 'Could not determine attribute string for ' \
              'object %s.' % (obj.__class__.__name__)

    mm = dvModule._module_manager

    # sanity check objEvals
    if type(objEvals) != type(()) and objEvals != None:
        raise TypeError, 'objEvals should be a tuple or None.'

    def commandCallable():
        # setup for and get values of all requested objEvals
        values = []

        if type(objEvals) == type(()):
            for objEval in objEvals:
                values.append(
                    eval('dvModule.%s.%s' % (objAttrString, objEval)))

        values = tuple(values)
        print values

        # do the actual callback
        mm.generic_progress_callback(getattr(dvModule, objAttrString),
            nameOfObject, getattr(dvModule, objAttrString).GetProgress(),
            progressText % values)

        # get rid of all bindings
        del values

    pc = itk.PyCommand.New()
    pc.SetCommandCallable(commandCallable)
    res = obj.AddObserver(itk.ProgressEvent(), pc.GetPointer())

    # we DON'T have to store a binding to the PyCommand; during AddObserver,
    # the ITK object makes its own copy.  The ITK object will also take care
    # of destroying the observer when it (the ITK object) is destroyed
    #obj.progressPyCommand = pc
<|endoftext|>


<commit_before>
from cStringIO import StringIO
from nose.tools import *

from tokenutils import Iterable2TokenStream, make_token

from libeeyore.functionvalues import *
from libeeyore.languagevalues import *
from libeeyore.values import *

from parse import EeyoreLexer
from parse import EeyoreParser
from parse import EeyoreTreeWalker

def _parse( tokens ):
    parser = EeyoreParser.Parser( Iterable2TokenStream( tokens ) )
    parser.program();
    walker = EeyoreTreeWalker.Walker()
    return walker.statement( parser.getAST() )

def test_hello_world():
    value = _parse( (
        make_token( "print",         EeyoreLexer.SYMBOL ),
        make_token( "(",             EeyoreLexer.LPAREN ),
        make_token( "Hello, world!", EeyoreLexer.STRING ),
        make_token( ")",             EeyoreLexer.RPAREN ),
        ) )

    assert_equal( value.__class__, EeyFunctionCall )
    assert_equal( value.func_name, "print" )

    func = value.func

    assert_equal( func.__class__, EeySymbol )
    assert_equal( func.symbol_name, "print" )

    args = value.args
    assert_equal( len( args ), 1 )

    assert_equal( args[0].__class__, EeyString )
    assert_equal( args[0].value, "Hello, world!" )

def test_import():
    value = _parse( (
        make_token( "import", EeyoreLexer.LITERAL_import ),
        make_token( "sys",    EeyoreLexer.SYMBOL ),
        ) )

    assert_equal( value.__class__, EeyImport )
    assert_equal( value.module_name, "sys" )

<commit_msg>Make the parse tests expect multiple values.<commit_after>
from cStringIO import StringIO
from nose.tools import *

from tokenutils import Iterable2TokenStream, make_token

from libeeyore.functionvalues import *
from libeeyore.languagevalues import *
from libeeyore.values import *

from parse import EeyoreLexer
from parse import EeyoreParser
from parse import EeyoreTreeWalker

def _parse( tokens ):
    parser = EeyoreParser.Parser( Iterable2TokenStream( tokens ) )
    parser.program();
    walker = EeyoreTreeWalker.Walker()
    return [walker.statement( parser.getAST() )]

def test_hello_world():
    values = _parse( (
        make_token( "print",         EeyoreLexer.SYMBOL ),
        make_token( "(",             EeyoreLexer.LPAREN ),
        make_token( "Hello, world!", EeyoreLexer.STRING ),
        make_token( ")",             EeyoreLexer.RPAREN ),
        ) )

    assert_equal( len( values ), 1 )
    value = values[0]

    assert_equal( value.__class__, EeyFunctionCall )
    assert_equal( value.func_name, "print" )

    func = value.func

    assert_equal( func.__class__, EeySymbol )
    assert_equal( func.symbol_name, "print" )

    args = value.args
    assert_equal( len( args ), 1 )

    assert_equal( args[0].__class__, EeyString )
    assert_equal( args[0].value, "Hello, world!" )

def test_import():
    values = _parse( (
        make_token( "import", EeyoreLexer.LITERAL_import ),
        make_token( "sys",    EeyoreLexer.SYMBOL ),
        ) )

    assert_equal( len( values ), 1 )
    value = values[0]

    assert_equal( value.__class__, EeyImport )
    assert_equal( value.module_name, "sys" )

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import logging
import os
import resource
import sys

import cv2
from joblib import Parallel, parallel_backend, delayed

logger = logging.getLogger(__name__)

abspath = os.path.abspath(os.path.dirname(__file__))
SENSOR = os.path.join(abspath, 'data', 'sensor_data.json')
BOW_PATH = os.path.join(abspath, 'data', 'bow')

# Handle different OpenCV versions
OPENCV3 = int(cv2.__version__.split('.')[0]) >= 3

if hasattr(cv2, 'flann_Index'):
    flann_Index = cv2.flann_Index
elif hasattr(cv2, 'flann') and hasattr(cv2.flann, 'Index'):
    flann_Index = cv2.flann.Index
else:
    logger.warning('Unable to find flann Index')
    flann_Index = None

# Parallel processes
def parallel_map(func, args, num_proc):
    """Run function for all arguments using multiple processes."""
    num_proc = min(num_proc, len(args))
    if num_proc <= 1:
        return list(map(func, args))
    else:
        with parallel_backend('loky', n_jobs=num_proc):
            batch_size = max(1, len(args)/num_proc)
            return Parallel(batch_size=batch_size)(delayed(func)(arg) for arg in args)

# Memory usage
if sys.platform == 'darwin':
    rusage_unit = 1
else:
    rusage_unit = 1024

def memory_available():
    """Available memory in MB.

    Only works on linux and returns None otherwise.
    """
    lines = os.popen('free -t -m').readlines()
    if not lines:
        return None
    available_mem = int(lines[1].split()[6])
    return available_mem

def current_memory_usage():
    return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * rusage_unit
<commit_msg>refactor: move it to a better place<commit_after># -*- coding: utf-8 -*-
import logging
import os
import resource
import sys

import cv2
from joblib import Parallel, parallel_backend, delayed

logger = logging.getLogger(__name__)

abspath = os.path.abspath(os.path.dirname(__file__))
SENSOR = os.path.join(abspath, 'data', 'sensor_data.json')
BOW_PATH = os.path.join(abspath, 'data', 'bow')

# Handle different OpenCV versions
OPENCV3 = int(cv2.__version__.split('.')[0]) >= 3

if hasattr(cv2, 'flann_Index'):
    flann_Index = cv2.flann_Index
elif hasattr(cv2, 'flann') and hasattr(cv2.flann, 'Index'):
    flann_Index = cv2.flann.Index
else:
    logger.warning('Unable to find flann Index')
    flann_Index = None

# Parallel processes
def parallel_map(func, args, num_proc):
    """Run function for all arguments using multiple processes."""
    num_proc = min(num_proc, len(args))
    if num_proc <= 1:
        return list(map(func, args))
    else:
        with parallel_backend('loky', n_jobs=num_proc):
            batch_size = max(1, len(args)/num_proc)
            return Parallel(batch_size=batch_size)(delayed(func)(arg) for arg in args)

# Memory usage
if sys.platform == 'darwin':
    rusage_unit = 1
else:
    rusage_unit = 1024

def memory_available():
    """Available memory in MB.

    Only works on linux and returns None otherwise.
    """
    lines = os.popen('free -t -m').readlines()
    if not lines:
        return None
    available_mem = int(lines[1].split()[6])
    return available_mem

def memory_available():
    """Available memory in MB.

    Only works on linux and returns None otherwise.
    """
    lines = os.popen('free -t -m').readlines()
    if not lines:
        return None
    available_mem = int(lines[1].split()[6])
    return available_mem

def current_memory_usage():
    return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * rusage_unit
<|endoftext|>


<commit_before>"""
Overview of all settings which can be customized.
"""
from django.conf import settings
from django.utils.translation import ugettext as _

FLUENT_DASHBOARD_ICON_THEME = getattr(settings, "FLUENT_DASHBOARD_ICON_THEME", "flaticons")

# The icon names are somewhat Oxygen specific,
# yet based on the Freedesktop Naming Naming Specification.

if FLUENT_DASHBOARD_ICON_THEME == 'flaticons':
    # New defaults with flaticons
    FLUENT_DASHBOARD_APP_ICONS = {
        'auth/group': 'multiple25.png',
        'auth/user': 'network60.png',
        'comments/comment': 'chat-1.png',
        'filebrowser/filebrowser': 'connectivity2.png',
        'fluent_blogs/entry': 'newspaper1.png',
        'fluent_faq/faqquestion': 'magnifier41.png',
        'fluent_pages/page': 'pencil34.png',
        'fluent_pages/pagelayout': 'layout.png',
        'fluent_comments/fluentcomment': 'chat-1.png',
        'fluentcms_contactform/contactformdata': 'chat.png',
        'fluentcms_googlemaps/marker': 'placeholder8.png',
        'fluentcms_googlemaps/markergroup': 'map35.png',
        'fluentcms_privatenotes/privatenotesitem': 'file.png',
        'redirects/redirect': 'right-arrow7.png',
        'sharedcontent/sharedcontent': 'puzzles3.png',
        'threadedcomments/threadedcomment': 'chat-1.png',
        'sites/site': 'global45.png',
        'users/user': 'network60.png',
    }

    FLUENT_DASHBOARD_DEFAULT_ICON = getattr(settings, "FLUENT_DASHBOARD_DEFAULT_ICON", 'unknown-document.png')
else:
    # Old defaults
    FLUENT_DASHBOARD_APP_ICONS = {
        'auth/group': 'resource-group.png',
        'auth/user':  'system-users.png',
        'cms/page': 'internet-web-browser.png',
        'comments/comment': 'kde-telepathy.png', #'irc-voice.png',
        'dashboardmods/rssdashboardmodule': 'feed-subscribe.png',
        'fluent_blogs/entry': 'view-calendar-journal.png',
        'fluent_pages/pagelayout': 'view-choose.png',
        'fluent_pages/page': 'internet-web-browser.png',
        'fluent_faq/faqquestion': 'system-help.png',
        'fluent_faq/faqcategory': 'custom-bookmarks-help.png',
        'fluent_comments/fluentcomment': 'kde-telepathy.png', #'irc-voice.png',
        'fluentcms_googlemaps/markergroup': 'marble.png',
        'fluentcms_googlemaps/marker': 'preferences-system-session-services.png',
        'fiber/contentitem': 'folder-txt.png',
        'fiber/file': 'folder.png',
        'fiber/image': 'folder-image.png',
        'fiber/page': 'internet-web-browser.png',
        'filebrowser/filebrowser': 'folder.png',
        'filer/folder': 'folder.png',
        'form_designer/formdefinition': 'mail-mark-task.png',
        'form_designer/formlog': 'view-calendar-journal.png',
        'google_analytics/analytics': 'view-statistics.png',
        'page/page': 'internet-web-browser.png',
        'media_tree/filenode': 'folder.png',
        'registration/registrationprofile': 'list-add-user.png',
        'sharedcontent/sharedcontent': 'x-office-document.png',
        'sites/site': 'applications-internet.png',
        'snippet/snippet': 'folder-txt.png',
        'tagging/tag': 'feed-subscribe.png',
        'tagging/taggeditem': 'feed-subscribe.png',
        'threadedcomments/threadedcomment': 'kde-telepathy.png', #'irc-voice.png',
        'zinnia/category': 'folder-bookmark.png',
        'zinnia/entry': 'view-calendar-journal.png',
    }

    FLUENT_DASHBOARD_DEFAULT_ICON = getattr(settings, "FLUENT_DASHBOARD_DEFAULT_ICON", 'unknown.png')

FLUENT_DASHBOARD_CMS_PAGE_MODEL = getattr(settings, "FLUENT_DASHBOARD_CMS_PAGE_MODEL", None)

FLUENT_DASHBOARD_CMS_APP_NAMES = getattr(settings, "FLUENT_DASHBOARD_CMS_APP_NAMES", (
    'cms.*',        # DjangoCMS
    'feincms.*',    # FeinCMS
    'fluent*',      # Fluent pages
    'fiber',        # Django-Fiber
    'filebrowser',  # django-filebrowser
    'media_tree',   # django-media-tree
))

FLUENT_DASHBOARD_CMS_MODEL_ORDER = getattr(settings, "FLUENT_DASHBOARD_CMS_MODEL_ORDER", {
    'page': 1,
    'object': 2,
    'layout': 3,
    'content': 4,
    'file': 5,
    'filebrowser': 5,
    'site': 99
})

FLUENT_DASHBOARD_APP_ICONS.update(getattr(settings, 'FLUENT_DASHBOARD_APP_ICONS', {}))

FLUENT_DASHBOARD_DEFAULT_MODULE = getattr(settings, 'FLUENT_DASHBOARD_DEFAULT_MODULE', 'admin_tools.dashboard.modules.AppList')

FLUENT_DASHBOARD_APP_GROUPS = getattr(settings, 'FLUENT_DASHBOARD_APP_GROUPS', (
    (_('CMS'), {
        'models': [
            "{0}.*".format(app) for app in FLUENT_DASHBOARD_CMS_APP_NAMES
        ],
        'module': 'fluent_dashboard.modules.CmsAppIconList',
        'collapsible': False,
    }),
    (_('Interactivity'), {
        'models': (
            'django.contrib.comments.*',
            'form_designer.*',
            'threadedcomments.*',
            'zinnia.*',
        ),
        'module': 'fluent_dashboard.modules.AppIconList',
        'collapsible': False,
    }),
    (_('Administration'), {
        'models': (
            'django.contrib.auth.*',
            'django.contrib.sites.*',
            '*.UserProfile',
            'registration.*',
            'dashboardmods.*',
            'google_analytics.*',
        ),
        'module': 'fluent_dashboard.modules.AppIconList',
        'collapsible': False,
    }),
    (_('Applications'), {
        'models': ('*',),
        'module': FLUENT_DASHBOARD_DEFAULT_MODULE,
        'collapsible': False,
    }),
    #(_('Developer tools'), ()),
))

# Provide defaults for some popular Django CMSes
if not FLUENT_DASHBOARD_CMS_PAGE_MODEL:
    if 'cms' in settings.INSTALLED_APPS:
        FLUENT_DASHBOARD_CMS_PAGE_MODEL = ('cms', 'page')
    elif 'feincms' in settings.INSTALLED_APPS:
        FLUENT_DASHBOARD_CMS_PAGE_MODEL = ('page', 'page')
    elif 'fluent_pages' in settings.INSTALLED_APPS:
        FLUENT_DASHBOARD_CMS_PAGE_MODEL = ('fluent_pages', 'page')
<commit_msg>Further improve the default dashboard layout<commit_after>"""
Overview of all settings which can be customized.
"""
from django.conf import settings
from django.utils.translation import ugettext as _

FLUENT_DASHBOARD_ICON_THEME = getattr(settings, "FLUENT_DASHBOARD_ICON_THEME", "flaticons")

# The icon names are somewhat Oxygen specific,
# yet based on the Freedesktop Naming Naming Specification.

if FLUENT_DASHBOARD_ICON_THEME == 'flaticons':
    # New defaults with flaticons
    FLUENT_DASHBOARD_APP_ICONS = {
        'auth/group': 'multiple25.png',
        'auth/user': 'network60.png',
        'comments/comment': 'chat-1.png',
        'filebrowser/filebrowser': 'connectivity2.png',
        'fluent_blogs/entry': 'newspaper1.png',
        'fluent_faq/faqquestion': 'magnifier41.png',
        'fluent_pages/page': 'pencil34.png',
        'fluent_pages/pagelayout': 'layout.png',
        'fluent_comments/fluentcomment': 'chat-1.png',
        'fluentcms_contactform/contactformdata': 'chat.png',
        'fluentcms_googlemaps/marker': 'placeholder8.png',
        'fluentcms_googlemaps/markergroup': 'map35.png',
        'fluentcms_privatenotes/privatenotesitem': 'file.png',
        'redirects/redirect': 'right-arrow7.png',
        'sharedcontent/sharedcontent': 'puzzles3.png',
        'threadedcomments/threadedcomment': 'chat-1.png',
        'sites/site': 'global45.png',
        'users/user': 'network60.png',

        # custom
        'blog/post': 'newspaper1.png',
        'news/news': 'newspaper1.png',
    }

    FLUENT_DASHBOARD_DEFAULT_ICON = getattr(settings, "FLUENT_DASHBOARD_DEFAULT_ICON", 'unknown-document.png')
else:
    # Old defaults
    FLUENT_DASHBOARD_APP_ICONS = {
        'auth/group': 'resource-group.png',
        'auth/user':  'system-users.png',
        'cms/page': 'internet-web-browser.png',
        'comments/comment': 'kde-telepathy.png', #'irc-voice.png',
        'dashboardmods/rssdashboardmodule': 'feed-subscribe.png',
        'fluent_blogs/entry': 'view-calendar-journal.png',
        'fluent_pages/pagelayout': 'view-choose.png',
        'fluent_pages/page': 'internet-web-browser.png',
        'fluent_faq/faqquestion': 'system-help.png',
        'fluent_faq/faqcategory': 'custom-bookmarks-help.png',
        'fluent_comments/fluentcomment': 'kde-telepathy.png', #'irc-voice.png',
        'fluentcms_googlemaps/markergroup': 'marble.png',
        'fluentcms_googlemaps/marker': 'preferences-system-session-services.png',
        'fiber/contentitem': 'folder-txt.png',
        'fiber/file': 'folder.png',
        'fiber/image': 'folder-image.png',
        'fiber/page': 'internet-web-browser.png',
        'filebrowser/filebrowser': 'folder.png',
        'filer/folder': 'folder.png',
        'form_designer/formdefinition': 'mail-mark-task.png',
        'form_designer/formlog': 'view-calendar-journal.png',
        'google_analytics/analytics': 'view-statistics.png',
        'page/page': 'internet-web-browser.png',
        'media_tree/filenode': 'folder.png',
        'registration/registrationprofile': 'list-add-user.png',
        'sharedcontent/sharedcontent': 'x-office-document.png',
        'sites/site': 'applications-internet.png',
        'snippet/snippet': 'folder-txt.png',
        'tagging/tag': 'feed-subscribe.png',
        'tagging/taggeditem': 'feed-subscribe.png',
        'threadedcomments/threadedcomment': 'kde-telepathy.png', #'irc-voice.png',
        'zinnia/category': 'folder-bookmark.png',
        'zinnia/entry': 'view-calendar-journal.png',
    }

    FLUENT_DASHBOARD_DEFAULT_ICON = getattr(settings, "FLUENT_DASHBOARD_DEFAULT_ICON", 'unknown.png')

FLUENT_DASHBOARD_CMS_PAGE_MODEL = getattr(settings, "FLUENT_DASHBOARD_CMS_PAGE_MODEL", None)

FLUENT_DASHBOARD_CMS_APP_NAMES = getattr(settings, "FLUENT_DASHBOARD_CMS_APP_NAMES", (
    'cms.*',        # DjangoCMS
    'feincms.*',    # FeinCMS
    'fluent*',      # Fluent pages, blogs, faq
    'fiber',        # Django-Fiber
    'django.contrib.flatpages.*',
    'django.contrib.redirects.*',
    'zinnia.models.entry.Entry',

    # Custom
    'blog.*',
    'faq.*',
    'news.*',
    '*.blog.*',
    '*.faq.*',
    '*.news.*',
))

FLUENT_DASHBOARD_CMS_MODEL_ORDER = getattr(settings, "FLUENT_DASHBOARD_CMS_MODEL_ORDER", {
    'page': 1,
    'object': 2,
    'layout': 3,
    'content': 4,
    'file': 5,
    'filebrowser': 5,
    'site': 99
})

FLUENT_DASHBOARD_APP_ICONS.update(getattr(settings, 'FLUENT_DASHBOARD_APP_ICONS', {}))

FLUENT_DASHBOARD_DEFAULT_MODULE = getattr(settings, 'FLUENT_DASHBOARD_DEFAULT_MODULE', 'admin_tools.dashboard.modules.AppList')

FLUENT_DASHBOARD_APP_GROUPS = getattr(settings, 'FLUENT_DASHBOARD_APP_GROUPS', (
    (_('CMS'), {
        # Mainly apps that produce an page + URL at the site.
        'models': [
            "{0}.*".format(app) for app in FLUENT_DASHBOARD_CMS_APP_NAMES
        ],
        'exclude': [
            # Show in other sections:
            'fluentcms_contactform.*',
            'fluent_contents.plugins.sharedcontent.*',
            # Less relevant, not producing pages:
            'fluent_pages.models.db.PageLayout',
        ],
        'module': 'fluent_dashboard.modules.CmsAppIconList',
        'collapsible': False,
    }),
    (_('Content Library'), {
        # Content that is linked by the pages in "CMS"
        'models': (
            'filebrowser.*',  # django-filebrowser
            'fluent_contents.plugins.sharedcontent.*',
            'form_designer.*',
            'forms_builder.*',
            'media_tree',  # django-media-tree
        ),
        'module': 'fluent_dashboard.modules.AppIconList',
        'collapsible': False,
    }),
    (_('Interactivity'), {
        # User comments, responses, etc..
        'models': (
            'django.contrib.comments.*',
            'fluentcms_contactform.*',
            'threadedcomments.*',
            'zinnia.*',
        ),
        'exclude': (
            'zinnia.models.entry.Entry',
        ),
        'module': 'fluent_dashboard.modules.AppIconList',
        'collapsible': False,
    }),
    (_('Administration'), {
        'models': (
            'django.contrib.auth.*',
            'django.contrib.sites.*',
            '*.UserProfile',
            'registration.*',
            'dashboardmods.*',
            'google_analytics.*',
        ),
        'module': 'fluent_dashboard.modules.AppIconList',
        'collapsible': False,
    }),
    (_('Applications'), {
        'models': ('*',),
        'module': FLUENT_DASHBOARD_DEFAULT_MODULE,
        'collapsible': False,
    }),
    #(_('Developer tools'), ()),
))

# Provide defaults for some popular Django CMSes
if not FLUENT_DASHBOARD_CMS_PAGE_MODEL:
    if 'cms' in settings.INSTALLED_APPS:
        FLUENT_DASHBOARD_CMS_PAGE_MODEL = ('cms', 'page')
    elif 'feincms' in settings.INSTALLED_APPS:
        FLUENT_DASHBOARD_CMS_PAGE_MODEL = ('page', 'page')
    elif 'fluent_pages' in settings.INSTALLED_APPS:
        FLUENT_DASHBOARD_CMS_PAGE_MODEL = ('fluent_pages', 'page')
<|endoftext|>


<commit_before>import os
from optparse import make_option
import subprocess
import sys
import threading
import time

from .run import Command as RunserverCommand

class Command(RunserverCommand):
    option_list = RunserverCommand.option_list + (
        make_option(
            '--wip',
            action='store_true',
            dest='wip',
            default=False,
            help='Run only tests marked with @wip'
        ),
    )

    def run_tests(self, *args):
        env = os.environ.copy()
        env['DJANGO_LIVE_TEST_SERVER_ADDRESS'] = 'localhost:8001'

        command = [
            'python',
            'manage.py',
            'test',
            'taskmanager',
        ]
        command.extend(args)

        return subprocess.call(command, env=env)

    def handle(self, *args, **kwargs):
        fnull = open(os.devnull, 'w')

        ember = threading.Thread(
            target=self.run_ember,
            kwargs={
                'stdout': fnull,
                'stderr': subprocess.STDOUT
            }
        )
        ember.daemon = True
        ember.start()

        time.sleep(10)

        test_args = []
        if kwargs['wip']:
            test_args.append('--behave_wip')

        try:
            result = self.run_tests(*test_args)
        except:
            result = 1
        self.teardown()
        sys.exit(result)
<commit_msg>Wait until Ember server is up before running tests.<commit_after>import os
from optparse import make_option
import socket
import subprocess
import sys
import threading
import time

from .run import Command as RunserverCommand

class Command(RunserverCommand):
    option_list = RunserverCommand.option_list + (
        make_option(
            '--wip',
            action='store_true',
            dest='wip',
            default=False,
            help='Run only tests marked with @wip'
        ),
    )

    def run_tests(self, *args):
        env = os.environ.copy()
        env['DJANGO_LIVE_TEST_SERVER_ADDRESS'] = 'localhost:8001'

        command = [
            'python',
            'manage.py',
            'test',
            'taskmanager',
        ]
        command.extend(args)

        return subprocess.call(command, env=env)

    def handle(self, *args, **kwargs):
        fnull = open(os.devnull, 'w')

        ember = threading.Thread(
            target=self.run_ember,
            kwargs={
                'stdout': fnull,
                'stderr': subprocess.STDOUT
            }
        )
        ember.daemon = True
        ember.start()

        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        while True:
            result = s.connect_ex(('127.0.0.1', 8000, ))
            if result == 0:
                s.close()
                break
            time.sleep(1)

        test_args = []
        if kwargs['wip']:
            test_args.append('--behave_wip')

        try:
            result = self.run_tests(*test_args)
        except:
            result = 1
        self.teardown()
        sys.exit(result)
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import numpy as np

def opensfm_to_upright(coords, width, height, orientation,
                       new_width=None, new_height=None):
    """
    Transform opensfm coordinates to upright coordinates, correcting for EXIF orientation.

    :param coords: Points in opensfm coordinate system.
    :param width: Width of original image in pixels unadjusted for orientation.
    :param height: Height of original image in pixels unadjusted for orientation.
    :param orientation: Orientation of original image.

    :return: Points in upright coordinate system.

    >>> sfm = np.array([[-0.5, -0.375],
    ...                 [-0.5,  0.375],
    ...                 [ 0.5, -0.375],
    ...                 [ 0.5,  0.375]])
    >>> opensfm_to_upright(sfm, 320, 240, 1)
    array([[ 0.,  0.],
           [ 0.,  320.],
           [ 240.,  0.],
           [ 240.,  320.]])
    """

    R_T = {
        1: np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]),
        3: np.array([[-1, 0, 0], [0, -1, 0], [0, 0, 1]]),
        6: np.array([[0, -1, 0], [1, 0, 0], [0, 0, 1]]),
        8: np.array([[0, 1, 0], [-1, 0, 0], [0, 0, 1]]),
    }

    w = float(width)
    h = float(height)
    s = max(w, h)

    H_inv = np.array([[s / w,     0, 0.5],
                      [    0, s / h, 0.5],
                      [    0,     0,   1]])

    T_inv = np.dot(R_T[orientation], H_inv)

    p = np.dot(coords, T_inv[:2, :2].T) + T_inv[:2, 2].reshape(1, 2)

    upright_width = width if orientation < 6 else height
    upright_height = height if orientation < 6 else width
    if new_width is not None:
        upright_width = new_width
    if new_height is not None:
        upright_height = new_height
    p[:, 0] = upright_width * p[:, 0]
    p[:, 1] = upright_height * p[:, 1]

    return p
<commit_msg>fix: fixed doctest (switched width and height)<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import numpy as np

def opensfm_to_upright(coords, width, height, orientation,
                       new_width=None, new_height=None):
    """
    Transform opensfm coordinates to upright coordinates, correcting for EXIF orientation.

    :param coords: Points in opensfm coordinate system.
    :param width: Width of original image in pixels unadjusted for orientation.
    :param height: Height of original image in pixels unadjusted for orientation.
    :param orientation: Orientation of original image.

    :return: Points in upright coordinate system.

    >>> sfm = np.array([[-0.5, -0.375],
    ...                 [-0.5,  0.375],
    ...                 [ 0.5, -0.375],
    ...                 [ 0.5,  0.375]])
    >>> opensfm_to_upright(sfm, 320, 240, 1)
    array([[  0.,   0.],
           [  0., 240.],
           [320.,   0.],
           [320., 240.]])
    """

    R_T = {
        1: np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]),
        3: np.array([[-1, 0, 0], [0, -1, 0], [0, 0, 1]]),
        6: np.array([[0, -1, 0], [1, 0, 0], [0, 0, 1]]),
        8: np.array([[0, 1, 0], [-1, 0, 0], [0, 0, 1]]),
    }

    w = float(width)
    h = float(height)
    s = max(w, h)

    H_inv = np.array([[s / w,     0, 0.5],
                      [    0, s / h, 0.5],
                      [    0,     0,   1]])

    T_inv = np.dot(R_T[orientation], H_inv)

    p = np.dot(coords, T_inv[:2, :2].T) + T_inv[:2, 2].reshape(1, 2)

    upright_width = width if orientation < 6 else height
    upright_height = height if orientation < 6 else width
    if new_width is not None:
        upright_width = new_width
    if new_height is not None:
        upright_height = new_height
    p[:, 0] = upright_width * p[:, 0]
    p[:, 1] = upright_height * p[:, 1]

    return p
<|endoftext|>


<commit_before>import types as t
from ..exceptions import CheckError

def is_subtype_of(check_obj, _type):
    try:
        assert issubclass(check_obj._val.__class__, _type)
        return check_obj
    except AssertionError:
        raise CheckError('{} is not subtype of {}'.format(check_obj._val, _type))

def is_not_subtype_of(check_obj, _type):
    try:
        assert issubclass(check_obj._val.__class__, _type)
        raise CheckError('{} is subtype of {}'.format(check_obj._val, _type))
    except AssertionError:
        return check_obj

def is_of_type(check_obj, _type):
    try:
        assert isinstance(check_obj._val, _type)
        return check_obj
    except AssertionError:
        raise CheckError('{} is not of type {}'.format(check_obj._val, _type))

def is_not_of_type(check_obj, _type):
    try:
        assert not isinstance(check_obj._val, _type)
        return check_obj
    except AssertionError:
        raise CheckError('{} is of type {}'.format(check_obj._val, _type))

def is_module(check_obj):
    try:
        assert isinstance(check_obj._val, t.ModuleType)
        return check_obj
    except AssertionError:
        raise CheckError('{} is not a module'.format(check_obj._val))

def is_not_module(check_obj):
    try:
        assert not isinstance(check_obj._val, t.ModuleType)
        return check_obj
    except AssertionError:
        raise CheckError('{} is a module'.format(check_obj._val))

def is_runnable(check_obj):
    try:
        assert isinstance(check_obj._val, types.FunctionType)
        return check_obj
    except AssertionError:
        raise CheckError('{} is not runnable'.format(check_obj._val))

def is_not_runnable(check_obj):
    try:
        assert not isinstance(check_obj._val, types.FunctionType)
        return check_obj
    except AssertionError:
        raise CheckError('{} is runnable'.format(check_obj._val))

# Any type

def equals(check_obj, expected):
    try:
        assert check_obj._val == expected
        return check_obj
    except AssertionError:
        raise CheckError('{} does not equal {}'.format(check_obj._val, expected))

def not_equals(check_obj, expected):
    try:
        assert check_obj._val == expected
        raise CheckError('{} equals {}'.format(check_obj._val, expected))
    except AssertionError:
        return check_obj<commit_msg>Fixed test failures<commit_after>import types as t
from ..exceptions import CheckError

def is_subtype_of(check_obj, _type):
    try:
        assert issubclass(check_obj._val.__class__, _type)
        return check_obj
    except AssertionError:
        raise CheckError('{} is not subtype of {}'.format(check_obj._val, _type))

def is_not_subtype_of(check_obj, _type):
    try:
        assert not issubclass(check_obj._val.__class__, _type)
        return check_obj
    except AssertionError:
        raise CheckError('{} is subtype of {}'.format(check_obj._val, _type))

def is_of_type(check_obj, _type):
    try:
        assert isinstance(check_obj._val, _type)
        return check_obj
    except AssertionError:
        raise CheckError('{} is not of type {}'.format(check_obj._val, _type))

def is_not_of_type(check_obj, _type):
    try:
        assert not isinstance(check_obj._val, _type)
        return check_obj
    except AssertionError:
        raise CheckError('{} is of type {}'.format(check_obj._val, _type))

def is_module(check_obj):
    try:
        assert isinstance(check_obj._val, t.ModuleType)
        return check_obj
    except AssertionError:
        raise CheckError('{} is not a module'.format(check_obj._val))

def is_not_module(check_obj):
    try:
        assert not isinstance(check_obj._val, t.ModuleType)
        return check_obj
    except AssertionError:
        raise CheckError('{} is a module'.format(check_obj._val))

def is_runnable(check_obj):
    try:
        assert isinstance(check_obj._val, types.FunctionType)
        return check_obj
    except AssertionError:
        raise CheckError('{} is not runnable'.format(check_obj._val))

def is_not_runnable(check_obj):
    try:
        assert not isinstance(check_obj._val, types.FunctionType)
        return check_obj
    except AssertionError:
        raise CheckError('{} is runnable'.format(check_obj._val))

# Any type

def equals(check_obj, expected):
    try:
        assert check_obj._val == expected
        return check_obj
    except AssertionError:
        raise CheckError('{} does not equal {}'.format(check_obj._val, expected))

def not_equals(check_obj, expected):
    try:
        assert check_obj._val == expected
        raise CheckError('{} equals {}'.format(check_obj._val, expected))
    except AssertionError:
        return check_obj<|endoftext|>


<commit_before>#!/usr/bin/env python

'''
@package ion.services.dm.utility.granule.record_dictionary
@file ion/services/dm/utility/granule/record_dictionary.py
@author Tim Giguere
@author Luke Campbell <LCampbell@ASAScience.com>
@brief https://confluence.oceanobservatories.org/display/CIDev/Record+Dictionary
'''

from pyon.core.exception import BadRequest
from pyon.core.object import IonObjectSerializer
from pyon.core.interceptor.encode import encode_ion
from pyon.util.arg_check import validate_equal
from pyon.util.log import log
from pyon.util.memoize import memoize_lru

from interface.services.dm.ipubsub_management_service import PubsubManagementServiceClient
from interface.objects import Granule

from coverage_model import ParameterDictionary, ConstantType, ConstantRangeType, get_value_class, SimpleDomainSet
from coverage_model.parameter_values import AbstractParameterValue, ConstantValue
from coverage_model.parameter_types import ParameterFunctionType

import numpy as np
import msgpack

class RecordDictionaryTool(object):
    """
    A record dictionary is a key/value store which contains records for a particular dataset. The keys are specified by
    a parameter dictionary which map to the fields in the dataset, the data types for the records as well as metadata about
    the fields. Each field in a record dictionary must contain the same number of records. A record can consist of a scalar
    value (typically a NumPy scalar) or it may contain an array or dictionary of values. The data type for each field is
    determined by the parameter dictionary. The length of the first assignment dictates the allowable size for the RDT -
    see the Tip below

    The record dictionary can contain an instance of the parameter dictionary itself or it may contain a reference to one
    by use of a stream definition. A stream definition is a resource persisted by the resource registry and contains the
    parameter dictionary. When the record dictionary is constructed the client will specify either a stream definition
    identifier or a parameter dictionary.

    ParameterDictionaries are inherently large and congest message traffic through RabbitMQ, therefore it is preferred
    to use stream definitions in lieu of parameter dictionaries directly.
    """

    _rd          = None
    _pdict       = None
    _shp         = None
    _locator     = None
    _stream_def  = None
    _dirty_shape = False

    def __init__(self,param_dictionary=None, stream_definition_id='', locator=None):
        """
        """
        if type(param_dictionary) == dict:
            self._pdict = ParameterDictionary.load(param_dictionary)

        elif isinstance(param_dictionary,ParameterDictionary):
            self._pdict = param_dictionary

        elif stream_definition_id:
            pdict = RecordDictionaryTool.pdict_from_stream_def(stream_definition_id)
            self._pdict = ParameterDictionary.load(pdict)
            self._stream_def = stream_definition_id

        else:
            raise BadRequest('Unable to create record dictionary with improper ParameterDictionary')

        if stream_definition_id:
            self._stream_def=stream_definition_id

        self._shp = None
        self._rd = {}
        self._locator = locator

        self._setup_params()

    @classmethod
    def load_from_granule(cls, g):
        if isinstance(g.param_dictionary, str):
            instance = cls(stream_definition_id=g.param_dictionary, locator=g.locator)
            pdict = RecordDictionaryTool.pdict_from_stream_def(g.param_dictionary)
            instance._pdict = ParameterDictionary.load(pdict)

        else:
            instance = cls(param_dictionary=g.param_dictionary, locator=g.locator)
            instance._pdict = ParameterDictionary.load(g.param_dictionary)

        if g.domain:
            instance._shp = (g.domain[0],)

        for k,v in g.record_dictionary.iteritems():
            key = instance._pdict.key_from_ord(k)
            if v is not None:
                ptype = instance._pdict.get_context(key).param_type
                paramval = get_value_class(ptype, domain_set = instance.domain)
                paramval[:] = v
                paramval.storage._storage.flags.writeable = False

                instance._rd[key] = paramval

        return instance

    def to_granule(self, data_producer_id='',provider_metadata_update={}):
        granule = Granule()
        granule.record_dictionary = {}

        for key,val in self._rd.iteritems():
            if val is not None:
                granule.record_dictionary[self._pdict.ord_from_key(key)] = val[:]
            else:
                granule.record_dictionary[self._pdict.ord_from_key(key)] = None

        granule.param_dictionary = self._stream_def or self._pdict.dump()
        granule.locator = self._locator
        granule.domain = self.domain.shape
        granule.data_producer_id=data_producer_id
        granule.provider_metadata_update=provider_metadata_update
        return granule

    def _setup_params(self):
        for param in self._pdict.keys():
            self._rd[param] = None

    @property
    def fields(self):
        return self._rd.keys()

    @property
    def domain(self):
        dom = SimpleDomainSet(self._shp)
        return dom

    @property
    def temporal_parameter(self):
        return self._pdict.temporal_parameter_name

    def __setitem__(self, name, vals):
        """
        Set a parameter
        """
        if name not in self._rd:
            raise KeyError(name)
        if vals is None:
            self._rd[name] = None
            return
        context = self._pdict.get_context(name)

        if self._shp is None and (isinstance(context.param_type, ConstantType) or isinstance(context.param_type, ConstantRangeType)):
            self._shp = (1,)
            self._dirty_shape = True

        elif self._shp is None or self._dirty_shape:
            if isinstance(vals, np.ndarray):
                self._shp = vals.shape
            elif isinstance(vals, list):
                self._shp = (len(vals),)
            else:
                raise BadRequest('No shape was defined')

            log.trace('Set shape to %s', self._shp)
            if self._dirty_shape:
                self._dirty_shape = False
                self._reshape_const()

        else:
            if isinstance(vals, np.ndarray):
                validate_equal(vals.shape, self._shp, 'Invalid shape on input (%s expecting %s)' % (vals.shape, self._shp))
            elif isinstance(vals, list):
                validate_equal(len(vals), self._shp[0], 'Invalid shape on input')

        dom = self.domain
        if isinstance(context.param_type, ParameterFunctionType):
            paramval = np.empty(self._shp, dtype='float32') #@TODO FIX THIS!!!
        else:
            paramval = get_value_class(context.param_type, domain_set = dom)
        paramval[:] = vals
        paramval.storage._storage.flags.writeable = False
        self._rd[name] = paramval

    def _reshape_const(self):
        for k in self.fields:
            if isinstance(self._rd[k], ConstantValue):
                self._rd[k].domain_set = self.domain

    def __getitem__(self, name):
        """
        Get an item by nick name from the record dictionary.
        """
        if self._rd[name] is not None:
            return self._rd[name][:]
        else:
            return None

    def iteritems(self):
        """ D.iteritems() -> an iterator over the (key, value) items of D """
        for k,v in self._rd.iteritems():
            if v is not None:
                yield k,v

    def iterkeys(self):
        """ D.iterkeys() -> an iterator over the keys of D """
        for k,v in self._rd.iteritems():
            if v is not None:
                yield k

    def itervalues(self):
        """ D.itervalues() -> an iterator over the values of D """
        for k,v in self._rd.iteritems():
            if v is not None:
                yield v

    def __contains__(self, key):
        """ D.__contains__(k) -> True if D has a key k, else False """
        return key in self._rd

    def __delitem__(self, y):
        """ x.__delitem__(y) <==> del x[y] """
        self._rd[y] = None

    def __iter__(self):
        """ x.__iter__() <==> iter(x) """
        for k in self._rd.iterkeys():
            yield k

    def __len__(self):
        """ x.__len__() <==> len(x) """
        if self._shp is None:
            return 0
        else:
            return self._shp[0]

    def __repr__(self):
        """ x.__repr__() <==> repr(x) """
        return self.pretty_print()

    def __str__(self):
        return 'Record Dictionary %s' % self._rd.keys()

    __hash__ = None

    def pretty_print(self):
        """
        @brief Pretty Print the record dictionary for debug or log purposes.
        """
        from pprint import pformat
        return pformat(self.__dict__)

    def __eq__(self, comp):
        if self._shp != comp._shp:
            return False
        if self._pdict != comp._pdict:
            return False

        for k,v in self._rd.iteritems():
            if v != comp._rd[k]:
                if isinstance(v, AbstractParameterValue) and isinstance(comp._rd[k], AbstractParameterValue):
                    if (v.content == comp._rd[k].content).all():
                        continue
                return False
        return True

    def __ne__(self, comp):
        return not (self == comp)

    def size(self):
        '''
        Truly poor way to calculate the size of a granule...
        returns the size in bytes.
        '''
        granule = self.to_granule()
        serializer = IonObjectSerializer()
        flat = serializer.serialize(granule)
        byte_stream = msgpack.packb(flat, default=encode_ion)
        return len(byte_stream)

    @classmethod
    def append(cls, rdt1, rdt2):
        assert rdt1.fields == rdt2.fields
        sd = rdt1._stream_def or rdt2._stream_def
        if sd:
            nrdt = cls(stream_definition_id=sd)
        else:
            nrdt = cls(param_dictionary=rdt1._pdict)

        for k in rdt1.fields:
            x = rdt1[k]
            y = rdt2[k]
            if x is None and y is None:
                continue

            if x is None:
                X = np.empty(rdt1._shp, dtype=rdt1._pdict.get_context(k).param_type.value_encoding)
                X.fill(rdt1._pdict.get_context(k).fill_value)
            else:
                X = x

            if y is None:
                Y = np.empty(rdt1._shp, dtype=rdt2._pdict.get_context(k).param_type.value_encoding)
                Y.fill(rdt2._pdict.get_context(k).fill_value)
            else:
                Y = y

            nrdt[k] = np.append(X,Y)

        return nrdt

    @staticmethod
    @memoize_lru(maxsize=100)
    def pdict_from_stream_def(stream_def_id):
        pubsub_cli = PubsubManagementServiceClient()
        stream_def_obj = pubsub_cli.read_stream_definition(stream_def_id)
        return stream_def_obj.parameter_dictionary

<commit_msg>RDT refactor<commit_after>#!/usr/bin/env python

'''
@package ion.services.dm.utility.granule.record_dictionary
@file ion/services/dm/utility/granule/record_dictionary.py
@author Tim Giguere
@author Luke Campbell <LCampbell@ASAScience.com>
@brief https://confluence.oceanobservatories.org/display/CIDev/Record+Dictionary
'''

from pyon.core.exception import BadRequest
from pyon.core.object import IonObjectSerializer
from pyon.core.interceptor.encode import encode_ion
from pyon.util.arg_check import validate_equal
from pyon.util.log import log
from pyon.util.memoize import memoize_lru

from interface.services.dm.ipubsub_management_service import PubsubManagementServiceClient
from interface.objects import Granule

from coverage_model import ParameterDictionary, ConstantType, ConstantRangeType, get_value_class, SimpleDomainSet
from coverage_model.parameter_values import AbstractParameterValue, ConstantValue
from coverage_model.parameter_types import ParameterFunctionType

import numpy as np
import msgpack

class RecordDictionaryTool(object):
    """
    A record dictionary is a key/value store which contains records for a particular dataset. The keys are specified by
    a parameter dictionary which map to the fields in the dataset, the data types for the records as well as metadata about
    the fields. Each field in a record dictionary must contain the same number of records. A record can consist of a scalar
    value (typically a NumPy scalar) or it may contain an array or dictionary of values. The data type for each field is
    determined by the parameter dictionary. The length of the first assignment dictates the allowable size for the RDT -
    see the Tip below

    The record dictionary can contain an instance of the parameter dictionary itself or it may contain a reference to one
    by use of a stream definition. A stream definition is a resource persisted by the resource registry and contains the
    parameter dictionary. When the record dictionary is constructed the client will specify either a stream definition
    identifier or a parameter dictionary.

    ParameterDictionaries are inherently large and congest message traffic through RabbitMQ, therefore it is preferred
    to use stream definitions in lieu of parameter dictionaries directly.
    """

    _rd          = None
    _pdict       = None
    _shp         = None
    _locator     = None
    _stream_def  = None
    _dirty_shape = False

    def __init__(self,param_dictionary=None, stream_definition_id='', locator=None):
        """
        """
        if type(param_dictionary) == dict:
            self._pdict = ParameterDictionary.load(param_dictionary)

        elif isinstance(param_dictionary,ParameterDictionary):
            self._pdict = param_dictionary

        elif stream_definition_id:
            pdict = RecordDictionaryTool.pdict_from_stream_def(stream_definition_id)
            self._pdict = ParameterDictionary.load(pdict)
            self._stream_def = stream_definition_id

        else:
            raise BadRequest('Unable to create record dictionary with improper ParameterDictionary')

        if stream_definition_id:
            self._stream_def=stream_definition_id

        self._shp = None
        self._rd = {}
        self._locator = locator

        self._setup_params()

    @classmethod
    def get_paramval(cls, ptype, domain, values):
        paramval = get_value_class(ptype, domain_set=domain)
        if isinstance(ptype,ParameterFunctionType):
            paramval.memoized_values = values
        else:
            paramval[:] = values
        paramval.storage._storage.flags.writeable = False
        return paramval

    @classmethod
    def load_from_granule(cls, g):
        if isinstance(g.param_dictionary, str):
            instance = cls(stream_definition_id=g.param_dictionary, locator=g.locator)
            pdict = RecordDictionaryTool.pdict_from_stream_def(g.param_dictionary)
            instance._pdict = ParameterDictionary.load(pdict)

        else:
            instance = cls(param_dictionary=g.param_dictionary, locator=g.locator)
            instance._pdict = ParameterDictionary.load(g.param_dictionary)

        if g.domain:
            instance._shp = (g.domain[0],)

        for k,v in g.record_dictionary.iteritems():
            key = instance._pdict.key_from_ord(k)
            if v is not None:
                ptype = instance._pdict.get_context(key).param_type
                paramval = cls.get_paramval(ptype, instance.domain, v)
                instance._rd[key] = paramval

        return instance

    def to_granule(self, data_producer_id='',provider_metadata_update={}):
        granule = Granule()
        granule.record_dictionary = {}

        for key,val in self._rd.iteritems():
            if val is not None:
                granule.record_dictionary[self._pdict.ord_from_key(key)] = val[:]
            else:
                granule.record_dictionary[self._pdict.ord_from_key(key)] = None

        granule.param_dictionary = self._stream_def or self._pdict.dump()
        granule.locator = self._locator
        granule.domain = self.domain.shape
        granule.data_producer_id=data_producer_id
        granule.provider_metadata_update=provider_metadata_update
        return granule

    def _setup_params(self):
        for param in self._pdict.keys():
            self._rd[param] = None

    @property
    def fields(self):
        return self._rd.keys()

    @property
    def domain(self):
        dom = SimpleDomainSet(self._shp)
        return dom

    @property
    def temporal_parameter(self):
        return self._pdict.temporal_parameter_name

    def __setitem__(self, name, vals):
        """
        Set a parameter
        """
        if name not in self._rd:
            raise KeyError(name)
        if vals is None:
            self._rd[name] = None
            return
        context = self._pdict.get_context(name)

        if self._shp is None and (isinstance(context.param_type, ConstantType) or isinstance(context.param_type, ConstantRangeType)):
            self._shp = (1,)
            self._dirty_shape = True

        elif self._shp is None or self._dirty_shape:
            if isinstance(vals, np.ndarray):
                self._shp = vals.shape
            elif isinstance(vals, list):
                self._shp = (len(vals),)
            else:
                raise BadRequest('No shape was defined')

            log.trace('Set shape to %s', self._shp)
            if self._dirty_shape:
                self._dirty_shape = False
                self._reshape_const()

        else:
            if isinstance(vals, np.ndarray):
                validate_equal(vals.shape, self._shp, 'Invalid shape on input (%s expecting %s)' % (vals.shape, self._shp))
            elif isinstance(vals, list):
                validate_equal(len(vals), self._shp[0], 'Invalid shape on input')

        dom = self.domain
        paramval = self.get_paramval(context.param_type, dom, vals)
        self._rd[name] = paramval

    def _reshape_const(self):
        for k in self.fields:
            if isinstance(self._rd[k], ConstantValue):
                self._rd[k].domain_set = self.domain

    def __getitem__(self, name):
        """
        Get an item by nick name from the record dictionary.
        """
        if self._rd[name] is not None:
            context = self._pdict.get_context(name)
            if isinstance(context.param_type, ParameterFunctionType):
                return self._rd[name].memoized_values[:]
            return self._rd[name][:]
        else:
            return None

    def iteritems(self):
        """ D.iteritems() -> an iterator over the (key, value) items of D """
        for k,v in self._rd.iteritems():
            if v is not None:
                yield k,v

    def iterkeys(self):
        """ D.iterkeys() -> an iterator over the keys of D """
        for k,v in self._rd.iteritems():
            if v is not None:
                yield k

    def itervalues(self):
        """ D.itervalues() -> an iterator over the values of D """
        for k,v in self._rd.iteritems():
            if v is not None:
                yield v

    def __contains__(self, key):
        """ D.__contains__(k) -> True if D has a key k, else False """
        return key in self._rd

    def __delitem__(self, y):
        """ x.__delitem__(y) <==> del x[y] """
        self._rd[y] = None

    def __iter__(self):
        """ x.__iter__() <==> iter(x) """
        for k in self._rd.iterkeys():
            yield k

    def __len__(self):
        """ x.__len__() <==> len(x) """
        if self._shp is None:
            return 0
        else:
            return self._shp[0]

    def __repr__(self):
        """ x.__repr__() <==> repr(x) """
        return self.pretty_print()

    def __str__(self):
        return 'Record Dictionary %s' % self._rd.keys()

    __hash__ = None

    def pretty_print(self):
        """
        @brief Pretty Print the record dictionary for debug or log purposes.
        """
        from pprint import pformat
        return pformat(self.__dict__)

    def __eq__(self, comp):
        if self._shp != comp._shp:
            return False
        if self._pdict != comp._pdict:
            return False

        for k,v in self._rd.iteritems():
            if v != comp._rd[k]:
                if isinstance(v, AbstractParameterValue) and isinstance(comp._rd[k], AbstractParameterValue):
                    if (v.content == comp._rd[k].content).all():
                        continue
                return False
        return True

    def __ne__(self, comp):
        return not (self == comp)

    def size(self):
        '''
        Truly poor way to calculate the size of a granule...
        returns the size in bytes.
        '''
        granule = self.to_granule()
        serializer = IonObjectSerializer()
        flat = serializer.serialize(granule)
        byte_stream = msgpack.packb(flat, default=encode_ion)
        return len(byte_stream)

    @classmethod
    def append(cls, rdt1, rdt2):
        assert rdt1.fields == rdt2.fields
        sd = rdt1._stream_def or rdt2._stream_def
        if sd:
            nrdt = cls(stream_definition_id=sd)
        else:
            nrdt = cls(param_dictionary=rdt1._pdict)

        for k in rdt1.fields:
            x = rdt1[k]
            y = rdt2[k]
            if x is None and y is None:
                continue

            if x is None:
                X = np.empty(rdt1._shp, dtype=rdt1._pdict.get_context(k).param_type.value_encoding)
                X.fill(rdt1._pdict.get_context(k).fill_value)
            else:
                X = x

            if y is None:
                Y = np.empty(rdt1._shp, dtype=rdt2._pdict.get_context(k).param_type.value_encoding)
                Y.fill(rdt2._pdict.get_context(k).fill_value)
            else:
                Y = y

            nrdt[k] = np.append(X,Y)

        return nrdt

    @staticmethod
    @memoize_lru(maxsize=100)
    def pdict_from_stream_def(stream_def_id):
        pubsub_cli = PubsubManagementServiceClient()
        stream_def_obj = pubsub_cli.read_stream_definition(stream_def_id)
        return stream_def_obj.parameter_dictionary

<|endoftext|>