<commit_before>#!/usr/bin/env python
# vim: set fileencoding=UTF-8 :

__author__ = 'Xitong Gao'
__email__ = 'xtg08@ic.ac.uk'

_OPERATORS = ['+', '*']

def _parse_r(s):
    s = s.strip()
    print s
    bracket_level = 0
    operator_pos = -1
    for i, v in enumerate(s):
        if v == '(':
            bracket_level += 1
        if v == ')':
            bracket_level -= 1
        if bracket_level == 1 and v in _OPERATORS:
            operator_pos = i
            break
    if operator_pos == -1:
        return s
    return (s[operator_pos],
            _parse_r(s[1:operator_pos].strip()),
            _parse_r(s[operator_pos + 1:-1].strip()))

def _unparse_r(t):
    if type(t) is str:
        return t
    operator, arg1, arg2 = t
    return '(' + _unparse_r(arg1) + operator + _unparse_r(arg2) + ')'

class ExprParser(object):

    def __init__(self, string_or_tree):
        if type(string_or_tree) is str:
            self.string = string_or_tree
        elif type(string_or_tree) is tuple:
            self.tree = string_or_tree

    @property
    def tree(self):
        return self._t

    @tree.setter
    def tree(self, t):
        self._t = t
        self._s = _unparse_r(t)

    @property
    def string(self):
        return self._s

    @string.setter
    def string(self, s):
        self._s = s
        self._t = _parse_r(self._s)

    def __str__(self):
        return self.string

class TreeTransformer(object):

    class TreeTransformerCore(object):

        def __init__(self, tree):
            super(TreeTransformer.TreeTransformerCore, self).__init__()
            self._t = tree

        def closure(self, f):
            trees = set([self._t])
            prev_trees = None
            while trees != prev_trees:
                prev_trees = trees
                trees = reduce(
                            lambda x, y: x.union(y),
                            [set(self._walk_r(t, f)) for t in trees])
            return trees

        def _walk_r(self, t, f):
            if type(t) is tuple:
                for e in set(f(t)):
                    yield e
                for e in set(self._walk_r(t[1], f)):
                    yield (t[0], e, t[2])
                for e in set(self._walk_r(t[2], f)):
                    yield (t[0], t[1], e)
            # make sure identity is not forgotten
            yield t

    def __init__(self, tree):
        super(TreeTransformer, self).__init__()
        self._t = tree

    def closure(self):
        pass

class ExprTreeTransformer(TreeTransformer):

    def __init__(self, tree):
        super(ExprTreeTransformer, self).__init__()
        pass

    def associativity(self, t):
        pass

    def distributivity(self, t):
        pass

    def commutativity(self, t):
        pass

if __name__ == '__main__':
    print ExprParser('((a + b) + c)').tree
<commit_msg>Add method to collate all transforms in subclass<commit_after>#!/usr/bin/env python
# vim: set fileencoding=UTF-8 :

__author__ = 'Xitong Gao'
__email__ = 'xtg08@ic.ac.uk'

import inspect

_OPERATORS = ['+', '*']

def _parse_r(s):
    s = s.strip()
    print s
    bracket_level = 0
    operator_pos = -1
    for i, v in enumerate(s):
        if v == '(':
            bracket_level += 1
        if v == ')':
            bracket_level -= 1
        if bracket_level == 1 and v in _OPERATORS:
            operator_pos = i
            break
    if operator_pos == -1:
        return s
    return (s[operator_pos],
            _parse_r(s[1:operator_pos].strip()),
            _parse_r(s[operator_pos + 1:-1].strip()))

def _unparse_r(t):
    if type(t) is str:
        return t
    operator, arg1, arg2 = t
    return '(' + _unparse_r(arg1) + operator + _unparse_r(arg2) + ')'

class ExprParser(object):

    def __init__(self, string_or_tree):
        if type(string_or_tree) is str:
            self.string = string_or_tree
        elif type(string_or_tree) is tuple:
            self.tree = string_or_tree

    @property
    def tree(self):
        return self._t

    @tree.setter
    def tree(self, t):
        self._t = t
        self._s = _unparse_r(t)

    @property
    def string(self):
        return self._s

    @string.setter
    def string(self, s):
        self._s = s
        self._t = _parse_r(self._s)

    def __str__(self):
        return self.string

class TreeTransformer(object):

    class TreeTransformerCore(object):

        def __init__(self, tree):
            super(TreeTransformer.TreeTransformerCore, self).__init__()
            self._t = tree

        def closure(self, f):
            trees = set([self._t])
            prev_trees = None
            while trees != prev_trees:
                prev_trees = trees
                trees = reduce(
                            lambda x, y: x.union(y),
                            [set(self._walk_r(t, f)) for t in trees])
            return trees

        def _walk_r(self, t, f):
            if type(t) is tuple:
                for e in set(f(t)):
                    yield e
                for e in set(self._walk_r(t[1], f)):
                    yield (t[0], e, t[2])
                for e in set(self._walk_r(t[2], f)):
                    yield (t[0], t[1], e)
            # make sure identity is not forgotten
            yield t

    def __init__(self, tree):
        super(TreeTransformer, self).__init__()
        self._t = tree

    def closure(self):
        """Perform transforms until transitive closure is reached.

        Returns:
            A set of trees after transform.
        """
        return TreeTransformer.TreeTransformerCore(self._t).closure(
                self._transform_collate)

    def _transform_collate(self, t):
        """Combines all transform methods into one.

        Args:
            t: A tree under transform.

        Returns:
            A set of trees after transform.
        """
        return reduce(
                lambda x, y: x.union(y),
                [set(f(t)) for f in self._transform_methods()])

    def _transform_methods(self):
        """Find all transform methods within the class

        Returns:
            A list of tuples containing method names and corresponding methods
            that can be called with a tree as the argument for each method.
        """
        methods = [member[0] for member in inspect.getmembers(
                self.__class__, predicate=inspect.ismethod)]
        return [(method, getattr(self, method))
                for method in methods if method.endswith('_transform')]

class ExprTreeTransformer(TreeTransformer):

    def __init__(self, tree):
        super(ExprTreeTransformer, self).__init__()
        pass

    def associativity(self, t):
        pass

    def distributivity(self, t):
        pass

    def commutativity(self, t):
        pass

if __name__ == '__main__':
    print ExprParser('((a + b) + c)').tree
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
#  test_referredby.py
#  referredby
#

import unittest
import urllib2

import referredby

class SearchEngineTestCase(unittest.TestCase):
    def test_google(self):
        url = 'http://www.google.com/aclk?sa=l&ai=c0564f78eeb0f4287b645d1be6d7fd5a&sig=81014def7eff1c830904851cbe03b917&ved=d53c790199a166e45ef51223996c1fe4&adurl=http://my.domain.com/%3Fad%26utm_medium%3Dcpc%26utm_source%3Dadwords%2Bsearch%26utm_campaign%3Dmycompany%2BBranded%2B-%2BUS%26utm_content%3Dterm1%2Bterm2%2B-%2BExact%26utm_creative%3D13752172593%26utm_target%3D%26utm_term%3Dterm1%2520term2%26utm_placement%3D&rct=j&q=term1+term2'
        url = urllib2.unquote(url)

        self.assertEqual(
                referredby.who(url),
                referredby.SearchEngine('Google', 'www.google.com',
                    ['term1', 'term2']),
            )

def suite():
    return unittest.makeSuite(SearchEngineTestCase)

if __name__ == '__main__':
    unittest.TextTestRunner(verbosity=1).run(suite())
<commit_msg>Add a test for a Yahoo search.<commit_after># -*- coding: utf-8 -*-
#
#  test_referredby.py
#  referredby
#

import unittest

import referredby

class SearchEngineTestCase(unittest.TestCase):
    def test_google(self):
        url = 'http://www.google.com/aclk?sa=l&ai=c0564f78eeb0f4287b645d1be6d7fd5a&sig=81014def7eff1c830904851cbe03b917&ved=d53c790199a166e45ef51223996c1fe4&adurl=http://my.domain.com/%3Fad%26utm_medium%3Dcpc%26utm_source%3Dadwords%2Bsearch%26utm_campaign%3Dmycompany%2BBranded%2B-%2BUS%26utm_content%3Dterm1%2Bterm2%2B-%2BExact%26utm_creative%3D13752172593%26utm_target%3D%26utm_term%3Dterm1%2520term2%26utm_placement%3D&rct=j&q=term1+term2'

        self.assertEqual(
                referredby.who(url),
                referredby.SearchEngine('Google', 'www.google.com',
                    ['term1', 'term2']),
            )

    def test_yahoo(self):
        url = 'http://id.search.yahoo.com/search?fr=mkg030&p=term1%20term2'

        self.assertEqual(
                referredby.who(url),
                referredby.SearchEngine('Yahoo', 'id.search.yahoo.com',
                    ['term1', 'term2']),
            )

def suite():
    return unittest.makeSuite(SearchEngineTestCase)

if __name__ == '__main__':
    unittest.TextTestRunner(verbosity=1).run(suite())
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function, unicode_literals

import os, sys, json
from datetime import datetime, timedelta

import boto3, requests

from . import register_parser
from .util import paginate, Timestamp
from .util.printing import format_table, page_output, tabulate, format_datetime
from .util.aws import region_name, offers_api, clients, instance_type_completer, get_products
from .util.compat import median

def pricing(args):
    if args.spot:
        args.columns = args.columns_spot
        paginator = clients.ec2.get_paginator("describe_spot_price_history")
        page_output(tabulate(paginate(paginator, StartTime=args.spot_start_time), args))
    elif args.service_code:
        if args.region is None:
            args.region = clients.ec2.meta.region_name
        args.columns += getattr(args, "columns_" + args.service_code, [])
        args.filters += getattr(args, "filters_" + args.service_code, [])
        if hasattr(args, "sort_by_" + args.service_code):
            args.sort_by = getattr(args, "sort_by_" + args.service_code)
        filters = [("location", region_name(args.region))] + args.filters
        table = get_products(args.service_code, region=args.region, filters=filters, terms=args.terms,
                             max_cache_age_days=args.max_cache_age_days)
        page_output(tabulate(table, args))
    else:
        client = boto3.client("pricing", region_name="us-east-1")
        args.columns = ["ServiceCode", "AttributeNames"]
        page_output(tabulate(paginate(client.get_paginator("describe_services")), args))

parser = register_parser(pricing, help="List AWS prices")
parser.add_argument("service_code", nargs="?", help="""
AWS product offer to list prices for. Run without this argument to see the list of available product service codes.""")
parser.add_argument("--columns", nargs="+")
parser.add_argument("--filters", nargs="+", metavar="NAME=VALUE", type=lambda x: x.split("=", 1), default=[])
parser.add_argument("--terms", nargs="+", default=["OnDemand"])
parser.add_argument("--sort-by")
parser.add_argument("--spot", action="store_true", help="Display AWS EC2 Spot Instance pricing history")
parser.add_argument("--spot-start-time", type=Timestamp, default=Timestamp("-1h"), metavar="START",
                    help="Time to start spot price history." + Timestamp.__doc__)
parser.add_argument("-columns-spot")
<commit_msg>Add completer for service names<commit_after>from __future__ import absolute_import, division, print_function, unicode_literals

import os, sys, json
from datetime import datetime, timedelta

import boto3, requests

from . import register_parser
from .util import paginate, Timestamp
from .util.printing import format_table, page_output, tabulate, format_datetime
from .util.aws import region_name, offers_api, clients, instance_type_completer, get_products
from .util.compat import median

def describe_services():
    client = boto3.client("pricing", region_name="us-east-1")
    return paginate(client.get_paginator("describe_services"))

def pricing(args):
    if args.spot:
        args.columns = args.columns_spot
        paginator = clients.ec2.get_paginator("describe_spot_price_history")
        page_output(tabulate(paginate(paginator, StartTime=args.spot_start_time), args))
    elif args.service_code:
        if args.region is None:
            args.region = clients.ec2.meta.region_name
        args.columns += getattr(args, "columns_" + args.service_code, [])
        args.filters += getattr(args, "filters_" + args.service_code, [])
        if hasattr(args, "sort_by_" + args.service_code):
            args.sort_by = getattr(args, "sort_by_" + args.service_code)
        filters = [("location", region_name(args.region))] + args.filters
        table = get_products(args.service_code, region=args.region, filters=filters, terms=args.terms,
                             max_cache_age_days=args.max_cache_age_days)
        page_output(tabulate(table, args))
    else:
        args.columns = ["ServiceCode", "AttributeNames"]
        page_output(tabulate(describe_services(), args))

parser = register_parser(pricing, help="List AWS prices")
pricing_arg = parser.add_argument("service_code", nargs="?", help="""
AWS product offer to list prices for. Run without this argument to see the list of available product service codes.""")
pricing_arg.completer = lambda **kwargs: [service["ServiceCode"] for service in describe_services()]
parser.add_argument("--columns", nargs="+")
parser.add_argument("--filters", nargs="+", metavar="NAME=VALUE", type=lambda x: x.split("=", 1), default=[])
parser.add_argument("--terms", nargs="+", default=["OnDemand"])
parser.add_argument("--sort-by")
parser.add_argument("--spot", action="store_true", help="Display AWS EC2 Spot Instance pricing history")
parser.add_argument("--spot-start-time", type=Timestamp, default=Timestamp("-1h"), metavar="START",
                    help="Time to start spot price history." + Timestamp.__doc__)
parser.add_argument("-columns-spot")
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import os
import Adafruit_DHT
import Adafruit_IO
import json
import requests
from lib import DHTStorage

# Parse command line parameters.
sensor_args = { '11': Adafruit_DHT.DHT11,
				'22': Adafruit_DHT.DHT22,
				'2302': Adafruit_DHT.AM2302 }
if len(sys.argv) == 3 and sys.argv[1] in sensor_args:
	sensor = sensor_args[sys.argv[1]]
	pin = sys.argv[2]
else:
	print 'usage: sudo ./Adafruit_DHT.py [11|22|2302] GPIOpin#'
	print 'example: sudo ./Adafruit_DHT.py 2302 4 - Read from an AM2302 connected to GPIO #4'
	sys.exit(1)

# Try to grab a sensor reading.  Use the read_retry method which will retry up
# to 15 times to get a sensor reading (waiting 2 seconds between each retry).
humidity, temperature = Adafruit_DHT.read_retry(sensor, pin)

# Un-comment the line below to convert the temperature to Fahrenheit.
# temperature = temperature * 9/5.0 + 32

# Note that sometimes you won't get a reading and
# the results will be null (because Linux can't
# guarantee the timing of calls to read the sensor).
# If this happens try again!

def send(client, humidity, temperature):
	client.send('humidity', '{0:0.3f}'.format(humidity))
	client.send('temperature',  '{0:0.3f}'.format(temperature))

def toadaio(client_key, humidity, temperature):
	try:
		aio = Adafruit_IO.Client(client_key)
		send(aio, humidity, temperature)
	except Adafruit_IO.errors.RequestError:
		pass
	except requests.exceptions.ConnectionError:
		pass

def todhtstorage(client_key, humidity, temperature):
	storage = DHTStorage(client_key)

	storage.add_temperature(temperature)
	storage.add_humidity(humidity)

if humidity is not None and temperature is not None:

	with open(os.path.join(os.path.dirname(__file__), 'config.json')) as data_file:
		config = json.load(data_file)

	client_key = config['key']
	todhtstorage(client_key, humidity, temperature)
	toadaio(client_key, humidity, temperature)
	# print 'Temp={0:0.1f}*  Humidity={1:0.1f}%'.format(temperature, humidity)
else:
	print 'Failed to get reading. Try again!'
	sys.exit(1)

# vim: set ts=4 sw=4 noet
<commit_msg>Ignore Adafruit_IO.errors.ThrottlingErro<commit_after>#!/usr/bin/env python

import sys
import os
import Adafruit_DHT
import Adafruit_IO
import json
import requests
from lib import DHTStorage

# Parse command line parameters.
sensor_args = { '11': Adafruit_DHT.DHT11,
				'22': Adafruit_DHT.DHT22,
				'2302': Adafruit_DHT.AM2302 }
if len(sys.argv) == 3 and sys.argv[1] in sensor_args:
	sensor = sensor_args[sys.argv[1]]
	pin = sys.argv[2]
else:
	print 'usage: sudo ./Adafruit_DHT.py [11|22|2302] GPIOpin#'
	print 'example: sudo ./Adafruit_DHT.py 2302 4 - Read from an AM2302 connected to GPIO #4'
	sys.exit(1)

# Try to grab a sensor reading.  Use the read_retry method which will retry up
# to 15 times to get a sensor reading (waiting 2 seconds between each retry).
humidity, temperature = Adafruit_DHT.read_retry(sensor, pin)

# Un-comment the line below to convert the temperature to Fahrenheit.
# temperature = temperature * 9/5.0 + 32

# Note that sometimes you won't get a reading and
# the results will be null (because Linux can't
# guarantee the timing of calls to read the sensor).
# If this happens try again!

def send(client, humidity, temperature):
	client.send('humidity', '{0:0.3f}'.format(humidity))
	client.send('temperature',  '{0:0.3f}'.format(temperature))

def toadaio(client_key, humidity, temperature):
	try:
		aio = Adafruit_IO.Client(client_key)
		send(aio, humidity, temperature)
	except Adafruit_IO.errors.RequestError:
		pass
	except requests.exceptions.ConnectionError:
		pass
	except Adafruit_IO.errors.ThrottlingError:
		pass

def todhtstorage(client_key, humidity, temperature):
	storage = DHTStorage(client_key)

	storage.add_temperature(temperature)
	storage.add_humidity(humidity)

if humidity is not None and temperature is not None:

	with open(os.path.join(os.path.dirname(__file__), 'config.json')) as data_file:
		config = json.load(data_file)

	client_key = config['key']
	todhtstorage(client_key, humidity, temperature)
	toadaio(client_key, humidity, temperature)
	# print 'Temp={0:0.1f}*  Humidity={1:0.1f}%'.format(temperature, humidity)
else:
	print 'Failed to get reading. Try again!'
	sys.exit(1)

# vim: set ts=4 sw=4 noet
<|endoftext|>


<commit_before>#
#    ICRAR - International Centre for Radio Astronomy Research
#    (c) UWA - The University of Western Australia, 2015
#    Copyright by UWA (in the framework of the ICRAR)
#    All rights reserved
#
#    This library is free software; you can redistribute it and/or
#    modify it under the terms of the GNU Lesser General Public
#    License as published by the Free Software Foundation; either
#    version 2.1 of the License, or (at your option) any later version.
#
#    This library is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with this library; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
#    MA 02111-1307  USA
#
import os
'''
Module containing docker-related applications and functions
'''

import logging
import time

from docker.client import AutoVersionClient

from dfms.data_object import BarrierAppDataObject, FileDataObject, \
    DirectoryContainer

logger = logging.getLogger(__name__)

DFMS_ROOT = '/dfms_root'

class DockerApp(BarrierAppDataObject):
    """
    A BarrierAppDataObject that represents a process running in a container
    hosted by a local docker daemon. Depending on the host system, the docker
    daemon might be automatically activated when a client tries to connect to
    it via its unix socket (like with systemd) or it needs to be brought up
    previous to any client operation (upstart). In any case, if the daemon is
    not present, this class will raise exceptions whenever it tries to connect
    to the server to perform some operation.

    Docker containers are built from docker images, which are pulled to the host
    where the docker daemon runs either explicitly (via `docker pull`) or less
    visibly (e.g., when running `docker run` using an image that has not been
    fetched yet). This DockerApp application will explicitly pull the image at
    `initialize` time, meaning that the docker images will become available at
    the time the physical graph (which this application is part of) is deployed.
    Docker containers also need a command to be run in them, which should be
    an available program inside the image.

    The inputs and outputs used by the dockerized application are made available
    by mapping host directories and files as "data volumes". Since non-existing
    files will appear as directories in the container filesystem, this class
    create any output FileDataObject first in the host filesystem with size 0,
    so it will also appear as a file in the container filesystem.

    Processes running inside docker container run as root by default. Before
    running the command that is meant to be container-ized, this application
    appends some commands to make sure that: a) a user exists in the container
    namespace with the same UID of the user running this application, and b)
    that the command is executed by such user. This way the output produced by
    the containerized application will belong to the host user.

    Since data volumes are a file-specific feature, this DockerApp application
    supports file-system based input/output DataObjects only, namely the
    FileDataObject and the DirectoryContainer types.

    Since the command to be run in the container receives most probably as
    arguments the paths of its inputs and outputs, and since these might not be
    known precisely until runtime, users should use placeholders for them in the
    command-line specification. Placeholders for input locations take the form
    of "%iX", where X starts from 0 and refers to the X-th input. Likewise,
    output locations are specified as "%oX".
    """

    def initialize(self, **kwargs):
        BarrierAppDataObject.initialize(self, **kwargs)

        self._image = self._getArg(kwargs, 'image', None)
        if not self._image:
            raise Exception('No docker image specified, cannot create DockerApp')

        if ":" not in self._image:
            logger.warn("Image %s is too generic since it doesn't specify a tag" % (self._image))

        self._command = self._getArg(kwargs, 'command', None)
        if not self._command:
            raise Exception("No command specified, cannot create DockerApp")

        if logger.isEnabledFor(logging.INFO):
            logger.info("DockerApp with image '%s' and command '%s' created" % (self._image, self._command))

        # Check if we have the image; otherwise pull it.
        c = AutoVersionClient()
        found = reduce(lambda a,b: a or self._image in b['RepoTags'], c.images(), False)

        if not found:
            if logger.isEnabledFor(logging.DEBUG):
                logger.debug("Image '%s' not found, pulling it" % (self._image))
            start = time.time()
            c.pull(self._image)
            end = time.time()
            if logger.isEnabledFor(logging.DEBUG):
                logger.debug("Took %.2f [s] to pull image '%s'" % ((end-start), self._image))
        else:
            if logger.isEnabledFor(logging.DEBUG):
                logger.debug("Image '%s' found, no need to pull it" % (self._image))

    def run(self):

        # Check inputs/outputs are of a valid type
        for i in self.inputs + self.outputs:
            if not isinstance(i, (FileDataObject, DirectoryContainer)):
                raise Exception("DataObject %r is not supported by the DockerApp" % (i))

        # We mount the inputs and outputs inside the docker under the
        # DFMS_ROOT directory, maintaining the rest of their original paths
        # The inputs are also mounted as RO to avoid any accidental write
        dockerInputs  = [DFMS_ROOT + i.path for i in self.inputs]
        dockerOutputs = [DFMS_ROOT + o.path for o in self.outputs]
        vols = dockerInputs + dockerOutputs
        binds  = [i.path + ":" + dockerInputs[x] + ":ro" for x,i in enumerate(self.inputs)]
        binds += [o.path + ":" + dockerOutputs[x]        for x,o in enumerate(self.outputs)]

        # Replace any placeholders that might be found in the command line
        # by the real path of the inputs and outputs
        cmd = self._command
        for x,i in enumerate(dockerInputs):
            cmd = cmd.replace("%%i%d" % (x), i)
        for x,o in enumerate(dockerOutputs):
            cmd = cmd.replace("%%o%d" % (x), o)

        if logger.isEnabledFor(logging.DEBUG):
            logger.debug("Command after placeholder replacement is: '%s'" % (cmd))

        # We want to run the container-ized program not as root, but as a user
        # with the same UID of the current user. This way the output produced
        # by the container user will be also owned by the host user, who won't
        # have any problem managing it.
        # We achieve this by creating a user with the same UID if one doesn't
        # exist, and running the command that user via "su"
        uid = os.getuid()
        cmd = "id -u {0} &> /dev/null || adduser --uid {0} r; cd; su - r -c /bin/bash -c '{1}'".format(uid, cmd)

        # Embed the command in bash
        cmd = '/bin/bash -c "%s"' % (cmd.replace('"','\\"'))

        if logger.isEnabledFor(logging.DEBUG):
            logger.debug("Command after user creation and wrapping is: '%s'" % (cmd))

        # If we're mapping files that don't exist yet (i.e., FileDataObjects
        # in the outputs) we need to touch them first; otherwise the mapping
        # will result in the container end being treated as a directory
        for f in filter(lambda do: isinstance(do, FileDataObject), self.outputs):
            f.ensureExists()

        c = AutoVersionClient()
        container_id = c.create_container(self._image, cmd, volumes=vols, host_config=c.create_host_config(binds=binds))

        if logger.isEnabledFor(logging.INFO):
            logger.info("Starting DockerApp with image '%s' and command '%s'" % (self._image, self._command))

        c.start(container_id)
        self._exitCode = c.wait(container_id)

        if logger.isEnabledFor(logging.INFO):
            logger.info("DockerApp with image '%s' and command '%s' finished with exit code %d" % (self._image, cmd, self._exitCode))<commit_msg>Comments on DockerApp, mainly TODOs<commit_after>#
#    ICRAR - International Centre for Radio Astronomy Research
#    (c) UWA - The University of Western Australia, 2015
#    Copyright by UWA (in the framework of the ICRAR)
#    All rights reserved
#
#    This library is free software; you can redistribute it and/or
#    modify it under the terms of the GNU Lesser General Public
#    License as published by the Free Software Foundation; either
#    version 2.1 of the License, or (at your option) any later version.
#
#    This library is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with this library; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
#    MA 02111-1307  USA
#
import os
'''
Module containing docker-related applications and functions
'''

import logging
import time

from docker.client import AutoVersionClient

from dfms.data_object import BarrierAppDataObject, FileDataObject, \
    DirectoryContainer

logger = logging.getLogger(__name__)

DFMS_ROOT = '/dfms_root'

class DockerApp(BarrierAppDataObject):
    """
    A BarrierAppDataObject that represents a process running in a container
    hosted by a local docker daemon. Depending on the host system, the docker
    daemon might be automatically activated when a client tries to connect to
    it via its unix socket (like with systemd) or it needs to be brought up
    previous to any client operation (upstart). In any case, if the daemon is
    not present, this class will raise exceptions whenever it tries to connect
    to the server to perform some operation.

    Docker containers are built from docker images, which are pulled to the host
    where the docker daemon runs either explicitly (via `docker pull`) or less
    visibly (e.g., when running `docker run` using an image that has not been
    fetched yet). This DockerApp application will explicitly pull the image at
    `initialize` time, meaning that the docker images will become available at
    the time the physical graph (which this application is part of) is deployed.
    Docker containers also need a command to be run in them, which should be
    an available program inside the image.

    Input and output
    ----------------

    The inputs and outputs used by the dockerized application are made available
    by mapping host directories and files as "data volumes". Since non-existing
    files will appear as directories in the container filesystem, this class
    create any output FileDataObject first in the host filesystem with size 0,
    so it will also appear as a file in the container filesystem.

    Processes running inside docker container run as root by default. Before
    running the command that is meant to be container-ized, this application
    appends some commands to make sure that: a) a user exists in the container
    namespace with the same UID of the user running this application, and b)
    that the command is executed by such user. This way the output produced by
    the containerized application will belong to the host user.

    Since data volumes are a file-specific feature, this DockerApp application
    supports file-system based input/output DataObjects only, namely the
    FileDataObject and the DirectoryContainer types.

    Since the command to be run in the container receives most probably as
    arguments the paths of its inputs and outputs, and since these might not be
    known precisely until runtime, users should use placeholders for them in the
    command-line specification. Placeholders for input locations take the form
    of "%iX", where X starts from 0 and refers to the X-th input. Likewise,
    output locations are specified as "%oX".

    Linking containers
    ------------------

    Although some containerized applications might run on their own, there are
    cases where applications need to talk to each other in order to advance
    (like in the case of client-server applications, or in the case of MPI
    applications). All containers started in the same host (and therefore, all
    applications running in them) belong by default to the same network, and
    therefore are already visible. The problem lies on getting them to know the
    hosts they should contact. See the TODO below.

    TODO:
    -----
    I manually tried to run an MPI program across more than one docker
    container and succeeded on it. In one container A I started simply an ssh
    server (the slave node), while on a second one I started the main mpiexec
    process (the master node). Thanks to this exercise a few things that need to
    be taken into account became clear:

    a) We need to have a placeholder mechanism to be able to refer to the IP of
    some other docker container, probably using the uid of the DockerApp we're
    referring to, something like:

    a = DockerApp('A', image='slave_image', cmd='sshd')
    b = DockerApp('B', image='master_image', cmd='mpiexec -H localhost,%ip{A} something')

    My current understanding is that there is no way to set a given IP on a
    newly created container, so manually assigning IPs to containers is not an
    option currently.

    b) This leads to a dependency problem on the startup process of the
    containers. In the example above, B needs to know A's IP address, and
    therefore A needs to be created first (although not necessarily run!), and
    in more complex examples there could be more complex dependencies involved.
    The framework would thus need to handle this scenario gracefully, starting
    the applications in the correct order, and letting them know information
    about each other (in this case the IPs of the containers they are starting).
    For this, we need to expose this start-up dependency explicitly at the
    AppDataObject level. In a dospec form, it could look something like:

    [
    {uid:'A', type='app', app='DockerApp', image='slave_image', cmd='sshd -D'},
    {uid:'B', type='app', app='DockerApp', image='master_image', cmd='mpiexec -H localhost,%ip{A} something', run_depends_on=['A']}
    ]

    c) Processes in containers might not always exit by themselves, and the
    containers will need to be manually stopped. This is the case for example of
    the slave containers, where the sshd daemon needs to be running before the
    master starts, but will not quit automatically once the master process has
    ended. Maybe the same "run_depends_on" attribute shown above can be reused
    for the purpose of automatically stopping containers, but on the other hand
    the 'starting A depends on B' and the 'B needs to be stopped after A'
    situations do not always correspond one to one, and thus one could think on
    a separate attribute to denote this relationship.
    """

    def initialize(self, **kwargs):
        BarrierAppDataObject.initialize(self, **kwargs)

        self._image = self._getArg(kwargs, 'image', None)
        if not self._image:
            raise Exception('No docker image specified, cannot create DockerApp')

        if ":" not in self._image:
            logger.warn("Image %s is too generic since it doesn't specify a tag" % (self._image))

        self._command = self._getArg(kwargs, 'command', None)
        if not self._command:
            raise Exception("No command specified, cannot create DockerApp")

        if logger.isEnabledFor(logging.INFO):
            logger.info("DockerApp with image '%s' and command '%s' created" % (self._image, self._command))

        # Check if we have the image; otherwise pull it.
        c = AutoVersionClient()
        found = reduce(lambda a,b: a or self._image in b['RepoTags'], c.images(), False)

        if not found:
            if logger.isEnabledFor(logging.DEBUG):
                logger.debug("Image '%s' not found, pulling it" % (self._image))
            start = time.time()
            c.pull(self._image)
            end = time.time()
            if logger.isEnabledFor(logging.DEBUG):
                logger.debug("Took %.2f [s] to pull image '%s'" % ((end-start), self._image))
        else:
            if logger.isEnabledFor(logging.DEBUG):
                logger.debug("Image '%s' found, no need to pull it" % (self._image))

    def run(self):

        # Check inputs/outputs are of a valid type
        for i in self.inputs + self.outputs:
            if not isinstance(i, (FileDataObject, DirectoryContainer)):
                raise Exception("DataObject %r is not supported by the DockerApp" % (i))

        # We mount the inputs and outputs inside the docker under the
        # DFMS_ROOT directory, maintaining the rest of their original paths
        # The inputs are also mounted as RO to avoid any accidental write
        dockerInputs  = [DFMS_ROOT + i.path for i in self.inputs]
        dockerOutputs = [DFMS_ROOT + o.path for o in self.outputs]
        vols = dockerInputs + dockerOutputs
        binds  = [i.path + ":" + dockerInputs[x] + ":ro" for x,i in enumerate(self.inputs)]
        binds += [o.path + ":" + dockerOutputs[x]        for x,o in enumerate(self.outputs)]

        # Replace any placeholders that might be found in the command line
        # by the real path of the inputs and outputs
        cmd = self._command
        for x,i in enumerate(dockerInputs):
            cmd = cmd.replace("%%i%d" % (x), i)
        for x,o in enumerate(dockerOutputs):
            cmd = cmd.replace("%%o%d" % (x), o)

        if logger.isEnabledFor(logging.DEBUG):
            logger.debug("Command after placeholder replacement is: '%s'" % (cmd))

        # We want to run the container-ized program not as root, but as a user
        # with the same UID of the current user. This way the output produced
        # by the container user will be also owned by the host user, who won't
        # have any problem managing it.
        # We achieve this by creating a user with the same UID if one doesn't
        # exist, and running the command that user via "su"
        uid = os.getuid()
        cmd = "id -u {0} &> /dev/null || adduser --uid {0} r; cd; su - r -c /bin/bash -c '{1}'".format(uid, cmd.replace("'","\\'"))

        # Embed the command in bash
        cmd = '/bin/bash -c "%s"' % (cmd.replace('"','\\"'))

        if logger.isEnabledFor(logging.DEBUG):
            logger.debug("Command after user creation and wrapping is: '%s'" % (cmd))

        # If we're mapping files that don't exist yet (i.e., FileDataObjects
        # in the outputs) we need to touch them first; otherwise the mapping
        # will result in the container end being treated as a directory
        for f in filter(lambda do: isinstance(do, FileDataObject), self.outputs):
            f.ensureExists()

        c = AutoVersionClient()
        container_id = c.create_container(self._image, cmd, volumes=vols, host_config=c.create_host_config(binds=binds))

        if logger.isEnabledFor(logging.INFO):
            logger.info("Starting DockerApp with image '%s' and command '%s'" % (self._image, self._command))

        c.start(container_id)
        self._exitCode = c.wait(container_id)

        if logger.isEnabledFor(logging.INFO):
            logger.info("DockerApp with image '%s' and command '%s' finished with exit code %d" % (self._image, cmd, self._exitCode))<|endoftext|>


<commit_before>import os, sys
from sqlalchemy import Column, Integer, ForeignKey, String, DateTime
from datetime import datetime
from sqlalchemy.orm import relationship
from multiprocessing import Process, JoinableQueue
base_path = os.path.dirname(os.path.abspath(__file__))
sys.path.append(os.path.join(base_path, 'lib'))
from waitlist.storage.database import Base, Shipfit, InvType, FitModule
from waitlist.base import db, manager

if __name__ == '___main__':
    manager.run()

class ConvertConsumer(Process):
    def __init__(self, tasks, name):
        Process.__init__(self)
        self.__tasks = tasks
        self.name = name

    def run(self):
        while True:
            work = self.__tasks.get()
            if work is None:
                break
            self.convert(work[0], work[1])

    def convert(self, offset, limit):
        fits = db.session.query(Shipfit).limit(limit).offset(offset).all()
        for fit in fits:
            if (fit.modules != None and fit.modules != ''):
                filtered_modules_string = ''
                for moduleDefStr in fit.modules.split(':'):
                    if (moduleDefStr == ''):
                        continue
                    try:
                        moduleDefArr = moduleDefStr.split(';')
                        if (len(moduleDefArr) != 2):
                            print("Skipping Module Fit ID=", fit.id, " Module Def Str:", moduleDefStr)
                            continue

                        # lets check here if that module exists
                        moduleDefArr[0] = int(moduleDefArr[0])
                        moduleDefArr[1] = int(moduleDefArr[1])
                        if moduleDefArr[1] > 2147483647 or moduleDefArr[1] < 0:
                            moduleDefArr[1] = 2147483647

                        module = db.session.query(InvType).get(moduleDefArr[0])
                        if (module == None):
                            print("No Module with ID=", str(moduleDefArr[0]))
                            continue

                        dbModule = FitModule(moduleID=moduleDefArr[0], amount=moduleDefArr[1])
                        fit.moduleslist.append(dbModule)
                        filtered_modules_string += str(moduleDefArr[0])+';'+str(moduleDefArr[1])+':'
                    except ValueError as e:
                        print("Fit ID=", str(fit.id), " Module Def Str:", moduleDefStr)
                        raise e
                    except IndexError as ie:
                        print("Fit ID=", str(fit.id), " Module Def Str:", moduleDefStr)
                        raise ie

                if filtered_modules_string == '' or filtered_modules_string == '::':
                    filtered_modules_string = ':'
                else:
                    filtered_modules_string += ':'
                if fit.modules != filtered_modules_string:
                    fit.modules = filtered_modules_string

        db.session.commit()
        db.session.close()

if __name__ == '__main__':
    consumer_threads = []
    task_queue = JoinableQueue(maxsize=100)
    for i in range(6):
        t = ConvertConsumer(task_queue, "dec"+str(i))
        t.start()
        consumer_threads.append(t)

    numberOfFits = db.session.query(Shipfit).count()
    stepSize = 1000
    for s in xrange(0, numberOfFits, stepSize):
        task_queue.put((s, stepSize))

    # send them all the signal to break
    for _ in range(6):
            task_queue.put(None)

'''
maybe to this whole thing directly in SQL
but writing it might be a real pain,
it should increase the speed A LOT through, I would guess
DELIMITER $$

DROP PROCEDURE IF EXISTS `insert_csv` $$
CREATE PROCEDURE `insert_csv`(_list MEDIUMTEXT)
BEGIN

DECLARE _next TEXT DEFAULT NULL;
DECLARE _nextlen INT DEFAULT NULL;
DECLARE _value TEXT DEFAULT NULL;

iterator:
LOOP
  -- exit the loop if the list seems empty or was null;
  -- this extra caution is necessary to avoid an endless loop in the proc.
  IF LENGTH(TRIM(_list)) = 0 OR _list IS NULL THEN
    LEAVE iterator;
  END IF;

  -- capture the next value from the list
  SET _next = SUBSTRING_INDEX(_list,',',1);

  -- save the length of the captured value; we will need to remove this
  -- many characters + 1 from the beginning of the string
  -- before the next iteration
  SET _nextlen = LENGTH(_next);

  -- trim the value of leading and trailing spaces, in case of sloppy CSV strings
  SET _value = TRIM(_next);

  -- insert the extracted value into the target table
  INSERT INTO t1 (c1) VALUES (_next);

  -- rewrite the original string using the `INSERT()` string function,
  -- args are original string, start position, how many characters to remove,
  -- and what to "insert" in their place (in this case, we "insert"
  -- an empty string, which removes _nextlen + 1 characters)
  SET _list = INSERT(_list,1,_nextlen + 1,'');
END LOOP;

END $$

DELIMITER ;
'''<commit_msg>removed some unused imports<commit_after>import os, sys
from multiprocessing import Process, JoinableQueue
base_path = os.path.dirname(os.path.abspath(__file__))
sys.path.append(os.path.join(base_path, 'lib'))
from waitlist.storage.database import Shipfit, InvType, FitModule
from waitlist.base import db, manager

if __name__ == '___main__':
    manager.run()

class ConvertConsumer(Process):
    def __init__(self, tasks, name):
        Process.__init__(self)
        self.__tasks = tasks
        self.name = name

    def run(self):
        while True:
            work = self.__tasks.get()
            if work is None:
                break
            self.convert(work[0], work[1])

    def convert(self, offset, limit):
        fits = db.session.query(Shipfit).limit(limit).offset(offset).all()
        for fit in fits:
            if (fit.modules != None and fit.modules != ''):
                filtered_modules_string = ''
                for moduleDefStr in fit.modules.split(':'):
                    if (moduleDefStr == ''):
                        continue
                    try:
                        moduleDefArr = moduleDefStr.split(';')
                        if (len(moduleDefArr) != 2):
                            print("Skipping Module Fit ID=", fit.id, " Module Def Str:", moduleDefStr)
                            continue

                        # lets check here if that module exists
                        moduleDefArr[0] = int(moduleDefArr[0])
                        moduleDefArr[1] = int(moduleDefArr[1])
                        if moduleDefArr[1] > 2147483647 or moduleDefArr[1] < 0:
                            moduleDefArr[1] = 2147483647

                        module = db.session.query(InvType).get(moduleDefArr[0])
                        if (module == None):
                            print("No Module with ID=", str(moduleDefArr[0]))
                            continue

                        dbModule = FitModule(moduleID=moduleDefArr[0], amount=moduleDefArr[1])
                        fit.moduleslist.append(dbModule)
                        filtered_modules_string += str(moduleDefArr[0])+';'+str(moduleDefArr[1])+':'
                    except ValueError as e:
                        print("Fit ID=", str(fit.id), " Module Def Str:", moduleDefStr)
                        raise e
                    except IndexError as ie:
                        print("Fit ID=", str(fit.id), " Module Def Str:", moduleDefStr)
                        raise ie

                if filtered_modules_string == '' or filtered_modules_string == '::':
                    filtered_modules_string = ':'
                else:
                    filtered_modules_string += ':'
                if fit.modules != filtered_modules_string:
                    fit.modules = filtered_modules_string

        db.session.commit()
        db.session.close()

if __name__ == '__main__':
    consumer_threads = []
    task_queue = JoinableQueue(maxsize=100)
    for i in range(6):
        t = ConvertConsumer(task_queue, "dec"+str(i))
        t.start()
        consumer_threads.append(t)

    numberOfFits = db.session.query(Shipfit).count()
    stepSize = 1000
    for s in xrange(0, numberOfFits, stepSize):
        task_queue.put((s, stepSize))

    # send them all the signal to break
    for _ in range(6):
            task_queue.put(None)

'''
maybe to this whole thing directly in SQL
but writing it might be a real pain,
it should increase the speed A LOT through, I would guess
DELIMITER $$

DROP PROCEDURE IF EXISTS `insert_csv` $$
CREATE PROCEDURE `insert_csv`(_list MEDIUMTEXT)
BEGIN

DECLARE _next TEXT DEFAULT NULL;
DECLARE _nextlen INT DEFAULT NULL;
DECLARE _value TEXT DEFAULT NULL;

iterator:
LOOP
  -- exit the loop if the list seems empty or was null;
  -- this extra caution is necessary to avoid an endless loop in the proc.
  IF LENGTH(TRIM(_list)) = 0 OR _list IS NULL THEN
    LEAVE iterator;
  END IF;

  -- capture the next value from the list
  SET _next = SUBSTRING_INDEX(_list,',',1);

  -- save the length of the captured value; we will need to remove this
  -- many characters + 1 from the beginning of the string
  -- before the next iteration
  SET _nextlen = LENGTH(_next);

  -- trim the value of leading and trailing spaces, in case of sloppy CSV strings
  SET _value = TRIM(_next);

  -- insert the extracted value into the target table
  INSERT INTO t1 (c1) VALUES (_next);

  -- rewrite the original string using the `INSERT()` string function,
  -- args are original string, start position, how many characters to remove,
  -- and what to "insert" in their place (in this case, we "insert"
  -- an empty string, which removes _nextlen + 1 characters)
  SET _list = INSERT(_list,1,_nextlen + 1,'');
END LOOP;

END $$

DELIMITER ;
'''<|endoftext|>


<commit_before>#!/usr/bin/python

"""
Running on local host, listening on https over tcp and http over unix domain socket
for authentication assertions and secure temporary storage of authorization assertions
"""
import os
import struct
import socket
import SocketServer
import ssl
import threading
import tempfile
import logging

SO_PEERCRED = 17

from SocketServer import TCPServer, UnixStreamServer, ThreadingMixIn, ThreadingTCPServer
from SimpleHTTPServer import SimpleHTTPRequestHandler

from lib.sock2proc import ProcInfo
from lib import libauthn, libauthz

LOCAL_PORT = 6443
LOCAL_PATH = tempfile.gettempdir() + "/cloudauth.sk"

IDP_SRVR_URL = "https://localhost:6443/roles"

SIG_KEY_FILE = "cnf/ssh_host_ecdsa_key"
SIG_CRT_FILE = "cnf/ssh_host_ecdsa_key.crt"
AuZ_KEY_FILE = "cnf/ssh_host_ecdsa_key"

TLS_KEY_FILE = "cnf/ssh_host_ecdsa_key"
TLS_CRT_FILE = "cnf/ssh_host_ecdsa_key.crt"

HTTP_RESP_HDRS = "HTTP/1.1 %(status)s\r\nContent-type:%(ctype)s\r\nContent-length:%(clen)s\r\n\r\n"

logger = logging.getLogger("authagent")

class CloudAuthHTTPReqHandler(SimpleHTTPRequestHandler):

    protocol_version = "HTTP/1.1"

    is_inet = True
    is_post = True

    def respond(self, status, ctype, clen, data) :

        self.send_response(status)

        self.send_header('Content-Type', ctype)
        self.send_header('Content-Length', clen)

        self.end_headers()

        self.wfile.write(data)

        self.wfile.flush()

    def service(self):

        #TODO request body processing

        logger.info("path = %s", self.path)

        qstr = self.req_qstr();
        body = self.req_body();

        if (self.path.startswith("/authn")) :
            # /authn?ttype=qst

            proc = self.peer_proc()

            authn = libauthn.assert_authn(proc, libauthn.file2buf(SIG_KEY_FILE), qstr, body)

            self.respond(200, "text/authn", str(len(authn)), authn)

        elif (self.path.startswith("/authz")) :
            # /authz?ttype=qst&srvs=foo
            proc = self.peer_proc()

            authn = libauthn.assert_authn(proc, libauthn.file2buf(SIG_KEY_FILE), qstr, body)

            cert = libauthn.file2buf(SIG_CRT_FILE)

            authz = libauthn.askfor_authz(authn, cert, IDP_SRVR_URL, qstr, body)

            self.respond(200, "text/authz", str(len(authz)), authz)

        elif (self.path.startswith("/cert")) :
            # /cert?subj=hostname

            cert = libauthn.file2buf(SIG_CRT_FILE)

            self.respond(200, "text/cert", str(len(cert)), cert)

        elif (self.path.startswith("/sign")) :
            # /sign?algo=ecs256&hash=b64urlsafe

            proc = self.peer_proc()

            sig = libauthn.hash_sign(proc, libauthn.file2buf(SIG_KEY_FILE), qstr, body)

            self.respond(200, "text/sign", str(len(sig)), sig)

        elif (self.path.startswith("/roles")) :
            # /roles?ttype=qst&<token>&srvs=foo
            # body is client/host cert for authn verification
            authz = libauthz.assert_authz(qstr, body, libauthn.file2buf(AuZ_KEY_FILE))

            self.respond(200, "text/authz-roles", str(len(authz)), authz)

    def peer_proc(self):

        proc = None

        if (isinstance(self.client_address, str)):
            self.is_inet = False
            creds = self.connection.getsockopt(socket.SOL_SOCKET, SO_PEERCRED, struct.calcsize('3i'))
            pid, euid, egid = struct.unpack('3i', creds)
            logger.info("client via AF_UNIX, pid %s, uid %s", pid, euid)
            proc = ProcInfo.pipe2proc(pid, euid)
        else:
            self.is_inet = True
            logger.info("client via AF_INET,  %s", self.client_address)
            proc = ProcInfo.sock2proc(self.client_address)

        return proc

    def do_GET(self):

        self.is_post = False
        self.service()

    def do_POST(self):

        self.is_post = True
        self.service()

    def req_body(self):

        size = self.headers.getheader('content-length')
        if (size == None):
            return ""

        size = int(size)

        return self.rfile.read(size)

    def req_qstr(self):

        idx = self.path.find("?")
        if (idx == -1) :
            return ""

        return self.path[idx + 1 :]

class HttpsThread (threading.Thread):

    def __init__(self, port, isTCP):

        threading.Thread.__init__(self)
        self.port = port
        self.isTCP = isTCP

    def run(self):

        handler = CloudAuthHTTPReqHandler

        httpd = None

        if (self.isTCP):

            class MyThreadingTCPServer(ThreadingTCPServer):
                 def server_bind(self):
                     self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
                     self.socket.bind(self.server_address)

            httpd = MyThreadingTCPServer(("", self.port), handler) #TODO "" -> localhost

            httpd.socket = ssl.wrap_socket (httpd.socket, keyfile = TLS_KEY_FILE, certfile=TLS_CRT_FILE, server_side=True)

        else :

            if (os.path.exists(self.port)):
                os.unlink(self.port)
            class ThreadingUsServer(ThreadingMixIn, UnixStreamServer): pass
            httpd = ThreadingUsServer(self.port, handler)

            os.chmod(self.port, 0777)

        logger.info("serving at port %s", self.port)

        httpd.serve_forever()

# Entrance for stand-alone execution
def main():

    logging.basicConfig(format='%(asctime)s %(levelname)s %(name)s %(message)s', level=logging.INFO)

    logger.info("cloudauth is running.")

    threads = []

    threads.append(HttpsThread(LOCAL_PORT, True));
    threads.append(HttpsThread(LOCAL_PATH, False));

    #TODO create unix domain thread

    # Wait for all threads to complete
    for t in threads:
        t.start()

    # Wait for all threads to complete
    for t in threads:
        t.join() #TODO restart if returned

    logger.info ("cloudauth is shutdown")

if __name__ == "__main__":

    main()

<commit_msg>work around bug of BaseHTTPRequestHandler.log_message() in < 2.7.3 for unix domain socket<commit_after>#!/usr/bin/python

"""
Running on local host, listening on https over tcp and http over unix domain socket
for authentication assertions and secure temporary storage of authorization assertions
"""
import os
import sys
import struct
import socket
import SocketServer
import ssl
import threading
import tempfile
import logging

SO_PEERCRED = 17

from SocketServer import TCPServer, UnixStreamServer, ThreadingMixIn, ThreadingTCPServer
from SimpleHTTPServer import SimpleHTTPRequestHandler

from lib.sock2proc import ProcInfo
from lib import libauthn, libauthz

LOCAL_PORT = 6443
LOCAL_PATH = tempfile.gettempdir() + "/cloudauth.sk"

IDP_SRVR_URL = "https://localhost:6443/roles"

SIG_KEY_FILE = "cnf/ssh_host_ecdsa_key"
SIG_CRT_FILE = "cnf/ssh_host_ecdsa_key.crt"
AuZ_KEY_FILE = "cnf/ssh_host_ecdsa_key"

TLS_KEY_FILE = "cnf/ssh_host_ecdsa_key"
TLS_CRT_FILE = "cnf/ssh_host_ecdsa_key.crt"

HTTP_RESP_HDRS = "HTTP/1.1 %(status)s\r\nContent-type:%(ctype)s\r\nContent-length:%(clen)s\r\n\r\n"

logger = logging.getLogger("authagent")

class CloudAuthHTTPReqHandler(SimpleHTTPRequestHandler):

    protocol_version = "HTTP/1.1"

    is_inet = True
    is_post = True

    def log_message(self, format, *args):
        pass
        """
        sys.stderr.write("%s - - [%s] %s\n" %
                         (self.client_address[0],
                          self.log_date_time_string(),
                          format%args))
        """
    def respond(self, status, ctype, clen, data) :

        self.send_response(status)

        self.send_header('Content-Type', ctype)
        self.send_header('Content-Length', clen)

        self.end_headers()

        self.wfile.write(data)

        self.wfile.flush()

    def service(self):

        #TODO request body processing

        logger.info("path = %s", self.path)

        qstr = self.req_qstr();
        body = self.req_body();

        if (self.path.startswith("/authn")) :
            # /authn?ttype=qst

            proc = self.peer_proc()

            authn = libauthn.assert_authn(proc, libauthn.file2buf(SIG_KEY_FILE), qstr, body)

            self.respond(200, "text/authn", str(len(authn)), authn)

        elif (self.path.startswith("/authz")) :
            # /authz?ttype=qst&srvs=foo
            proc = self.peer_proc()

            authn = libauthn.assert_authn(proc, libauthn.file2buf(SIG_KEY_FILE), qstr, body)

            cert = libauthn.file2buf(SIG_CRT_FILE)

            authz = libauthn.askfor_authz(authn, cert, IDP_SRVR_URL, qstr, body)

            self.respond(200, "text/authz", str(len(authz)), authz)

        elif (self.path.startswith("/cert")) :
            # /cert?subj=hostname

            cert = libauthn.file2buf(SIG_CRT_FILE)

            self.respond(200, "text/cert", str(len(cert)), cert)

        elif (self.path.startswith("/sign")) :
            # /sign?algo=ecs256&hash=b64urlsafe

            proc = self.peer_proc()

            sig = libauthn.hash_sign(proc, libauthn.file2buf(SIG_KEY_FILE), qstr, body)

            self.respond(200, "text/sign", str(len(sig)), sig)

        elif (self.path.startswith("/roles")) :
            # /roles?ttype=qst&<token>&srvs=foo
            # body is client/host cert for authn verification
            authz = libauthz.assert_authz(qstr, body, libauthn.file2buf(AuZ_KEY_FILE))

            self.respond(200, "text/authz-roles", str(len(authz)), authz)

    def peer_proc(self):

        proc = None

        if (isinstance(self.client_address, str)):
            self.is_inet = False
            creds = self.connection.getsockopt(socket.SOL_SOCKET, SO_PEERCRED, struct.calcsize('3i'))
            pid, euid, egid = struct.unpack('3i', creds)
            logger.info("client via AF_UNIX, pid %s, uid %s", pid, euid)
            proc = ProcInfo.pipe2proc(pid, euid)
        else:
            self.is_inet = True
            logger.info("client via AF_INET,  %s", self.client_address)
            proc = ProcInfo.sock2proc(self.client_address)

        return proc

    def do_GET(self):

        self.is_post = False
        self.service()

    def do_POST(self):

        self.is_post = True
        self.service()

    def req_body(self):

        size = self.headers.getheader('content-length')
        if (size == None):
            return ""

        size = int(size)

        return self.rfile.read(size)

    def req_qstr(self):

        idx = self.path.find("?")
        if (idx == -1) :
            return ""

        return self.path[idx + 1 :]

class HttpsThread (threading.Thread):

    def __init__(self, port, isTCP):

        threading.Thread.__init__(self)
        self.port = port
        self.isTCP = isTCP

    def run(self):

        handler = CloudAuthHTTPReqHandler

        httpd = None

        if (self.isTCP):

            class MyThreadingTCPServer(ThreadingTCPServer):
                 def server_bind(self):
                     self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
                     self.socket.bind(self.server_address)

            httpd = MyThreadingTCPServer(("", self.port), handler) #TODO "" -> localhost

            httpd.socket = ssl.wrap_socket (httpd.socket, keyfile = TLS_KEY_FILE, certfile=TLS_CRT_FILE, server_side=True)

        else :

            if (os.path.exists(self.port)):
                os.unlink(self.port)
            class ThreadingUsServer(ThreadingMixIn, UnixStreamServer): pass
            httpd = ThreadingUsServer(self.port, handler)

            os.chmod(self.port, 0777)

        logger.info("serving at port %s", self.port)

        httpd.serve_forever()

# Entrance for stand-alone execution
def main():

    logging.basicConfig(format='%(asctime)s %(levelname)s %(name)s %(message)s', level=logging.INFO)

    logger.info("cloudauth is running.")

    threads = []

    threads.append(HttpsThread(LOCAL_PORT, True));
    threads.append(HttpsThread(LOCAL_PATH, False));

    #TODO create unix domain thread

    # Wait for all threads to complete
    for t in threads:
        t.start()

    # Wait for all threads to complete
    for t in threads:
        t.join() #TODO restart if returned

    logger.info ("cloudauth is shutdown")

if __name__ == "__main__":

    main()

<|endoftext|>


<commit_before># This is a python copy of the original five.js
# Found at https://github.com/jackdcrawford/five.git

import random
import math
import time

# d8888b.  .d8b.  d8b   db d8888b.  .d88b.  .88b  d88.
# 88  `8D d8' `8b 888o  88 88  `8D .8P  Y8. 88'YbdP`88
# 88oobY' 88ooo88 88V8o 88 88   88 88    88 88  88  88
# 88`8b   88~~~88 88 V8o88 88   88 88    88 88  88  88
# 88 `88. 88   88 88  V888 88  .8D `8b  d8' 88  88  88
# 88   YD YP   YP VP   V8P Y8888D'  `Y88P'  YP  YP  YP
#
#

def five():
    print 5
def roman():
    print 'V'
def string():
    print "five"
def ru():
    print "russian five"

# .88b  d88.  .d8b.  d888888b db   db .d8888.
# 88'YbdP`88 d8' `8b `~~88~~' 88   88 88'  YP
# 88  88  88 88ooo88    88    88ooo88 `8bo.
# 88  88  88 88~~~88    88    88~~~88   `Y8b.
# 88  88  88 88   88    88    88   88 db   8D
# YP  YP  YP YP   YP    YP    YP   YP `8888Y'
#
#

def plus(int):
    sum = 5 + int
    print sum
def minus(int):
    sum = 5 - int
    print sum
def times(int):
    sum = 5 * int
    print sum
def divide(int):
    sum = 5 / int
    print sum
def power(int):
    power = math.pow(5,int)
    print power
<commit_msg>added bool stuff<commit_after># This is a python copy of the original five.js
# Found at https://github.com/jackdcrawford/five.git

import random
import math
import time

# d8888b.  .d8b.  d8b   db d8888b.  .d88b.  .88b  d88.
# 88  `8D d8' `8b 888o  88 88  `8D .8P  Y8. 88'YbdP`88
# 88oobY' 88ooo88 88V8o 88 88   88 88    88 88  88  88
# 88`8b   88~~~88 88 V8o88 88   88 88    88 88  88  88
# 88 `88. 88   88 88  V888 88  .8D `8b  d8' 88  88  88
# 88   YD YP   YP VP   V8P Y8888D'  `Y88P'  YP  YP  YP

def five():
    print 5
def roman():
    print 'V'
def string():
    print "five"
def ru():
    print "пять"

# .88b  d88.  .d8b.  d888888b db   db .d8888.
# 88'YbdP`88 d8' `8b `~~88~~' 88   88 88'  YP
# 88  88  88 88ooo88    88    88ooo88 `8bo.
# 88  88  88 88~~~88    88    88~~~88   `Y8b.
# 88  88  88 88   88    88    88   88 db   8D
# YP  YP  YP YP   YP    YP    YP   YP `8888Y'

def plus(int):
    sum = 5 + int
    print sum
def minus(int):
    sum = 5 - int
    print sum
def times(int):
    sum = 5 * int
    print sum
def divide(int):
    sum = 5 / int
    print sum
def power(int):
    power = math.pow(5,int)
    print power
<|endoftext|>


<commit_before># (c) Nelen & Schuurmans.  MIT licensed, see LICENSE.rst.
from __future__ import unicode_literals

from rest_framework import fields, serializers
from rest_framework.reverse import reverse
from django.conf import settings
from django.contrib.gis.geos import Point

COLNAME_FORMAT_MS = '%Y-%m-%dT%H:%M:%S.%fZ' # supports milliseconds

class AquoField(fields.Field):
    class Meta:
        fields = ('id', 'code', 'description')

    def to_native(self, obj):
        return dict([(k, getattr(obj, k, None)) for k in self.Meta.fields])

class DateTimeField(fields.Field):
    def field_to_native(self, obj, field_name):
        field = getattr(obj, field_name, None)
        if field:
            return field.strftime(COLNAME_FORMAT_MS)

class RelatedField(serializers.ModelField):
    def field_to_native(self, obj, field_name):
        relation = getattr(obj, field_name)
        return getattr(relation, self.model_field, None)

class ManyRelatedField(serializers.ModelField):
    def field_to_native(self, obj, field_name):
        manager = getattr(obj, field_name)
        return [getattr(rel, self.model_field, None) for rel in manager.all()]

class HyperlinkedRelatedMethod(serializers.HyperlinkedRelatedField):
    def field_to_native(self, obj, field_name):
        method = getattr(obj, field_name)
        try:
            return self.to_native(method())
        except:
            pass

class ManyHyperlinkedRelatedMethod(serializers.HyperlinkedRelatedField):
    def field_to_native(self, obj, field_name):
        method = getattr(obj, field_name)
        try:
            return [self.to_native(item) for item in method()]
        except:
            pass

class ManyHyperlinkedParents(serializers.HyperlinkedRelatedField):
    def field_to_native(self, obj, field_name):
        manager = getattr(obj, field_name)
        return [self.to_native(item.parent) for item in manager.all()]

class ManyHyperlinkedChilds(serializers.HyperlinkedRelatedField):
    def field_to_native(self, obj, field_name):
        manager = getattr(obj, field_name)
        return [self.to_native(item.child) for item in manager.all()]

class LatestValue(serializers.HyperlinkedIdentityField):
    def field_to_native(self, obj, field_name):
        if obj.is_file():
            latest_value = obj.latest_value_file()
            if latest_value:
                return reverse('event-detail', args=[obj.uuid, latest_value],
                    request=self.context['request'])
            return None
        return obj.latest_value()

class OpenDAPLink(serializers.HyperlinkedIdentityField):
    def field_to_native(self, obj, field_name):
        opendap_url = getattr(settings, 'OPENDAP_BASE_URL', '')
        opendap_format = 'ascii'
        return "%s/%s.%s" % (opendap_url, obj.uuid, opendap_format)

class DictChoiceField(serializers.ChoiceField):
    def to_native(self, value):
        #get display value
        choices_dict = dict(self._choices)
        if value in choices_dict:
            return choices_dict[value]
        else:
            return 'unknown choice (%s)'%str(value)

    def from_native(self, value):
        #get value from display value (assuming that display values are unique)
        choices_dict = dict([(a[1],a[0]) for a in self._choices])
        return choices_dict[value]

class GeometryPointField(serializers.Field):

    def field_from_native(self, data, files, field_name, into):
        """
        get geometry object
        """
        value = data.getlist(field_name, None)
        srid = int(data.get('srid', 4258))
        if value and len(value) > 0 and len(value[0]) > 0:
            if len(value) < 2:
                value = value[0].split(',')
            values = [float(v) for v in value]
            geo_input = Point(*values, srid=srid)
            if srid != 4258:
                geo_input = geo_input.transform(4258, clone=True)
            into[field_name] = geo_input
            return geo_input
        else:
            return ''
<commit_msg>Solve 28992 transformation problem<commit_after># (c) Nelen & Schuurmans.  MIT licensed, see LICENSE.rst.
from __future__ import unicode_literals

from django.conf import settings
from django.contrib.gis.geos import Point
from rest_framework import fields, serializers
from rest_framework.reverse import reverse

from ddsc_core.utils import transform

COLNAME_FORMAT_MS = '%Y-%m-%dT%H:%M:%S.%fZ'  # supports milliseconds

class AquoField(fields.Field):
    class Meta:
        fields = ('id', 'code', 'description')

    def to_native(self, obj):
        return dict([(k, getattr(obj, k, None)) for k in self.Meta.fields])

class DateTimeField(fields.Field):
    def field_to_native(self, obj, field_name):
        field = getattr(obj, field_name, None)
        if field:
            return field.strftime(COLNAME_FORMAT_MS)

class RelatedField(serializers.ModelField):
    def field_to_native(self, obj, field_name):
        relation = getattr(obj, field_name)
        return getattr(relation, self.model_field, None)

class ManyRelatedField(serializers.ModelField):
    def field_to_native(self, obj, field_name):
        manager = getattr(obj, field_name)
        return [getattr(rel, self.model_field, None) for rel in manager.all()]

class HyperlinkedRelatedMethod(serializers.HyperlinkedRelatedField):
    def field_to_native(self, obj, field_name):
        method = getattr(obj, field_name)
        try:
            return self.to_native(method())
        except:
            pass

class ManyHyperlinkedRelatedMethod(serializers.HyperlinkedRelatedField):
    def field_to_native(self, obj, field_name):
        method = getattr(obj, field_name)
        try:
            return [self.to_native(item) for item in method()]
        except:
            pass

class ManyHyperlinkedParents(serializers.HyperlinkedRelatedField):
    def field_to_native(self, obj, field_name):
        manager = getattr(obj, field_name)
        return [self.to_native(item.parent) for item in manager.all()]

class ManyHyperlinkedChilds(serializers.HyperlinkedRelatedField):
    def field_to_native(self, obj, field_name):
        manager = getattr(obj, field_name)
        return [self.to_native(item.child) for item in manager.all()]

class LatestValue(serializers.HyperlinkedIdentityField):
    def field_to_native(self, obj, field_name):
        if obj.is_file():
            latest_value = obj.latest_value_file()
            if latest_value:
                return reverse('event-detail', args=[obj.uuid, latest_value],
                    request=self.context['request'])
            return None
        return obj.latest_value()

class OpenDAPLink(serializers.HyperlinkedIdentityField):
    def field_to_native(self, obj, field_name):
        opendap_url = getattr(settings, 'OPENDAP_BASE_URL', '')
        opendap_format = 'ascii'
        return "%s/%s.%s" % (opendap_url, obj.uuid, opendap_format)

class DictChoiceField(serializers.ChoiceField):
    def to_native(self, value):
        #get display value
        choices_dict = dict(self._choices)
        if value in choices_dict:
            return choices_dict[value]
        else:
            return 'unknown choice (%s)' % str(value)

    def from_native(self, value):
        #get value from display value (assuming that display values are unique)
        choices_dict = dict([(a[1], a[0]) for a in self._choices])
        return choices_dict[value]

class GeometryPointField(serializers.Field):

    def field_from_native(self, data, files, field_name, into):
        """
        get geometry object
        """
        value = data.getlist(field_name, None)
        srid = int(data.get('srid', 4258))
        if value and len(value) > 0 and len(value[0]) > 0:
            if len(value) < 2:
                value = value[0].split(',')
            values = [float(v) for v in value]
            geo_input = Point(*values, srid=srid)
            if srid != 4258:
                geo_input = transform(geo_input, 4258, clone=True)
            into[field_name] = geo_input
            return geo_input
        else:
            return ''
<|endoftext|>


<commit_before>from django import template

register = template.Library()

@register.filter
def title_for_barchart(formula, locations):
    title = "%s for %s" % (formula.indicator.name, title_for_location_type(locations))
    return title

@register.filter
def location_names(hierarchial_data):
    names = [str(location.name) for location in hierarchial_data.keys()]
    return names

@register.filter
def title_for_location_type(locations):
    next_type_in_hierarchy = locations.next_type_in_hierarchy()
    if next_type_in_hierarchy:
        return next_type_in_hierarchy.name

@register.filter
def get_values(hierarchial_data):
    return hierarchial_data.values()

@register.filter
def is_float(value):
    return type(value) == float

@register.filter
def can_display_location_based_chart(locations):
    return title_for_location_type(locations) != None

@register.filter
def household_names(households):
    names = [str(household.get_head().surname) for household in households.keys()]
    return names

@register.filter
def get_numerator_denominator_values(households_data, formula):
    aggregated_data = {formula.numerator.text: [], formula.denominator.text: []}
    for household, answers in households_data.items():
        for question, answer in answers.items():
            aggregated_data[question.text].append(answer)
    result = []
    for name, data in aggregated_data.items():
        result.append({
            'name': str(name),
            'data': data
        })
    return result

@register.filter
def multi_choice_options(computed_value):
    return [str(key) for key in computed_value.keys()]

@register.filter
def get_computational_value_by_answer(hierarchial_data):
    options = {}
    for location, answers in hierarchial_data.items():
        for option, answer in answers.items():
            if not options.has_key(option):
                options[option] = []
            options[option].append(answer)
    data = []
    for key, value in options.items():
        data.append({
            'name': str(key),
            'data': value
        })
    return data

@register.filter
def get_numerator(answer, formula):
    return answer[formula.numerator].text

@register.filter
def get_denominator(answer, formula):
    return answer[formula.denominator]<commit_msg>removed unused methods in chart_templage_tags<commit_after>from django import template

register = template.Library()

@register.filter
def get_computational_value_by_answer(hierarchial_data):
    options = {}
    for location, answers in hierarchial_data.items():
        for option, answer in answers.items():
            if not options.has_key(option):
                options[option] = []
            options[option].append(answer)
    data = []
    for key, value in options.items():
        data.append({
            'name': str(key),
            'data': value
        })
    return data<|endoftext|>


<commit_before>import numpy as np
from dipy.reconst.cache import Cache
from dipy.reconst.multi_voxel import multi_voxel_fit
from scipy.special import hermite, gamma
from scipy.misc import factorial, factorial2
import dipy.reconst.dti as dti
from ..utils.optpkg import optional_package

cvxopt, have_cvxopt, _ = optional_package("cvxopt")

class MapmriModel(Cache):

    def __init__(self, gtab, radial_order=4, lambd=0, eap_cons = False, anisotropic_scaling = True):

        self.bvals = gtab.bvals
        self.bvecs = gtab.bvecs
        self.gtab = gtab
        self.radial_order = radial_order
        self.lambd = lambd
        self.eap_cons = eap_cons
        self.anisotropic_scaling=anisotropic_scaling
        if (gtab.big_delta is None) or (gtab.small_delta is None):
            self.tau = 1 / (4 * np.pi ** 2)
        else:
            self.tau = gtab.big_delta - gtab.small_delta / 3.0

    @multi_voxel_fit
    def fit(self, data):
        ind=self.gtab.bvals<=2000
        gtab2 = gradient_table(self.gtab.bvals[ind], self.gtab.bvecs[ind,:])
        tenmodel=dti.TensorModel(gtab2)
        tenfit=tenmodel.fit(data[...,ind])

        evals = tenfit.evals
        R=tenfit.evecs

        ind_evals = np.argsort(evals)[::-1]
        evals = evals[ind_evals]
        R = R[ind_evals,:]

        evals = np.clip(evals,1e-04,evals.max())

        if self.anisotropic_scaling:
            mu = np.sqrt(evals*2*self.tau)

        else:
            mumean=np.sqrt(evals.mean()*2*self.tau)
            mu=np.array([mumean,mumean,mumean])

        qvals=np.sqrt(self.gtab.bvals/self.tau) / (2 * np.pi)
        qvecs=np.dot(self.gtab.bvecs,R)
        q=qvecs*qvals[:,None]

        M = mapmri_phi_matrix(self.radial_order, mu, q.T)

        ind_mat = mapmri_index_matrix(self.radial_order)

        if self.eap_cons:
            if not have_cvxopt:
                raise ValueError(
                    'CVXOPT package needed to enforce constraints')
            import cvxopt.solvers
            # rmax is linear in mu with rmax \aprox 0.3 for mu = 1/(2*pi*sqrt(700))
            rmax = 0.35 * self.mu * (2 * np.pi * np.sqrt(700))
            rgrad = gen_rgrid(rmax = rmax, Nstep = 10)
            K = mapmri_psi_matrix(self.radial_order,  mu, rgrad, self.tau)

            Q = cvxopt.matrix(np.dot(M.T,M))
            p = cvxopt.matrix(-1*np.dot(M.T,data))
            G = cvxopt.matrix(-1*K)
            h = cvxopt.matrix(np.zeros((K.shape[0])),(K.shape[0],1))
            solvers.options['show_progress'] = False
            sol = solvers.qp(Q, p, G, h)
            if sol['status'] != 'optimal':
                warn('Optimization did not find a solution')

            coef = np.array(sol['x'])[:,0]
        else:
            I = np.eye(M.shape[1])
            pseudoInv = np.dot(np.linalg.inv(np.dot(M.T, M) + self.lambd * I), M.T)
            coef = np.dot(pseudoInv, data)

        Bm=Bmat(ind_mat)

        E0 = 0
        for i in range(ind_mat.shape[0]):
            E0 = E0 + coef[i] * Bm[i]
        coef = coef / E0

        return MapmriFit(self, coef, mu, R, ind_mat, Bm)

class MapmriFit():

    def __init__(self, model, mapmri_coef, mu, R, ind_mat):
        """ Calculates diffusion properties for a single voxel

        Parameters
        ----------
        model : object,
            AnalyticalModel
        mapmri_coef : 1d ndarray,
            mapmri coefficients
        """

        self.model = model
        self._mapmri_coef = mapmri_coef
        self.gtab = model.gtab
        self.radial_order = model.radial_order
        self.mu = mu
        self.R = R
        self.ind_mat = ind_mat

    @property
    def mapmri_mu(self):
        """The SHORE coefficients
        """
        return self.mu
    @property
    def mapmri_R(self):
        """The SHORE coefficients
        """
        return self.R
    @property
    def mapmri_coeff(self):
        """The SHORE coefficients
        """
        return self._mapmri_coef

    def odf(self, sphere, smoment=0):
        r""" Calculates the real analytical odf for a given discrete sphere.

        Eq.32
        """
        v_ = sphere.vertices
        v = np.dot(v_,self.R)
        I_s = mapmri_odf_matrix(self.radial_order,self.mu, s, v)
        odf = np.dot(I_s, self._mapmri_coef)
        return np.clip(odf,0,odf.max())

def mapmri_index_matrix(radial_order):

    index_matrix = []
    for n in range(0, radial_order + 1, 2):
        for i in range(0, n + 1):
            for j in range(0, n - i + 1):
                index_matrix.append([i, j, n - i - j])

    return np.array(index_matrix)

def Bmat(ind_mat):
    B = np.zeros(ind_mat.shape[0])
    for i in range (ind_mat.shape[0]):
        n1, n2, n3 = ind_mat[i]
        K = int(not(n1%2) and not(n2%2) and not(n3%2))
        B[i] = K * np.sqrt(factorial(n1)*factorial(n2)*factorial(n3)) /(factorial2(n1)*factorial2(n2)*factorial2(n3))

    return B

def mapmri_phi_1d(n, q, mu):
    """
    Eq. 4
    """

    qn = 2 * np.pi * mu * q
    H = hermite(n)(qn)
    i = np.complex(0, 1)
    f = factorial(n)

    k = i ** (-n) / np.sqrt(2 ** (n) * f)
    phi = k * np.exp(- qn ** 2 / 2) * H

    return phi

def mapmri_phi_3d(n, q, mu):
    """
    Eq. 23
    """

    n1, n2, n3 = n
    qx, qy, qz = q
    mux, muy, muz = mu

    phi = mapmri_phi_1d
    return np.real(phi(n1, qx, mux) * phi(n2, qy, muy) * phi(n3, qz, muz))

def mapmri_phi_matrix(radial_order, mu, q_gradients):

    ind_mat = mapmri_index_matrix(radial_order)

    n_elem = ind_mat.shape[0]

    n_qgrad = q_gradients.shape[1]

    M = np.zeros((n_qgrad, n_elem))

    for j in range(n_elem):
        M[:, j] = mapmri_phi_3d(ind_mat[j], q_gradients, mu)

    return M

def mapmri_psi_1d(n, x, mu):
    """
    Eq. 10
    """

    H = hermite(n)(x / mu)
    f = factorial(n)

    k = 1 / (np.sqrt(2 ** (n + 1) * np.pi * f) * mu)
    psi = k * np.exp(- x ** 2 / (2 * mu ** 2)) * H

    return psi

def mapmri_psi_3d(n, r, mu):
    """
    Eq. 22
    """

    n1, n2, n3 = n
    x, y, z = r.T
    mux, muy, muz = mu

    psi = mapmri_psi_1d
    return psi(n1, x, mux) * psi(n2, y, muy) * psi(n3, z, muz)

def mapmri_psi_matrix(radial_order, mu, rgrad, tau):

    ind_mat = mapmri_index_matrix(radial_order)

    n_elem = ind_mat.shape[0]

    n_rgrad = rgrad.shape[0]

    K = np.zeros((n_rgrad, n_elem))

    for j in range(n_elem):
        K[:, j] = mapmri_psi_3d(ind_mat[j], rgrad, mu)

    return K

def mapmri_odf_matrix(radial_order, mu, s, vertices):
    """
    Eq. 33 (choose ux=uy=uz)
    """

    ind_mat = mapmri_index_matrix(radial_order)

    n_vert = vertices.shape[0]

    n_elem = ind_mat.shape[0]

    odf_mat = np.zeros((n_vert, n_elem))

    rho = mu
    mux,muy,muz = mu
    rho=1.0/np.sqrt((vertices[:,0]/mux)**2 + (vertices[:,1]/muy)**2 + (vertices[:,2]/muz)**2)
    alpha = 2 * rho * (vertices[:,0]/mux)
    beta = 2 * rho * (vertices[:,1]/muy)
    gamma = 2 * rho * (vertices[:,2]/muz)
    const= rho ** (3 + s) / np.sqrt(2 ** (2 - s) * np.pi ** 3 * (mux ** 2 * muy ** 2 * muz ** 2))

    for j in range(n_elem):
        n1, n2, n3 = ind_mat[j]
        f = np.sqrt(factorial(n1) * factorial(n2) * factorial(n3))
        odf_mat[:, j] = const * f * _odf_cfunc(n1, n2, n3, alpha, beta, gamma, s)

    return odf_mat

def _odf_cfunc(n1, n2, n3, a, b, g, s):
    """
    Eq. 34
    """

    f = factorial
    f2 = factorial2

    sumc = 0
    for i in range(0, n1 + 1, 2):
        for j in range(0, n2 + 1, 2):
            for k in range(0, n3 + 1, 2):

                nn = n1 + n2 + n3 - i - j - k

                gam = (-1) ** ((i + j + k) / 2.0) * gamma((3 + s + nn) / 2.0)

                num1 = a ** (n1 - i)

                num2 = b ** (n2 - j)

                num3 = g ** (n3 - k)

                num = gam * num1 * num2 * num3

                denom = f(n1 - i) * f(n2 - j) * f(
                    n3 - k) * f2(i) * f2(j) * f2(k)

                sumc += num / denom

    return sumc

def gen_rgrid(rmax, Nstep = 10):
    rgrad = []
    # Build a regular grid of Nstep**3 points in (R^2 X R+)
    gridmax = rmax / np.sqrt(3)
    for xx in np.linspace(-gridmax,gridmax,Nstep):
        for yy in np.linspace(-gridmax,gridmax,Nstep):
            for zz in np.linspace(0,gridmax,Nstep):
                rgrad.append([xx, yy, zz])
    return np.array(rgrad)

def mapmri_evaluate_E(radial_order, coeff, qlist, mu):

    ind_mat = mapmri_index_matrix(radial_order)

    n_elem = ind_mat.shape[0]

    n_qgrad = qlist.shape[0]

    data_out = np.zeros(n_qgrad)

    for i in range(n_qgrad):
        for j in range(n_elem):
            data_out[i] += coeff[j] * mapmri_phi_3d(ind_mat[j], qlist[i], mu)

    return data_out

def mapmri_evaluate_EAP(radial_order, coeff, rlist, mu):

    ind_mat = mapmri_index_matrix(radial_order)

    n_elem = ind_mat.shape[0]

    n_rgrad = rlist.shape[0]

    data_out = np.zeros(n_rgrad)

    for i in range(n_rgrad):
        for j in range(n_elem):
            data_out[i] += coeff[j] * mapmri_psi_3d(ind_mat[j], rlist[i], mu)

    return data_out

<commit_msg>fix bugs in mapmri draft<commit_after>import numpy as np
from dipy.reconst.cache import Cache
from dipy.reconst.multi_voxel import multi_voxel_fit
from scipy.special import hermite, gamma
from scipy.misc import factorial, factorial2
import dipy.reconst.dti as dti
from warnings import warn
from dipy.core.gradients import gradient_table
from ..utils.optpkg import optional_package

cvxopt, have_cvxopt, _ = optional_package("cvxopt")

class MapmriModel(Cache):

    def __init__(self, gtab, radial_order=4, lambd=0, eap_cons = False, anisotropic_scaling = True):

        self.bvals = gtab.bvals
        self.bvecs = gtab.bvecs
        self.gtab = gtab
        self.radial_order = radial_order
        self.lambd = lambd
        self.eap_cons = eap_cons
        self.anisotropic_scaling=anisotropic_scaling
        if (gtab.big_delta is None) or (gtab.small_delta is None):
            self.tau = 1 / (4 * np.pi ** 2)
        else:
            self.tau = gtab.big_delta - gtab.small_delta / 3.0

    @multi_voxel_fit
    def fit(self, data):
        ind=self.gtab.bvals<=2000
        gtab2 = gradient_table(self.gtab.bvals[ind], self.gtab.bvecs[ind,:])
        tenmodel=dti.TensorModel(gtab2)
        tenfit=tenmodel.fit(data[...,ind])

        evals = tenfit.evals
        R=tenfit.evecs

        ind_evals = np.argsort(evals)[::-1]
        evals = evals[ind_evals]
        R = R[ind_evals,:]

        evals = np.clip(evals,1e-04,evals.max())

        if self.anisotropic_scaling:
            mu = np.sqrt(evals*2*self.tau)

        else:
            mumean=np.sqrt(evals.mean()*2*self.tau)
            mu=np.array([mumean,mumean,mumean])

        qvals=np.sqrt(self.gtab.bvals/self.tau) / (2 * np.pi)
        qvecs=np.dot(self.gtab.bvecs,R)
        q=qvecs*qvals[:,None]

        M = mapmri_phi_matrix(self.radial_order, mu, q.T)

        ind_mat = mapmri_index_matrix(self.radial_order)

        I = np.diag(ind_mat.sum(1))

        if self.eap_cons:
            if not have_cvxopt:
                raise ValueError(
                    'CVXOPT package needed to enforce constraints')
            import cvxopt.solvers
            # rmax is linear in mu with rmax \aprox 0.3 for mu = 1/(2*pi*sqrt(700))
            rmax = 0.35 * mu.mean()* (2 * np.pi * np.sqrt(700))
            rgrad = gen_rgrid(rmax = rmax, Nstep = 10)
            K = mapmri_psi_matrix(self.radial_order,  mu, rgrad, self.tau)

            Q = cvxopt.matrix(np.dot(M.T,M)+ self.lambd * I)
            p = cvxopt.matrix(-1*np.dot(M.T,data))
            G = cvxopt.matrix(-1*K)
            h = cvxopt.matrix(np.zeros((K.shape[0])),(K.shape[0],1))
            cvxopt.solvers.options['show_progress'] = False
            sol = cvxopt.solvers.qp(Q, p, G, h)
            if sol['status'] != 'optimal':
                warn('Optimization did not find a solution')

            coef = np.array(sol['x'])[:,0]
        else:
            pseudoInv = np.dot(np.linalg.inv(np.dot(M.T, M) + self.lambd * I), M.T)
            coef = np.dot(pseudoInv, data)

        Bm=Bmat(ind_mat)

        E0 = 0
        for i in range(ind_mat.shape[0]):
            E0 = E0 + coef[i] * Bm[i]
        coef = coef / E0

        return MapmriFit(self, coef, mu, R, ind_mat)

class MapmriFit():

    def __init__(self, model, mapmri_coef, mu, R, ind_mat):
        """ Calculates diffusion properties for a single voxel

        Parameters
        ----------
        model : object,
            AnalyticalModel
        mapmri_coef : 1d ndarray,
            mapmri coefficients
        """

        self.model = model
        self._mapmri_coef = mapmri_coef
        self.gtab = model.gtab
        self.radial_order = model.radial_order
        self.mu = mu
        self.R = R
        self.ind_mat = ind_mat

    @property
    def mapmri_mu(self):
        """The SHORE coefficients
        """
        return self.mu
    @property
    def mapmri_R(self):
        """The SHORE coefficients
        """
        return self.R
    @property
    def mapmri_coeff(self):
        """The SHORE coefficients
        """
        return self._mapmri_coef

    def odf(self, sphere, s=0):
        r""" Calculates the real analytical odf for a given discrete sphere.

        Eq.32
        """
        v_ = sphere.vertices
        v = np.dot(v_,self.R)
        I_s = mapmri_odf_matrix(self.radial_order,self.mu, s, v)
        odf = np.dot(I_s, self._mapmri_coef)
        return np.clip(odf,0,odf.max())

def mapmri_index_matrix(radial_order):

    index_matrix = []
    for n in range(0, radial_order + 1, 2):
        for i in range(0, n + 1):
            for j in range(0, n - i + 1):
                index_matrix.append([i, j, n - i - j])

    return np.array(index_matrix)

def Bmat(ind_mat):
    B = np.zeros(ind_mat.shape[0])
    for i in range (ind_mat.shape[0]):
        n1, n2, n3 = ind_mat[i]
        K = int(not(n1%2) and not(n2%2) and not(n3%2))
        B[i] = K * np.sqrt(factorial(n1)*factorial(n2)*factorial(n3)) /(factorial2(n1)*factorial2(n2)*factorial2(n3))

    return B

def mapmri_phi_1d(n, q, mu):
    """
    Eq. 4
    """

    qn = 2 * np.pi * mu * q
    H = hermite(n)(qn)
    i = np.complex(0, 1)
    f = factorial(n)

    k = i ** (-n) / np.sqrt(2 ** (n) * f)
    phi = k * np.exp(- qn ** 2 / 2) * H

    return phi

def mapmri_phi_3d(n, q, mu):
    """
    Eq. 23
    """

    n1, n2, n3 = n
    qx, qy, qz = q
    mux, muy, muz = mu

    phi = mapmri_phi_1d
    return np.real(phi(n1, qx, mux) * phi(n2, qy, muy) * phi(n3, qz, muz))

def mapmri_phi_matrix(radial_order, mu, q_gradients):

    ind_mat = mapmri_index_matrix(radial_order)

    n_elem = ind_mat.shape[0]

    n_qgrad = q_gradients.shape[1]

    M = np.zeros((n_qgrad, n_elem))

    for j in range(n_elem):
        M[:, j] = mapmri_phi_3d(ind_mat[j], q_gradients, mu)

    return M

def mapmri_psi_1d(n, x, mu):
    """
    Eq. 10
    """

    H = hermite(n)(x / mu)
    f = factorial(n)

    k = 1 / (np.sqrt(2 ** (n + 1) * np.pi * f) * mu)
    psi = k * np.exp(- x ** 2 / (2 * mu ** 2)) * H

    return psi

def mapmri_psi_3d(n, r, mu):
    """
    Eq. 22
    """

    n1, n2, n3 = n
    x, y, z = r.T
    mux, muy, muz = mu

    psi = mapmri_psi_1d
    return psi(n1, x, mux) * psi(n2, y, muy) * psi(n3, z, muz)

def mapmri_psi_matrix(radial_order, mu, rgrad, tau):

    ind_mat = mapmri_index_matrix(radial_order)

    n_elem = ind_mat.shape[0]

    n_rgrad = rgrad.shape[0]

    K = np.zeros((n_rgrad, n_elem))

    for j in range(n_elem):
        K[:, j] = mapmri_psi_3d(ind_mat[j], rgrad, mu)

    return K

def mapmri_odf_matrix(radial_order, mu, s, vertices):
    """
    Eq. 33 (choose ux=uy=uz)
    """

    ind_mat = mapmri_index_matrix(radial_order)

    n_vert = vertices.shape[0]

    n_elem = ind_mat.shape[0]

    odf_mat = np.zeros((n_vert, n_elem))

    rho = mu
    mux,muy,muz = mu
    rho=1.0/np.sqrt((vertices[:,0]/mux)**2 + (vertices[:,1]/muy)**2 + (vertices[:,2]/muz)**2)
    alpha = 2 * rho * (vertices[:,0]/mux)
    beta = 2 * rho * (vertices[:,1]/muy)
    gamma = 2 * rho * (vertices[:,2]/muz)
    const= rho ** (3 + s) / np.sqrt(2 ** (2 - s) * np.pi ** 3 * (mux ** 2 * muy ** 2 * muz ** 2))

    for j in range(n_elem):
        n1, n2, n3 = ind_mat[j]
        f = np.sqrt(factorial(n1) * factorial(n2) * factorial(n3))
        odf_mat[:, j] = const * f * _odf_cfunc(n1, n2, n3, alpha, beta, gamma, s)

    return odf_mat

def _odf_cfunc(n1, n2, n3, a, b, g, s):
    """
    Eq. 34
    """

    f = factorial
    f2 = factorial2

    sumc = 0
    for i in range(0, n1 + 1, 2):
        for j in range(0, n2 + 1, 2):
            for k in range(0, n3 + 1, 2):

                nn = n1 + n2 + n3 - i - j - k

                gam = (-1) ** ((i + j + k) / 2.0) * gamma((3 + s + nn) / 2.0)

                num1 = a ** (n1 - i)

                num2 = b ** (n2 - j)

                num3 = g ** (n3 - k)

                num = gam * num1 * num2 * num3

                denom = f(n1 - i) * f(n2 - j) * f(
                    n3 - k) * f2(i) * f2(j) * f2(k)

                sumc += num / denom

    return sumc

def gen_rgrid(rmax, Nstep = 10):
    rgrad = []
    # Build a regular grid of Nstep**3 points in (R^2 X R+)
    gridmax = rmax / np.sqrt(3)
    for xx in np.linspace(-gridmax,gridmax,Nstep):
        for yy in np.linspace(-gridmax,gridmax,Nstep):
            for zz in np.linspace(0,gridmax,Nstep):
                rgrad.append([xx, yy, zz])
    return np.array(rgrad)

def mapmri_evaluate_E(radial_order, coeff, qlist, mu):

    ind_mat = mapmri_index_matrix(radial_order)

    n_elem = ind_mat.shape[0]

    n_qgrad = qlist.shape[0]

    data_out = np.zeros(n_qgrad)

    for i in range(n_qgrad):
        for j in range(n_elem):
            data_out[i] += coeff[j] * mapmri_phi_3d(ind_mat[j], qlist[i], mu)

    return data_out

def mapmri_evaluate_EAP(radial_order, coeff, rlist, mu):

    ind_mat = mapmri_index_matrix(radial_order)

    n_elem = ind_mat.shape[0]

    n_rgrad = rlist.shape[0]

    data_out = np.zeros(n_rgrad)

    for i in range(n_rgrad):
        for j in range(n_elem):
            data_out[i] += coeff[j] * mapmri_psi_3d(ind_mat[j], rlist[i], mu)

    return data_out

<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

def quicksort(a_list):
    """Quick sort algortihm by recursion with list comprehension.

    Procedure:
      - Pick a pivot which ideally is a median pf the list.
      - Arrange half elements which are smaller than pivot to left,
        and the other half ones that are bigger than pivot to right.
      - Then to each half, revursively apply quicksort().

    Time complexity: O(n*logn).
    Space complexity: O(n).
    """
    if len(a_list) <= 1:
        return a_list
    pivot = a_list[len(a_list) // 2]
    left_list = [x for x in a_list if x < pivot]
    middle_list = [x for x in a_list if x == pivot]
    right_list = [x for x in a_list if x > pivot]
    return quicksort(left_list) + middle_list + quicksort(right_list)

def _partition(a_list, left, right, pivot):
    """Get partition point for patition."""
    pivot = a_list[pivot]

    while left <= right:
        while a_list[left] < pivot:
            # Find left element which is bigger than pivot.
            left += 1

        while a_list[right] > pivot:
            # Find right element which is smaller than pivot.
            right -= 1

        if left <= right:
            # If "bigger" left element is on left side of "smaller" right,
            # swap the two elements, and then keep moving.
            a_list[right], a_list[left] = a_list[left], a_list[right]
            left += 1
            right -= 1

    return left

def _quicksort_recur(a_list, left, right):
    if left < right:
        pivot = left + (right - left) // 2
        partition = _partition(a_list, left, right, pivot)
        _quicksort_recur(a_list, left, partition - 1)
        _quicksort_recur(a_list, partition + 1, right)

def quicksort_raw(a_list):
    """Quick sort algortihm with raw recursion.

    Time complexity: O(n*logn).
    Space complexity: O(n).
    """
    _quicksort_recur(a_list, 0, len(a_list) - 1)
    return a_list

def main():
    import time
    import random

    # a_list = [54, 26, 93, 17, 77, 31, 44, 55, 20]
    a_list = range(100)
    random.shuffle(a_list)

    start_time = time.time()
    print(quicksort(a_list))
    print('Time for quick sort by recursion with list comprehension:: {}'
          .format(time.time() - start_time))

    start_time = time.time()
    print(quicksort_raw(a_list))
    print('Time for quick sort by raw recursion:: {}'
          .format(time.time() - start_time))

if __name__ == '__main__':
    main()
<commit_msg>Revise var names<commit_after>from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

def quicksort(nums):
    """Quick sort algortihm by recursion with list comprehension.

    Procedure:
      - Pick a pivot which ideally is a median pf the list.
      - Arrange half elements which are smaller than pivot to left,
        and the other half ones that are bigger than pivot to right.
      - Then to each half, revursively apply quicksort().

    Time complexity: O(n*logn).
    Space complexity: O(n).
    """
    if len(nums) <= 1:
        return nums
    pivot = nums[len(nums) // 2]

    small_nums = [x for x in nums if x < pivot]
    mid_nums = [x for x in nums if x == pivot]
    large_nums = [x for x in nums if x > pivot]

    return quicksort(small_nums) + mid_nums + quicksort(large_nums)

def _partition(nums, left, right, mid):
    """Get partition point."""
    pivot = nums[mid]

    while left <= right:
        while nums[left] < pivot:
            # Find left element which is bigger than pivot.
            left += 1

        while nums[right] > pivot:
            # Find right element which is smaller than pivot.
            right -= 1

        if left <= right:
            # If "bigger" left element is on left side of "smaller" right,
            # swap the two elements, and then keep moving.
            nums[right], nums[left] = nums[left], nums[right]
            left += 1
            right -= 1

    return left

def _quicksort_recur(nums, left, right):
    """Quick sort by recursion."""
    if left < right:
        mid = left + (right - left) // 2
        partition = _partition(nums, left, right, mid)
        _quicksort_recur(nums, left, partition - 1)
        _quicksort_recur(nums, partition + 1, right)

def quicksort_raw(nums):
    """Raw quick sort algortihm with recursion.

    Time complexity: O(n*logn).
    Space complexity: O(n).
    """
    _quicksort_recur(nums, 0, len(nums) - 1)
    return nums

def main():
    import time
    import random

    # nums = [54, 26, 93, 17, 77, 31, 44, 55, 20]
    nums = range(100)
    random.shuffle(nums)

    start_time = time.time()
    print(quicksort(nums))
    print('Time for quick sort by recursion: {}'
          .format(time.time() - start_time))

    start_time = time.time()
    print(quicksort_raw(nums))
    print('Time for quick sort by "raw" recursion: {}'
          .format(time.time() - start_time))

if __name__ == '__main__':
    main()
<|endoftext|>