<commit_before>"""
   =======
   License
   =======
   Copyright (c) 2017 Thomas Lehmann

   Permission is hereby granted, free of charge, to any person obtaining a copy of this
   software and associated documentation files (the "Software"), to deal in the Software
   without restriction, including without limitation the rights to use, copy, modify, merge,
   publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons
   to whom the Software is furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in all copies
   or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
   INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
   IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
   DAMAGES OR OTHER LIABILITY,
   WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
# pylint: disable=R0201
import math
import unittest
from hamcrest import assert_that, equal_to, calling, raises
from concept.math.vector import Vector2d
from concept.math.point import Point2d
from concept.math.line import Line2d
from concept.errors.exceptions import PointIsNotOnTheGivenLine

class TestLine2d(unittest.TestCase):
    """ Testing math 2d line. """

    def test_init(self):
        """Testing of method Line2d.__init__."""
        line = Line2d(Point2d(1.0, 2.0), Vector2d(3.0, 4.0))
        assert_that(line.position, equal_to(Point2d(1.0, 2.0)))
        assert_that(line.direction, equal_to(Vector2d(3.0, 4.0)))

    def test_repr(self):
        """Testing of method Line2d.__repr__."""
        line = Line2d(Point2d(1.0, 2.0), Vector2d(3.0, 4.0))
        expected = "Line2d(position=Point2d(x=1, y=2), direction=Vector2d(x=3, y=4))"
        assert_that(str(line), equal_to(expected))

    def test_turned_left(self):
        """Testing of method Line2d.turned_left."""
        line = Line2d(Point2d(1.0, 2.0), Vector2d(1.0, 0.0)).turned_left()
        assert_that(line.position, equal_to(Point2d(1.0, 2.0)))
        assert_that(line.direction, equal_to(Vector2d(0.0, 1.0)))

    def test_turned_right(self):
        """Testing of method Line2d.turned_right."""
        line = Line2d(Point2d(1.0, 2.0), Vector2d(1.0, 0.0)).turned_right()
        assert_that(line.position, equal_to(Point2d(1.0, 2.0)))
        assert_that(line.direction, equal_to(Vector2d(0.0, -1.0)))

    def test_point(self):
        """Testing of method Line2d.point."""
        line = Line2d(Point2d(1.0, 2.0), Vector2d(3.0, 4.0))
        assert_that(line.point(0.0), equal_to(Point2d(1.0, 2.0)))
        assert_that(line.point(0.5), equal_to(Point2d(2.5, 4.0)))
        assert_that(line.point(1.0), equal_to(Point2d(4.0, 6.0)))

    def test_length(self):
        """Testing of method Line2d.length."""
        line = Line2d(Point2d(1.0, 2.0), Vector2d(3.0, 4.0))
        assert_that(line.length(), equal_to(Vector2d(3.0, 4.0).length()))

    def test_is_parallel(self):
        """Testing of method Line2d.is_parallel."""
        line_a = Line2d(Point2d(1.0, 0.0), Vector2d(3.0, 0.0))
        line_b = Line2d(Point2d(2.0, 0.0), Vector2d(6.0, 0.0))
        line_c = Line2d(Point2d(2.0, 0.0), Vector2d(6.0, 1.0))
        assert_that(line_a.is_parallel(line_a), equal_to(True))
        assert_that(line_a.is_parallel(line_b), equal_to(True))
        assert_that(line_a.is_parallel(line_c), equal_to(False))

    def test_angle(self):
        """Testing of method Line2d.angle."""
        line_a = Line2d(Point2d(1.0, 2.0), Vector2d(1.0, 0.0))
        line_b = Line2d(Point2d(3.0, 4.0), Vector2d(0.0, 1.0))
        # should be -90 degree
        angle = line_a.angle(line_b) * 180.0 / math.pi
        assert_that(abs(angle+90.0) < 1e-10, equal_to(True))

    def test_factor(self):
        """Testing of method Line2d.factor."""
        line = Line2d(Point2d(1.0, 1.0), Vector2d(1.0, 1.0))
        assert_that(line.factor(Point2d(1.0, 1.0)), equal_to(0.0))
        assert_that(line.factor(Point2d(1.5, 1.5)), equal_to(0.5))
        assert_that(line.factor(Point2d(2.0, 2.0)), equal_to(1.0))
        assert_that(line.factor(Point2d(0.0, 0.0)), equal_to(-1.0))
        assert_that(line.factor(Point2d(3.0, 3.0)), equal_to(+2.0))
        assert_that(calling(line.factor).with_args(Point2d(0.0, 1.0)),
                    raises(PointIsNotOnTheGivenLine))

        line = Line2d(Point2d(1.0, 1.0), Vector2d(0.0, 1.0))
        assert_that(line.factor(Point2d(1.0, 1.5)), equal_to(0.5))

    def test_valid_intersection(self):
        """Testing of method Line2d.intersection."""
        line_a = Line2d(Point2d(0.0, 2.0), Vector2d(4.0, 0.0))
        line_b = Line2d(Point2d(3.0, 0.0), Vector2d(0.0, 4.0))
        assert_that(line_a.intersection(line_b), equal_to(Point2d(3.0, 2.0)))
<commit_msg>coverage improved<commit_after>"""
   =======
   License
   =======
   Copyright (c) 2017 Thomas Lehmann

   Permission is hereby granted, free of charge, to any person obtaining a copy of this
   software and associated documentation files (the "Software"), to deal in the Software
   without restriction, including without limitation the rights to use, copy, modify, merge,
   publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons
   to whom the Software is furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in all copies
   or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
   INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
   IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
   DAMAGES OR OTHER LIABILITY,
   WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
# pylint: disable=R0201
import math
import unittest
from hamcrest import assert_that, equal_to, is_not, calling, raises
from concept.math.vector import Vector2d
from concept.math.point import Point2d
from concept.math.line import Line2d
from concept.errors.exceptions import PointIsNotOnTheGivenLine, UnsupportedOperation,\
                                      NoLineIntersection

class TestLine2d(unittest.TestCase):
    """ Testing math 2d line. """

    def test_init(self):
        """Testing of method Line2d.__init__."""
        line = Line2d(Point2d(1.0, 2.0), Vector2d(3.0, 4.0))
        assert_that(line.position, equal_to(Point2d(1.0, 2.0)))
        assert_that(line.direction, equal_to(Vector2d(3.0, 4.0)))

    def test_repr(self):
        """Testing of method Line2d.__repr__."""
        line = Line2d(Point2d(1.0, 2.0), Vector2d(3.0, 4.0))
        expected = "Line2d(position=Point2d(x=1, y=2), direction=Vector2d(x=3, y=4))"
        assert_that(str(line), equal_to(expected))

    def test_turned_left(self):
        """Testing of method Line2d.turned_left."""
        line = Line2d(Point2d(1.0, 2.0), Vector2d(1.0, 0.0)).turned_left()
        assert_that(line.position, equal_to(Point2d(1.0, 2.0)))
        assert_that(line.direction, equal_to(Vector2d(0.0, 1.0)))

    def test_turned_right(self):
        """Testing of method Line2d.turned_right."""
        line = Line2d(Point2d(1.0, 2.0), Vector2d(1.0, 0.0)).turned_right()
        assert_that(line.position, equal_to(Point2d(1.0, 2.0)))
        assert_that(line.direction, equal_to(Vector2d(0.0, -1.0)))

    def test_point(self):
        """Testing of method Line2d.point."""
        line = Line2d(Point2d(1.0, 2.0), Vector2d(3.0, 4.0))
        assert_that(line.point(0.0), equal_to(Point2d(1.0, 2.0)))
        assert_that(line.point(0.5), equal_to(Point2d(2.5, 4.0)))
        assert_that(line.point(1.0), equal_to(Point2d(4.0, 6.0)))

    def test_length(self):
        """Testing of method Line2d.length."""
        line = Line2d(Point2d(1.0, 2.0), Vector2d(3.0, 4.0))
        assert_that(line.length(), equal_to(Vector2d(3.0, 4.0).length()))

    def test_is_parallel(self):
        """Testing of method Line2d.is_parallel."""
        line_a = Line2d(Point2d(1.0, 0.0), Vector2d(3.0, 0.0))
        line_b = Line2d(Point2d(2.0, 0.0), Vector2d(6.0, 0.0))
        line_c = Line2d(Point2d(2.0, 0.0), Vector2d(6.0, 1.0))
        assert_that(line_a.is_parallel(line_a), equal_to(True))
        assert_that(line_a.is_parallel(line_b), equal_to(True))
        assert_that(line_a.is_parallel(line_c), equal_to(False))
        assert_that(calling(line_a.is_parallel).with_args(1234),
                    raises(UnsupportedOperation))

    def test_angle(self):
        """Testing of method Line2d.angle."""
        line_a = Line2d(Point2d(1.0, 2.0), Vector2d(1.0, 0.0))
        line_b = Line2d(Point2d(3.0, 4.0), Vector2d(0.0, 1.0))
        # should be -90 degree
        angle = line_a.angle(line_b) * 180.0 / math.pi
        assert_that(abs(angle+90.0) < 1e-10, equal_to(True))

    def test_factor(self):
        """Testing of method Line2d.factor."""
        line = Line2d(Point2d(1.0, 1.0), Vector2d(1.0, 1.0))
        assert_that(line.factor(Point2d(1.0, 1.0)), equal_to(0.0))
        assert_that(line.factor(Point2d(1.5, 1.5)), equal_to(0.5))
        assert_that(line.factor(Point2d(2.0, 2.0)), equal_to(1.0))
        assert_that(line.factor(Point2d(0.0, 0.0)), equal_to(-1.0))
        assert_that(line.factor(Point2d(3.0, 3.0)), equal_to(+2.0))
        assert_that(calling(line.factor).with_args(Point2d(0.0, 1.0)),
                    raises(PointIsNotOnTheGivenLine))

        line = Line2d(Point2d(1.0, 1.0), Vector2d(0.0, 1.0))
        assert_that(line.factor(Point2d(1.0, 1.5)), equal_to(0.5))

    def test_valid_intersection(self):
        """Testing of method Line2d.intersection."""
        line_a = Line2d(Point2d(0.0, 2.0), Vector2d(4.0, 0.0))
        line_b = Line2d(Point2d(3.0, 0.0), Vector2d(0.0, 4.0))
        assert_that(line_a.intersection(line_b), equal_to(Point2d(3.0, 2.0)))

    def test_no_intersection(self):
        """Testing of method Line2d.intersection."""
        line_a = Line2d(Point2d(0.0, 2.0), Vector2d(4.0, 0.0))
        line_b = Line2d(Point2d(0.0, 1.0), Vector2d(4.0, 0.0))
        line_c = Line2d(Point2d(2.0, 0.0), Vector2d(0.0, 1.0))
        # raised because lines are parallel
        assert_that(calling(line_a.intersection).with_args(line_b),
                    raises(NoLineIntersection))
        # factor of line_c is not betwee 0 and 1
        assert_that(calling(line_a.intersection).with_args(line_c),
                    raises(NoLineIntersection))

    def test_equal(self):
        """Testing of method Line2d.__eq__."""
        line_a = Line2d(Point2d(0.0, 2.0), Vector2d(4.0, 0.0))
        line_b = Line2d(Point2d(0.0, 2.0), Vector2d(4.0, 0.0))
        line_c = Line2d(Point2d(3.0, 0.0), Vector2d(0.0, 4.0))
        assert_that(line_a, equal_to(line_b))
        assert_that(line_a, is_not(equal_to(line_c)))
        assert_that(line_a.__eq__(1234), equal_to(False))
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Generic python script.
"""
__author__ = "Alex Drlica-Wagner"
import numpy as np
import fitsio
from ugali.utils.logger import logger
logger.setLevel(logger.WARN)

CONFIG='ugali/config/config_test.yaml'
LON = RA = 53.92
LAT = DEC = -54.05
IDX = [1,2537,9000]

def test_loglike():
    """ Test ugali.analysis.loglike """
    import ugali.analysis.loglike
    loglike = ugali.analysis.loglike.createLoglike(CONFIG,lon=LON,lat=LAT)

    source = loglike.source
    np.testing.assert_equal(source.richness,1000.)

    # Probability calculations
    np.testing.assert_allclose(loglike.f,0.1945461,rtol=1e-6)
    np.testing.assert_allclose(loglike.u[IDX],
                               [1.11721569e-11, 3.67872840e-03, 3.25216604e-02])
    np.testing.assert_allclose(loglike.b[IDX],
                               [4215.31143651, 9149.29106545, 1698.22182173])
    np.testing.assert_allclose(loglike.p[IDX],
                               [2.65037519e-12, 4.01916371e-04, 1.87905722e-02])
    np.testing.assert_allclose(loglike(),390.153891)
    np.testing.assert_allclose(loglike.ts(),780.30778)
    np.testing.assert_allclose(loglike.nobs,194.546134)

    # Fit the richness
    interval = loglike.richness_interval()
    np.testing.assert_allclose(interval,(1713.4082, 1969.1216))
    lnl,rich,para = loglike.fit_richness()
    np.testing.assert_allclose(lnl,418.666213)
    np.testing.assert_allclose(rich,1839.005237)
    np.testing.assert_allclose(loglike.source.richness,1839.005237)

    # Write membership
    filename = 'test-membership.fits'
    loglike.write_membership(filename)
    mem,hdr = fitsio.read(filename,header=True)

    # Testing output data
    np.testing.assert_allclose(loglike.p,mem['PROB'])
    np.testing.assert_allclose(loglike.catalog.ra,mem['RA'])
    np.testing.assert_allclose(loglike.catalog.dec,mem['DEC'])
    np.testing.assert_allclose(loglike.catalog.color,mem['COLOR'])

    # Testing output metadata
    np.testing.assert_allclose(loglike.ts(),hdr['TS'])
    np.testing.assert_allclose(source.richness,hdr['RICHNESS'])
    np.testing.assert_allclose(source.age,hdr['AGE'])
    np.testing.assert_allclose(source.z,hdr['METALLICITY'])
    np.testing.assert_allclose(source.distance_modulus,hdr['DISTANCE_MODULUS'])
    np.testing.assert_allclose(source.lon,hdr['LON'])
    np.testing.assert_allclose(source.lat,hdr['LAT'])

if __name__ == "__main__":
    import argparse
    description = __doc__
    parser = argparse.ArgumentParser(description=description)
    args = parser.parse_args()

    test_loglike()
<commit_msg>rtol in loglike tests<commit_after>#!/usr/bin/env python
"""
Generic python script.
"""
__author__ = "Alex Drlica-Wagner"
import numpy as np
import fitsio
from ugali.utils.logger import logger
logger.setLevel(logger.WARN)

CONFIG='ugali/config/config_test.yaml'
LON = RA = 53.92
LAT = DEC = -54.05
IDX = [1,2537,9000]

def test_loglike():
    """ Test ugali.analysis.loglike """
    import ugali.analysis.loglike
    loglike = ugali.analysis.loglike.createLoglike(CONFIG,lon=LON,lat=LAT)

    source = loglike.source
    np.testing.assert_equal(source.richness,1000.)

    # Probability calculations
    np.testing.assert_allclose(loglike.f,0.1945461,rtol=1e-6)
    np.testing.assert_allclose(loglike.u[IDX],
                               [1.11721569e-11, 3.67872840e-03, 3.25216604e-02],
                               rtol=1e-6)
    np.testing.assert_allclose(loglike.b[IDX],
                               [4215.31143651, 9149.29106545, 1698.22182173],
                               rtol=1e-6)
    np.testing.assert_allclose(loglike.p[IDX],
                               [2.65037519e-12, 4.01916371e-04, 1.87905722e-02],
                               rtol=1e-6)
    np.testing.assert_allclose(loglike(),390.153891)
    np.testing.assert_allclose(loglike.ts(),780.30778)
    np.testing.assert_allclose(loglike.nobs,194.546134)

    # Fit the richness
    interval = loglike.richness_interval()
    np.testing.assert_allclose(interval,(1713.4082, 1969.1216))
    lnl,rich,para = loglike.fit_richness()
    np.testing.assert_allclose(lnl,418.666213)
    np.testing.assert_allclose(rich,1839.005237)
    np.testing.assert_allclose(loglike.source.richness,1839.005237)

    # Write membership
    filename = 'test-membership.fits'
    loglike.write_membership(filename)
    mem,hdr = fitsio.read(filename,header=True)

    # Testing output data
    np.testing.assert_allclose(loglike.p,mem['PROB'])
    np.testing.assert_allclose(loglike.catalog.ra,mem['RA'])
    np.testing.assert_allclose(loglike.catalog.dec,mem['DEC'])
    np.testing.assert_allclose(loglike.catalog.color,mem['COLOR'])

    # Testing output metadata
    np.testing.assert_allclose(loglike.ts(),hdr['TS'])
    np.testing.assert_allclose(source.richness,hdr['RICHNESS'])
    np.testing.assert_allclose(source.age,hdr['AGE'])
    np.testing.assert_allclose(source.z,hdr['METALLICITY'])
    np.testing.assert_allclose(source.distance_modulus,hdr['DISTANCE_MODULUS'])
    np.testing.assert_allclose(source.lon,hdr['LON'])
    np.testing.assert_allclose(source.lat,hdr['LAT'])

if __name__ == "__main__":
    import argparse
    description = __doc__
    parser = argparse.ArgumentParser(description=description)
    args = parser.parse_args()

    test_loglike()
<|endoftext|>


<commit_before>from octopus.lib import dates
from copy import deepcopy
import locale

class DataSchemaException(Exception):
    pass

class DataObj(object):
    """
    Class which provides services to other classes which store their internal data
    as a python data structure in the self.data field.
    """

    SCHEMA = None

    def __init__(self, raw=None):
        self.data = {} if raw is None else raw
        self.validate()

    def validate(self):
        if self.SCHEMA is not None:
            validate(self.data, self.SCHEMA)
        return True

    def _get_path(self, path, default):
        parts = path.split(".")
        context = self.data

        for i in range(len(parts)):
            p = parts[i]
            d = {} if i < len(parts) - 1 else default
            context = context.get(p, d)
        return context

    def _set_path(self, path, val):
        parts = path.split(".")
        context = self.data

        for i in range(len(parts)):
            p = parts[i]

            if p not in context and i < len(parts) - 1:
                context[p] = {}
                context = context[p]
            elif p in context and i < len(parts) - 1:
                context = context[p]
            else:
                context[p] = val

    def _delete_from_list(self, path, val=None, matchsub=None, prune=True):
        l = self._get_list(path)

        removes = []
        i = 0
        for entry in l:
            if val is not None:
                if entry == val:
                    removes.append(i)
            elif matchsub is not None:
                matches = 0
                for k, v in matchsub.iteritems():
                    if entry.get(k) == v:
                        matches += 1
                if matches == len(matchsub.keys()):
                    removes.append(i)
            i += 1

        removes.sort(reverse=True)
        for r in removes:
            del l[r]

        if len(l) == 0 and prune:
            self._delete(path, prune)

    def _delete(self, path, prune=True):
        parts = path.split(".")
        context = self.data

        stack = []
        for i in range(len(parts)):
            p = parts[i]
            if p in context:
                if i < len(parts) - 1:
                    stack.append(context[p])
                    context = context[p]
                else:
                    del context[p]
                    if prune and len(stack) > 0:
                        stack.pop() # the last element was just deleted
                        self._prune_stack(stack)

    def _prune_stack(self, stack):
        while len(stack) > 0:
            context = stack.pop()
            todelete = []
            for k, v in context.iteritems():
                if isinstance(v, dict) and len(v.keys()) == 0:
                    todelete.append(k)
            for d in todelete:
                del context[d]

    def _coerce(self, val, cast, accept_failure=False):
        try:
            return cast(val)
        except (ValueError, TypeError):
            if accept_failure:
                return val
            raise DataSchemaException("Cast with " + str(cast) + " failed on " + str(val))

    def _get_single(self, path, coerce=None, default=None, allow_coerce_failure=True):
        # get the value at the point in the object
        val = self._get_path(path, default)

        if coerce is not None and val is not None:
            # if you want to coerce and there is something to coerce do it
            return self._coerce(val, coerce, accept_failure=allow_coerce_failure)
        else:
            # otherwise return the value
            return val

    def _get_list(self, path, coerce=None, by_reference=True, allow_coerce_failure=True):
        # get the value at the point in the object
        val = self._get_path(path, None)

        # if there is no value and we want to do by reference, then create it, bind it and return it
        if val is None and by_reference:
            mylist = []
            self._set_single(path, mylist)
            return mylist

        # otherwise, default is an empty list
        elif val is None and not by_reference:
            return []

        # check that the val is actually a list
        if not isinstance(val, list):
            raise DataSchemaException("Expecting a list at " + path + " but found " + str(type(val)))

        # if there is a value, do we want to coerce each of them
        if coerce is not None:
            coerced = [self._coerce(v, coerce, accept_failure=allow_coerce_failure) for v in val]
            if by_reference:
                self._set_single(path, coerced)
            return coerced
        else:
            if by_reference:
                return val
            else:
                return deepcopy(val)

    def _set_single(self, path, val, coerce=None, allow_coerce_failure=False, allowed_values=None, allowed_range=None,
                    allow_none=True, ignore_none=False):

        if val is None and ignore_none:
            return

        if val is None and not allow_none:
            raise DataSchemaException("NoneType is not allowed at " + path)

        # first see if we need to coerce the value (and don't coerce None)
        if coerce is not None and val is not None:
            val = self._coerce(val, coerce, accept_failure=allow_coerce_failure)

        if allowed_values is not None and val not in allowed_values:
            raise DataSchemaException("Value " + str(val) + " is not permitted at " + path)

        if allowed_range is not None:
            lower, upper = allowed_range
            if (lower is not None and val < lower) or (upper is not None and val > upper):
                raise DataSchemaException("Value " + str(val) + " is outside the allowed range: " + str(lower) + " - " + str(upper))

        # now set it at the path point in the object
        self._set_path(path, val)

    def _set_list(self, path, val, coerce=None, allow_coerce_failure=False):
        # first ensure that the value is a list
        if not isinstance(val, list):
            val = [val]

        # now coerce each of the values
        val = [self._coerce(v, coerce, accept_failure=allow_coerce_failure) for v in val]

        # now set it on the path
        self._set_path(path, val)

    def _add_to_list(self, path, val, coerce=None, allow_coerce_failure=False, allow_none=False, ignore_none=True):
        if val is None and ignore_none:
            return

        if val is None and not allow_none:
            raise DataSchemaException("NoneType is not allowed in list at " + path)

        # first coerce the value
        if coerce is not None:
            val = self._coerce(val, coerce, accept_failure=allow_coerce_failure)
        current = self._get_list(path, by_reference=True)
        current.append(val)

    def _utf8_unicode(self):
        def to_utf8_unicode(val):
            if isinstance(val, unicode):
                return val
            elif isinstance(val, basestring):
                try:
                    return val.decode("utf8", "strict")
                except UnicodeDecodeError:
                    raise ValueError("Could not decode string")
            else:
                return unicode(val)

        return to_utf8_unicode

    def _int(self):
        def intify(val):
            # try the straight cast
            try:
                return int(val)
            except ValueError:
                pass

            # could have commas in it, so try stripping them
            try:
                return int(val.replace(",", ""))
            except ValueError:
                pass

            # try the locale-specific approach
            try:
                return locale.atoi(val)
            except ValueError:
                pass

            raise ValueError("Could not convert string to int: {x}".format(x=val))

        return intify

    def _float(self):
        def floatify(val):
            # try the straight cast
            try:
                return float(val)
            except ValueError:
                pass

            # could have commas in it, so try stripping them
            try:
                return float(val.replace(",", ""))
            except ValueError:
                pass

            # try the locale-specific approach
            try:
                return locale.atof(val)
            except ValueError:
                pass

            raise ValueError("Could not convert string to float: {x}".format(x=val))

        return floatify

    def _date_str(self, in_format=None, out_format=None):
        def datify(val):
            return dates.reformat(val, in_format=in_format, out_format=out_format)

        return datify

class ObjectSchemaValidationError(Exception):
    pass

def validate(obj, schema):
    # all fields
    allowed = schema.get("bools", []) + schema.get("fields", []) + schema.get("lists", []) + schema.get("objects", [])

    for k, v in obj.iteritems():
        # is k allowed at all
        if k not in allowed:
            raise ObjectSchemaValidationError("object contains key " + k + " which is not permitted by schema")

        # check the bools are bools
        if k in schema.get("bools", []):
            if type(v) != bool:
                raise ObjectSchemaValidationError("object contains " + k + " = " + str(v) + " but expected boolean")

        # check that the fields are plain old strings
        if k in schema.get("fields", []):
            if type(v) != str and type(v) != unicode and type(v) != int and type(v) != float:
                raise ObjectSchemaValidationError("object contains " + k + " = " + str(v) + " but expected string, unicode or a number")

        # check that the lists are really lists
        if k in schema.get("lists", []):
            if type(v) != list:
                raise ObjectSchemaValidationError("object contains " + k + " = " + str(v) + " but expected list")
            # if it is a list, then for each member validate
            entry_schema = schema.get("list_entries", {}).get(k)
            if entry_schema is None:
                # validate the entries as fields
                for e in v:
                    if type(e) != str and type(e) != unicode and type(e) != int and type(e) != float:
                        raise ObjectSchemaValidationError("list in object contains " + str(type(e)) + " but expected string, unicode or a number in " + k)
            else:
                # validate each entry against the schema
                for e in v:
                    validate(e, entry_schema)

        # check that the objects are objects
        if k in schema.get("objects", []):
            if type(v) != dict:
                raise ObjectSchemaValidationError("object contains " + k + " = " + str(v) + " but expected object/dict")
            # if it is an object, then validate
            object_schema = schema.get("object_entries", {}).get(k)
            if object_schema is None:
                #raise ObjectSchemaValidationError("no object entry for object " + k)
                pass # we are not imposing a schema on this object
            else:
                validate(v, object_schema)
<commit_msg>refactor error string formatting for unicode support<commit_after>from octopus.lib import dates
from copy import deepcopy
import locale

class DataSchemaException(Exception):
    pass

class DataObj(object):
    """
    Class which provides services to other classes which store their internal data
    as a python data structure in the self.data field.
    """

    SCHEMA = None

    def __init__(self, raw=None):
        self.data = {} if raw is None else raw
        self.validate()

    def validate(self):
        if self.SCHEMA is not None:
            validate(self.data, self.SCHEMA)
        return True

    def _get_path(self, path, default):
        parts = path.split(".")
        context = self.data

        for i in range(len(parts)):
            p = parts[i]
            d = {} if i < len(parts) - 1 else default
            context = context.get(p, d)
        return context

    def _set_path(self, path, val):
        parts = path.split(".")
        context = self.data

        for i in range(len(parts)):
            p = parts[i]

            if p not in context and i < len(parts) - 1:
                context[p] = {}
                context = context[p]
            elif p in context and i < len(parts) - 1:
                context = context[p]
            else:
                context[p] = val

    def _delete_from_list(self, path, val=None, matchsub=None, prune=True):
        l = self._get_list(path)

        removes = []
        i = 0
        for entry in l:
            if val is not None:
                if entry == val:
                    removes.append(i)
            elif matchsub is not None:
                matches = 0
                for k, v in matchsub.iteritems():
                    if entry.get(k) == v:
                        matches += 1
                if matches == len(matchsub.keys()):
                    removes.append(i)
            i += 1

        removes.sort(reverse=True)
        for r in removes:
            del l[r]

        if len(l) == 0 and prune:
            self._delete(path, prune)

    def _delete(self, path, prune=True):
        parts = path.split(".")
        context = self.data

        stack = []
        for i in range(len(parts)):
            p = parts[i]
            if p in context:
                if i < len(parts) - 1:
                    stack.append(context[p])
                    context = context[p]
                else:
                    del context[p]
                    if prune and len(stack) > 0:
                        stack.pop() # the last element was just deleted
                        self._prune_stack(stack)

    def _prune_stack(self, stack):
        while len(stack) > 0:
            context = stack.pop()
            todelete = []
            for k, v in context.iteritems():
                if isinstance(v, dict) and len(v.keys()) == 0:
                    todelete.append(k)
            for d in todelete:
                del context[d]

    def _coerce(self, val, cast, accept_failure=False):
        try:
            return cast(val)
        except (ValueError, TypeError):
            if accept_failure:
                return val
            raise DataSchemaException(u"Cast with {x} failed on {y}".format(x=cast, y=val))

    def _get_single(self, path, coerce=None, default=None, allow_coerce_failure=True):
        # get the value at the point in the object
        val = self._get_path(path, default)

        if coerce is not None and val is not None:
            # if you want to coerce and there is something to coerce do it
            return self._coerce(val, coerce, accept_failure=allow_coerce_failure)
        else:
            # otherwise return the value
            return val

    def _get_list(self, path, coerce=None, by_reference=True, allow_coerce_failure=True):
        # get the value at the point in the object
        val = self._get_path(path, None)

        # if there is no value and we want to do by reference, then create it, bind it and return it
        if val is None and by_reference:
            mylist = []
            self._set_single(path, mylist)
            return mylist

        # otherwise, default is an empty list
        elif val is None and not by_reference:
            return []

        # check that the val is actually a list
        if not isinstance(val, list):
            raise DataSchemaException(u"Expecting a list at {x} but found {y}".format(x=path, y=val))

        # if there is a value, do we want to coerce each of them
        if coerce is not None:
            coerced = [self._coerce(v, coerce, accept_failure=allow_coerce_failure) for v in val]
            if by_reference:
                self._set_single(path, coerced)
            return coerced
        else:
            if by_reference:
                return val
            else:
                return deepcopy(val)

    def _set_single(self, path, val, coerce=None, allow_coerce_failure=False, allowed_values=None, allowed_range=None,
                    allow_none=True, ignore_none=False):

        if val is None and ignore_none:
            return

        if val is None and not allow_none:
            raise DataSchemaException(u"NoneType is not allowed at {x}".format(x=path))

        # first see if we need to coerce the value (and don't coerce None)
        if coerce is not None and val is not None:
            val = self._coerce(val, coerce, accept_failure=allow_coerce_failure)

        if allowed_values is not None and val not in allowed_values:
            raise DataSchemaException(u"Value {x} is not permitted at {y}".format(x=val, y=path))

        if allowed_range is not None:
            lower, upper = allowed_range
            if (lower is not None and val < lower) or (upper is not None and val > upper):
                raise DataSchemaException("Value {x} is outside the allowed range: {l} - {u}".format(x=val, l=lower, u=upper))

        # now set it at the path point in the object
        self._set_path(path, val)

    def _set_list(self, path, val, coerce=None, allow_coerce_failure=False):
        # first ensure that the value is a list
        if not isinstance(val, list):
            val = [val]

        # now coerce each of the values
        val = [self._coerce(v, coerce, accept_failure=allow_coerce_failure) for v in val]

        # now set it on the path
        self._set_path(path, val)

    def _add_to_list(self, path, val, coerce=None, allow_coerce_failure=False, allow_none=False, ignore_none=True):
        if val is None and ignore_none:
            return

        if val is None and not allow_none:
            raise DataSchemaException(u"NoneType is not allowed in list at {x}".format(x=path))

        # first coerce the value
        if coerce is not None:
            val = self._coerce(val, coerce, accept_failure=allow_coerce_failure)
        current = self._get_list(path, by_reference=True)
        current.append(val)

    def _utf8_unicode(self):
        def to_utf8_unicode(val):
            if isinstance(val, unicode):
                return val
            elif isinstance(val, basestring):
                try:
                    return val.decode("utf8", "strict")
                except UnicodeDecodeError:
                    raise ValueError(u"Could not decode string")
            else:
                return unicode(val)

        return to_utf8_unicode

    def _int(self):
        def intify(val):
            # strip any characters that are outside the ascii range - they won't make up the float anyway
            # and this will get rid of things like strange currency marks
            if isinstance(val, unicode):
                val = val.encode("ascii", errors="ignore")

            # try the straight cast
            try:
                return int(val)
            except ValueError:
                pass

            # could have commas in it, so try stripping them
            try:
                return int(val.replace(",", ""))
            except ValueError:
                pass

            # try the locale-specific approach
            try:
                return locale.atoi(val)
            except ValueError:
                pass

            raise ValueError(u"Could not convert string to int: {x}".format(x=val))

        return intify

    def _float(self):
        def floatify(val):
            # strip any characters that are outside the ascii range - they won't make up the float anyway
            # and this will get rid of things like strange currency marks
            if isinstance(val, unicode):
                val = val.encode("ascii", errors="ignore")

            # try the straight cast
            try:
                return float(val)
            except ValueError:
                pass

            # could have commas in it, so try stripping them
            try:
                return float(val.replace(",", ""))
            except ValueError:
                pass

            # try the locale-specific approach
            try:
                return locale.atof(val)
            except ValueError:
                pass

            raise ValueError(u"Could not convert string to float: {x}".format(x=val))

        return floatify

    def _date_str(self, in_format=None, out_format=None):
        def datify(val):
            return dates.reformat(val, in_format=in_format, out_format=out_format)

        return datify

class ObjectSchemaValidationError(Exception):
    pass

def validate(obj, schema):
    # all fields
    allowed = schema.get("bools", []) + schema.get("fields", []) + schema.get("lists", []) + schema.get("objects", [])

    for k, v in obj.iteritems():
        # is k allowed at all
        if k not in allowed:
            raise ObjectSchemaValidationError("object contains key " + k + " which is not permitted by schema")

        # check the bools are bools
        if k in schema.get("bools", []):
            if type(v) != bool:
                raise ObjectSchemaValidationError("object contains " + k + " = " + str(v) + " but expected boolean")

        # check that the fields are plain old strings
        if k in schema.get("fields", []):
            if type(v) != str and type(v) != unicode and type(v) != int and type(v) != float:
                raise ObjectSchemaValidationError("object contains " + k + " = " + str(v) + " but expected string, unicode or a number")

        # check that the lists are really lists
        if k in schema.get("lists", []):
            if type(v) != list:
                raise ObjectSchemaValidationError("object contains " + k + " = " + str(v) + " but expected list")
            # if it is a list, then for each member validate
            entry_schema = schema.get("list_entries", {}).get(k)
            if entry_schema is None:
                # validate the entries as fields
                for e in v:
                    if type(e) != str and type(e) != unicode and type(e) != int and type(e) != float:
                        raise ObjectSchemaValidationError("list in object contains " + str(type(e)) + " but expected string, unicode or a number in " + k)
            else:
                # validate each entry against the schema
                for e in v:
                    validate(e, entry_schema)

        # check that the objects are objects
        if k in schema.get("objects", []):
            if type(v) != dict:
                raise ObjectSchemaValidationError("object contains " + k + " = " + str(v) + " but expected object/dict")
            # if it is an object, then validate
            object_schema = schema.get("object_entries", {}).get(k)
            if object_schema is None:
                #raise ObjectSchemaValidationError("no object entry for object " + k)
                pass # we are not imposing a schema on this object
            else:
                validate(v, object_schema)
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function, division

import os
import glob
import gzip

from base import webqtlConfig

def get_samplelist(file_type, geno_file):
    if file_type == "geno":
        return get_samplelist_from_geno(geno_file)
    elif file_type == "plink":
        return get_samplelist_from_plink(geno_file)

def get_samplelist_from_geno(genofilename):
    if os.path.isfile(genofilename + '.gz'):
        genofilename += '.gz'
        genofile = gzip.open(genofilename)
    else:
        genofile = open(genofilename)

    for line in genofile:
        line = line.strip()
        if not line:
            continue
        if line.startswith(("#", "@")):
            continue
        break

    headers = line.split()

    if headers[3] == "Mb":
        samplelist = headers[4:]
    else:
        samplelist = headers[3:]
    return samplelist

def get_samplelist_from_plink(genofilename):
    genofile = open(genofilename)

    samplelist = []
    for line in genofile:
        line = line.split(" ")
        samplelist.append(line[1])

    return samplelist
<commit_msg>Fixed minor issue that caused the sample list to be fetched incorrectly for a group groups<commit_after>from __future__ import absolute_import, print_function, division

import os
import glob
import gzip

from base import webqtlConfig

def get_samplelist(file_type, geno_file):
    if file_type == "geno":
        return get_samplelist_from_geno(geno_file)
    elif file_type == "plink":
        return get_samplelist_from_plink(geno_file)

def get_samplelist_from_geno(genofilename):
    if os.path.isfile(genofilename + '.gz'):
        genofilename += '.gz'
        genofile = gzip.open(genofilename)
    else:
        genofile = open(genofilename)

    for line in genofile:
        line = line.strip()
        if not line:
            continue
        if line.startswith(("#", "@")):
            continue
        break

    headers = line.split("\t")

    if headers[3] == "Mb":
        samplelist = headers[4:]
    else:
        samplelist = headers[3:]
    return samplelist

def get_samplelist_from_plink(genofilename):
    genofile = open(genofilename)

    samplelist = []
    for line in genofile:
        line = line.split(" ")
        samplelist.append(line[1])

    return samplelist
<|endoftext|>


<commit_before><commit_msg>[FIX] server.py: stop the FSWatcher when stopping the server<commit_after><|endoftext|>


<commit_before>import sys
import unittest
import json
from . import unittestsetup
from .unittestsetup import environment as environment
from .unittestsetup import fetchTestData
import requests_mock

try:
    from nose_parameterized import parameterized
except:
    print("*** Please install 'nose_parameterized' to run these tests ***")
    exit(0)

import oandapyV20
from oandapyV20 import API
from oandapyV20.exceptions import V20Error, StreamTerminated
import oandapyV20.endpoints.pricing as pricing
from oandapyV20.endpoints.pricing import responses

access_token = None
accountID = None
account_cur = None
api = None

class TestPricing(unittest.TestCase):
    """Tests regarding the pricing endpoints."""

    def setUp(self):
        """setup for all tests."""
        global access_token
        global accountID
        global account_cur
        global api
        # self.maxDiff = None
        try:
            accountID, account_cur, access_token = unittestsetup.auth()
            setattr(sys.modules["oandapyV20.oandapyV20"],
                    "TRADING_ENVIRONMENTS",
                    {"practice": {
                     "stream": "https://test.com",
                     "api": "https://test.com",
                     }})
            api = API(environment=environment,
                      access_token=access_token)
            api.api_url = 'https://test.com'
        except Exception as e:
            print("%s" % e)
            exit(0)

    @requests_mock.Mocker()
    def test__pricing(self, mock_get):
        """get the pricing information for instruments."""
        tid = "_v3_accounts_accountID_pricing"
        resp, data, params = fetchTestData(responses, tid)
        r = pricing.PricingInfo(accountID, params=params)
        mock_get.register_uri('GET',
                              "{}/{}".format(api.api_url, r),
                              text=json.dumps(resp))
        result = api.request(r)
        instr = params["instruments"].split(",")
        self.assertTrue(result["prices"][0]["instrument"] == instr[0] and
                        result["prices"][1]["instrument"] == instr[1])

    @requests_mock.Mocker()
    def test__pricing_stream(self, mock_get):
        """get the streaming pricing information for instruments."""
        tid = "_v3_accounts_accountID_pricing_stream"
        resp, data, params = fetchTestData(responses, tid)
        text = "\n".join([json.dumps(t) for t in resp])
        r = pricing.PricingStream(accountID, params=params)
        mock_get.register_uri('GET',
                              "{}/{}".format(api.api_url, r),
                              text=text)
        result = []
        n = 0
        m = 3
        with self.assertRaises(StreamTerminated) as oErr:
            for rec in api.request(r):
                result.append(rec)
                n += 1
                # terminate when we have m response lines
                if n == m:
                    r.terminate()

        # the result containing m items, should equal the first m items
        self.assertTrue(result == resp[0:m])

    def test__pricing_stream_termination_1(self):
        """terminate a stream that does not exist."""
        params = {"instruments": "EUR_USD,EUR_JPY"}
        r = pricing.PricingStream(accountID, params=params)
        with self.assertRaises(ValueError) as oErr:
            r.terminate()

if __name__ == "__main__":

    unittest.main()
<commit_msg>flake8 corrections<commit_after>import sys
import unittest
import json
from . import unittestsetup
from .unittestsetup import environment as environment
from .unittestsetup import fetchTestData
import requests_mock

from oandapyV20 import API
from oandapyV20.exceptions import StreamTerminated
import oandapyV20.endpoints.pricing as pricing
from oandapyV20.endpoints.pricing import responses

access_token = None
accountID = None
account_cur = None
api = None

class TestPricing(unittest.TestCase):
    """Tests regarding the pricing endpoints."""

    def setUp(self):
        """setup for all tests."""
        global access_token
        global accountID
        global account_cur
        global api
        # self.maxDiff = None
        try:
            accountID, account_cur, access_token = unittestsetup.auth()
            setattr(sys.modules["oandapyV20.oandapyV20"],
                    "TRADING_ENVIRONMENTS",
                    {"practice": {
                     "stream": "https://test.com",
                     "api": "https://test.com",
                     }})
            api = API(environment=environment,
                      access_token=access_token)
            api.api_url = 'https://test.com'
        except Exception as e:
            print("%s" % e)
            exit(0)

    @requests_mock.Mocker()
    def test__pricing(self, mock_get):
        """get the pricing information for instruments."""
        tid = "_v3_accounts_accountID_pricing"
        resp, data, params = fetchTestData(responses, tid)
        r = pricing.PricingInfo(accountID, params=params)
        mock_get.register_uri('GET',
                              "{}/{}".format(api.api_url, r),
                              text=json.dumps(resp))
        result = api.request(r)
        instr = params["instruments"].split(",")
        self.assertTrue(result["prices"][0]["instrument"] == instr[0] and
                        result["prices"][1]["instrument"] == instr[1])

    @requests_mock.Mocker()
    def test__pricing_stream(self, mock_get):
        """get the streaming pricing information for instruments."""
        tid = "_v3_accounts_accountID_pricing_stream"
        resp, data, params = fetchTestData(responses, tid)
        text = "\n".join([json.dumps(t) for t in resp])
        r = pricing.PricingStream(accountID, params=params)
        mock_get.register_uri('GET',
                              "{}/{}".format(api.api_url, r),
                              text=text)
        result = []
        n = 0
        m = 3
        with self.assertRaises(StreamTerminated):
            for rec in api.request(r):
                result.append(rec)
                n += 1
                # terminate when we have m response lines
                if n == m:
                    r.terminate()

        # the result containing m items, should equal the first m items
        self.assertTrue(result == resp[0:m])

    def test__pricing_stream_termination_1(self):
        """terminate a stream that does not exist."""
        params = {"instruments": "EUR_USD,EUR_JPY"}
        r = pricing.PricingStream(accountID, params=params)
        with self.assertRaises(ValueError):
            r.terminate()

if __name__ == "__main__":

    unittest.main()
<|endoftext|>


<commit_before>"""
@brief      test tree node (time=10s)
"""

import sys
import os
import unittest
import re
import shutil
import warnings

try:
    import src
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..")))
    if path not in sys.path:
        sys.path.append(path)
    import src

from src.pyquickhelper import fLOG, get_temp_folder
from src.pyquickhelper.pycode.build_helper import get_build_script, get_script_command, get_extra_script_command, _default_nofolder
from src.pyquickhelper.pycode.setup_helper import write_pyproj

class TestBuildScript(unittest.TestCase):

    def test_build_script(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        if sys.platform.startswith("win"):
            sc = get_build_script("pyquickhelper")
            # fLOG(sc)
            assert "c:\\Python34_x64vir%virtual_env_suffix%\\install" in sc
            lines = sc.split("\n")
            for line in lines:
                if "__" in line and _default_nofolder not in line:
                    raise Exception("issue with __ in:\n" + line)

            scc = get_script_command(
                "unittest", "pyquickhelper", requirements=[])
            assert "setup.py" in scc
            assert "__" not in scc

            sccc = get_extra_script_command(
                "local_pypi", "pyquickhelper", port=8067, requirements=[])
            assert "python" in sccc
            if "__" in sccc:
                raise Exception(sccc)
        else:
            # not yet implemented for this platform
            return

    def test_build_script_all(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        project_var_name = "pyquickhelper"
        requirements = None
        port = 8067

        if sys.platform.startswith("win"):

            for c in {"build_script", "clean_space",
                      "write_version", "clean_pyd",
                      "build_sphinx", "unittests",
                      "unittests_LONG", "unittests_SKIP",
                      "copy27", "test_local_pypi",
                      "setup_hook"}:
                sc = get_script_command(
                    c, project_var_name, requirements=requirements, port=port)
                assert len(sc) > 0
                assert "__" not in sc

            for c in {"notebook", "publish", "publish_doc", "local_pypi", "run27",
                      "build27", "setupdep", "copy_dist",
                      "any_setup_command", "build_dist"}:
                sc = get_extra_script_command(
                    c, project_var_name, requirements=requirements, port=port)
                assert len(sc) > 0
                assert "__" not in sc
                if c == "run27":
                    if "nosetest" not in sc:
                        raise Exception(sc)
        else:
            # not yet implemented for this platform
            return

    def test_build_pyproj(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_pyproj")
        root = os.path.normpath(os.path.join(temp, "..", "..", ".."))

        if sys.version_info[0] == 2:
            # not available
            return

        write_pyproj(root, temp)

        with open(os.path.join(temp, "ptvs_project.pyproj"), "r", encoding="utf8") as f:
            content = f.read()
        assert "build\\" not in content
        assert "setup_helper.py" in content

if __name__ == "__main__":
    unittest.main()
<commit_msg>update a failing unit test on Windows<commit_after>"""
@brief      test tree node (time=10s)
"""

import sys
import os
import unittest
import re
import shutil
import warnings

try:
    import src
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..")))
    if path not in sys.path:
        sys.path.append(path)
    import src

from src.pyquickhelper import fLOG, get_temp_folder
from src.pyquickhelper.pycode.build_helper import get_build_script, get_script_command, get_extra_script_command, _default_nofolder
from src.pyquickhelper.pycode.setup_helper import write_pyproj

class TestBuildScript(unittest.TestCase):

    def test_build_script(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        if sys.platform.startswith("win"):
            sc = get_build_script("pyquickhelper")
            # fLOG(sc)
            assert "c:\\Python34_x64vir%virtual_env_suffix%\\install" in sc
            lines = sc.split("\n")
            for line in lines:
                if "__" in line and _default_nofolder not in line:
                    raise Exception("issue with __ in:\n" + line)

            scc = get_script_command(
                "unittest", "pyquickhelper", requirements=[])
            assert "setup.py" in scc
            assert "__" not in scc

            sccc = get_extra_script_command(
                "local_pypi", "pyquickhelper", port=8067, requirements=[])
            assert "python" in sccc
            if "__" in sccc:
                raise Exception(sccc)
        else:
            # not yet implemented for this platform
            return

    def test_build_script_all(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        project_var_name = "pyquickhelper"
        requirements = None
        port = 8067

        if sys.platform.startswith("win"):

            for c in {"build_script", "clean_space",
                      "write_version", "clean_pyd",
                      "build_sphinx", "unittests",
                      "unittests_LONG", "unittests_SKIP",
                      "copy27", "test_local_pypi",
                      "setup_hook"}:
                sc = get_script_command(
                    c, project_var_name, requirements=requirements, port=port)
                assert len(sc) > 0
                assert "__" not in sc

            unit_test_folder = os.path.abspath(
                os.path.join(os.path.dirname(__file__), ".."))
            for c in {"notebook", "publish", "publish_doc", "local_pypi", "run27",
                      "build27", "setupdep", "copy_dist",
                      "any_setup_command", "build_dist"}:
                sc = get_extra_script_command(
                    c, project_var_name, requirements=requirements, port=port,
                    unit_test_folder=unit_test_folder)
                assert len(sc) > 0
                assert "__" not in sc
                if c == "run27":
                    if "nosetest" not in sc:
                        raise Exception(sc)
        else:
            # not yet implemented for this platform
            return

    def test_build_pyproj(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        temp = get_temp_folder(__file__, "temp_pyproj")
        root = os.path.normpath(os.path.join(temp, "..", "..", ".."))

        if sys.version_info[0] == 2:
            # not available
            return

        write_pyproj(root, temp)

        with open(os.path.join(temp, "ptvs_project.pyproj"), "r", encoding="utf8") as f:
            content = f.read()
        assert "build\\" not in content
        assert "setup_helper.py" in content

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/python
# coding=utf-8

import sys
sys.path.insert(0, '../')
import unittest
import lib.process as process
import socket
import sys
import subprocess
import os
import random
import tempfile
import time

class PortPoolTestCase(unittest.TestCase):
    def setUp(self):
        self.hostname = process.HOSTNAME
        self.pp = process.PortPool()
        self.pp.change_range(min_port=1025, max_port=1030)
        self.sockets = {}

    def tearDown(self):
        for s in self.sockets:
            self.sockets[s].close()

    def listen_port(self, port, max_connection=0):
        if self.sockets.get(port, None):
            self.sockets[port].close()

        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.bind((process.HOSTNAME, port))
        s.listen(max_connection)
        self.sockets[port] = s

    def test_singleton(self):
        pp2 = process.PortPool(min_port=1025, max_port=1038)
        self.failUnlessEqual(id(self.pp), id(pp2))

    def test_port_sequence(self):
        ports = set([1025, 1026, 1027, 1028, 30, 28, 22, 45])
        self.pp.change_range(port_sequence=ports)
        _ports = self.pp._PortPool__closed.union(self.pp._PortPool__ports)
        self.assertEqual(ports, _ports)

    def test_find_port(self):
        self.pp.change_range(1040, 1040)
        port = self.pp.port()
        self.assertTrue(port > 0)
        self.listen_port(port)
        self.assertRaises(IndexError, self.pp.port)

    def test_port_with_check(self):
        port1, port2 = self.pp.port(check=True), self.pp.port(check=True)
        self.pp.change_range(port_sequence=[port1, port2])
        self.listen_port(port1, 0)
        self.assertTrue(port2 == self.pp.port(check=True))

    def test_check_port(self):
        port = self.pp.port(check=True)
        self.assertTrue(self.pp._PortPool__check_port(port))
        self.listen_port(port)
        self.assertFalse(self.pp._PortPool__check_port(port))

    def test_release_port(self):
        port = self.pp.port(check=True)
        self.assertTrue(port in self.pp._PortPool__closed)
        self.pp.release_port(port)
        self.assertFalse(port in self.pp._PortPool__closed)

    def test_refresh(self):
        ports = set([random.randint(1025, 2000) for i in xrange(15)])
        self.pp.change_range(port_sequence=ports)
        ports_opened = self.pp._PortPool__ports.copy()
        test_port = ports_opened.pop()
        self.assertTrue(test_port in self.pp._PortPool__ports)
        self.assertTrue(len(self.pp._PortPool__ports) > 1)
        for port in ports:
            if port != test_port:
                try:
                    self.listen_port(port)
                except (socket.error):
                    pass

        self.pp.refresh()
        self.assertTrue(len(self.pp._PortPool__ports) == 1)

    def test_refresh_only_closed(self):
        ports = set([random.randint(1025, 2000) for i in xrange(15)])
        self.pp.change_range(port_sequence=ports)
        closed_num = len(self.pp._PortPool__closed)
        self.pp.port(), self.pp.port()
        self.assertTrue(closed_num + 2 == len(self.pp._PortPool__closed))

        ports_opened = self.pp._PortPool__ports.copy()
        test_port = ports_opened.pop()
        self.listen_port(test_port)
        self.pp.refresh(only_closed=True)
        self.assertTrue(closed_num == len(self.pp._PortPool__closed))

        self.pp.refresh()
        self.assertTrue(closed_num + 1 == len(self.pp._PortPool__closed))

    def test_change_range(self):
        self.pp.change_range(min_port=1025, max_port=1033)
        ports = self.pp._PortPool__closed.union(self.pp._PortPool__ports)
        self.assertTrue(ports == set(range(1025, 1033 + 1)))

        random_ports = set([random.randint(1025, 2000) for i in xrange(15)])
        self.pp.change_range(port_sequence=random_ports)
        ports = self.pp._PortPool__closed.union(self.pp._PortPool__ports)
        self.assertTrue(ports == random_ports)

class ProcessTestCase(unittest.TestCase):
    def setUp(self):
        self.hostname = process.HOSTNAME
        self.s = None
        self.executable = sys.executable
        self.pp = process.PortPool(min_port=1025, max_port=2000)
        self.sockets = {}

    def tearDown(self):
        for s in self.sockets:
            self.sockets[s].close()

    def listen_port(self, port, max_connection=0):
        if self.sockets.get(port, None):
            self.sockets[port].close()

        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.bind((process.HOSTNAME, port))
        s.listen(max_connection)
        self.sockets[port] = s

    def test_wait_for(self):
        port = self.pp.port(check=True)
        self.listen_port(port, max_connection=1)
        self.assertTrue(process.wait_for(port, 1))
        self.sockets.pop(port).close()
        self.assertFalse(process.wait_for(port, 1))

    def test_mprocess(self):
        self.assertRaises(OSError, process.mprocess, 'fake_process_', '')
        port = self.pp.port()
        self.listen_port(port, max_connection=0)
        pid, host = process.mprocess(self.executable, '', port=port, timeout=2)
        self.assertTrue(pid > 0)
        self.assertEqual(host, self.hostname + ':' + str(port))
        self.sockets.pop(port).close()
        self.assertRaises(OSError, process.mprocess, self.executable, '', port, 2)

    def test_kill_mprocess(self):
        p = subprocess.Popen([self.executable])
        pid = p.pid
        self.assertTrue(process.proc_alive(pid))
        process.kill_mprocess(pid)
        self.assertFalse(process.proc_alive(pid))

    def test_cleanup_process(self):
        fd_cfg, config_path = tempfile.mkstemp()
        fd_key, key_file = tempfile.mkstemp()
        fd_log, log_path = tempfile.mkstemp()
        db_path = tempfile.mkdtemp()
        self.assertTrue(os.path.exists(config_path))
        self.assertTrue(os.path.exists(key_file))
        self.assertTrue(os.path.exists(log_path))
        self.assertTrue(os.path.exists(db_path))
        os.fdopen(fd_cfg, 'w').write("keyFile={key_file}\nlogPath={log_path}\ndbpath={db_path}".format(**locals()))
        cfg = {'keyFile': key_file, 'logPath': log_path, 'dbpath': db_path}
        process.cleanup_mprocess(config_path, cfg)
        self.assertFalse(os.path.exists(config_path))
        self.assertFalse(os.path.exists(key_file))
        self.assertFalse(os.path.exists(log_path))
        self.assertFalse(os.path.exists(db_path))

    def test_remove_path(self):
        fd, file_path = tempfile.mkstemp()
        self.assertTrue(os.path.exists(file_path))
        process.remove_path(file_path)
        self.assertFalse(os.path.exists(file_path))

        dir_path = tempfile.mkdtemp()
        fd, file_path = tempfile.mkstemp(dir=dir_path)
        process.remove_path(dir_path)
        self.assertFalse(os.path.exists(file_path))
        self.assertFalse(os.path.exists(dir_path))

    def test_write_config(self):
        cfg = {'port': 27017, 'objcheck': 'true'}
        config_path = process.write_config(cfg)
        self.assertTrue(os.path.exists(config_path))
        config_data = open(config_path, 'r').read()
        self.assertTrue('port=27017' in config_data)
        self.assertTrue('objcheck=true' in config_data)
        process.cleanup_mprocess(config_path, cfg)

    def test_proc_alive(self):
        p = subprocess.Popen([self.executable])
        pid = p.pid
        self.assertTrue(process.proc_alive(pid))
        p.kill(), os.wait()
        self.assertFalse(process.proc_alive(pid))

        self.assertFalse(process.proc_alive(None))
        self.assertFalse(process.proc_alive('2333'))

if __name__ == '__main__':
    unittest.main()
<commit_msg>add test_write_config_with_specify_config_path<commit_after>#!/usr/bin/python
# coding=utf-8

import sys
sys.path.insert(0, '../')
import unittest
import lib.process as process
import socket
import sys
import subprocess
import os
import random
import tempfile

class PortPoolTestCase(unittest.TestCase):
    def setUp(self):
        self.hostname = process.HOSTNAME
        self.pp = process.PortPool()
        self.pp.change_range(min_port=1025, max_port=1030)
        self.sockets = {}

    def tearDown(self):
        for s in self.sockets:
            self.sockets[s].close()

    def listen_port(self, port, max_connection=0):
        if self.sockets.get(port, None):
            self.sockets[port].close()

        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.bind((process.HOSTNAME, port))
        s.listen(max_connection)
        self.sockets[port] = s

    def test_singleton(self):
        pp2 = process.PortPool(min_port=1025, max_port=1038)
        self.failUnlessEqual(id(self.pp), id(pp2))

    def test_port_sequence(self):
        ports = set([1025, 1026, 1027, 1028, 30, 28, 22, 45])
        self.pp.change_range(port_sequence=ports)
        _ports = self.pp._PortPool__closed.union(self.pp._PortPool__ports)
        self.assertEqual(ports, _ports)

    def test_find_port(self):
        self.pp.change_range(1040, 1040)
        port = self.pp.port()
        self.assertTrue(port > 0)
        self.listen_port(port)
        self.assertRaises(IndexError, self.pp.port)

    def test_port_with_check(self):
        port1, port2 = self.pp.port(check=True), self.pp.port(check=True)
        self.pp.change_range(port_sequence=[port1, port2])
        self.listen_port(port1, 0)
        self.assertTrue(port2 == self.pp.port(check=True))

    def test_check_port(self):
        port = self.pp.port(check=True)
        self.assertTrue(self.pp._PortPool__check_port(port))
        self.listen_port(port)
        self.assertFalse(self.pp._PortPool__check_port(port))

    def test_release_port(self):
        port = self.pp.port(check=True)
        self.assertTrue(port in self.pp._PortPool__closed)
        self.pp.release_port(port)
        self.assertFalse(port in self.pp._PortPool__closed)

    def test_refresh(self):
        ports = set([random.randint(1025, 2000) for i in xrange(15)])
        self.pp.change_range(port_sequence=ports)
        ports_opened = self.pp._PortPool__ports.copy()
        test_port = ports_opened.pop()
        self.assertTrue(test_port in self.pp._PortPool__ports)
        self.assertTrue(len(self.pp._PortPool__ports) > 1)
        for port in ports:
            if port != test_port:
                try:
                    self.listen_port(port)
                except (socket.error):
                    pass

        self.pp.refresh()
        self.assertTrue(len(self.pp._PortPool__ports) == 1)

    def test_refresh_only_closed(self):
        ports = set([random.randint(1025, 2000) for i in xrange(15)])
        self.pp.change_range(port_sequence=ports)
        closed_num = len(self.pp._PortPool__closed)
        self.pp.port(), self.pp.port()
        self.assertTrue(closed_num + 2 == len(self.pp._PortPool__closed))

        ports_opened = self.pp._PortPool__ports.copy()
        test_port = ports_opened.pop()
        self.listen_port(test_port)
        self.pp.refresh(only_closed=True)
        self.assertTrue(closed_num == len(self.pp._PortPool__closed))

        self.pp.refresh()
        self.assertTrue(closed_num + 1 == len(self.pp._PortPool__closed))

    def test_change_range(self):
        self.pp.change_range(min_port=1025, max_port=1033)
        ports = self.pp._PortPool__closed.union(self.pp._PortPool__ports)
        self.assertTrue(ports == set(range(1025, 1033 + 1)))

        random_ports = set([random.randint(1025, 2000) for i in xrange(15)])
        self.pp.change_range(port_sequence=random_ports)
        ports = self.pp._PortPool__closed.union(self.pp._PortPool__ports)
        self.assertTrue(ports == random_ports)

class ProcessTestCase(unittest.TestCase):
    def setUp(self):
        self.hostname = process.HOSTNAME
        self.s = None
        self.executable = sys.executable
        self.pp = process.PortPool(min_port=1025, max_port=2000)
        self.sockets = {}

    def tearDown(self):
        for s in self.sockets:
            self.sockets[s].close()

    def listen_port(self, port, max_connection=0):
        if self.sockets.get(port, None):
            self.sockets[port].close()

        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.bind((process.HOSTNAME, port))
        s.listen(max_connection)
        self.sockets[port] = s

    def test_wait_for(self):
        port = self.pp.port(check=True)
        self.listen_port(port, max_connection=1)
        self.assertTrue(process.wait_for(port, 1))
        self.sockets.pop(port).close()
        self.assertFalse(process.wait_for(port, 1))

    def test_mprocess(self):
        self.assertRaises(OSError, process.mprocess, 'fake_process_', '')
        port = self.pp.port()
        self.listen_port(port, max_connection=0)
        pid, host = process.mprocess(self.executable, '', port=port, timeout=2)
        self.assertTrue(pid > 0)
        self.assertEqual(host, self.hostname + ':' + str(port))
        self.sockets.pop(port).close()
        self.assertRaises(OSError, process.mprocess, self.executable, '', port, 1)

    def test_kill_mprocess(self):
        p = subprocess.Popen([self.executable])
        pid = p.pid
        self.assertTrue(process.proc_alive(pid))
        process.kill_mprocess(pid)
        self.assertFalse(process.proc_alive(pid))

    def test_cleanup_process(self):
        fd_cfg, config_path = tempfile.mkstemp()
        fd_key, key_file = tempfile.mkstemp()
        fd_log, log_path = tempfile.mkstemp()
        db_path = tempfile.mkdtemp()
        self.assertTrue(os.path.exists(config_path))
        self.assertTrue(os.path.exists(key_file))
        self.assertTrue(os.path.exists(log_path))
        self.assertTrue(os.path.exists(db_path))
        os.fdopen(fd_cfg, 'w').write("keyFile={key_file}\nlogPath={log_path}\ndbpath={db_path}".format(**locals()))
        cfg = {'keyFile': key_file, 'logPath': log_path, 'dbpath': db_path}
        process.cleanup_mprocess(config_path, cfg)
        self.assertFalse(os.path.exists(config_path))
        self.assertFalse(os.path.exists(key_file))
        self.assertFalse(os.path.exists(log_path))
        self.assertFalse(os.path.exists(db_path))

    def test_remove_path(self):
        fd, file_path = tempfile.mkstemp()
        self.assertTrue(os.path.exists(file_path))
        process.remove_path(file_path)
        self.assertFalse(os.path.exists(file_path))

        dir_path = tempfile.mkdtemp()
        fd, file_path = tempfile.mkstemp(dir=dir_path)
        process.remove_path(dir_path)
        self.assertFalse(os.path.exists(file_path))
        self.assertFalse(os.path.exists(dir_path))

    def test_write_config(self):
        cfg = {'port': 27017, 'objcheck': 'true'}
        config_path = process.write_config(cfg)
        self.assertTrue(os.path.exists(config_path))
        config_data = open(config_path, 'r').read()
        self.assertTrue('port=27017' in config_data)
        self.assertTrue('objcheck=true' in config_data)
        process.cleanup_mprocess(config_path, cfg)

    def test_write_config_with_specify_config_path(self):
        cfg = {'port': 27017, 'objcheck': 'true'}
        fd_key, file_path = tempfile.mkstemp()
        config_path = process.write_config(cfg, file_path)
        self.assertEqual(file_path, config_path)
        process.cleanup_mprocess(config_path, cfg)

    def test_proc_alive(self):
        p = subprocess.Popen([self.executable])
        pid = p.pid
        self.assertTrue(process.proc_alive(pid))
        p.kill(), os.wait()
        self.assertFalse(process.proc_alive(pid))

        self.assertFalse(process.proc_alive(None))
        self.assertFalse(process.proc_alive('2333'))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import fiona

def test_profile():
    with fiona.open('tests/data/coutwildrnp.shp') as src:
        assert src.meta['crs_wkt'] == 'GEOGCS["GCS_WGS_1984",DATUM["WGS_1984",SPHEROID["WGS_84",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295],AUTHORITY["EPSG","4326"]]'

def test_profile_creation_wkt(tmpdir):
    outfilename = str(tmpdir.join('test.shp'))
    with fiona.open('tests/data/coutwildrnp.shp') as src:
        profile = src.meta
        profile['crs'] = 'bogus'
        with fiona.open(outfilename, 'w', **profile) as dst:
            assert dst.crs == {'init': 'epsg:4326'}
            assert dst.crs_wkt == 'GEOGCS["GCS_WGS_1984",DATUM["WGS_1984",SPHEROID["WGS_84",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295],AUTHORITY["EPSG","4326"]]'
<commit_msg>Rework tmpdir for nose (no pytest)<commit_after>import os
import tempfile

import fiona

def test_profile():
    with fiona.open('tests/data/coutwildrnp.shp') as src:
        assert src.meta['crs_wkt'] == 'GEOGCS["GCS_WGS_1984",DATUM["WGS_1984",SPHEROID["WGS_84",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295],AUTHORITY["EPSG","4326"]]'

def test_profile_creation_wkt():
    tmpdir = tempfile.mkdtemp()
    outfilename = os.path.join(tmpdir, 'test.shp')
    with fiona.open('tests/data/coutwildrnp.shp') as src:
        profile = src.meta
        profile['crs'] = 'bogus'
        with fiona.open(outfilename, 'w', **profile) as dst:
            assert dst.crs == {'init': 'epsg:4326'}
            assert dst.crs_wkt == 'GEOGCS["GCS_WGS_1984",DATUM["WGS_1984",SPHEROID["WGS_84",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295],AUTHORITY["EPSG","4326"]]'
<|endoftext|>


<commit_before>"""
okeanidanalysis.lib
===================

Generally useful methods that span across submodules.

"""

import time, datetime, pytz, itertools
import numpy as np
import scipy as sp
import scipy.interpolate
import matplotlib as mpl
import matplotlib.patches

UNIX_EPOCH = datetime.datetime(1970, 1, 1, tzinfo=pytz.UTC)

golden_ratio = 1.61803398875

def utime(t, convention=None):
    """Convert time to mircoseconds since epoch.

    Expects time given as a datetime object.

    """
#    typet = type(t)
#    if typet is datetime.datetime: # t = t.timetuple()
#        return int(time.mktime(t.timetuple())*1e6 + t.microsecond)
#    else:
#        raise TypeError('time type {} not recognized'.format(typet))
#TODO something to handle matlab datenum convention easily
    return [ue * 1e6 for ue in python_datetime_to_unix_epoch(t)]

def python_datetime_to_unix_epoch(dto):
    if type(dto) is not list: dto = [dto] # TODO is there a more elegant way?
    try:
        return [(dt - UNIX_EPOCH).total_seconds for dt in dto]
    except TypeError: # can't subtract offset-naive and offset-aware datetimes
        dto = (pytz.UTC.localize(dt) for dt in dto)
        return [(dt - UNIX_EPOCH).total_seconds() for dt in dto]

def matlab_datenum_to_python_datetime(dn):
    day = (datetime.datetime.fromordinal(int(n)) for n in dn)
    frac = (datetime.timedelta(days=n%1) for n in dn)
    shift = datetime.timedelta(days = 366)
    return [d + f - shift for d, f in itertools.izip(day, frac)]

def injectlocals(l, skip=['self','args','kwargs'], **kwargs):
    """Update a dictionary with another, skipping specified keys."""
    if l.has_key('kwargs') : kwargs.update(l['kwargs'])
    kwargs.update(dict((k, v) for k, v in l.items() if k not in skip))
    return kwargs

def make_multiplier(n): return lambda x: x*n

def make_divider(n): return lambda x: x/n

def indexclip(s, e, *a): return [b[int(s):int(e)] for b in a]

def angle_difference(a, b, degrees=False):
    if degrees: p = 180.0
    else: p = np.pi
    return np.mod(np.unwrap(a, p) - np.unwrap(b, p) + p, 2 * p) - p

def rad2deg360(x):
    """

    """
    d = np.rad2deg(x)
    d[d < 0] += 360
    return d

def rmnans(x, *a):
    """Remove values from array(s), using NaNs in key array.

    Examples
    --------
    >>> b = np.array([0,1,nan,3])
    >>> c = np.ones(b.shape)
    >>> bn, cn = rmnans(b,c)
    >>> bn
    array([ 0.,  1.,  3.])
    >>> cn
    array([ 1.,  1.,  1.])

    """
    mask = np.isnan(x)
    y = [x[~mask]]
    for b in a: y.append(b[~mask])
    return y

def plot_date_blocks(t, v, axes, colormap=None, unit_height=False, **kw):
    #TODO logic to check edges & heights for sanity
#    if kw.has_key('ec') or kw.has_key('edgecolor'):
#        pass
#    else:
#        kw.update(dict(edgecolor='red'))
    d = np.diff(v)
    left = np.hstack((t[0],t[d.nonzero()]))
    right = np.hstack((t[d.nonzero()],mpl.dates.num2date(axes.get_xlim()[-1])))
    height = np.hstack((v[d.nonzero()], v[-1]))
    print height
    norm_height = height / np.float(np.max(height))
    for l, r, h, nh  in itertools.izip(left, right, height, norm_height):
        # draw a rectangle with the appropriate color
        if colormap:
            kw.update(dict(color = colormap(nh)))
        if unit_height:
            h = 1 # do this _after_ colormap
        ll = (mpl.dates.date2num(l),0)
        w = mpl.dates.date2num(r) - mpl.dates.date2num(l)
        axes.add_patch(mpl.patches.Rectangle(ll, w, h, **kw))
    pt = np.vstack((left, right))
    pv = np.vstack((height, height))
    print pt.shape, pv.shape
    axes.plot_date(pt.ravel('F'), pv.ravel('F'), 'k-')

# TODO modify to use PatchCollection and set colors that way
# http://matplotlib.org/examples/api/patch_collection.html

def slant_range_and_depths_to_horizontal_range(r, d1, d2):
    """
    Compute the radius of a circle on a sphere (half-chord on a circle)
    using the Pythagorean theorem, the sphere radius, and the apothem (in
    this case, the difference in depths).

    Sphere radius must be greater than difference in depths.

    Examples
    --------
    >>> depth_corrected_range(10, 5, 9)
    # TODO
    """
    return (r**2 - (d1 - d2)**2) **0.5

def loadmtx(filename):
    """Read matrix from `.mtx` file

    The first two elements in a `.mtx` file are matrix dimensions encoded as
    32-bit integers, and the rest of the file is a column major matrix of
    single precision (i.e., 32-bit float).

    Note that the endianness may be different depending on platform, so
    `.mtx` files may not be portable. This method is primarily intended to
    handle files sent to and from vehicles in the field over low-badwidth
    communication channels (e.g., Iridium SBD), and for development on
    techniques that use those files. Please use HDF5, NetCDF, MATLAB, or
    numpy.save methods for any other use cases.

    Parameters
    ----------
    filename : str
        The file to load the matrix from.

    Returns
    -------
    a : array
        A numpy array containing the matrix saved in the file.

    Examples
    --------
    TODO

    See also: savemtx, numpy.fromfile, numpy.save, numpy.savez

    """
    with open(filename, 'rb') as f:
        s = np.fromfile(file=f, dtype=np.int32, count=2)
        return np.fromfile(file=f, dtype=np.float32).reshape(s, order='F').T

def savemtx(a, filename):
    """Save array as a matrix in a `.mtx` file

    The first two elements in a `.mtx` file are matrix dimensions encoded as
    32-bit integers, and the rest of the file is a column major matrix of
    single precision (i.e., 32-bit float).

    Note that the endianness may be different depending on platform, so
    `.mtx` files may not be portable. This method is primarily intended to
    handle files sent to and from vehicles in the field over low-badwidth
    communication channels (e.g., Iridium SBD), and for development on
    techniques that use those files. Please use HDF5, NetCDF, MATLAB, or
    numpy.save methods for any other use cases.

    Parameters
    ----------
    a : array-like
        The array to be saved.
    filename : str
        The file to load the matrix from.

    Examples
    --------
    TODO

    See also: loadmtx, numpy.fromfile, numpy.save, numpy.savez

    """
    with open(filename, 'wb') as f:
        f.write(np.array(a.shape, dtype=np.int32).tostring(order='F'))
        f.write(a.astype(np.float32).tostring(order='F'))

<commit_msg>Revert this change because it didn't actually fix anything. 1.U.mtx and 1.ens_mean.mtx still have incompatible shapes unless you transpose one of them.<commit_after>"""
okeanidanalysis.lib
===================

Generally useful methods that span across submodules.

"""

import time, datetime, pytz, itertools
import numpy as np
import scipy as sp
import scipy.interpolate
import matplotlib as mpl
import matplotlib.patches

UNIX_EPOCH = datetime.datetime(1970, 1, 1, tzinfo=pytz.UTC)

golden_ratio = 1.61803398875

def utime(t, convention=None):
    """Convert time to mircoseconds since epoch.

    Expects time given as a datetime object.

    """
#    typet = type(t)
#    if typet is datetime.datetime: # t = t.timetuple()
#        return int(time.mktime(t.timetuple())*1e6 + t.microsecond)
#    else:
#        raise TypeError('time type {} not recognized'.format(typet))
#TODO something to handle matlab datenum convention easily
    return [ue * 1e6 for ue in python_datetime_to_unix_epoch(t)]

def python_datetime_to_unix_epoch(dto):
    if type(dto) is not list: dto = [dto] # TODO is there a more elegant way?
    try:
        return [(dt - UNIX_EPOCH).total_seconds for dt in dto]
    except TypeError: # can't subtract offset-naive and offset-aware datetimes
        dto = (pytz.UTC.localize(dt) for dt in dto)
        return [(dt - UNIX_EPOCH).total_seconds() for dt in dto]

def matlab_datenum_to_python_datetime(dn):
    day = (datetime.datetime.fromordinal(int(n)) for n in dn)
    frac = (datetime.timedelta(days=n%1) for n in dn)
    shift = datetime.timedelta(days = 366)
    return [d + f - shift for d, f in itertools.izip(day, frac)]

def injectlocals(l, skip=['self','args','kwargs'], **kwargs):
    """Update a dictionary with another, skipping specified keys."""
    if l.has_key('kwargs') : kwargs.update(l['kwargs'])
    kwargs.update(dict((k, v) for k, v in l.items() if k not in skip))
    return kwargs

def make_multiplier(n): return lambda x: x*n

def make_divider(n): return lambda x: x/n

def indexclip(s, e, *a): return [b[int(s):int(e)] for b in a]

def angle_difference(a, b, degrees=False):
    if degrees: p = 180.0
    else: p = np.pi
    return np.mod(np.unwrap(a, p) - np.unwrap(b, p) + p, 2 * p) - p

def rad2deg360(x):
    """

    """
    d = np.rad2deg(x)
    d[d < 0] += 360
    return d

def rmnans(x, *a):
    """Remove values from array(s), using NaNs in key array.

    Examples
    --------
    >>> b = np.array([0,1,nan,3])
    >>> c = np.ones(b.shape)
    >>> bn, cn = rmnans(b,c)
    >>> bn
    array([ 0.,  1.,  3.])
    >>> cn
    array([ 1.,  1.,  1.])

    """
    mask = np.isnan(x)
    y = [x[~mask]]
    for b in a: y.append(b[~mask])
    return y

def plot_date_blocks(t, v, axes, colormap=None, unit_height=False, **kw):
    #TODO logic to check edges & heights for sanity
#    if kw.has_key('ec') or kw.has_key('edgecolor'):
#        pass
#    else:
#        kw.update(dict(edgecolor='red'))
    d = np.diff(v)
    left = np.hstack((t[0],t[d.nonzero()]))
    right = np.hstack((t[d.nonzero()],mpl.dates.num2date(axes.get_xlim()[-1])))
    height = np.hstack((v[d.nonzero()], v[-1]))
    print height
    norm_height = height / np.float(np.max(height))
    for l, r, h, nh  in itertools.izip(left, right, height, norm_height):
        # draw a rectangle with the appropriate color
        if colormap:
            kw.update(dict(color = colormap(nh)))
        if unit_height:
            h = 1 # do this _after_ colormap
        ll = (mpl.dates.date2num(l),0)
        w = mpl.dates.date2num(r) - mpl.dates.date2num(l)
        axes.add_patch(mpl.patches.Rectangle(ll, w, h, **kw))
    pt = np.vstack((left, right))
    pv = np.vstack((height, height))
    print pt.shape, pv.shape
    axes.plot_date(pt.ravel('F'), pv.ravel('F'), 'k-')

# TODO modify to use PatchCollection and set colors that way
# http://matplotlib.org/examples/api/patch_collection.html

def slant_range_and_depths_to_horizontal_range(r, d1, d2):
    """
    Compute the radius of a circle on a sphere (half-chord on a circle)
    using the Pythagorean theorem, the sphere radius, and the apothem (in
    this case, the difference in depths).

    Sphere radius must be greater than difference in depths.

    Examples
    --------
    >>> depth_corrected_range(10, 5, 9)
    # TODO
    """
    return (r**2 - (d1 - d2)**2) **0.5

def loadmtx(filename):
    """Read matrix from `.mtx` file

    The first two elements in a `.mtx` file are matrix dimensions encoded as
    32-bit integers, and the rest of the file is a column major matrix of
    single precision (i.e., 32-bit float).

    Note that the endianness may be different depending on platform, so
    `.mtx` files may not be portable. This method is primarily intended to
    handle files sent to and from vehicles in the field over low-badwidth
    communication channels (e.g., Iridium SBD), and for development on
    techniques that use those files. Please use HDF5, NetCDF, MATLAB, or
    numpy.save methods for any other use cases.

    Parameters
    ----------
    filename : str
        The file to load the matrix from.

    Returns
    -------
    a : array
        A numpy array containing the matrix saved in the file.

    Examples
    --------
    TODO

    See also: savemtx, numpy.fromfile, numpy.save, numpy.savez

    """
    with open(filename, 'rb') as f:
        shape = np.fromfile(file=f, dtype=np.int32, count=2)[::-1]
        return np.fromfile(file=f, dtype=np.float32).reshape(shape).T

def savemtx(a, filename):
    """Save array as a matrix in a `.mtx` file

    The first two elements in a `.mtx` file are matrix dimensions encoded as
    32-bit integers, and the rest of the file is a column major matrix of
    single precision (i.e., 32-bit float).

    Note that the endianness may be different depending on platform, so
    `.mtx` files may not be portable. This method is primarily intended to
    handle files sent to and from vehicles in the field over low-badwidth
    communication channels (e.g., Iridium SBD), and for development on
    techniques that use those files. Please use HDF5, NetCDF, MATLAB, or
    numpy.save methods for any other use cases.

    Parameters
    ----------
    a : array-like
        The array to be saved.
    filename : str
        The file to load the matrix from.

    Examples
    --------
    TODO

    See also: loadmtx, numpy.fromfile, numpy.save, numpy.savez

    """
    with open(filename, 'wb') as f:
        f.write(np.array(a.shape, dtype=np.int32).tostring(order='F'))
        f.write(a.astype(np.float32).tostring(order='F'))

<|endoftext|>