<commit_before>import pytest

from populus.migrations.migration import (
    get_migration_classes_for_execution,
)
from populus.project import Project

@pytest.fixture()
def project(request):
    # This should probably be configurable using the `request` fixture but it's
    # unclear what needs to be configurable.
    return Project()

@pytest.yield_fixture()
def unmigrated_chain(request, project):
    # This should probably allow you to specify the test chain to be used based
    # on the `request` object.  It's unclear what the best way to do this is
    # so... punt!
    chain = project.get_chain('testrpc')

    # TODO: this should run migrations.  If `testrpc` it should be snapshotted.
    # In the future we should be able to snapshot the `geth` chains too and
    # save them for faster test runs.

    with chain:
        yield chain

@pytest.fixture()
def chain(unmigrated_chain):
    # Determine if we have any migrations to run.
    migrations_to_execute = get_migration_classes_for_execution(
        unmigrated_chain.project.migrations,
        unmigrated_chain,
    )

    for migration in migrations_to_execute:
        migration.execute()

    return unmigrated_chain

@pytest.fixture()
def web3(chain):
    return chain.web3

@pytest.fixture()
def contracts(chain):
    return chain.contract_factories

@pytest.fixture()
def accounts(web3):
    return web3.eth.accounts
<commit_msg>try to fix chain migrations for fixture<commit_after>import pytest

from populus.migrations.migration import (
    get_migration_classes_for_execution,
)
from populus.project import Project

@pytest.fixture()
def project(request):
    # This should probably be configurable using the `request` fixture but it's
    # unclear what needs to be configurable.
    return Project()

@pytest.yield_fixture()
def unmigrated_chain(request, project):
    # This should probably allow you to specify the test chain to be used based
    # on the `request` object.  It's unclear what the best way to do this is
    # so... punt!
    chain = project.get_chain('testrpc')

    # TODO: this should run migrations.  If `testrpc` it should be snapshotted.
    # In the future we should be able to snapshot the `geth` chains too and
    # save them for faster test runs.

    with chain:
        yield chain

@pytest.fixture()
def chain(unmigrated_chain):
    # Determine if we have any migrations to run.
    migrations_to_execute = get_migration_classes_for_execution(
        unmigrated_chain.project.migrations,
        unmigrated_chain,
    )

    for migration in migrations_to_execute:
        migration.execute()

    return unmigrated_chain

@pytest.fixture()
def web3(unmigrated_chain):
    return chain.web3

@pytest.fixture()
def contracts(unmigrated_chain):
    return chain.contract_factories

@pytest.fixture()
def accounts(web3):
    return web3.eth.accounts
<|endoftext|>


<commit_before>"""
Descriptors are definitions for functions.
"""
import markdown
from django.template.defaultfilters import slugify
from urlparse import parse_qs
from django.http import QueryDict
import logging

log = logging.getLogger('sharrock')

class MalformedDescriptor(Exception):
    """
    A programming error indicating a function descriptor that has been improperly
    formatted.
    """
    pass

class ParamRequired(Exception):
    """
    An exception indicating that a required param has not been supplied.
    """
    def __init__(self,param_name):
        self.param = param_name

    def __unicode__(self):
        return u'%s is a required parameter and has not been specified.' % unicode(self.param)

    def __str__(self):
        return unicode(self)

class Param(object):
    """
    A parameter for a function.
    """
    def __init__(self,name,required=False,default=None,description=None):
        self.name = name
        self.required = required
        self.default = default
        self.description = description

    def get_from_dict(self,raw_dict):
        """
        Gets param from raw dict.  If param is missing and required, raises
        ParamRequired, otherwise returns None.
        """
        raw_value = raw_dict.get(self.name,self.default)
        if raw_value is None and self.required:
            raise ParamRequired(self.name)
        elif raw_value is None:
            return None
        else:
            return self.process(raw_value)

    def process(self,raw):
        """
        Processes the raw value, returning the new value.
        """
        raise NotImplemented

#######################################################
### Below are Parameters to be used in Descriptors. ###
#######################################################

class UnicodeParam(Param):
    """
    A parameter with a unicode payload.
    """
    def process(self,raw):
        return unicode(raw)

    @property
    def type(self):
        return 'Unicode'

class IntegerParam(Param):
    """
    A param with an integer value.
    """
    def process(self,raw):
        return int(raw)

    @property
    def type(self):
        return 'Integer'

class FloatParam(Param):
    """
    A param with a float value.
    """
    def process(self,raw):
        return float(raw)

    @property
    def type(self):
        return 'Float'

class ListParam(Param):
    """
    A param with a list of values.
    """
    def __init__(self,name,item_param,**kwargs):
        super(ListParam,self).__init__(name,**kwargs)
        self.item_param = item_param

    def process(self,raw):
        return [self.item_param.get(None,raw_item) for raw_item in raw]

    @property
    def type(self):
        return 'List'

class DictParam(Param):
    """
    A param with a dictionary of values.
    """
    def __init__(self,name,params={},**kwargs):
        super(DictParam,self).__init__(name,**kwargs)
        self.param_dict = params.copy() # always copy mutable arguments

    def process(self,raw):
        log.debug('DictParam processing raw value: %s' % raw)
        if not self.param_dict:
            # if no params have been defined, treat as a wildcard dictionary - any key/value pairs are acceptable
            return raw.copy()
        else:
            return_dict = {}
            for param_key, param in self.param_dict.items():
                return_dict[param_key] = param.get_from_dict(raw)
            return return_dict

    @property
    def type(self):
        return 'Dictionary'

###################
### Serializers ###
###################
class UnsupportedSerializationFormat(Exception):
    """
    Indicates an unsupported serialization format.
    """
    pass

class Serializer(object):
    """
    Base class for serializers.
    """
    def serialize(self,python_object):
        """
        Serializes the object.  Returns the serialized form of the object.
        """
        raise NotImplemented

    def deserialize(self,serialized_object):
        """
        Deserialized the object.  Returns python object.
        """
        raise NotImplemented

import json

class JSONSerializer(Serializer):
    """
    JSON Serializer.
    """
    name = 'json'

    def serialize(self,python_object):
        if python_object:
            return json.dumps(python_object)
        else:
            return python_object

    def deserialize(self,serialized_object):
        log.debug('JSON Serializer loading serialized objects:%s' % serialized_object)
        if serialized_object:
            return json.loads(serialized_object)
        else:
            return serialized_object

################
### Security ###
################
class AccessDenied(Exception):
    """
    Indicates the user does not have permission to perform the task.
    """
    pass

class SecurityCheck(object):
    """
    Base class for security check.  This base class will allow any user, essentially
    making a public API.
    """
    def __init__(self,*permissions):
        self.permissions = permissions

    def check(self,request):
        """
        Checks the request for permission to access the function.  No return value.
        Raises an AccessDenied exception if the request fails the permission check.
        """
        pass # anyone allowed in base class.

###################
### Descriptors ###
###################

import re

def space_out_camel_case(stringAsCamelCase):
    """
    By Simon Hartley, posted on http://refactormycode.com/codes/675-camelcase-to-camel-case-python-newbie

    Adds spaces to a camel case string.  Failure to space out string returns the original string.
    >>> space_out_camel_case('DMLSServicesOtherBSTextLLC')
    'DMLS Services Other BS Text LLC'
    """

    if stringAsCamelCase is None:
        return None

    pattern = re.compile('([A-Z][A-Z][a-z])|([a-z][A-Z])')
    return pattern.sub(lambda m: m.group()[:1] + " " + m.group()[1:], stringAsCamelCase)

class DescriptorMetaclass(type):
    """
    Metaclass for descriptors.
    """
    def __new__(cls,name,bases,attrs):
        """
        Constructs a new descriptor class.
        """
        if name != 'Descriptor': # don't operate on the base class
            # New attribute dictionary
            new_attrs = {'serializer_dict':{},'params':[]}

            # Name
            if 'verbose_name' in attrs:
                new_attrs['service_name'] = attrs['verbose_name']
            else:
                new_attrs['service_name'] = space_out_camel_case(name)

            # Params group params
            for attribute_name, attribute_value in attrs.items():
                if isinstance(attribute_value,Param):
                    new_attrs['params'].append(attribute_value)
                elif isinstance(attribute_value,Serializer):
                    new_attrs['serializer_dict'][attribute_value.name] = attribute_value

            # If no serializers has been set, set default
            if not new_attrs['serializer_dict']:
                new_attrs['serializer_dict']['json'] = JSONSerializer()

            # If no security has been set, set default
            if not 'security' in attrs:
                new_attrs['security'] = SecurityCheck()

            # If data parsing flag has not been set, set to False
            if not 'data_parsing' in attrs:
                new_attrs['data_parsing'] = False

            # Deprecated message
            if not 'deprecated' in attrs:
                new_attrs['deprecated'] = None

            attrs.update(new_attrs)

        return type.__new__(cls,name,bases,attrs)

class Descriptor(object):
    """
    Base class for function descriptors.

    Specific descriptor classes should subclass this base class.  They can then
    define the following
    """
    # Based on metaclass activity - add:
    # serializer/deserializers
    # params, required and optional, default values
    # security hooks
    # API version

    __metaclass__ = DescriptorMetaclass # class factory mounts here

    def __init__(self,is_deprecated=None):
        """
        Constructor.  is_deprecated flag indicates if this is a deprecated API, either through
        declaration in the local class, or as inherited from a parent entity.
        """
        self.is_deprecated = is_deprecated # deprecation from parent entity

        if self.deprecated:
            self.is_deprecated = self.deprecated # deprecation occuring at local class level

    def serialize(self,python_object,format):
        """
        Serializes the object.
        """
        if python_object is None:
            return None
        try:
            serializer = self.serializer_dict[format]
            return serializer.serialize(python_object)
        except KeyError:
            raise UnsupportedSerializationFormat

    def deserialize(self,serialized_object,format):
        """
        Deserializes the object.
        """
        if serialized_object is None:
            return None
        try:
            serializer = self.serializer_dict[format]
            return serializer.deserialize(serialized_object)
        except KeyError:
            raise UnsupportedSerializationFormat

    def execute(self,request,data,params):
        """
        Executes the function.
        """
        raise NotImplemented # Subclasses implement

    def extract_kwargs(self,request):
        """
        Attempts to extract keyword args from the raw data.  Returns None
        if no kwargs are to be had.
        """
        if self.data_parsing or not self.params: # data parsing descriptors ignore keyword args
            return {} # no params means no keyword args

        if request.GET:
            return request.GET.copy()
        elif request.POST:
            return request.POST.copy()
        elif parse_qs(request.raw_post_data):
            return QueryDict(request.raw_post_data)
        else:
            return {}

    def http_service(self,request,format='json'):
        """
        Services the request.
        """
        # 1. Check security
        self.security.check(request)

        # 2. Deserialize incoming data
        data = self.deserialize(request.raw_post_data,format)
        data = data or {} # set to empty dictionary if serializer returned nothing

        # 3. Get kwargs
        kwargs = self.extract_kwargs(request)

        # 4. Process params
        param_data = {}
        for param in self.params:
            if self.data_parsing:
                param_data[param.name] = param.get_from_dict(data) # extract params from data
            else:
                param_data[param.name] = param.get_from_dict(kwargs) # extract params from kwargs

        # 5. Execute service
        result = self.execute(request,data,param_data)

        # 6. Serialize result
        return self.serialize(result,format)

    @property
    def name(self):
        return self.__class__.__name__

    @property
    def slug(self):
        return slugify(self.__class__.__name__)

    @property
    def docs(self):
        lines = self.__doc__.splitlines(True)
        docstring = ''
        for line in lines:
            if line.isspace():
                docstring += line
            else:
                docstring += line.lstrip()

        return markdown.markdown(docstring)

    @property
    def docs_plain(self):
        return self.__doc__

#################
### Resources ###
#################
class MethodNotAllowed(Exception):
    """
    Represents a HTTP 405 Method Not Allowed response.
    Raised when an attempt to access an unimplemented method on
    a resource.
    """

class Resource(object):
    """
    A resource is a roll-up of services to provide RESTful functionality.
    This can be useful to clients that expect REST behavior.
    """
    response_codes = {'get':200,'post':201,'delete':200,'put':200}
    headers = {'json':{'Content-type':'application/json'},'xml':{'Content-type':'application/xml'}}

    def __init__(self,is_deprecated=None):
        """
        Constructor, set deprecation on member methods.
        """
        self.is_deprecated = is_deprecated
        if not self.is_deprecated:
            if hasattr(self,'deprecated'):
                self.is_deprecated = self.deprecated

        if self.is_deprecated: # deprecation from parent entity or local declaration
            for method_name in self.response_codes.keys():
                if hasattr(self,method_name):
                    getattr(self,method_name).is_deprecated = self.is_deprecated

    @property
    def name(self):
        return self.__class__.__name__

    @property
    def slug(self):
        return slugify(self.__class__.__name__)

    def http_service(self,request,format='json'):
        """
        Services the request.
        """
        # Check implementation
        self.check_method(request)

        # Get action method
        action_method_name = request.method.lower()
        action_method = getattr(self,action_method_name)

        # Action method executes http service
        serialized_result = action_method.http_service(request,format=format)

        # response headers and status
        response_headers = self.response_headers(request,format)
        status_code = self.status_code(request)

        return (status_code,response_headers,serialized_result)

    def check_method(self,request):
        """
        Checks if the HTTP method has been implemented.
        """
        method = request.method.lower()
        if not hasattr(self,method):
            raise MethodNotAllowed

    def response_headers(self,request,format='json'):
        """
        Generates a list of appropriate response headers for the http response.
        """
        return Resource.headers[format]

    def status_code(self,request):
        """
        Generates the appropriate success status code for the http response.
        """
        return Resource.response_codes[request.method.lower()]

# =======================
# = Miscellaneous Erros =
# =======================

class Conflict(Exception):
    """
    An exception indicating there is a user-resolvable problem with the function or resource being addressed.
    Should contain enough information so that the user can resolve the problem.
    """
<commit_msg>Fixing issue with list param processing.<commit_after>"""
Descriptors are definitions for functions.
"""
import markdown
from django.template.defaultfilters import slugify
from urlparse import parse_qs
from django.http import QueryDict
import logging

log = logging.getLogger('sharrock')

class MalformedDescriptor(Exception):
    """
    A programming error indicating a function descriptor that has been improperly
    formatted.
    """
    pass

class ParamRequired(Exception):
    """
    An exception indicating that a required param has not been supplied.
    """
    def __init__(self,param_name):
        self.param = param_name

    def __unicode__(self):
        return u'%s is a required parameter and has not been specified.' % unicode(self.param)

    def __str__(self):
        return unicode(self)

class Param(object):
    """
    A parameter for a function.
    """
    def __init__(self,name,required=False,default=None,description=None):
        self.name = name
        self.required = required
        self.default = default
        self.description = description

    def get_from_dict(self,raw_dict):
        """
        Gets param from raw dict.  If param is missing and required, raises
        ParamRequired, otherwise returns None.
        """
        raw_value = raw_dict.get(self.name,self.default)
        if raw_value is None and self.required:
            raise ParamRequired(self.name)
        elif raw_value is None:
            return None
        else:
            return self.process(raw_value)

    def process(self,raw):
        """
        Processes the raw value, returning the new value.
        """
        raise NotImplemented

#######################################################
### Below are Parameters to be used in Descriptors. ###
#######################################################

class UnicodeParam(Param):
    """
    A parameter with a unicode payload.
    """
    def process(self,raw):
        return unicode(raw)

    @property
    def type(self):
        return 'Unicode'

class IntegerParam(Param):
    """
    A param with an integer value.
    """
    def process(self,raw):
        return int(raw)

    @property
    def type(self):
        return 'Integer'

class FloatParam(Param):
    """
    A param with a float value.
    """
    def process(self,raw):
        return float(raw)

    @property
    def type(self):
        return 'Float'

class ListParam(Param):
    """
    A param with a list of values.
    """
    def __init__(self,name,item_param,**kwargs):
        super(ListParam,self).__init__(name,**kwargs)
        self.item_param = item_param

    def process(self,raw):
        return [self.item_param.process(raw_item) for raw_item in raw]

    @property
    def type(self):
        return 'List'

class DictParam(Param):
    """
    A param with a dictionary of values.
    """
    def __init__(self,name,params={},**kwargs):
        super(DictParam,self).__init__(name,**kwargs)
        self.param_dict = params.copy() # always copy mutable arguments

    def process(self,raw):
        log.debug('DictParam processing raw value: %s' % raw)
        if not self.param_dict:
            # if no params have been defined, treat as a wildcard dictionary - any key/value pairs are acceptable
            return raw.copy()
        else:
            return_dict = {}
            for param_key, param in self.param_dict.items():
                return_dict[param_key] = param.get_from_dict(raw)
            return return_dict

    @property
    def type(self):
        return 'Dictionary'

###################
### Serializers ###
###################
class UnsupportedSerializationFormat(Exception):
    """
    Indicates an unsupported serialization format.
    """
    pass

class Serializer(object):
    """
    Base class for serializers.
    """
    def serialize(self,python_object):
        """
        Serializes the object.  Returns the serialized form of the object.
        """
        raise NotImplemented

    def deserialize(self,serialized_object):
        """
        Deserialized the object.  Returns python object.
        """
        raise NotImplemented

import json

class JSONSerializer(Serializer):
    """
    JSON Serializer.
    """
    name = 'json'

    def serialize(self,python_object):
        if python_object:
            return json.dumps(python_object)
        else:
            return python_object

    def deserialize(self,serialized_object):
        log.debug('JSON Serializer loading serialized objects:%s' % serialized_object)
        if serialized_object:
            return json.loads(serialized_object)
        else:
            return serialized_object

################
### Security ###
################
class AccessDenied(Exception):
    """
    Indicates the user does not have permission to perform the task.
    """
    pass

class SecurityCheck(object):
    """
    Base class for security check.  This base class will allow any user, essentially
    making a public API.
    """
    def __init__(self,*permissions):
        self.permissions = permissions

    def check(self,request):
        """
        Checks the request for permission to access the function.  No return value.
        Raises an AccessDenied exception if the request fails the permission check.
        """
        pass # anyone allowed in base class.

###################
### Descriptors ###
###################

import re

def space_out_camel_case(stringAsCamelCase):
    """
    By Simon Hartley, posted on http://refactormycode.com/codes/675-camelcase-to-camel-case-python-newbie

    Adds spaces to a camel case string.  Failure to space out string returns the original string.
    >>> space_out_camel_case('DMLSServicesOtherBSTextLLC')
    'DMLS Services Other BS Text LLC'
    """

    if stringAsCamelCase is None:
        return None

    pattern = re.compile('([A-Z][A-Z][a-z])|([a-z][A-Z])')
    return pattern.sub(lambda m: m.group()[:1] + " " + m.group()[1:], stringAsCamelCase)

class DescriptorMetaclass(type):
    """
    Metaclass for descriptors.
    """
    def __new__(cls,name,bases,attrs):
        """
        Constructs a new descriptor class.
        """
        if name != 'Descriptor': # don't operate on the base class
            # New attribute dictionary
            new_attrs = {'serializer_dict':{},'params':[]}

            # Name
            if 'verbose_name' in attrs:
                new_attrs['service_name'] = attrs['verbose_name']
            else:
                new_attrs['service_name'] = space_out_camel_case(name)

            # Params group params
            for attribute_name, attribute_value in attrs.items():
                if isinstance(attribute_value,Param):
                    new_attrs['params'].append(attribute_value)
                elif isinstance(attribute_value,Serializer):
                    new_attrs['serializer_dict'][attribute_value.name] = attribute_value

            # If no serializers has been set, set default
            if not new_attrs['serializer_dict']:
                new_attrs['serializer_dict']['json'] = JSONSerializer()

            # If no security has been set, set default
            if not 'security' in attrs:
                new_attrs['security'] = SecurityCheck()

            # If data parsing flag has not been set, set to False
            if not 'data_parsing' in attrs:
                new_attrs['data_parsing'] = False

            # Deprecated message
            if not 'deprecated' in attrs:
                new_attrs['deprecated'] = None

            attrs.update(new_attrs)

        return type.__new__(cls,name,bases,attrs)

class Descriptor(object):
    """
    Base class for function descriptors.

    Specific descriptor classes should subclass this base class.  They can then
    define the following
    """
    # Based on metaclass activity - add:
    # serializer/deserializers
    # params, required and optional, default values
    # security hooks
    # API version

    __metaclass__ = DescriptorMetaclass # class factory mounts here

    def __init__(self,is_deprecated=None):
        """
        Constructor.  is_deprecated flag indicates if this is a deprecated API, either through
        declaration in the local class, or as inherited from a parent entity.
        """
        self.is_deprecated = is_deprecated # deprecation from parent entity

        if self.deprecated:
            self.is_deprecated = self.deprecated # deprecation occuring at local class level

    def serialize(self,python_object,format):
        """
        Serializes the object.
        """
        if python_object is None:
            return None
        try:
            serializer = self.serializer_dict[format]
            return serializer.serialize(python_object)
        except KeyError:
            raise UnsupportedSerializationFormat

    def deserialize(self,serialized_object,format):
        """
        Deserializes the object.
        """
        if serialized_object is None:
            return None
        try:
            serializer = self.serializer_dict[format]
            return serializer.deserialize(serialized_object)
        except KeyError:
            raise UnsupportedSerializationFormat

    def execute(self,request,data,params):
        """
        Executes the function.
        """
        raise NotImplemented # Subclasses implement

    def extract_kwargs(self,request):
        """
        Attempts to extract keyword args from the raw data.  Returns None
        if no kwargs are to be had.
        """
        if self.data_parsing or not self.params: # data parsing descriptors ignore keyword args
            return {} # no params means no keyword args

        if request.GET:
            return request.GET.copy()
        elif request.POST:
            return request.POST.copy()
        elif parse_qs(request.raw_post_data):
            return QueryDict(request.raw_post_data)
        else:
            return {}

    def http_service(self,request,format='json'):
        """
        Services the request.
        """
        # 1. Check security
        self.security.check(request)

        # 2. Deserialize incoming data
        data = self.deserialize(request.raw_post_data,format)
        data = data or {} # set to empty dictionary if serializer returned nothing

        # 3. Get kwargs
        kwargs = self.extract_kwargs(request)

        # 4. Process params
        param_data = {}
        for param in self.params:
            if self.data_parsing:
                param_data[param.name] = param.get_from_dict(data) # extract params from data
            else:
                param_data[param.name] = param.get_from_dict(kwargs) # extract params from kwargs

        # 5. Execute service
        result = self.execute(request,data,param_data)

        # 6. Serialize result
        return self.serialize(result,format)

    @property
    def name(self):
        return self.__class__.__name__

    @property
    def slug(self):
        return slugify(self.__class__.__name__)

    @property
    def docs(self):
        lines = self.__doc__.splitlines(True)
        docstring = ''
        for line in lines:
            if line.isspace():
                docstring += line
            else:
                docstring += line.lstrip()

        return markdown.markdown(docstring)

    @property
    def docs_plain(self):
        return self.__doc__

#################
### Resources ###
#################
class MethodNotAllowed(Exception):
    """
    Represents a HTTP 405 Method Not Allowed response.
    Raised when an attempt to access an unimplemented method on
    a resource.
    """

class Resource(object):
    """
    A resource is a roll-up of services to provide RESTful functionality.
    This can be useful to clients that expect REST behavior.
    """
    response_codes = {'get':200,'post':201,'delete':200,'put':200}
    headers = {'json':{'Content-type':'application/json'},'xml':{'Content-type':'application/xml'}}

    def __init__(self,is_deprecated=None):
        """
        Constructor, set deprecation on member methods.
        """
        self.is_deprecated = is_deprecated
        if not self.is_deprecated:
            if hasattr(self,'deprecated'):
                self.is_deprecated = self.deprecated

        if self.is_deprecated: # deprecation from parent entity or local declaration
            for method_name in self.response_codes.keys():
                if hasattr(self,method_name):
                    getattr(self,method_name).is_deprecated = self.is_deprecated

    @property
    def name(self):
        return self.__class__.__name__

    @property
    def slug(self):
        return slugify(self.__class__.__name__)

    def http_service(self,request,format='json'):
        """
        Services the request.
        """
        # Check implementation
        self.check_method(request)

        # Get action method
        action_method_name = request.method.lower()
        action_method = getattr(self,action_method_name)

        # Action method executes http service
        serialized_result = action_method.http_service(request,format=format)

        # response headers and status
        response_headers = self.response_headers(request,format)
        status_code = self.status_code(request)

        return (status_code,response_headers,serialized_result)

    def check_method(self,request):
        """
        Checks if the HTTP method has been implemented.
        """
        method = request.method.lower()
        if not hasattr(self,method):
            raise MethodNotAllowed

    def response_headers(self,request,format='json'):
        """
        Generates a list of appropriate response headers for the http response.
        """
        return Resource.headers[format]

    def status_code(self,request):
        """
        Generates the appropriate success status code for the http response.
        """
        return Resource.response_codes[request.method.lower()]

# =======================
# = Miscellaneous Erros =
# =======================

class Conflict(Exception):
    """
    An exception indicating there is a user-resolvable problem with the function or resource being addressed.
    Should contain enough information so that the user can resolve the problem.
    """
<|endoftext|>


<commit_before>import os, requests, json
from functools import wraps

from flask import Flask
from flask import request, redirect, url_for, flash, render_template

from portality import settings
from portality.error_handler import setup_error_logging
from flask.ext.login import LoginManager, current_user
login_manager = LoginManager()

def create_app():
    app = Flask(__name__)
    configure_app(app)
    if app.config['INITIALISE_INDEX']: initialise_index(app)
    setup_error_logging(app)
    setup_jinja(app)
    login_manager.setup_app(app)
    return app

def configure_app(app):
    app.config.from_object(settings)
    # parent directory
    here = os.path.dirname(os.path.abspath( __file__ ))
    config_path = os.path.join(os.path.dirname(here), 'app.cfg')
    if os.path.exists(config_path):
        app.config.from_pyfile(config_path)

    app.config['DOAJENV'] = get_app_env(app)
    config_path = os.path.join(os.path.dirname(here), app.config['DOAJENV'] + '.cfg')
    print 'Running in ' + app.config['DOAJENV']  # the app.logger is not set up yet (?)
    if os.path.exists(config_path):
        app.config.from_pyfile(config_path)
        print 'Loaded final config from ' + config_path

def get_app_env(app):
    if not app.config.get('VALID_ENVIRONMENTS'):
        raise Exception('VALID_ENVIRONMENTS must be set in the config. There shouldn\'t be a reason to change it in different set ups, or not have it.')

    env = os.getenv('DOAJENV')
    if not env:
        env = app.config.get('DOAJENV')

    if not env:
        raise Exception(
"""
Set the DOAJENV environment variable when running the app please, guessing is futile and fraught with peril.
DOAJENV=test python portality/app.py
to run the app will do.
Or use the supervisord options - put this in the config: environment= DOAJENV="test" .

Finally, the least preferred approach is to put DOAJENV="dev" in app.cfg in the root of the repo.
Only do this for dev environments so you don't have to bother specifying it each time.

Valid values are: {valid_doajenv_vals}

You can put environment-specific secret settings in <environment>.cfg , e.g. dev.cfg .

The environment specified in the DOAJENV environment variable will override that specified in the
application configuration (settings.py or app.cfg).
""".format(valid_doajenv_vals=', '.join(app.config['VALID_ENVIRONMENTS']))
        )

    return env

def initialise_index(app):
    mappings = app.config["MAPPINGS"]
    i = str(app.config['ELASTIC_SEARCH_HOST']).rstrip('/')
    i += '/' + app.config['ELASTIC_SEARCH_DB']
    for key, mapping in mappings.iteritems():
        im = i + '/' + key + '/_mapping'
        exists = requests.get(im)
        if exists.status_code != 200:
            ri = requests.post(i)
            r = requests.put(im, json.dumps(mapping))
            print key, r.status_code

def setup_jinja(app):
    '''Add jinja extensions and other init-time config as needed.'''

    app.jinja_env.add_extension('jinja2.ext.do')
    app.jinja_env.add_extension('jinja2.ext.loopcontrols')
    app.jinja_env.globals['getattr'] = getattr
    app.jinja_env.globals['type'] = type

    # a jinja filter that prints to the Flask log
    def jinja_debug(text):
        print text
        return ''
    app.jinja_env.filters['debug']=jinja_debug

app = create_app()

# misc stuff available for importing anywhere

# a decorator to be used elsewhere (or in this file) in the app,
# anywhere where a view f() should be served only over SSL
def ssl_required(fn):
    @wraps(fn)
    def decorated_view(*args, **kwargs):
        if app.config.get("SSL"):
            if request.is_secure:
                return fn(*args, **kwargs)
            else:
                return redirect(request.url.replace("http://", "https://"))

        return fn(*args, **kwargs)

    return decorated_view

def restrict_to_role(role):
    if current_user.is_anonymous():
        flash('You are trying to access a protected area. Please log in first.', 'error')
        return redirect(url_for('account.login', next=request.url))

    if not current_user.has_role(role):
        flash('You do not have permission to access this area of the site.', 'error')
        return redirect(url_for('doaj.home'))

def write_required(fn):
    @wraps(fn)
    def decorated_view(*args, **kwargs):
        if app.config.get("READ_ONLY_MODE", False):
            return render_template("doaj/readonly.html")

        return fn(*args, **kwargs)

    return decorated_view
<commit_msg>prevent initialise_index from touching the index during startup if in read-only mode<commit_after>import os, requests, json
from functools import wraps

from flask import Flask
from flask import request, redirect, url_for, flash, render_template

from portality import settings
from portality.error_handler import setup_error_logging
from flask.ext.login import LoginManager, current_user
login_manager = LoginManager()

def create_app():
    app = Flask(__name__)
    configure_app(app)
    if app.config['INITIALISE_INDEX']: initialise_index(app)
    setup_error_logging(app)
    setup_jinja(app)
    login_manager.setup_app(app)
    return app

def configure_app(app):
    app.config.from_object(settings)
    # parent directory
    here = os.path.dirname(os.path.abspath( __file__ ))
    config_path = os.path.join(os.path.dirname(here), 'app.cfg')
    if os.path.exists(config_path):
        app.config.from_pyfile(config_path)

    app.config['DOAJENV'] = get_app_env(app)
    config_path = os.path.join(os.path.dirname(here), app.config['DOAJENV'] + '.cfg')
    print 'Running in ' + app.config['DOAJENV']  # the app.logger is not set up yet (?)
    if os.path.exists(config_path):
        app.config.from_pyfile(config_path)
        print 'Loaded final config from ' + config_path

def get_app_env(app):
    if not app.config.get('VALID_ENVIRONMENTS'):
        raise Exception('VALID_ENVIRONMENTS must be set in the config. There shouldn\'t be a reason to change it in different set ups, or not have it.')

    env = os.getenv('DOAJENV')
    if not env:
        env = app.config.get('DOAJENV')

    if not env:
        raise Exception(
"""
Set the DOAJENV environment variable when running the app please, guessing is futile and fraught with peril.
DOAJENV=test python portality/app.py
to run the app will do.
Or use the supervisord options - put this in the config: environment= DOAJENV="test" .

Finally, the least preferred approach is to put DOAJENV="dev" in app.cfg in the root of the repo.
Only do this for dev environments so you don't have to bother specifying it each time.

Valid values are: {valid_doajenv_vals}

You can put environment-specific secret settings in <environment>.cfg , e.g. dev.cfg .

The environment specified in the DOAJENV environment variable will override that specified in the
application configuration (settings.py or app.cfg).
""".format(valid_doajenv_vals=', '.join(app.config['VALID_ENVIRONMENTS']))
        )

    return env

def initialise_index(app):
    if app.config.get("READ_ONLY_MODE", False):
        app.logger.warn("System is in READ-ONLY mode, initialise_index command cannot run")
        return

    mappings = app.config["MAPPINGS"]
    i = str(app.config['ELASTIC_SEARCH_HOST']).rstrip('/')
    i += '/' + app.config['ELASTIC_SEARCH_DB']
    for key, mapping in mappings.iteritems():
        im = i + '/' + key + '/_mapping'
        exists = requests.get(im)
        if exists.status_code != 200:
            ri = requests.post(i)
            r = requests.put(im, json.dumps(mapping))
            print key, r.status_code

def setup_jinja(app):
    '''Add jinja extensions and other init-time config as needed.'''

    app.jinja_env.add_extension('jinja2.ext.do')
    app.jinja_env.add_extension('jinja2.ext.loopcontrols')
    app.jinja_env.globals['getattr'] = getattr
    app.jinja_env.globals['type'] = type

    # a jinja filter that prints to the Flask log
    def jinja_debug(text):
        print text
        return ''
    app.jinja_env.filters['debug']=jinja_debug

app = create_app()

# misc stuff available for importing anywhere

# a decorator to be used elsewhere (or in this file) in the app,
# anywhere where a view f() should be served only over SSL
def ssl_required(fn):
    @wraps(fn)
    def decorated_view(*args, **kwargs):
        if app.config.get("SSL"):
            if request.is_secure:
                return fn(*args, **kwargs)
            else:
                return redirect(request.url.replace("http://", "https://"))

        return fn(*args, **kwargs)

    return decorated_view

def restrict_to_role(role):
    if current_user.is_anonymous():
        flash('You are trying to access a protected area. Please log in first.', 'error')
        return redirect(url_for('account.login', next=request.url))

    if not current_user.has_role(role):
        flash('You do not have permission to access this area of the site.', 'error')
        return redirect(url_for('doaj.home'))

def write_required(fn):
    @wraps(fn)
    def decorated_view(*args, **kwargs):
        if app.config.get("READ_ONLY_MODE", False):
            return render_template("doaj/readonly.html")

        return fn(*args, **kwargs)

    return decorated_view
<|endoftext|>


<commit_before># (C) Datadog, Inc. 2018-present
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)

from codecs import open
from os import path

from setuptools import setup

HERE = path.abspath(path.dirname(__file__))

# Get version info
ABOUT = {}
with open(path.join(HERE, "datadog_checks", "postgres", "__about__.py")) as f:
    exec(f.read(), ABOUT)

# Get the long description from the README file
with open(path.join(HERE, 'README.md'), encoding='utf-8') as f:
    long_description = f.read()

def get_dependencies():
    dep_file = path.join(HERE, 'requirements.in')
    if not path.isfile(dep_file):
        return []

    with open(dep_file, encoding='utf-8') as f:
        return f.readlines()

CHECKS_BASE_REQ = 'datadog-checks-base>=16.5.0'

setup(
    name='datadog-postgres',
    version=ABOUT["__version__"],
    description='The Postgres check',
    long_description=long_description,
    long_description_content_type='text/markdown',
    keywords='datadog agent postgres check',
    url='https://github.com/DataDog/integrations-core',
    author='Datadog',
    author_email='packages@datadoghq.com',
    license='BSD',
    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Developers',
        'Intended Audience :: System Administrators',
        'Topic :: System :: Monitoring',
        'License :: OSI Approved :: BSD License',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.7',
    ],
    packages=['datadog_checks.postgres'],
    # Run-time dependencies
    install_requires=[CHECKS_BASE_REQ],
    extras_require={'deps': get_dependencies()},
    # Extra files to ship with the wheel package
    include_package_data=True,
)
<commit_msg>postgres: bump datadog_checks_base min version (#8777)<commit_after># (C) Datadog, Inc. 2018-present
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)

from codecs import open
from os import path

from setuptools import setup

HERE = path.abspath(path.dirname(__file__))

# Get version info
ABOUT = {}
with open(path.join(HERE, "datadog_checks", "postgres", "__about__.py")) as f:
    exec(f.read(), ABOUT)

# Get the long description from the README file
with open(path.join(HERE, 'README.md'), encoding='utf-8') as f:
    long_description = f.read()

def get_dependencies():
    dep_file = path.join(HERE, 'requirements.in')
    if not path.isfile(dep_file):
        return []

    with open(dep_file, encoding='utf-8') as f:
        return f.readlines()

CHECKS_BASE_REQ = 'datadog-checks-base>=16.6.0'

setup(
    name='datadog-postgres',
    version=ABOUT["__version__"],
    description='The Postgres check',
    long_description=long_description,
    long_description_content_type='text/markdown',
    keywords='datadog agent postgres check',
    url='https://github.com/DataDog/integrations-core',
    author='Datadog',
    author_email='packages@datadoghq.com',
    license='BSD',
    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Developers',
        'Intended Audience :: System Administrators',
        'Topic :: System :: Monitoring',
        'License :: OSI Approved :: BSD License',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.7',
    ],
    packages=['datadog_checks.postgres'],
    # Run-time dependencies
    install_requires=[CHECKS_BASE_REQ],
    extras_require={'deps': get_dependencies()},
    # Extra files to ship with the wheel package
    include_package_data=True,
)
<|endoftext|>


<commit_before># define a function that performs the post-processing of the posterior values and creates a results page
import numpy  as np

# a function to get the credible intervals using a greedy binning method
def credible_interval(dsamples, ci):
   n, binedges = np.histogram(dsamples, bins=250)
   dbins = binedges[1]-binedges[0] # width of a histogram bin
   bins = binedges[0:-1]+dbins/2. # centres of bins

   histIndices=np.argsort(n)[::-1]  # indices of the points of the histogram in decreasing order

   frac = 0.0
   j = 0
   for i in histIndices:
     frac += float(n[i])/float(len(dsamples))
     j = j+1
     if frac >= ci:
       break

   return (np.min(bins[histIndices[:j]]), np.max(bins[histIndices[:j]]))

def postprocessing(postsamples, variables, abscissa, data, email, outdir):
  # import the triangle plot code
  import triangle

  # import matplotlib
  from matplotlib import pyplot as pl

  # the format text to include in the page
  fm = {}

  # the string containing the webpage
  htmlpage = """
<!DOCTYPE HTML>

MCMC output page

Marginalised posteriors

{posteriorfig}

Best fit model distribution

{bestfitfig}

"""

  # create triangle plot
  labels = ['$%s$' % var for var in variables.split(',')] # LaTeX labels
  nvars = len(variables.split(','))
  levels = 1.-np.exp(-0.5*np.array([1., 2.])**2) # plot 1 and 2 sigma contours

  fig = triangle.corner(postsamples[:nvars,:], labels=labels, levels=levels)

  # output the figure
  postfigfile = 'posterior_plots.png'
  fig.savefig(postfigfile)

  fm['posteriorfig'] = ''

  # get the 68% and 95% credible intervals
  inter68 = []
  inter95 = []
  for i in range(nvars):
    inter68.append(credible_interval(postsamples[i,:], 0.68))
    inter95.append(credible_interval(postsamples[i,:], 0.95))

  # get the correlation coefficient matrix
  corrcoef = np.corrcoef(postsamples[:nvars,:])

  # create plot of data along with distribution of best fit models
  from mymodel import mymodel

  # load data
  datafile = 'data_file.txt'
  try:
    data = np.loadtxt(datafile)
  except:
    try:
      data = np.loadtxt(datafile)
    except:
      # this should work as the file must have been read in for the MCMC to run

  # load abscissa
  absfile = 'abscissa_file.txt'
  try:
    xdata = np.loadtxt(absfile)
  except:
    try:
      xdata = np.loadtxt(absfile)
    except:
      # this should work as the file must have been read in for the MCMC to run

  fig2 = pl.figure()
  pl.plot(xdata, data, 'k.', ms=1, label='Data')
  varidxs = range(nvars)
  if 'sigma' in variables:
    sigmaidx = variables.index('sigma')
    varidxs.remove(sigmaidx)
  varidxs = np.array(varidxs)

  randidxs = np.random.permutation(len(postsamples))[0:100]

  # overplot models for 100 random draws from the posterior
  for i in range(100):
    thesevars = postsamples[varidxs, randidxs[i]].tolist()
    thesevars.append(xdata)
    thismodel = mymodel(*thesevars) # unpack list as arguments of model function
    pl.plot(xdata, thismodel, '-', color='mediumblue', lw=3, alpha=0.05)

  pl.legend(loc='best')
  modelplot = 'model_plot.png'
  fig2.savefig(modelplot)

  fm['bestfitfig'] = ''

  # output page
  ppfile = 'postprocessing.html'
  fp = open(ppfile)
  fp.write(htmlpage.format(**fm))
  fp.close()

  # email the page
  import emailrepsonse
  emailresponse.emailrepsonse(email, outdir, ppfile)<commit_msg>postprocessing.py: add ouptut of more statistics<commit_after># define a function that performs the post-processing of the posterior values and creates a results page
import numpy  as np

# convert a floating point number into a string in X.X x 10^Z format
def exp_str(f, p=1):
  if p > 16:
    print >> sys.stderr, "Precision must be less than 16 d.p."
    p = 16

  s = '%.16e' % f
  ssplit = s.split('e')
  return '%.*f×10%d' % (p, float(ssplit[0]), int(ssplit[1]))

# a function to get the credible intervals using a greedy binning method
def credible_interval(dsamples, ci):
   n, binedges = np.histogram(dsamples, bins=250)
   dbins = binedges[1]-binedges[0] # width of a histogram bin
   bins = binedges[0:-1]+dbins/2. # centres of bins

   histIndices=np.argsort(n)[::-1]  # indices of the points of the histogram in decreasing order

   frac = 0.0
   j = 0
   for i in histIndices:
     frac += float(n[i])/float(len(dsamples))
     j = j+1
     if frac >= ci:
       break

   return (np.min(bins[histIndices[:j]]), np.max(bins[histIndices[:j]]))

def postprocessing(postsamples, variables, abscissa, data, email, outdir):
  # import the triangle plot code
  import triangle

  # import matplotlib
  from matplotlib import pyplot as pl

  # the format text to include in the page
  fm = {}

  # the string containing the webpage
  htmlpage = """
<!DOCTYPE HTML>

MCMC output page

Marginalised posteriors

{posteriorfig}

Best fit values

{resultstable}

{corrcoeftable}

Best fit model distribution

{bestfitfig}

"""

  varnames = variables.split(',')

  # create triangle plot
  labels = ['$%s$' % var for var in variables.split(',')] # LaTeX labels
  nvars = len(variables.split(','))
  levels = 1.-np.exp(-0.5*np.array([1., 2.])**2) # plot 1 and 2 sigma contours

  fig = triangle.corner(postsamples[:nvars,:], labels=labels, levels=levels)

  # output the figure
  postfigfile = 'posterior_plots.png'
  fig.savefig(postfigfile)

  fm['posteriorfig'] = ''

  # get the 68% and 95% credible intervals
  inter68 = []
  inter95 = []
  for i in range(nvars):
    inter68.append(credible_interval(postsamples[i,:], 0.68))
    inter95.append(credible_interval(postsamples[i,:], 0.95))

  # output the results table
  resultstable = "
\n
Variable	Mean	Median	Mode	σ	68%% CI	95%% CI
\n"
  for i in range(nvars):
    resstrs = [varnames[i]]

    meanv = np.mean(postsamples[i,:])
    if meanv > 1e3 or meanv < 1e-2:
      meanstr = exp_str(meanv)
    else:
      meanstr = '%.1f' % meanv
    resstrs.append(menastr)

    medianv = np.median(postsamples[i,:])
    if medianv > 1e3 or medianv < 1e-2:
      medianstr = exp_str(medianv)
    else:
      medianstr = '%.1f' % medianv
    resstrs.append(medianstr)

    modev = postsamples[i,np.argmax(postsamples[i,:])]
    if modev > 1e3 or modev < 1e-2:
      modestr = exp_str(modev)
    else:
      modestr = '%.1f' % modev
    resstrs.append(modestr)

    sigmav = np.std(postsamples[i,:])
    if sigmav > 1e3 or sigmav < 1e-2:
      sigmastr = exp_str(sigmav)
    else:
      sigmastr = '%.1f' % sigmav
    resstrs.append(sigmastr)

    ci68str = "[{0}, {1}]"
    if inter68[i][0] > 1e3 or inter68[i][0] < 1e-2:
      cis1 = exp_str(inter68[i][0])
    else:
      cis1 = '%.1f' % inter68[i][1]
    if inter68[i][1] > 1e3 or inter68[i][1] < 1e-2:
      cis2 = exp_str(inter68[i][1])
    else:
      cis2 = '%.1f' % inter68[i][1]
    resstrs.append(ci68str.format(cis1, cis2))

    ci95str = "[{0}, {1}]"
    if inter95[i][0] > 1e3 or inter95[i][0] < 1e-2:
      cis1 = exp_str(inter95[i][0])
    else:
      cis1 = '%.1f' % inter95[i][1]
    if inter95[i][1] > 1e3 or inter95[i][1] < 1e-2:
      cis2 = exp_str(inter95[i][1])
    else:
      cis2 = '%.1f' % inter95[i][1]
    resstrs.append(ci95str.format(cis1, cis2))

    resultstable += "
{0}	{1}	{2}	{3}	{4}	{5}	{6}
".format(resstrs)
  resultstable += "
\n"

  fm['resultstable'] = resultstable

  # get the correlation coefficient matrix
  corrcoef = np.corrcoef(postsamples[:nvars,:])

  corrcoeftable = "
	"
  for i in range(nvars):
    corrcoeftable += "%s	" % varnames[i]
  corrcoeftable += "
\n"
  for i in range(nvars):
    corrcoeftable += "
%s	" % varnames[i]
    for j in range(nvars):
      corrcoeftable += "%.2f	" % corrcoef[i,j]
    corrcoeftable += "
\n"
  corrcoeftable += "
\n"

  # create plot of data along with distribution of best fit models
  from mymodel import mymodel

  # load data
  datafile = 'data_file.txt'
  try:
    data = np.loadtxt(datafile)
  except:
    try:
      data = np.loadtxt(datafile)
    except:
      # this should work as the file must have been read in for the MCMC to run

  # load abscissa
  absfile = 'abscissa_file.txt'
  try:
    xdata = np.loadtxt(absfile)
  except:
    try:
      xdata = np.loadtxt(absfile)
    except:
      # this should work as the file must have been read in for the MCMC to run

  fig2 = pl.figure()
  pl.plot(xdata, data, 'k.', ms=1, label='Data')
  varidxs = range(nvars)
  if 'sigma' in variables:
    sigmaidx = variables.index('sigma')
    varidxs.remove(sigmaidx)
  varidxs = np.array(varidxs)

  randidxs = np.random.permutation(len(postsamples))[0:100]

  # overplot models for 100 random draws from the posterior
  for i in range(100):
    thesevars = postsamples[varidxs, randidxs[i]].tolist()
    thesevars.append(xdata)
    thismodel = mymodel(*thesevars) # unpack list as arguments of model function
    pl.plot(xdata, thismodel, '-', color='mediumblue', lw=3, alpha=0.05)

  pl.legend(loc='best')
  modelplot = 'model_plot.png'
  fig2.savefig(modelplot)

  fm['bestfitfig'] = ''

  # output page
  ppfile = 'postprocessing.html'
  fp = open(ppfile)
  fp.write(htmlpage.format(**fm))
  fp.close()

  # email the page
  import emailrepsonse
  emailresponse.emailrepsonse(email, outdir, ppfile)<|endoftext|>


<commit_before>import json
from .coda_aware import CODA_Aware
from datetime import datetime

def datetime_to_filter_string(dt, cmp):
    """Create filter string from datetime."""
    ret = ""
    if dt is not None:
        ret = "IngestionDate " + cmp + dt.strftime(" datetime'%Y-%m-%dT%I:%M:%S'")
    return ret

def geom_to_filter_string(bbox):
    """Create filter string from GML polygon."""
    if bbox is not None:
        res = 'footprint:"Intersects({})"'.format(bbox)
    else:
        res = ""
    return ""

class Query(CODA_Aware):
    """Find CODA products based on search criteria."""

    def __init__(self, time_start, time_end, intersect_geom, prod_type):
        """
        Construct OData REST query for CODA to find matching products.

        :param time_start: datetime of earliest data to be returned
        :param time_end: datetime of most recent data to be returned
        :param intersect_geom: Intersecting geometry in WGS84. Disabled.
        :param prod_type: Product type substring as defined in CODA user guide

        intersect_geom is currently not used due to questions concerning the
        API.
        """
        self.time_start = time_start
        self.time_end = time_end
        self.prod_type = prod_type
        self.geom_filter = intersect_geom
        if not self._prod_type_is_valid():
            raise ()
        self.uuids = []

    def _prod_type_is_valid(self):
        return True

    def get_uuids(self):
        product_filter = "substringof('" + self.prod_type + "',Name)"

        ge_filter = datetime_to_filter_string(self.time_start, "ge")
        le_filter = datetime_to_filter_string(self.time_end, "le")
        if ge_filter and le_filter:
            time_filter = ge_filter + " and " + le_filter
        elif ge_filter or le_filter:
            time_filter = ge_filter + le_filter
        if time_filter:
            time_filter = " and " + time_filter

        geom_filter = geom_to_filter_string(self.geom_filter)
        if geom_filter:
            geom_filter = " and " + geom_filter

        query = "Products?&$filter=" + product_filter + time_filter\
                + geom_filter + "&$format=json"
        print(query)
        found_results = [product["Id"] for product in json.loads(self.query(
            query).text)["d"]["results"]]
        return found_results
    # iterate over results and return when empty becaues of query limits

if __name__ == '__main__':
    q1 = Query(datetime(2017, 2, 2, 1), datetime(2017, 2, 2, 2), "foo", "SR_2_WAT")
    q2 = Query(datetime(2017, 2, 1), datetime(2017, 2, 2), "SR_2_WAT")
    print(len(q1.get_uuids()))
    print(len(q2.get_uuids()))

    q1 = Query(datetime(2017, 2, 2, 1), datetime(2017, 2, 2, 2), None, "SR_2_WAT")
    q1 = Query(datetime(2017, 2, 2, 1), datetime(2017, 2, 2, 2), polysad, "SR_2_WAT")
    print(len(q1.get_uuids()))
<commit_msg>Allow finding all UUIDs for query<commit_after>import json
from .coda_aware import CODA_Aware
from datetime import datetime

def datetime_to_filter_string(dt, cmp):
    """Create filter string from datetime."""
    ret = ""
    if dt is not None:
        ret = "IngestionDate " + cmp + dt.strftime(" datetime'%Y-%m-%dT%I:%M:%S'")
    return ret

def geom_to_filter_string(bbox):
    """Create filter string from GML polygon."""
    if bbox is not None:
        res = 'footprint:"Intersects({})"'.format(bbox)
    else:
        res = ""
    return ""

class Query(CODA_Aware):
    """Find CODA products based on search criteria."""

    def __init__(self, prod_type, time_start, time_end, intersect_geom):
        """
        Construct OData REST query for CODA to find matching products.

        :param time_start: datetime of earliest data to be returned
        :param time_end: datetime of most recent data to be returned
        :param intersect_geom: Intersecting geometry in WGS84. Disabled.
        :param prod_type: Product type substring as defined in CODA user guide

        intersect_geom is currently not used due to questions concerning the
        API.
        """
        self.time_start = time_start
        self.time_end = time_end
        self.prod_type = prod_type
        self.geom_filter = intersect_geom
        self.uuids = []

    def get_uuids(self, limit=None):
        product_filter = "substringof('" + self.prod_type + "',Name)"

        ge_filter = datetime_to_filter_string(self.time_start, "ge")
        le_filter = datetime_to_filter_string(self.time_end, "le")
        if ge_filter and le_filter:
            time_filter = ge_filter + " and " + le_filter
        elif ge_filter or le_filter:
            time_filter = ge_filter + le_filter
        if time_filter:
            time_filter = " and " + time_filter

        geom_filter = geom_to_filter_string(self.geom_filter)
        if geom_filter:
            geom_filter = " and " + geom_filter

        filters = product_filter + time_filter + geom_filter

        results = []
        found_results = [True]
        skip = 0
        # We iterate because API only returns limited number of UUIDs
        while found_results:
            query = "Products?$skip={}&$filter={}&$format=json".format(
                skip, filters)
            json_results = json.loads(self.query(query).text)
            try:
                found_results = [product["Id"] for product in json_results[
                    "d"]["results"]]
            except KeyError:
                found_results = []
            results += found_results
            skip += len(found_results)
            if limit is not None:
                if skip > limit:
                    found_results = []
        return results

if __name__ == '__main__':
    q = Query("SR_2_WAT", datetime(2017, 2, 1), datetime(2017, 2, 2), None)
    q = Query("OL_1", datetime(2017, 2, 1), datetime(2017, 2, 3), None)
    q.get_uuids()
<|endoftext|>


<commit_before>from requests import Session as _Session
from requests.exceptions import ConnectionError, ChunkedEncodingError, Timeout, HTTPError
from requests.adapters import HTTPAdapter
import logging
import time
from .cookiejar import ClientCookieJar

try:
    from fake_useragent import UserAgent
except ImportError:
    UserAgent = None
    ua = None
    ua_str = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' \
        'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
else:
    ua = UserAgent(fallback='Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
                   'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36')
    ua_str = ua.chrome

session_logger = logging.getLogger('showroom.session')

class ClientSession(_Session):
    """
    Wrapper for requests.Session.

    Mainly used to catch temporary errors and set a Timeout

    Overrides requests.Session.get() and increases max pool size

    Raises:
        May raise TimeoutError, ConnectionError, HTTPError, or ChunkedEncodingError
        if retries are exceeded.
    """

    # TODO: set pool_maxsize based on config
    def __init__(self, pool_maxsize=100):
        super().__init__()
        self.cookies = ClientCookieJar()
        https_adapter = HTTPAdapter(pool_maxsize=pool_maxsize)
        self.mount('https://www.showroom-live.com', https_adapter)
        self.headers = {"User-Agent": ua_str}

    # TODO: post
    def get(self, url, params=None, max_delay=30.0, max_retries=20, **kwargs):
        error_count = 0
        wait = 0
        timeouts = 0
        while True:
            try:
                r = super().get(url, params=params, timeout=(3.0, 15.0), **kwargs)
                r.raise_for_status()
            except Timeout as e:
                session_logger.debug('Timeout while fetching {}: {}'.format(url, e))
                timeouts += 1
                wait = min(2 * 1.5 ** timeouts, max_delay)

                if timeouts > max_retries:
                    session_logger.error('Max timeouts exceeded while fetching {}: {}'.format(url, e))
                    raise
                elif timeouts > max_retries // 2:
                    session_logger.warning('{} timeouts while fetching {}: {}'.format(timeouts, url, e))

            except ChunkedEncodingError as e:
                session_logger.debug('Chunked encoding error while fetching {}: {}'.format(url, e))
                error_count += 1
                wait = min(wait + error_count, max_delay)

                if error_count > max_retries:
                    session_logger.warning('Max retries exceeded while fetching {}: {}'.format(url, e))
                    raise

            except HTTPError as e:
                status_code = e.response.status_code
                session_logger.debug('{} while fetching {}: {}'.format(status_code, url, e))

                error_count += 1
                wait = min(wait + 2 + error_count, max_delay)

                # Some of these aren't recoverable
                if status_code == 404:
                    session_logger.error('Getting {} failed permanently: 404 page not found'.format(url))
                    raise  # PageNotFoundError(e)  # ?
                elif status_code == 403:
                    session_logger.error('Getting {} failed permanently: 403 permission denied'.format(url))
                    raise  # specific error?
                elif status_code == 402:
                    session_logger.error('Getting {} failed permanently: '
                                         '401 auth required (not implemented)'.format(url))
                    raise
                elif status_code == 429:
                    session_logger.error('Too many requests while getting {}: {}'.format(url, e))
                    wait += 5 * 60.0
                elif 400 <= status_code < 500:
                    session_logger.error('Getting {} failed permanently: {}'.format(url, e))
                    raise

                if error_count > max_retries:
                    session_logger.warning('Max retries exceeded while fetching {}: {}'.format(url, e))
                    raise

            except ConnectionError as e:
                session_logger.debug('ConnectionError while accessing {}: {}'.format(url, e))

                error_count += 1
                wait = min(wait + 2 * error_count, max_delay)

                # ConnectionErrors are assumed to be always recoverable
                # if error_count > max_retries:
                #     session_logger.warning('Max retries exceeded while fetching {}: {}'.format(url, e))
                #     raise

            else:
                return r

            session_logger.debug('Retrying in {} seconds...'.format(wait))
            time.sleep(wait)<commit_msg>Updating agent<commit_after>from requests import Session as _Session
from requests.exceptions import ConnectionError, ChunkedEncodingError, Timeout, HTTPError
from requests.adapters import HTTPAdapter
import logging
import time
from .cookiejar import ClientCookieJar

#try:
#    from fake_useragent import UserAgent
#except ImportError:
#    UserAgent = None
#    ua = None
ua_str = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
#else:
#    ua = UserAgent(fallback='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36')
#    ua_str = ua.chrome

session_logger = logging.getLogger('showroom.session')

class ClientSession(_Session):
    """
    Wrapper for requests.Session.

    Mainly used to catch temporary errors and set a Timeout

    Overrides requests.Session.get() and increases max pool size

    Raises:
        May raise TimeoutError, ConnectionError, HTTPError, or ChunkedEncodingError
        if retries are exceeded.
    """

    # TODO: set pool_maxsize based on config
    def __init__(self, pool_maxsize=100):
        super().__init__()
        self.cookies = ClientCookieJar()
        https_adapter = HTTPAdapter(pool_maxsize=pool_maxsize)
        self.mount('https://www.showroom-live.com', https_adapter)
        self.headers = {"User-Agent": ua_str}

    # TODO: post
    def get(self, url, params=None, max_delay=30.0, max_retries=20, **kwargs):
        error_count = 0
        wait = 0
        timeouts = 0
        while True:
            try:
                r = super().get(url, params=params, timeout=(3.0, 15.0), **kwargs)
                r.raise_for_status()
            except Timeout as e:
                session_logger.debug('Timeout while fetching {}: {}'.format(url, e))
                timeouts += 1
                wait = min(2 * 1.5 ** timeouts, max_delay)

                if timeouts > max_retries:
                    session_logger.error('Max timeouts exceeded while fetching {}: {}'.format(url, e))
                    raise
                elif timeouts > max_retries // 2:
                    session_logger.warning('{} timeouts while fetching {}: {}'.format(timeouts, url, e))

            except ChunkedEncodingError as e:
                session_logger.debug('Chunked encoding error while fetching {}: {}'.format(url, e))
                error_count += 1
                wait = min(wait + error_count, max_delay)

                if error_count > max_retries:
                    session_logger.warning('Max retries exceeded while fetching {}: {}'.format(url, e))
                    raise

            except HTTPError as e:
                status_code = e.response.status_code
                session_logger.debug('{} while fetching {}: {}'.format(status_code, url, e))

                error_count += 1
                wait = min(wait + 2 + error_count, max_delay)

                # Some of these aren't recoverable
                if status_code == 404:
                    session_logger.error('Getting {} failed permanently: 404 page not found'.format(url))
                    raise  # PageNotFoundError(e)  # ?
                elif status_code == 403:
                    session_logger.error('Getting {} failed permanently: 403 permission denied'.format(url))
                    raise  # specific error?
                elif status_code == 402:
                    session_logger.error('Getting {} failed permanently: '
                                         '401 auth required (not implemented)'.format(url))
                    raise
                elif status_code == 429:
                    session_logger.error('Too many requests while getting {}: {}'.format(url, e))
                    wait += 5 * 60.0
                elif 400 <= status_code < 500:
                    session_logger.error('Getting {} failed permanently: {}'.format(url, e))
                    raise

                if error_count > max_retries:
                    session_logger.warning('Max retries exceeded while fetching {}: {}'.format(url, e))
                    raise

            except ConnectionError as e:
                session_logger.debug('ConnectionError while accessing {}: {}'.format(url, e))

                error_count += 1
                wait = min(wait + 2 * error_count, max_delay)

                # ConnectionErrors are assumed to be always recoverable
                # if error_count > max_retries:
                #     session_logger.warning('Max retries exceeded while fetching {}: {}'.format(url, e))
                #     raise

            else:
                return r

            session_logger.debug('Retrying in {} seconds...'.format(wait))
            time.sleep(wait)<|endoftext|>


<commit_before>from __future__ import print_function

from contextlib import contextmanager

import six

from .adaptiverun import adaptiverun
from .datatypes import ArgsAndSetup
from .report import generate_table
from .utils import repr_is_constructor

_stimeit_current_function = None
_dummy = object()

@contextmanager
def current_function(f):
    global _stimeit_current_function
    _stimeit_current_function = f
    yield
    _stimeit_current_function = None

class SimpleTimeIt:

    def __init__(self):
        """Initializes a SimpleTimeIt object."""
        self._functions = []
        self._args_setups = []

    def call_with(self, args, setup='pass'):
        """Resisters a string as the argument list to the functions
        to be called as part of the performance comparisons. For instance, if
        a SimpleTimeIt is specified as follows:

            from simpletimeit.stimeit import SimpleTimeIt

            st = SimpleTimeIt()
            st.call_with("'Eric', 3, x=10")

            @st.time_this
            foo(name, n, x=15):
                ...

        Then `st.run()` will call (the equivalent of)

            exec('foo({args})'.format(args="'Eric', 3, x=10"))

        If 'args' is not a string, `call_with` will try to "do the right
        thing" and convert it to a string. If that string, when executed, will
        not render value equal to 'args', this method will throw an error. So,
        for instance, `10` and `{a: 10, b: 15}` will work because
        `10 == eval('10')` and `{a: 10, b: 15} == eval('{a: 10, b: 15}')`, but
        `range(10)` will not work because `range(10) != eval('range(10)').

        Takes an optional 'setup' argument. This string will be evaluated
        before the timing runs, as with the 'setup' argument to Timer. This
        value is 'pass' by default.
        """
        valid = (isinstance(args, six.string_types)
                 or repr_is_constructor(args))
        if not valid:
            raise ValueError(
                ('{args} will be passed to a format string, which will then '
                 'be executed as Python code. Thus, arguments must either be '
                 'a string to be evaluated as the arguments to the timed '
                 'function, or be a value whose string representation '
                 'constructs an identical object. see the `call_with` '
                 'documentation for more details.').format(args=args))
        if not (isinstance(setup, six.string_types) or callable(setup)):
            raise ValueError(
                "'setup' argument must be a string or callable.")

        self._args_setups.append(ArgsAndSetup(args=str(args), setup=setup))

    def time_this(self, f):
        """A decorator. Registers the decorated function as a TimedFunction
        with this SimpleTimeIt, leaving the function unchanged.
        """
        self._functions.append(f)
        return f

    def _prepared_setup(self, setup, func):
        """Generates the setup call for a given timing run."""
        setup_template = (
            'from simpletimeit.stimeit import _stimeit_current_function\n'
            '{setup}')
        if isinstance(setup, six.string_types):
            return setup_template.format(setup=setup)
        elif callable(setup):
            def prepared_setup_callable():
                global _stimeit_current_function
                _stimeit_current_function = func
                setup()
            return prepared_setup_callable
        else:
            raise ValueError("'setup' must be a string or callable")

    def _run_setup_and_func_with_args(self, setup, func, args):
        # assumes args == eval(str(args)) or that args is a string
        # (this property checked in call_with)
        stmt_template = '_stimeit_current_function({args})'
        with current_function(func):
            return adaptiverun(stmt=stmt_template.format(args=args),
                               setup=self._prepared_setup(setup, func),
                               title=repr(args))

    def run(self, report_function=generate_table, as_string=False):
        """Runs each of the functions registered with this SimpleTimeIt using
        each arguments-setup pair registered with this SimpleTimeIt.

        report_function should take a list of objects conforming to the
        Report API and return a string reporting on the comparison.

        If as_string is True, this function returns the table or tables
        generated as a string. Otherwise, it prints the tables to stdout and
        returns None."""
        out = six.StringIO() if as_string else None

        for x in self._args_setups:
            args, setup = x.args, x.setup
            results = []
            title = 'args: {args}'.format(args=args)

            for func in self._functions:
                r = self._run_setup_and_func_with_args(setup, func, args)
                results.append(r._replace(timedfunction=func))
            print(report_function(results, title=title) + '\n', file=out)
        return out.getvalue() if as_string else None

    @property
    def functions():
        for x in self._fuctions:
            yield x

    @property
    def args_setups():
        for x in self._args_setups:
            yield x

_module_instance = SimpleTimeIt()

def reset():
    global _module_instance
    _module_instance = SimpleTimeIt()

time_this = _module_instance.time_this
call_with = _module_instance.call_with
run = _module_instance.run
<commit_msg>improves documentation<commit_after>from __future__ import print_function

from contextlib import contextmanager

import six

from .adaptiverun import adaptiverun
from .datatypes import ArgsAndSetup
from .report import generate_table
from .utils import repr_is_constructor

_stimeit_current_function = None
_dummy = object()

@contextmanager
def current_function(f):
    global _stimeit_current_function
    _stimeit_current_function = f
    yield
    _stimeit_current_function = None

class SimpleTimeIt:
    """A class for running simple performance comparisons between functions.

    Typically, you will use call_with to add a number of arguments and
    optional setup routines, and time_this to register a number of functions to
    be compared. Then, calling run will run each function with each argument
    list and print a table comparing the functions on each argument."""

    def __init__(self):
        """Initializes a SimpleTimeIt object."""
        self._functions = []
        self._args_setups = []

    def call_with(self, args, setup='pass'):
        """Resisters a string as the argument list to the functions
        to be called as part of the performance comparisons. For instance, if
        a SimpleTimeIt is specified as follows:

            from simpletimeit.stimeit import SimpleTimeIt

            st = SimpleTimeIt()
            st.call_with("'Eric', 3, x=10")

            @st.time_this
            foo(name, n, x=15):
                pass

        Then `st.run()` will call (the equivalent of)

            exec('foo({args})'.format(args="'Eric', 3, x=10"))

        If 'args' is not a string, `call_with` will try to "do the right
        thing" and convert it to a string. If that string, when executed, will
        not render value equal to 'args', this method will throw an error. So,
        for instance, `10` and `{a: 10, b: 15}` will work because
        `10 == eval('10')` and `{a: 10, b: 15} == eval('{a: 10, b: 15}')`, but
        `range(10)` will not work because `range(10) != eval('range(10)').

        Takes an optional 'setup' argument. This string will be evaluated
        before the timing runs, as with the 'setup' argument to Timer. This
        value is 'pass' by default.
        """
        valid = (isinstance(args, six.string_types)
                 or repr_is_constructor(args))
        if not valid:
            raise ValueError(
                ('{args} will be passed to a format string, which will then '
                 'be executed as Python code. Thus, arguments must either be '
                 'a string to be evaluated as the arguments to the timed '
                 'function, or be a value whose string representation '
                 'constructs an identical object. see the `call_with` '
                 'documentation for more details.').format(args=args))
        if not (isinstance(setup, six.string_types) or callable(setup)):
            raise ValueError(
                "'setup' argument must be a string or callable.")

        self._args_setups.append(ArgsAndSetup(args=str(args), setup=setup))

    def time_this(self, f):
        """A decorator. Registers the decorated function as a TimedFunction
        with this SimpleTimeIt, leaving the function unchanged.
        """
        self._functions.append(f)
        return f

    def _prepared_setup(self, setup, func):
        """Generates the setup routine for a given timing run."""
        setup_template = (
            'from simpletimeit.stimeit import _stimeit_current_function\n'
            '{setup}')
        if isinstance(setup, six.string_types):
            return setup_template.format(setup=setup)
        elif callable(setup):
            def prepared_setup_callable():
                global _stimeit_current_function
                _stimeit_current_function = func
                setup()
            return prepared_setup_callable
        else:
            raise ValueError("'setup' must be a string or callable")

    def _run_setup_and_func_with_args(self, setup, func, args):
        # assumes args == eval(str(args)) or that args is a string
        # (this property checked in call_with)
        stmt_template = '_stimeit_current_function({args})'
        with current_function(func):
            return adaptiverun(stmt=stmt_template.format(args=args),
                               setup=self._prepared_setup(setup, func),
                               title=repr(args))

    def run(self, report_function=generate_table, as_string=False):
        """Runs each of the functions registered with this SimpleTimeIt using
        each arguments-setup pair registered with this SimpleTimeIt.

        report_function should take a list of objects conforming to the
        Report API and return a string reporting on the comparison.

        If as_string is True, this function returns the table or tables
        generated as a string. Otherwise, it prints the tables to stdout and
        returns None."""
        out = six.StringIO() if as_string else None

        for x in self._args_setups:
            args, setup = x.args, x.setup
            results = []
            title = 'args: {args}'.format(args=args)

            for func in self._functions:
                r = self._run_setup_and_func_with_args(setup, func, args)
                results.append(r._replace(timedfunction=func))
            print(report_function(results, title=title) + '\n', file=out)
        return out.getvalue() if as_string else None

    @property
    def functions():
        for x in self._fuctions:
            yield x

    @property
    def args_setups():
        for x in self._args_setups:
            yield x

_module_instance = SimpleTimeIt()

def reset():
    global _module_instance
    _module_instance = SimpleTimeIt()

time_this = _module_instance.time_this
call_with = _module_instance.call_with
run = _module_instance.run
<|endoftext|>


<commit_before>import plotly as py
import plotly.graph_objs as go
from plotly import tools
import numpy as np

color_vector = ['r', 'g', 'b', 'c', 'm', 'y', 'k']
color = ['red', 'green', 'blue', 'c', 'm', 'y', 'k']

def plotStations(baseStations):
    traces = []
    for i, bs in enumerate(baseStations):
        #plt.plot(bs[0][0], bs[0][1], marker=(4, 0, bs[1]), markersize=40)
        cords = np.array(bs[0])
        slope = np.tan(np.radians(bs[1][0]))
        eq = to_coef(slope, cords)
        traces.append( plotLine(eq, bs[0], bs[1][0], i) )
    return traces

def plotStations3D(baseStations, fig):
    for i, bs in enumerate(baseStations):
        cords = np.array(bs[0])
        slope = np.tan(np.radians(bs[1][0]))
        eq = to_coef(slope, cords)
        plotPlane(eq, bs[0], bs[1][0], fig, color_vector[i%len(color_vector)])

def plotLine(eq, center, ang, idx):
    if ang % 180 < 45 or ang % 180 > 135:
        x1 = center[0] + .2
        x2 = center[0] - .2
        y1 =  (eq[1] - eq[0][0]*x1)/(eq[0][1])
        y2 =  (eq[1] - eq[0][0]*x2)/(eq[0][1])
    else:
        y1 = center[1] + .2
        y2 = center[1] - .2
        x1 =  (eq[1] - eq[0][1]*y1)/(eq[0][0])
        x2 =  (eq[1] - eq[0][1]*y2)/(eq[0][0])

    trace = go.Scatter(
        x=[x1,x2],
        y=[y1,y2],
        line = dict(
            color = color[idx%len(color)],
            width = 10
        )
        #mode='markers',
        #marker=dict(
        #    size=20,
        #)
    )

    return trace

def plotPlane(eq, center, ang, fig, color):
    if ang % 180 < 45 or ang % 180 > 135:
        x1 = center[0] + .2
        x2 = center[0] - .2
        y1 =  (eq[1] - eq[0][0]*x1)/(eq[0][1])
        y2 =  (eq[1] - eq[0][0]*x2)/(eq[0][1])
    else:
        y1 = center[1] + .2
        y2 = center[1] - .2
        x1 =  (eq[1] - eq[0][1]*y1)/(eq[0][0])
        x2 =  (eq[1] - eq[0][1]*y2)/(eq[0][0])
    x = [x1,x2,x2,x1]
    y = [y1,y2,y2,y1]
    z = [-1,-1,1,1]
    verts = [zip(x,y,z)]
    ax = fig.gca(projection='3d')
    p3c = Poly3DCollection(verts)
    p3c.set_facecolor(color)
    ax.add_collection3d(p3c)

def to_coef(m, p):
    #conversion from point/slope form of line to standard form. Do on paper if confused.
    const = p[1]-m*p[0]
    #return the left side coefficients and the right side constant. these represent the equation.
    return ([-m, 1], const)

def rescale(data):
    #data = np.nan_to_num(data) # TODO: FIX THIS!!!! YIKES
    return (data - np.min(data))/(np.max(data) - np.min(data))

def plot_scatter(positions, error, title):

    trace = go.Scatter(
        x=positions[:,0],
        y=positions[:,1],
        mode='markers',
        marker=dict(
            size=8,
            #color=rescale(error),
            color=error,
            colorscale='Jet',
            showscale=True,
            opacity=0.8
        )
    )

    return trace

def plot_scatter3D(positions, error, title):

    trace = go.Scatter3d(
        x=positions[:,0],
        y=positions[:,1],
        z=positions[:,2],
        mode='markers',
        marker=dict(
            size=8,
            color=rescale(error),
            #color=error,
            colorscale='Jet',
            showscale=True,
            opacity=0.5
        )
    )

    return trace

# positions (x,y,z?) as numpy array
def plot_error(true_pos, predicted_pos, error, bases, title, saveflag, dir_name, iter_number):
    # 2D case
    if true_pos.shape[1] == 2:
        fig = tools.make_subplots(rows=1, cols=2)
        trace = plot_scatter(true_pos, error, title)
        fig.append_trace(trace, 1,1)

        trace = plot_scatter(predicted_pos, error, title)
        fig.append_trace(trace, 1,2)
        ax_range = [-4.5,4.5]
        fig['layout']['xaxis1'].update( range=ax_range)
        fig['layout']['xaxis2'].update( range=ax_range)
        fig['layout']['yaxis1'].update( range=ax_range)
        fig['layout']['yaxis2'].update( range=ax_range)
        fig['layout'].update(showlegend=False)

        for t in plotStations(bases):
            fig.append_trace(t, 1,1)
            fig.append_trace(t, 1,2)

        if saveflag:
            py.offline.plot(fig, filename=dir_name + 'error-fig-iteration%d.html' % (iter_number))

    # 3D case
    if true_pos.shape[1] == 3:
        fig = tools.make_subplots(rows=1, cols=2, specs=[[{'is_3d': True}, {'is_3d': True}]])
        data = plot_scatter3D(true_pos, error, title)
        fig.append_trace(data, 1,1)

        data = plot_scatter3D(predicted_pos, error, title)
        fig.append_trace(data, 1,2)
        fig['layout']['scene1']['aspectmode'] = 'cube'
        fig['layout']['scene2']['aspectmode'] = 'cube'

        if saveflag:
            py.offline.plot(fig, filename=dir_name + 'error-fig-iteration%d.html' % (iter_number))

<commit_msg>updating plotting for 2x2 for rescales<commit_after>import plotly as py
import plotly.graph_objs as go
from plotly import tools
import numpy as np

color_vector = ['r', 'g', 'b', 'c', 'm', 'y', 'k']
color = ['red', 'green', 'blue', 'c', 'm', 'y', 'k']

def plotStations(baseStations):
    traces = []
    for i, bs in enumerate(baseStations):
        #plt.plot(bs[0][0], bs[0][1], marker=(4, 0, bs[1]), markersize=40)
        cords = np.array(bs[0])
        slope = np.tan(np.radians(bs[1][0]))
        eq = to_coef(slope, cords)
        traces.append( plotLine(eq, bs[0], bs[1][0], i) )
    return traces

def plotStations3D(baseStations, fig):
    for i, bs in enumerate(baseStations):
        cords = np.array(bs[0])
        slope = np.tan(np.radians(bs[1][0]))
        eq = to_coef(slope, cords)
        plotPlane(eq, bs[0], bs[1][0], fig, color_vector[i%len(color_vector)])

def plotLine(eq, center, ang, idx):
    if ang % 180 < 45 or ang % 180 > 135:
        x1 = center[0] + .2
        x2 = center[0] - .2
        y1 =  (eq[1] - eq[0][0]*x1)/(eq[0][1])
        y2 =  (eq[1] - eq[0][0]*x2)/(eq[0][1])
    else:
        y1 = center[1] + .2
        y2 = center[1] - .2
        x1 =  (eq[1] - eq[0][1]*y1)/(eq[0][0])
        x2 =  (eq[1] - eq[0][1]*y2)/(eq[0][0])

    trace = go.Scatter(
        x=[x1,x2],
        y=[y1,y2],
        line = dict(
            color = color[idx%len(color)],
            width = 10
        )
        #mode='markers',
        #marker=dict(
        #    size=20,
        #)
    )

    return trace

def plotPlane(eq, center, ang, fig, color):
    if ang % 180 < 45 or ang % 180 > 135:
        x1 = center[0] + .2
        x2 = center[0] - .2
        y1 =  (eq[1] - eq[0][0]*x1)/(eq[0][1])
        y2 =  (eq[1] - eq[0][0]*x2)/(eq[0][1])
    else:
        y1 = center[1] + .2
        y2 = center[1] - .2
        x1 =  (eq[1] - eq[0][1]*y1)/(eq[0][0])
        x2 =  (eq[1] - eq[0][1]*y2)/(eq[0][0])
    x = [x1,x2,x2,x1]
    y = [y1,y2,y2,y1]
    z = [-1,-1,1,1]
    verts = [zip(x,y,z)]
    ax = fig.gca(projection='3d')
    p3c = Poly3DCollection(verts)
    p3c.set_facecolor(color)
    ax.add_collection3d(p3c)

def to_coef(m, p):
    #conversion from point/slope form of line to standard form. Do on paper if confused.
    const = p[1]-m*p[0]
    #return the left side coefficients and the right side constant. these represent the equation.
    return ([-m, 1], const)

def rescale(data):
    #data = np.nan_to_num(data) # TODO: FIX THIS!!!! YIKES
    return (data - np.min(data))/(np.max(data) - np.min(data))

def plot_scatter(positions, error, title, to_rescale=False):

    if to_rescale:
        trace = go.Scatter(
            x=positions[:,0],
            y=positions[:,1],
            mode='markers',
            marker=dict(
                size=8,
                color=rescale(error),
                colorscale='Jet',
                colorbar=go.ColorBar(
                    title='Colorbar',
                    yanchor='bottom'
                    ),
                showscale=True,
                opacity=0.8
            )
        )
    else:
        trace = go.Scatter(
            x=positions[:,0],
            y=positions[:,1],
            mode='markers',
            marker=dict(
                size=8,
                color=error,
                colorscale='Jet',
                showscale=True,
                colorbar=go.ColorBar(
                    title='Colorbar',
                    yanchor='top'
                    ),
                opacity=0.8
            )
        )

    return trace

def plot_scatter3D(positions, error, title):

    trace = go.Scatter3d(
        x=positions[:,0],
        y=positions[:,1],
        z=positions[:,2],
        mode='markers',
        marker=dict(
            size=8,
            color=rescale(error),
            #color=error,
            colorscale='Jet',
            showscale=True,
            opacity=0.5
        )
    )

    return trace

# positions (x,y,z?) as numpy array
def plot_error(true_pos, predicted_pos, error, bases, title, saveflag, dir_name, iter_number):
    # 2D case
    if true_pos.shape[1] == 2:
        fig = tools.make_subplots(rows=2, cols=2)
        trace = plot_scatter(true_pos, error, title)
        fig.append_trace(trace, 1,1)
        trace = plot_scatter(true_pos, error, title, True)
        fig.append_trace(trace, 2,1)

        trace = plot_scatter(predicted_pos, error, title)
        fig.append_trace(trace, 1,2)
        trace = plot_scatter(predicted_pos, error, title, True)
        fig.append_trace(trace, 2,2)
        ax_range = [-4.5,4.5]
        fig['layout']['xaxis1'].update( range=ax_range)
        fig['layout']['xaxis2'].update( range=ax_range)
        fig['layout']['xaxis3'].update( range=ax_range)
        fig['layout']['xaxis4'].update( range=ax_range)
        fig['layout']['yaxis1'].update( range=ax_range)
        fig['layout']['yaxis2'].update( range=ax_range)
        fig['layout']['yaxis3'].update( range=ax_range)
        fig['layout']['yaxis4'].update( range=ax_range)
        fig['layout'].update(showlegend=False)

        for t in plotStations(bases):
            fig.append_trace(t, 1,1)
            fig.append_trace(t, 1,2)

        if saveflag:
            py.offline.plot(fig, filename=dir_name + 'error-fig-iteration%d.html' % (iter_number))

    # 3D case
    if true_pos.shape[1] == 3:
        fig = tools.make_subplots(rows=1, cols=2, specs=[[{'is_3d': True}, {'is_3d': True}]])
        data = plot_scatter3D(true_pos, error, title)
        fig.append_trace(data, 1,1)

        data = plot_scatter3D(predicted_pos, error, title)
        fig.append_trace(data, 1,2)
        fig['layout']['scene1']['aspectmode'] = 'cube'
        fig['layout']['scene2']['aspectmode'] = 'cube'

        if saveflag:
            py.offline.plot(fig, filename=dir_name + 'error-fig-iteration%d.html' % (iter_number))

<|endoftext|>