<commit_before>"""
Contains help methods and classes to perform tests.
"""
import base64
import random
import re
import sys
import tempfile

from Crypto.PublicKey import RSA
from saml2 import server, BINDING_HTTP_POST, BINDING_HTTP_REDIRECT
from saml2.authn_context import AuthnBroker, authn_context_class_ref, PASSWORD
from saml2.cert import OpenSSLWrapper
from saml2.client import Saml2Client
from saml2.config import config_factory, Config
from saml2.metadata import entity_descriptor
from saml2.saml import name_id_from_string, NAMEID_FORMAT_TRANSIENT, NAMEID_FORMAT_PERSISTENT
from saml2.samlp import NameIDPolicy

from satosa.backends.base import BackendModule
from satosa.frontends.base import FrontendModule

class FakeSP(Saml2Client):
    """
    A SAML service provider that can be used to perform tests.
    """

    def __init__(self, config_module, config=None):
        """
        :type config_module: str
        :type config: {dict}

        :param config_module: Path to a file containing the SP SAML configuration.
        :param config: SP SAML configuration.
        """
        if config is None:
            config = config_factory('sp', config_module)
        Saml2Client.__init__(self, config)

    def make_auth_req(self, entity_id):
        """
        :type entity_id: str
        :rtype: str

        :param entity_id: SAML entity id
        :return: Authentication URL.
        """
        # Picks a binding to use for sending the Request to the IDP
        _binding, destination = self.pick_binding(
                'single_sign_on_service',
                [BINDING_HTTP_REDIRECT, BINDING_HTTP_POST], 'idpsso',
                entity_id=entity_id)
        # Binding here is the response binding that is which binding the
        # IDP shou  ld use to return the response.
        acs = self.config.getattr('endpoints', 'sp')[
            'assertion_consumer_service']
        # just pick one
        return_binding = None
        for i in range(len(acs)):
            endp, return_binding = acs[i]
            if return_binding == _binding:
                break

        req_id, req = self.create_authn_request(destination,
                                                binding=return_binding)
        ht_args = self.apply_binding(_binding, '%s' % req, destination,
                                     relay_state='hello')

        url = ht_args['headers'][0][1]
        return url

class FakeIdP(server.Server):
    """
    A SAML IdP that can be used to perform tests.
    """

    def __init__(self, user_db, config):
        """
        :type user_db: {dict}
        :type config: {dict}

        :param user_db: A dictionary with the user id as key and parameter dictionary as value.
        :param config: IdP SAML configuration.
        """
        server.Server.__init__(self, config=config)
        self.user_db = user_db

    def handle_auth_req(self, saml_request, relay_state, binding, userid):
        """
        Handles a SAML request, validates and creates a SAML response.
        :type saml_request: str
        :type relay_state: str
        :type binding: str
        :type userid: str
        :rtype:

        :param saml_request:
        :param relay_state: RelayState is a parameter used by some SAML protocol implementations to
        identify the specific resource at the resource provider in an IDP initiated single sign on
        scenario.
        :param binding:
        :param userid: The user identification.
        :return: A tuple with
        """
        auth_req = self.parse_authn_request(saml_request, binding)
        binding_out, destination = self.pick_binding(
                'assertion_consumer_service',
                entity_id=auth_req.message.issuer.text, request=auth_req.message)

        resp_args = self.response_args(auth_req.message)
        authn_broker = AuthnBroker()
        authn_broker.add(authn_context_class_ref(PASSWORD), lambda: None, 10,
                         'unittest_idp.xml')
        authn_broker.get_authn_by_accr(PASSWORD)
        resp_args['authn'] = authn_broker.get_authn_by_accr(PASSWORD)

        _resp = self.create_authn_response(self.user_db[userid],
                                           userid=userid,
                                           **resp_args)

        http_args = self.apply_binding(BINDING_HTTP_POST, '%s' % _resp,
                                       destination, relay_state, response=True)
        url = http_args['url']
        saml_response = base64.b64encode(str(_resp).encode("utf-8"))
        resp = {'SAMLResponse': saml_response, 'RelayState': relay_state}
        return url, resp

    def get_post_action_body(self, form):
        form_str = form
        if isinstance(form, list):
            form_str = ""
            for item in form:
                form_str += item
        resp = re.split(r'([^=, ]+)="([^" ]+|[^," ]+)" ?', form_str)
        count = 0
        action = None
        saml_response = None
        relay_state = None
        for value in resp:
            if value == "action":
                action = resp[count + 1]
            if value == 'SAMLRequest':
                saml_response = resp[count + 3]
            if value == "RelayState":
                relay_state = resp[count + 3]
            count += 1
        body = {"SAMLRequest": saml_response, "RelayState": relay_state}
        return action, body

def create_metadata_from_config_dict(config):
    nspair = {"xs": "http://www.w3.org/2001/XMLSchema"}
    conf = Config().load(config, metadata_construction=True)
    return entity_descriptor(conf).to_string(nspair).decode("utf-8")

class FileGenerator(object):
    """
    Creates different types of temporary files that is useful for testing.
    """
    _instance = None

    def __init__(self):
        if FileGenerator._instance:
            raise TypeError('Singletons must be accessed through `get_instance()`.')
        else:
            FileGenerator._instance = self
        self.generate_certs = {}
        self.metadata = {}

    @staticmethod
    def get_instance():
        """
        :rtype: FileGenerator

        :return: A singleton instance of the class.
        """
        if FileGenerator._instance is None:
            FileGenerator._instance = FileGenerator()
        return FileGenerator._instance

    def generate_cert(self, code=None):
        """
        Will generate a certificate and key. If code is used the same certificate and key will
        always be returned for the same code.
        :type code: str
        :rtype: (tempfile._TemporaryFileWrapper, tempfile._TemporaryFileWrapper)

        :param: code: A unique code to represent a certificate and key.
        :return: A certificate and key temporary file.
        """
        if code in self.generate_certs:
            return self.generate_certs[code]
        sn = random.randint(1, sys.maxsize)
        cert_info = {
            "cn": "localhost",
            "country_code": "se",
            "state": "ac",
            "city": "Umea",
            "organization": "ITS",
            "organization_unit": "DIRG"
        }
        osw = OpenSSLWrapper()
        cert_str, key_str = osw.create_certificate(cert_info, request=False, sn=sn, key_length=2048)
        cert_file = tempfile.NamedTemporaryFile()
        cert_file.write(cert_str)
        cert_file.flush()
        key_file = tempfile.NamedTemporaryFile()
        key_file.write(key_str)
        key_file.flush()
        if code is not None:
            self.generate_certs[code] = cert_file, key_file
        return cert_file, key_file

    def create_metadata(self, config, code=None):
        """
        Will generate a metadata file. If code is used the same metadata file will
        always be returned for the same code.
        :type config: {dict}
        :type code: str

        :param config: A SAML configuration.
        :param code: A unique code to represent a certificate and key.
        """
        if code in self.metadata:
            return self.metadata[code]

        desc = create_metadata_from_config_dict(config)

        tmp_file = tempfile.NamedTemporaryFile()
        tmp_file.write(desc.encode("utf-8"))
        tmp_file.flush()
        if code:
            self.metadata[code] = tmp_file
        return tmp_file

def private_to_public_key(pk_file):
    f = open(pk_file, 'r')
    pk = RSA.importKey(f.read())
    return pk.publickey().exportKey('PEM')

def create_name_id():
    """
    :rtype: str

    :return: Returns a SAML nameid as XML string.
    """
    test_name_id = """<?xml version="1.0" encoding="utf-8"?>
<NameID xmlns="urn:oasis:names:tc:SAML:2.0:assertion"
  Format="urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress"
  SPProvidedID="sp provided id">
  tmatsuo@example.com
</NameID>
"""
    return name_id_from_string(test_name_id)

def create_name_id_policy_transient():
    """
    Creates a transient name id policy.
    :return:
    """
    nameid_format = NAMEID_FORMAT_TRANSIENT
    name_id_policy = NameIDPolicy(format=nameid_format)
    return name_id_policy

def create_name_id_policy_persistent():
    """
    Creates a transient name id policy.
    :return:
    """
    nameid_format = NAMEID_FORMAT_PERSISTENT
    name_id_policy = NameIDPolicy(format=nameid_format)
    return name_id_policy

class FakeBackend(BackendModule):
    """
    TODO comment
    """

    def __init__(self, start_auth_func=None, internal_attributes=None,
                 register_endpoints_func=None):
        super(FakeBackend, self).__init__(None, internal_attributes)

        self.start_auth_func = start_auth_func
        self.register_endpoints_func = register_endpoints_func

    def start_auth(self, context, request_info, state):
        """
        TODO comment
        :type context: TODO comment
        :type request_info: TODO comment
        :type state: TODO comment

        :param context: TODO comment
        :param request_info: TODO comment
        :param state: TODO comment
        """
        if self.start_auth:
            return self.start_auth(context, request_info, state)
        return None

    def register_endpoints(self):
        """
        TODO comment
        """
        if self.register_endpoints_func:
            return self.register_endpoints_func()
        return None

class FakeFrontend(FrontendModule):
    """
    TODO comment
    """

    def __init__(self, handle_authn_request_func=None, internal_attributes=None,
                 handle_authn_response_func=None,
                 register_endpoints_func=None):
        super(FakeFrontend, self).__init__(None, internal_attributes)
        self.handle_authn_request_func = handle_authn_request_func
        self.handle_authn_response_func = handle_authn_response_func
        self.register_endpoints_func = register_endpoints_func

    def handle_authn_request(self, context, binding_in):
        """
        TODO comment

        :type context:
        :type binding_in:

        :param context:
        :param binding_in:
        :return:
        """
        if self.handle_authn_request_func:
            return self.handle_authn_request_func(context, binding_in)
        return None

    def handle_authn_response(self, context, internal_response, state):
        """
        TODO comment
        :type context: TODO comment
        :type internal_response: TODO comment
        :type state: TODO comment

        :param context: TODO  comment
        :param internal_response: TODO comment
        :param state: TODO comment
        :return: TODO comment
        """
        if self.handle_authn_response_func:
            return self.handle_authn_response_func(context, internal_response, state)
        return None

    def register_endpoints(self, providers):
        """
        TODO comment

        :type providers: TODO comment

        :param providers: TODO comment
        :return: TODO comment
        """
        if self.register_endpoints_func:
            return self.register_endpoints_func(providers)
<commit_msg>Add function for generating cert and private key in memory.<commit_after>"""
Contains help methods and classes to perform tests.
"""
import base64
import re
import tempfile

from Crypto.PublicKey import RSA
from saml2 import server, BINDING_HTTP_POST, BINDING_HTTP_REDIRECT
from saml2.authn_context import AuthnBroker, authn_context_class_ref, PASSWORD
from saml2.cert import OpenSSLWrapper
from saml2.client import Saml2Client
from saml2.config import config_factory, Config
from saml2.metadata import entity_descriptor
from saml2.saml import name_id_from_string, NAMEID_FORMAT_TRANSIENT, NAMEID_FORMAT_PERSISTENT
from saml2.samlp import NameIDPolicy

from satosa.backends.base import BackendModule
from satosa.frontends.base import FrontendModule

class FakeSP(Saml2Client):
    """
    A SAML service provider that can be used to perform tests.
    """

    def __init__(self, config_module, config=None):
        """
        :type config_module: str
        :type config: {dict}

        :param config_module: Path to a file containing the SP SAML configuration.
        :param config: SP SAML configuration.
        """
        if config is None:
            config = config_factory('sp', config_module)
        Saml2Client.__init__(self, config)

    def make_auth_req(self, entity_id):
        """
        :type entity_id: str
        :rtype: str

        :param entity_id: SAML entity id
        :return: Authentication URL.
        """
        # Picks a binding to use for sending the Request to the IDP
        _binding, destination = self.pick_binding(
                'single_sign_on_service',
                [BINDING_HTTP_REDIRECT, BINDING_HTTP_POST], 'idpsso',
                entity_id=entity_id)
        # Binding here is the response binding that is which binding the
        # IDP shou  ld use to return the response.
        acs = self.config.getattr('endpoints', 'sp')[
            'assertion_consumer_service']
        # just pick one
        return_binding = None
        for i in range(len(acs)):
            endp, return_binding = acs[i]
            if return_binding == _binding:
                break

        req_id, req = self.create_authn_request(destination,
                                                binding=return_binding)
        ht_args = self.apply_binding(_binding, '%s' % req, destination,
                                     relay_state='hello')

        url = ht_args['headers'][0][1]
        return url

class FakeIdP(server.Server):
    """
    A SAML IdP that can be used to perform tests.
    """

    def __init__(self, user_db, config):
        """
        :type user_db: {dict}
        :type config: {dict}

        :param user_db: A dictionary with the user id as key and parameter dictionary as value.
        :param config: IdP SAML configuration.
        """
        server.Server.__init__(self, config=config)
        self.user_db = user_db

    def handle_auth_req(self, saml_request, relay_state, binding, userid):
        """
        Handles a SAML request, validates and creates a SAML response.
        :type saml_request: str
        :type relay_state: str
        :type binding: str
        :type userid: str
        :rtype:

        :param saml_request:
        :param relay_state: RelayState is a parameter used by some SAML protocol implementations to
        identify the specific resource at the resource provider in an IDP initiated single sign on
        scenario.
        :param binding:
        :param userid: The user identification.
        :return: A tuple with
        """
        auth_req = self.parse_authn_request(saml_request, binding)
        binding_out, destination = self.pick_binding(
                'assertion_consumer_service',
                entity_id=auth_req.message.issuer.text, request=auth_req.message)

        resp_args = self.response_args(auth_req.message)
        authn_broker = AuthnBroker()
        authn_broker.add(authn_context_class_ref(PASSWORD), lambda: None, 10,
                         'unittest_idp.xml')
        authn_broker.get_authn_by_accr(PASSWORD)
        resp_args['authn'] = authn_broker.get_authn_by_accr(PASSWORD)

        _resp = self.create_authn_response(self.user_db[userid],
                                           userid=userid,
                                           **resp_args)

        http_args = self.apply_binding(BINDING_HTTP_POST, '%s' % _resp,
                                       destination, relay_state, response=True)
        url = http_args['url']
        saml_response = base64.b64encode(str(_resp).encode("utf-8"))
        resp = {'SAMLResponse': saml_response, 'RelayState': relay_state}
        return url, resp

    def get_post_action_body(self, form):
        form_str = form
        if isinstance(form, list):
            form_str = ""
            for item in form:
                form_str += item
        resp = re.split(r'([^=, ]+)="([^" ]+|[^," ]+)" ?', form_str)
        count = 0
        action = None
        saml_response = None
        relay_state = None
        for value in resp:
            if value == "action":
                action = resp[count + 1]
            if value == 'SAMLRequest':
                saml_response = resp[count + 3]
            if value == "RelayState":
                relay_state = resp[count + 3]
            count += 1
        body = {"SAMLRequest": saml_response, "RelayState": relay_state}
        return action, body

def create_metadata_from_config_dict(config):
    nspair = {"xs": "http://www.w3.org/2001/XMLSchema"}
    conf = Config().load(config, metadata_construction=True)
    return entity_descriptor(conf).to_string(nspair).decode("utf-8")

def generate_cert():
    cert_info = {
        "cn": "localhost",
        "country_code": "se",
        "state": "ac",
        "city": "Umea",
        "organization": "ITS",
        "organization_unit": "DIRG"
    }
    osw = OpenSSLWrapper()
    cert_str, key_str = osw.create_certificate(cert_info, request=False)
    return cert_str, key_str

class FileGenerator(object):
    """
    Creates different types of temporary files that is useful for testing.
    """
    _instance = None

    def __init__(self):
        if FileGenerator._instance:
            raise TypeError('Singletons must be accessed through `get_instance()`.')
        else:
            FileGenerator._instance = self
        self.generate_certs = {}
        self.metadata = {}

    @staticmethod
    def get_instance():
        """
        :rtype: FileGenerator

        :return: A singleton instance of the class.
        """
        if FileGenerator._instance is None:
            FileGenerator._instance = FileGenerator()
        return FileGenerator._instance

    def generate_cert(self, code=None):
        """
        Will generate a certificate and key. If code is used the same certificate and key will
        always be returned for the same code.
        :type code: str
        :rtype: (tempfile._TemporaryFileWrapper, tempfile._TemporaryFileWrapper)

        :param: code: A unique code to represent a certificate and key.
        :return: A certificate and key temporary file.
        """
        if code in self.generate_certs:
            return self.generate_certs[code]

        cert_str, key_str = generate_cert()

        cert_file = tempfile.NamedTemporaryFile()
        cert_file.write(cert_str)
        cert_file.flush()
        key_file = tempfile.NamedTemporaryFile()
        key_file.write(key_str)
        key_file.flush()
        if code is not None:
            self.generate_certs[code] = cert_file, key_file
        return cert_file, key_file

    def create_metadata(self, config, code=None):
        """
        Will generate a metadata file. If code is used the same metadata file will
        always be returned for the same code.
        :type config: {dict}
        :type code: str

        :param config: A SAML configuration.
        :param code: A unique code to represent a certificate and key.
        """
        if code in self.metadata:
            return self.metadata[code]

        desc = create_metadata_from_config_dict(config)

        tmp_file = tempfile.NamedTemporaryFile()
        tmp_file.write(desc.encode("utf-8"))
        tmp_file.flush()
        if code:
            self.metadata[code] = tmp_file
        return tmp_file

def private_to_public_key(pk_file):
    f = open(pk_file, 'r')
    pk = RSA.importKey(f.read())
    return pk.publickey().exportKey('PEM')

def create_name_id():
    """
    :rtype: str

    :return: Returns a SAML nameid as XML string.
    """
    test_name_id = """<?xml version="1.0" encoding="utf-8"?>
<NameID xmlns="urn:oasis:names:tc:SAML:2.0:assertion"
  Format="urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress"
  SPProvidedID="sp provided id">
  tmatsuo@example.com
</NameID>
"""
    return name_id_from_string(test_name_id)

def create_name_id_policy_transient():
    """
    Creates a transient name id policy.
    :return:
    """
    nameid_format = NAMEID_FORMAT_TRANSIENT
    name_id_policy = NameIDPolicy(format=nameid_format)
    return name_id_policy

def create_name_id_policy_persistent():
    """
    Creates a transient name id policy.
    :return:
    """
    nameid_format = NAMEID_FORMAT_PERSISTENT
    name_id_policy = NameIDPolicy(format=nameid_format)
    return name_id_policy

class FakeBackend(BackendModule):
    """
    TODO comment
    """

    def __init__(self, start_auth_func=None, internal_attributes=None,
                 register_endpoints_func=None):
        super(FakeBackend, self).__init__(None, internal_attributes)

        self.start_auth_func = start_auth_func
        self.register_endpoints_func = register_endpoints_func

    def start_auth(self, context, request_info, state):
        """
        TODO comment
        :type context: TODO comment
        :type request_info: TODO comment
        :type state: TODO comment

        :param context: TODO comment
        :param request_info: TODO comment
        :param state: TODO comment
        """
        if self.start_auth:
            return self.start_auth(context, request_info, state)
        return None

    def register_endpoints(self):
        """
        TODO comment
        """
        if self.register_endpoints_func:
            return self.register_endpoints_func()
        return None

class FakeFrontend(FrontendModule):
    """
    TODO comment
    """

    def __init__(self, handle_authn_request_func=None, internal_attributes=None,
                 handle_authn_response_func=None,
                 register_endpoints_func=None):
        super(FakeFrontend, self).__init__(None, internal_attributes)
        self.handle_authn_request_func = handle_authn_request_func
        self.handle_authn_response_func = handle_authn_response_func
        self.register_endpoints_func = register_endpoints_func

    def handle_authn_request(self, context, binding_in):
        """
        TODO comment

        :type context:
        :type binding_in:

        :param context:
        :param binding_in:
        :return:
        """
        if self.handle_authn_request_func:
            return self.handle_authn_request_func(context, binding_in)
        return None

    def handle_authn_response(self, context, internal_response, state):
        """
        TODO comment
        :type context: TODO comment
        :type internal_response: TODO comment
        :type state: TODO comment

        :param context: TODO  comment
        :param internal_response: TODO comment
        :param state: TODO comment
        :return: TODO comment
        """
        if self.handle_authn_response_func:
            return self.handle_authn_response_func(context, internal_response, state)
        return None

    def register_endpoints(self, providers):
        """
        TODO comment

        :type providers: TODO comment

        :param providers: TODO comment
        :return: TODO comment
        """
        if self.register_endpoints_func:
            return self.register_endpoints_func(providers)
<|endoftext|>


<commit_before>import arcpy
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import datetime as dt
import time

speed_data = []
time_data = []

def plot(input_shp):
    searchCursor = arcpy.SearchCursor(input_shp)

    for row in searchCursor:
        speed_data.append(row.getValue("bttlNcr"))
        fmt = "%Y-%m-%d_%H-%M-%S"
        date = dt.datetime.strptime(row.getValue("date"), fmt)
        time_data.append(date)

    displayPlot()
    del searchCursor

def displayPlot():
    fig, ax1 = plt.subplots(nrows = 1, sharex = True)
    #secondLocator = mdates.SecondLocator()
    #ax1.xaxis.set_minor_locator(secondLocator)
    ax1.plot(time_data, speed_data, 'r-')

    plt.grid(True)
    plt.show()

<commit_msg>Added Histogram, Accuracy Plot<commit_after>import arcpy
import os
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import matplotlib.patches as mpatches
import datetime as dt
import numpy as np
import time

speed_data = []
accuracy_data = []
time_data = []

def init(input_shp):

    searchCursor = arcpy.SearchCursor(input_shp)

    for row in searchCursor:
        speed_data.append(row.getValue("bttlNcr"))
        accuracy_data.append(row.getValue("accuracy"))
        fmt = "%Y-%m-%d_%H-%M-%S"
        date = dt.datetime.strptime(row.getValue("date"), fmt)
        time_data.append(date)
    del searchCursor

    displaySpeedPlot()
    displayAccuracyPlot()
    displaySpeedHist()

def displaySpeedPlot():
    fig, ax1 = plt.subplots(nrows = 1, sharex = True)

    mean = np.mean(speed_data)
    mean_data = [mean] * len(time_data)

    # Styling
    xfmt = mdates.DateFormatter('%H:%M:%S')
    ax1.xaxis.set_major_formatter(xfmt)
    plt.grid(True)
    ax1.plot(time_data, speed_data, 'r-')
    ax1.plot(time_data, mean_data, 'b-')

    # Titles
    fig.suptitle("Speed over Time")
    ax1.set_xlabel("t")
    ax1.set_ylabel("km / h")

    # Legend

    speedPatch = mpatches.Patch(color='red', label='Speed in km/h')

    labelString = "Average Speed: %s km/h" % round(mean,3)
    averagePatch = mpatches.Patch(color='blue', label=labelString)

    plt.legend(handles=[speedPatch,averagePatch])

    #save as png
    imageOut = arcpy.env.scratchFolder + os.sep + "speed.png"
    fig.savefig(imageOut, dpi=fig.dpi)

def displayAccuracyPlot():
    fig, ax1 = plt.subplots(nrows = 1, sharex = True)

    mean = np.mean(accuracy_data)
    mean_data = [mean] * len(time_data)

    # Styling
    xfmt = mdates.DateFormatter('%H:%M:%S')
    ax1.xaxis.set_major_formatter(xfmt)
    plt.grid(True)
    ax1.plot(time_data, accuracy_data, 'r-')
    ax1.plot(time_data, mean_data, 'b-')

    # Titles
    fig.suptitle("Speed over Time")
    ax1.set_xlabel("t")
    ax1.set_ylabel("km / h")

    # Legend
    accuracyPatch = mpatches.Patch(color='red', label="Accuracy in meters")
    averagePatch = mpatches.Patch(color='blue', label="Average Accuracy")

    plt.legend(handles=[accuracyPatch,averagePatch])

    #save as png
    imageOut = arcpy.env.scratchFolder + os.sep + "accuracy.png"
    fig.savefig(imageOut, dpi=fig.dpi)

def displaySpeedHist():

    fig = plt.figure()
    plt.hist(speed_data)

    plt.xlabel('Speed Values')
    plt.ylabel('Frequency')

    plt.title('Frequency of Speed Values')

    imageOut = arcpy.env.scratchFolder + os.sep + "hist.png"
    fig.savefig(imageOut, dpi=fig.dpi)
<|endoftext|>


<commit_before>__author__ = 'msamia'
__version__ = '3.0.52'

from .lib import *
<commit_msg>new major version<commit_after>__author__ = 'msamia'
__version__ = '4.0.0'

from .lib import *
<|endoftext|>


<commit_before>import six
from compose.cli.command import get_project

from .containers import Container, HostAddress, Service

class DockerClientStub(object):
    STATES = {
        'STOPPED': 'STOPPED',
        'UP': 'UP',
    }
    CREATED_STATES = {
        'NOT_CREATED': 'NOT_CREATED',
        'CREATED': 'CREATED'
    }

    def __init__(self, services=None):
        self.created_state = self.CREATED_STATES['NOT_CREATED']
        self.state = self.STATES['STOPPED']
        if services is not None:
            self._services = services
        else:
            self._services = {}

    def up(self):
        self.created_state = self.CREATED_STATES['CREATED']
        self.state = self.STATES['UP']

    def stop(self):
        self.created_state = self.CREATED_STATES['CREATED']
        self.state = self.STATES['STOPPED']

    def remove(self):
        self.created_state = self.CREATED_STATES['NOT_CREATED']
        self.state = self.STATES['STOPPED']

    @property
    def services(self):
        return self._services

class DockerClient(object):
    def __init__(self, project_name, base_dir, config_path):
        self.project = get_project(
            base_dir,
            project_name=project_name,
            config_path=[config_path]
        )

    def up(self):
        self.project.up()

    def stop(self):
        self.project.stop()

    def remove(self):
        self.project.remove_stopped()

    @property
    def services(self):
        return {
            service.name: Service(
                name=service.name,
                containers=list(map(create_container, service.containers())),
            )
            for service in self.project.services
        }

    def scale(self, service_name, n):
        self.project.get_service(service_name).scale(n)

def create_container(container):
    return Container(
        name=container.name,
        port_mappings=port_mappings_from_container(container)
    )

def internal_port_from_docker(docker_port):
    return int(docker_port.split('/')[0])

def port_mappings_from_container(docker_container):
    return {
        internal_port_from_docker(internal): [
            HostAddress(
                ip=address['HostIp'],
                port=int(address['HostPort'])
            )
            for address in addresses
        ]
        for internal, addresses in six.iteritems(docker_container.ports)
    }
<commit_msg>fix NoneType is not interable in case of no port mapping<commit_after>import six
from compose.cli.command import get_project

from .containers import Container, HostAddress, Service

class DockerClientStub(object):
    STATES = {
        'STOPPED': 'STOPPED',
        'UP': 'UP',
    }
    CREATED_STATES = {
        'NOT_CREATED': 'NOT_CREATED',
        'CREATED': 'CREATED'
    }

    def __init__(self, services=None):
        self.created_state = self.CREATED_STATES['NOT_CREATED']
        self.state = self.STATES['STOPPED']
        if services is not None:
            self._services = services
        else:
            self._services = {}

    def up(self):
        self.created_state = self.CREATED_STATES['CREATED']
        self.state = self.STATES['UP']

    def stop(self):
        self.created_state = self.CREATED_STATES['CREATED']
        self.state = self.STATES['STOPPED']

    def remove(self):
        self.created_state = self.CREATED_STATES['NOT_CREATED']
        self.state = self.STATES['STOPPED']

    @property
    def services(self):
        return self._services

class DockerClient(object):
    def __init__(self, project_name, base_dir, config_path):
        self.project = get_project(
            base_dir,
            project_name=project_name,
            config_path=[config_path]
        )

    def up(self):
        self.project.up()

    def stop(self):
        self.project.stop()

    def remove(self):
        self.project.remove_stopped()

    @property
    def services(self):
        return {
            service.name: Service(
                name=service.name,
                containers=list(map(create_container, service.containers())),
            )
            for service in self.project.services
        }

    def scale(self, service_name, n):
        self.project.get_service(service_name).scale(n)

def create_container(container):
    return Container(
        name=container.name,
        port_mappings=port_mappings_from_container(container)
    )

def internal_port_from_docker(docker_port):
    return int(docker_port.split('/')[0])

def port_mappings_from_container(docker_container):
    mappings = {}
    for internal, addresses in six.iteritems(docker_container.ports):
        if not addresses:
            mappings[internal_port_from_docker(internal)] = []
        else:
            mappings[internal_port_from_docker(internal)] = [
                HostAddress(
                    ip=address['HostIp'],
                    port=int(address['HostPort'])
                )
                for address in addresses
        ]
    return mappings
<|endoftext|>


<commit_before>#
# Angel Farguell, CU Denver
#

from __future__ import absolute_import
import glob, re, datetime, logging, requests
import os.path as osp
import numpy as np
from six.moves.urllib.request import urlopen
from cmr import GranuleQuery
from utils import Dict, utc_to_utcf, duplicates
from .downloader import download_url, DownloadError
from six.moves import filter
from six.moves import range

class SatError(Exception):
	"""
	Raised when a SatSource cannot retrieve satellite data.
	"""
	pass

class SatSource(object):
	"""
	The parent class of all satellite sources that implements common functionality, for example

	- local satellite validation (file size check)
	- Satellite retrieval with retries (smart check whether server implements http-range)
	"""

	def __init__(self, js):
		"""
		Initialize satellite source with ingest directory (where satellite files are stored).

		:param js: job structure with at least ingest_path root of satellite storage and sys_install_path
		"""
		self.ingest_dir=osp.abspath(osp.join(js.get('ingest_path','ingest'),self.prefix))
		self.cache_dir=osp.abspath(js.get('cache_path','cache'))
		self.sys_dir=osp.abspath(js.get('sys_install_path',None))
		self.appkey=js.get('appkey',None)
		if not appkey:
			try:
				tokens = json.load(open('etc/tokens.json'))
				self.appkey = tokens.get('appkey',None)
			except:
				logging.warning('Any etc/tokens.json specified, any token is going to be used.')

	def available_locally_sat(self, path):
		"""
		Check if a satellite file is available locally and if it's file size checks out.

		:param path: the satellite file path
		"""
		info_path = path + '.size'
		if osp.exists(path) and osp.exists(info_path):
			content_size = int(open(info_path).read())
			return osp.getsize(path) == content_size
		else:
			return False

	def search_api_sat(self, sname, bbox, time, version=None):
		"""
		API search of the different satellite granules and return metadata dictionary

		:param sname: short name satellite product, ex: 'MOD03'
		:param bbox: polygon with the search bounding box
		:param time: time interval (init_time_iso,final_time_iso)

		:return metas: a dictionary with all the metadata for the API search
		"""
		maxg=100
		time_utcf=(utc_to_utcf(time[0]),utc_to_utcf(time[1]))
		api = GranuleQuery()
		if not version:
			search = api.parameters(
					short_name=sname,
					downloadable=True,
					polygon=bbox,
					temporal=time_utcf
				)
		else:
			search = api.parameters(
					short_name=sname,
					downloadable=True,
					polygon=bbox,
					temporal=time_utcf,
					version=version
				)
		sh=search.hits()
		logging.info("CMR API: %s gets %s hits in this range" % (sname,sh))
		if sh>maxg:
			logging.warning("The number of hits %s is larger than the limit %s." % (sh,maxg))
			logging.warning("Any satellite data with prefix %s used." % self.prefix)
			logging.warning("Use a reduced bounding box or a reduced time interval.")
			metas = []
		else:
			metas = api.get(sh)
		return metas

	def search_archive_sat(self, prod, time, geo_metas):
		"""
		Archive search of the different satellite granules

		:param prod: string of short name product, ex: 'VNP09'
		:param time: time interval (init_time_datetime,final_time_datetime)
		:param geo_metas: granules of the geolocation metadata
		:return metas: dictionary with the metadata of the search
		"""
		ids=['.'.join(gm['producer_granule_id'].split('.')[1:3]) for gm in geo_metas] # satellite ids in bounding box
		metas=[]
		delta=time[1]-time[0]
		nh=int(delta.total_seconds()/3600)
		dates=[time[0]+datetime.timedelta(seconds=3600*k) for k in range(1,nh+1)]
		fold=np.unique(['%d/%03d' % (date.timetuple().tm_year,date.timetuple().tm_yday) for date in dates])
		urls=[self.url+'/'+prod+'/'+f for f in fold]
		for u in urls:
			js=requests.get(u+'.json').json()
			for j in js:
				arg=np.argwhere(np.array(ids)=='.'.join(j['name'].split('.')[1:3]))
				if arg.size:
					ar=arg[0][0]
					g=Dict(j)
					g.links=[{'href': u+'/'+g.name}]
					g.time_start=geo_metas[ar]['time_start']
					g.time_end=geo_metas[ar]['time_end']
					g.dataset_id='Archive downloaded: unknown dataset id'
					g.producer_granule_id=j['name']
					metas.append(g)
		logging.info('Archive: %s gets %s hits in this range' % (self.prefix+prod,len(metas)))
		return metas

	def get_metas_sat(self, bbox, time, burned=False):
		"""
		Get all the meta data for all the necessary products

		:param bbox: polygon with the search bounding box
		:param time: time interval (init_time_datetime,final_time_datetime)
		:param maxg: max number of granules to process
		:return granules: dictionary with the metadata of all the products
		"""
		metas=Dict([])
		metas.geo=self.search_api_sat(self.geo_prefix,bbox,time,self.version)
		if not metas.geo:
			if not self.pre_geo_prefix:
				logging.warning('any geolocation data matches the search.')
				return metas
			else:
				pre_geo=self.search_api_sat(self.pre_geo_prefix,bbox,time,self.version)
				metas.geo=self.search_archive_sat(self.geo_prefix,time,pre_geo)
		fire=self.search_api_sat(self.fire_prefix,bbox,time)
		if fire:
			# eliminate NRT products (duplicated)
			nlist=[m for m in fire if m['data_center']!='LANCEMODIS']
			logging.info('eliminating %d NRT products (duplicated)' % int(abs(len(fire)-len(nlist))))
			metas.fire=nlist
		else:
			metas.fire=self.search_archive_sat(self.fire_prefix,time,metas.geo)
		if burned:
			metas.ref=self.search_api_sat(self.ref_prefix,bbox,time)
			if not metas.ref:
				metas.ref=self.search_archive_sat(self.ref_prefix,time,metas.geo)
		return metas

	def download_sat(self, url, appkey):
		"""
		Download a satellite file from a satellite service

		:param url: the URL of the file
		:param appkey: key to use for the download or None if not
		"""
		logging.info('downloading %s satellite data from %s' % (self.prefix, url))
		sat_name = osp.basename(url)
		sat_path = osp.join(self.ingest_dir,sat_name)
		if self.available_locally_sat(sat_path):
			logging.info('%s is available locally' % sat_path)
			return {'url': url,'local_path': sat_path}
		else:
			try:
				download_url(url, sat_path, appkey=appkey)
				return {'url': url,'local_path': sat_path,'downloaded': datetime.datetime.now}
			except DownloadError as e:
				logging.error('%s cannot download satellite file %s' % (self.prefix, url))
				logging.warning('Please check %s for %s' % (self.info_url, self.info))
				return {}
				raise SatError('SatSource: failed to download file %s' % url)

	def retrieve_data_sat(self, bounds, from_utc, to_utc):
		"""
		Retrieve satellite data in a bounding box coordinates and time interval

		:param bounds: polygon with the search bounding box
		:param time: time interval (init_time_iso,final_time_iso)
		:return data: dictonary with all the data
		"""
		lonmin,lonmax,latmin,latmax = bounds
		bbox = [(lonmin,latmax),(lonmin,latmin),(lonmax,latmin),(lonmax,latmax),(lonmin,latmax)]
		time = (from_utc, to_utc)

		metas = self.get_metas_sat(bbox,time)

		logging.info('retrieve_data_sat: saved %s metas for %s satellite service' % (sum([len(m) for m in metas.items()]),self.prefix))

		manifest = Dict({})

		for k, meta in metas.items():
			logging.info('downloading %s products' % k)
			for m in meta:
				id = osp.splitext(m['producer_granule_id'])[0]
				url = m['links'][0]['href']
				dc = m['data_center']
				m.update(self.download_sat(url,self.datacenter_to_appkey(dc)))
				if m:
				    manifest.update({id: m})

		group_manifest = self.group_files_sat(manifest)

		return group_manifest

	def group_files_sat(self, manifest):
		"""
                Group all the satellite files from downloaded manifest

                :param manifest: satellite manifest from retrieving
		:return result: groupped and cleanned manifest
                """
		if not manifest:
			return manifest
		result = Dict({})
		geore = re.compile(r'%s' % self.geo_prefix)
		pregeore = re.compile(r'%s' % self.pre_geo_prefix)
		firere = re.compile(r'%s' % self.fire_prefix)
		keys = np.array(list(manifest.keys()))
		labels = np.array([''.join(k.split('.')[1:3]) for k in keys])
		indexes = duplicates(labels)
		for k,ind in indexes.items():
			lenind = len(ind)
			if lenind != 2:
				logging.warning('group_files_sat: number of geo and fire granules %d different than 2' % lenind)
				if lenind < 2:
					logging.error('group_files_sat: geo or fire file is missing, number of granules %d different than 2' % lenind)
					continue
			geo = list(filter(geore.search,keys[ind]))
			pregeo = list(filter(pregeore.search,keys[ind]))
			fire = list(filter(firere.search,keys[ind]))
			if not geo:
				if pregeo:
					geok = pregeo[0]
				else:
					logging.error('group_files_sat: no geo data in the manifest')
					continue
			else:
				geok = geo[0]

			if fire:
				firek = fire[0]
			else:
				logging.error('group_files_sat: no fire data in the manifest')
				continue

			logging.info('group_files_sat: %s - geo %s and fire %s' % (k,geok,firek))
			try:
			        r = Dict({
					'time_start_iso' : manifest[geok]['time_start'],
					'time_end_iso' : manifest[geok]['time_end'],
					'geo_url' : manifest[geok]['url'],
					'geo_local_path' : manifest[geok]['local_path'],
					'geo_description' : manifest[geok]['dataset_id'],
					'fire_url' : manifest[firek]['url'],
					'fire_local_path' : manifest[firek]['local_path'],
					'fire_description' : manifest[firek]['dataset_id']
				})
			        result.update({k: r})
			except Exception as e:
				logging.error('group_files_sat: when creating manifest with error %s' % str(e))
				continue
		return result

	def read_sat(self, files, metas, bounds):
		"""
		Read all the satellite files from a specific satellite service

		:param file: dictionary with geolocation (03), fire (14) (, and reflectance (09)) file paths for satellite service
		:param bounds: spatial bounds tuple (lonmin,lonmax,latmin,latmax)
		:return ret: dictionary with Latitude, Longitude and fire mask arrays read
		"""
		pass

	def read_files_sat(self, file, bounds):
		"""
		Read a specific satellite files from a specific satellite service

		:param file: dictionary with geolocation (03), fire (14) (, and reflectance (09)) file paths for satellite service
		:param bounds: spatial bounds tuple (lonmin,lonmax,latmin,latmax)
		:return ret: dictionary with Latitude, Longitude and fire mask arrays read
		"""
		pass

	def datacenter_to_appkey(self,data_center):
		"""
		From data center to appkey to use for that data center

		:param data_center: string with the data center information
		"""
		return {'LAADS': self.appkey,
				'LPDAAC_ECS': None,
				'LANCEMODIS': self.appkey
			}.get(dataset_id)

	# instance variables
	id=None
	info_url=None
	info=None
	url=None
	prefix=None
	pre_geo_prefix=None
	geo_prefix=None
	fire_prefix=None
	ref_prefix=None
	platform=None
	version=None

<commit_msg>fixing a typo<commit_after>#
# Angel Farguell, CU Denver
#

from __future__ import absolute_import
import glob, re, datetime, logging, requests
import os.path as osp
import numpy as np
from six.moves.urllib.request import urlopen
from cmr import GranuleQuery
from utils import Dict, utc_to_utcf, duplicates
from .downloader import download_url, DownloadError
from six.moves import filter
from six.moves import range

class SatError(Exception):
	"""
	Raised when a SatSource cannot retrieve satellite data.
	"""
	pass

class SatSource(object):
	"""
	The parent class of all satellite sources that implements common functionality, for example

	- local satellite validation (file size check)
	- Satellite retrieval with retries (smart check whether server implements http-range)
	"""

	def __init__(self, js):
		"""
		Initialize satellite source with ingest directory (where satellite files are stored).

		:param js: job structure with at least ingest_path root of satellite storage and sys_install_path
		"""
		self.ingest_dir=osp.abspath(osp.join(js.get('ingest_path','ingest'),self.prefix))
		self.cache_dir=osp.abspath(js.get('cache_path','cache'))
		self.sys_dir=osp.abspath(js.get('sys_install_path',None))
		self.appkey=js.get('appkey',None)
		if not self.appkey:
			try:
				tokens = json.load(open('etc/tokens.json'))
				self.appkey = tokens.get('appkey',None)
			except:
				logging.warning('Any etc/tokens.json specified, any token is going to be used.')

	def available_locally_sat(self, path):
		"""
		Check if a satellite file is available locally and if it's file size checks out.

		:param path: the satellite file path
		"""
		info_path = path + '.size'
		if osp.exists(path) and osp.exists(info_path):
			content_size = int(open(info_path).read())
			return osp.getsize(path) == content_size
		else:
			return False

	def search_api_sat(self, sname, bbox, time, version=None):
		"""
		API search of the different satellite granules and return metadata dictionary

		:param sname: short name satellite product, ex: 'MOD03'
		:param bbox: polygon with the search bounding box
		:param time: time interval (init_time_iso,final_time_iso)

		:return metas: a dictionary with all the metadata for the API search
		"""
		maxg=100
		time_utcf=(utc_to_utcf(time[0]),utc_to_utcf(time[1]))
		api = GranuleQuery()
		if not version:
			search = api.parameters(
					short_name=sname,
					downloadable=True,
					polygon=bbox,
					temporal=time_utcf
				)
		else:
			search = api.parameters(
					short_name=sname,
					downloadable=True,
					polygon=bbox,
					temporal=time_utcf,
					version=version
				)
		sh=search.hits()
		logging.info("CMR API: %s gets %s hits in this range" % (sname,sh))
		if sh>maxg:
			logging.warning("The number of hits %s is larger than the limit %s." % (sh,maxg))
			logging.warning("Any satellite data with prefix %s used." % self.prefix)
			logging.warning("Use a reduced bounding box or a reduced time interval.")
			metas = []
		else:
			metas = api.get(sh)
		return metas

	def search_archive_sat(self, prod, time, geo_metas):
		"""
		Archive search of the different satellite granules

		:param prod: string of short name product, ex: 'VNP09'
		:param time: time interval (init_time_datetime,final_time_datetime)
		:param geo_metas: granules of the geolocation metadata
		:return metas: dictionary with the metadata of the search
		"""
		ids=['.'.join(gm['producer_granule_id'].split('.')[1:3]) for gm in geo_metas] # satellite ids in bounding box
		metas=[]
		delta=time[1]-time[0]
		nh=int(delta.total_seconds()/3600)
		dates=[time[0]+datetime.timedelta(seconds=3600*k) for k in range(1,nh+1)]
		fold=np.unique(['%d/%03d' % (date.timetuple().tm_year,date.timetuple().tm_yday) for date in dates])
		urls=[self.url+'/'+prod+'/'+f for f in fold]
		for u in urls:
			js=requests.get(u+'.json').json()
			for j in js:
				arg=np.argwhere(np.array(ids)=='.'.join(j['name'].split('.')[1:3]))
				if arg.size:
					ar=arg[0][0]
					g=Dict(j)
					g.links=[{'href': u+'/'+g.name}]
					g.time_start=geo_metas[ar]['time_start']
					g.time_end=geo_metas[ar]['time_end']
					g.dataset_id='Archive downloaded: unknown dataset id'
					g.producer_granule_id=j['name']
					metas.append(g)
		logging.info('Archive: %s gets %s hits in this range' % (self.prefix+prod,len(metas)))
		return metas

	def get_metas_sat(self, bbox, time, burned=False):
		"""
		Get all the meta data for all the necessary products

		:param bbox: polygon with the search bounding box
		:param time: time interval (init_time_datetime,final_time_datetime)
		:param maxg: max number of granules to process
		:return granules: dictionary with the metadata of all the products
		"""
		metas=Dict([])
		metas.geo=self.search_api_sat(self.geo_prefix,bbox,time,self.version)
		if not metas.geo:
			if not self.pre_geo_prefix:
				logging.warning('any geolocation data matches the search.')
				return metas
			else:
				pre_geo=self.search_api_sat(self.pre_geo_prefix,bbox,time,self.version)
				metas.geo=self.search_archive_sat(self.geo_prefix,time,pre_geo)
		fire=self.search_api_sat(self.fire_prefix,bbox,time)
		if fire:
			# eliminate NRT products (duplicated)
			nlist=[m for m in fire if m['data_center']!='LANCEMODIS']
			logging.info('eliminating %d NRT products (duplicated)' % int(abs(len(fire)-len(nlist))))
			metas.fire=nlist
		else:
			metas.fire=self.search_archive_sat(self.fire_prefix,time,metas.geo)
		if burned:
			metas.ref=self.search_api_sat(self.ref_prefix,bbox,time)
			if not metas.ref:
				metas.ref=self.search_archive_sat(self.ref_prefix,time,metas.geo)
		return metas

	def download_sat(self, url, appkey):
		"""
		Download a satellite file from a satellite service

		:param url: the URL of the file
		:param appkey: key to use for the download or None if not
		"""
		logging.info('downloading %s satellite data from %s' % (self.prefix, url))
		sat_name = osp.basename(url)
		sat_path = osp.join(self.ingest_dir,sat_name)
		if self.available_locally_sat(sat_path):
			logging.info('%s is available locally' % sat_path)
			return {'url': url,'local_path': sat_path}
		else:
			try:
				download_url(url, sat_path, appkey=appkey)
				return {'url': url,'local_path': sat_path,'downloaded': datetime.datetime.now}
			except DownloadError as e:
				logging.error('%s cannot download satellite file %s' % (self.prefix, url))
				logging.warning('Please check %s for %s' % (self.info_url, self.info))
				return {}
				raise SatError('SatSource: failed to download file %s' % url)

	def retrieve_data_sat(self, bounds, from_utc, to_utc):
		"""
		Retrieve satellite data in a bounding box coordinates and time interval

		:param bounds: polygon with the search bounding box
		:param time: time interval (init_time_iso,final_time_iso)
		:return data: dictonary with all the data
		"""
		lonmin,lonmax,latmin,latmax = bounds
		bbox = [(lonmin,latmax),(lonmin,latmin),(lonmax,latmin),(lonmax,latmax),(lonmin,latmax)]
		time = (from_utc, to_utc)

		metas = self.get_metas_sat(bbox,time)

		logging.info('retrieve_data_sat: saved %s metas for %s satellite service' % (sum([len(m) for m in metas.items()]),self.prefix))

		manifest = Dict({})

		for k, meta in metas.items():
			logging.info('downloading %s products' % k)
			for m in meta:
				id = osp.splitext(m['producer_granule_id'])[0]
				url = m['links'][0]['href']
				dc = m['data_center']
				m.update(self.download_sat(url,self.datacenter_to_appkey(dc)))
				if m:
				    manifest.update({id: m})

		group_manifest = self.group_files_sat(manifest)

		return group_manifest

	def group_files_sat(self, manifest):
		"""
                Group all the satellite files from downloaded manifest

                :param manifest: satellite manifest from retrieving
		:return result: groupped and cleanned manifest
                """
		if not manifest:
			return manifest
		result = Dict({})
		geore = re.compile(r'%s' % self.geo_prefix)
		pregeore = re.compile(r'%s' % self.pre_geo_prefix)
		firere = re.compile(r'%s' % self.fire_prefix)
		keys = np.array(list(manifest.keys()))
		labels = np.array([''.join(k.split('.')[1:3]) for k in keys])
		indexes = duplicates(labels)
		for k,ind in indexes.items():
			lenind = len(ind)
			if lenind != 2:
				logging.warning('group_files_sat: number of geo and fire granules %d different than 2' % lenind)
				if lenind < 2:
					logging.error('group_files_sat: geo or fire file is missing, number of granules %d different than 2' % lenind)
					continue
			geo = list(filter(geore.search,keys[ind]))
			pregeo = list(filter(pregeore.search,keys[ind]))
			fire = list(filter(firere.search,keys[ind]))
			if not geo:
				if pregeo:
					geok = pregeo[0]
				else:
					logging.error('group_files_sat: no geo data in the manifest')
					continue
			else:
				geok = geo[0]

			if fire:
				firek = fire[0]
			else:
				logging.error('group_files_sat: no fire data in the manifest')
				continue

			logging.info('group_files_sat: %s - geo %s and fire %s' % (k,geok,firek))
			try:
			        r = Dict({
					'time_start_iso' : manifest[geok]['time_start'],
					'time_end_iso' : manifest[geok]['time_end'],
					'geo_url' : manifest[geok]['url'],
					'geo_local_path' : manifest[geok]['local_path'],
					'geo_description' : manifest[geok]['dataset_id'],
					'fire_url' : manifest[firek]['url'],
					'fire_local_path' : manifest[firek]['local_path'],
					'fire_description' : manifest[firek]['dataset_id']
				})
			        result.update({k: r})
			except Exception as e:
				logging.error('group_files_sat: when creating manifest with error %s' % str(e))
				continue
		return result

	def read_sat(self, files, metas, bounds):
		"""
		Read all the satellite files from a specific satellite service

		:param file: dictionary with geolocation (03), fire (14) (, and reflectance (09)) file paths for satellite service
		:param bounds: spatial bounds tuple (lonmin,lonmax,latmin,latmax)
		:return ret: dictionary with Latitude, Longitude and fire mask arrays read
		"""
		pass

	def read_files_sat(self, file, bounds):
		"""
		Read a specific satellite files from a specific satellite service

		:param file: dictionary with geolocation (03), fire (14) (, and reflectance (09)) file paths for satellite service
		:param bounds: spatial bounds tuple (lonmin,lonmax,latmin,latmax)
		:return ret: dictionary with Latitude, Longitude and fire mask arrays read
		"""
		pass

	def datacenter_to_appkey(self,data_center):
		"""
		From data center to appkey to use for that data center

		:param data_center: string with the data center information
		"""
		return {'LAADS': self.appkey,
				'LPDAAC_ECS': None,
				'LANCEMODIS': self.appkey
			}.get(dataset_id)

	# instance variables
	id=None
	info_url=None
	info=None
	url=None
	prefix=None
	pre_geo_prefix=None
	geo_prefix=None
	fire_prefix=None
	ref_prefix=None
	platform=None
	version=None

<|endoftext|>


<commit_before>from collections import OrderedDict
import re

import sublime
from sublime_plugin import TextCommand

from . import util

interfaces = {}
subclasses = []

class Interface():

    interface_type = ""
    read_only = True
    syntax_file = ""
    word_wrap = False

    dedent = 0
    skip_first_line = False

    regions = {}
    template = ""

    _initialized = False

    def __new__(cls, repo_path=None, **kwargs):
        """
        Search for intended interface in active window - if found, bring it
        to focus and return it instead of creating a new interface.
        """
        window = sublime.active_window()
        for view in window.views():
            vset = view.settings()
            if vset.get("git_savvy.interface") == cls.interface_type and \
               vset.get("git_savvy.repo_path") == repo_path:
                window.focus_view(view)
                return interfaces[view.id()]

        return super().__new__(cls)

    def __init__(self, repo_path=None, view=None):
        if self._initialized:
            return
        self._initialized = True

        subclass_attrs = (getattr(self, attr) for attr in vars(self.__class__).keys())

        self.partials = {
            attr.key: attr
            for attr in subclass_attrs
            if callable(attr) and hasattr(attr, "key")
            }

        if self.skip_first_line:
            self.template = self.template[self.template.find("\n") + 1:]
        if self.dedent:
            for attr in vars(self.__class__).keys():
                if attr.startswith("template"):
                    setattr(self, attr, "\n".join(
                        line[self.dedent:] if len(line) >= self.dedent else line
                        for line in getattr(self, attr).split("\n")
                        ))

        if view:
            self.view = view
            self.render(nuke_cursors=False)
        else:
            self.view = self.create_view(repo_path)

        interfaces[self.view.id()] = self

    def create_view(self, repo_path):
        window = sublime.active_window()
        self.view = window.new_file()

        self.view.settings().set("git_savvy.repo_path", repo_path)
        self.view.set_name(self.title())
        self.view.settings().set("git_savvy.{}_view".format(self.interface_type), True)
        self.view.settings().set("git_savvy.interface", self.interface_type)
        self.view.settings().set("word_wrap", self.word_wrap)
        self.view.set_syntax_file(self.syntax_file)
        self.view.set_scratch(True)
        self.view.set_read_only(self.read_only)
        util.view.disable_other_plugins(self.view)

        self.render()
        window.focus_view(self.view)

        return self.view

    def render(self, nuke_cursors=True):
        self.clear_regions()
        if hasattr(self, "pre_render"):
            self.pre_render()

        rendered = self.template

        keyed_content = self.get_keyed_content()
        for key, new_content in keyed_content.items():
            new_content_len = len(new_content)
            pattern = re.compile(r"\{(<+ )?" + key + r"\}")

            match = pattern.search(rendered)
            while match:
                start, end = match.span()
                backspace_group = match.groups()[0]
                backspaces = backspace_group.count("<") if backspace_group else 0
                start -= backspaces

                rendered = rendered[:start] + new_content + rendered[end:]

                self.adjust(start, end - start, new_content_len)
                if new_content_len:
                    self.regions[key] = [start, start+new_content_len]

                match = pattern.search(rendered)

        self.view.run_command("gs_new_content_and_regions", {
            "content": rendered,
            "regions": self.regions,
            "nuke_cursors": nuke_cursors
            })

    def adjust(self, idx, orig_len, new_len):
        """
        When interpolating template variables, update region ranges for previously-evaluated
        variables, that are situated later on in the output/template string.
        """
        shift = new_len - orig_len
        for key, region in self.regions.items():
            if region[0] > idx:
                region[0] += shift
                region[1] += shift
            elif region[1] > idx or region[0] == idx:
                region[1] += shift

    def get_keyed_content(self):
        keyed_content = OrderedDict(
            (key, render_fn())
            for key, render_fn in self.partials.items()
            )

        for key in keyed_content:
            output = keyed_content[key]
            if isinstance(output, tuple):
                sub_template, complex_partials = output
                keyed_content[key] = sub_template

                for render_fn in complex_partials:
                    keyed_content[render_fn.key] = render_fn()

        return keyed_content

    def update(self, key, content):
        self.view.run_command("gs_update_region", {
            "key": "git_savvy_interface." + key,
            "content": content
            })

    def clear_regions(self):
        for key in self.regions.keys():
            self.view.erase_regions("git_savvy_interface." + key)
        self.regions = {}

    def get_view_regions(self, key):
        return self.view.get_regions("git_savvy_interface." + key)

    def get_selection_line(self):
        selections = self.view.sel()
        if not selections or len(selections) > 1:
            sublime.status_message("Please make a selection.")
            return None

        selection = selections[0]
        return selection, util.view.get_lines_from_regions(self.view, [selection])[0]

def partial(key):
    def decorator(fn):
        fn.key = key
        return fn
    return decorator

class GsNewContentAndRegionsCommand(TextCommand):

    def run(self, edit, content, regions, nuke_cursors=False):
        cursors_num = len(self.view.sel())
        is_read_only = self.view.is_read_only()
        self.view.set_read_only(False)
        self.view.replace(edit, sublime.Region(0, self.view.size()), content)
        self.view.set_read_only(is_read_only)

        if not cursors_num or nuke_cursors:
            selections = self.view.sel()
            selections.clear()
            pt = sublime.Region(0, 0)
            selections.add(pt)

        for key, region_range in regions.items():
            a, b = region_range
            self.view.add_regions("git_savvy_interface." + key, [sublime.Region(a, b)])

class GsUpdateRegionCommand(TextCommand):

    def run(self, edit, key, content):
        is_read_only = self.view.is_read_only()
        self.view.set_read_only(False)
        for region in self.view.get_regions(key):
            self.view.replace(edit, region, content)
        self.view.set_read_only(is_read_only)

def register_listeners(InterfaceClass):
    subclasses.append(InterfaceClass)

def get_interface(view_id):
    return interfaces.get(view_id, None)

class GsInterfaceCloseCommand(TextCommand):

    """
    Clean up references to interfaces for closed views.
    """

    def run(self, edit):
        sublime.set_timeout_async(self.run_async, 0)

    def run_async(self):
        view_id = self.view.id()
        if view_id in interfaces:
            del interfaces[view_id]

class GsInterfaceRefreshCommand(TextCommand):

    """
    Re-render GitSavvy interface view.
    """

    def run(self, edit):
        sublime.set_timeout_async(self.run_async, 0)

    def run_async(self):
        interface_type = self.view.settings().get("git_savvy.interface")
        for InterfaceSubclass in subclasses:
            if InterfaceSubclass.interface_type == interface_type:
                existing_interface = interfaces.get(self.view.id(), None)
                if existing_interface:
                    existing_interface.render(nuke_cursors=False)
                else:
                    interface = InterfaceSubclass(view=self.view)
                    interfaces[interface.view.id()] = interface
<commit_msg>Fix: Cursor behavior not as expected on re-render.<commit_after>from collections import OrderedDict
import re

import sublime
from sublime_plugin import TextCommand

from . import util

interfaces = {}
subclasses = []

class Interface():

    interface_type = ""
    read_only = True
    syntax_file = ""
    word_wrap = False

    dedent = 0
    skip_first_line = False

    regions = {}
    template = ""

    _initialized = False

    def __new__(cls, repo_path=None, **kwargs):
        """
        Search for intended interface in active window - if found, bring it
        to focus and return it instead of creating a new interface.
        """
        window = sublime.active_window()
        for view in window.views():
            vset = view.settings()
            if vset.get("git_savvy.interface") == cls.interface_type and \
               vset.get("git_savvy.repo_path") == repo_path:
                window.focus_view(view)
                return interfaces[view.id()]

        return super().__new__(cls)

    def __init__(self, repo_path=None, view=None):
        if self._initialized:
            return
        self._initialized = True

        subclass_attrs = (getattr(self, attr) for attr in vars(self.__class__).keys())

        self.partials = {
            attr.key: attr
            for attr in subclass_attrs
            if callable(attr) and hasattr(attr, "key")
            }

        if self.skip_first_line:
            self.template = self.template[self.template.find("\n") + 1:]
        if self.dedent:
            for attr in vars(self.__class__).keys():
                if attr.startswith("template"):
                    setattr(self, attr, "\n".join(
                        line[self.dedent:] if len(line) >= self.dedent else line
                        for line in getattr(self, attr).split("\n")
                        ))

        if view:
            self.view = view
            self.render(nuke_cursors=False)
        else:
            self.view = self.create_view(repo_path)

        interfaces[self.view.id()] = self

    def create_view(self, repo_path):
        window = sublime.active_window()
        self.view = window.new_file()

        self.view.settings().set("git_savvy.repo_path", repo_path)
        self.view.set_name(self.title())
        self.view.settings().set("git_savvy.{}_view".format(self.interface_type), True)
        self.view.settings().set("git_savvy.interface", self.interface_type)
        self.view.settings().set("word_wrap", self.word_wrap)
        self.view.set_syntax_file(self.syntax_file)
        self.view.set_scratch(True)
        self.view.set_read_only(self.read_only)
        util.view.disable_other_plugins(self.view)

        self.render()
        window.focus_view(self.view)

        return self.view

    def render(self, nuke_cursors=False):
        self.clear_regions()
        if hasattr(self, "pre_render"):
            self.pre_render()

        rendered = self.template

        keyed_content = self.get_keyed_content()
        for key, new_content in keyed_content.items():
            new_content_len = len(new_content)
            pattern = re.compile(r"\{(<+ )?" + key + r"\}")

            match = pattern.search(rendered)
            while match:
                start, end = match.span()
                backspace_group = match.groups()[0]
                backspaces = backspace_group.count("<") if backspace_group else 0
                start -= backspaces

                rendered = rendered[:start] + new_content + rendered[end:]

                self.adjust(start, end - start, new_content_len)
                if new_content_len:
                    self.regions[key] = [start, start+new_content_len]

                match = pattern.search(rendered)

        self.view.run_command("gs_new_content_and_regions", {
            "content": rendered,
            "regions": self.regions,
            "nuke_cursors": nuke_cursors
            })

    def adjust(self, idx, orig_len, new_len):
        """
        When interpolating template variables, update region ranges for previously-evaluated
        variables, that are situated later on in the output/template string.
        """
        shift = new_len - orig_len
        for key, region in self.regions.items():
            if region[0] > idx:
                region[0] += shift
                region[1] += shift
            elif region[1] > idx or region[0] == idx:
                region[1] += shift

    def get_keyed_content(self):
        keyed_content = OrderedDict(
            (key, render_fn())
            for key, render_fn in self.partials.items()
            )

        for key in keyed_content:
            output = keyed_content[key]
            if isinstance(output, tuple):
                sub_template, complex_partials = output
                keyed_content[key] = sub_template

                for render_fn in complex_partials:
                    keyed_content[render_fn.key] = render_fn()

        return keyed_content

    def update(self, key, content):
        self.view.run_command("gs_update_region", {
            "key": "git_savvy_interface." + key,
            "content": content
            })

    def clear_regions(self):
        for key in self.regions.keys():
            self.view.erase_regions("git_savvy_interface." + key)
        self.regions = {}

    def get_view_regions(self, key):
        return self.view.get_regions("git_savvy_interface." + key)

    def get_selection_line(self):
        selections = self.view.sel()
        if not selections or len(selections) > 1:
            sublime.status_message("Please make a selection.")
            return None

        selection = selections[0]
        return selection, util.view.get_lines_from_regions(self.view, [selection])[0]

def partial(key):
    def decorator(fn):
        fn.key = key
        return fn
    return decorator

class GsNewContentAndRegionsCommand(TextCommand):

    def run(self, edit, content, regions, nuke_cursors=False):
        selections = self.view.sel()

        if selections and not nuke_cursors:
            cursors_row_col = [self.view.rowcol(cursor.a) for cursor in selections]
        else:
            cursors_row_col = [(0, 0)]

        selections.clear()

        is_read_only = self.view.is_read_only()
        self.view.set_read_only(False)
        self.view.replace(edit, sublime.Region(0, self.view.size()), content)
        self.view.set_read_only(is_read_only)

        for row, col in cursors_row_col:
            pt = self.view.text_point(row, col)
            selections.add(sublime.Region(pt, pt))

        for key, region_range in regions.items():
            a, b = region_range
            self.view.add_regions("git_savvy_interface." + key, [sublime.Region(a, b)])

class GsUpdateRegionCommand(TextCommand):

    def run(self, edit, key, content):
        is_read_only = self.view.is_read_only()
        self.view.set_read_only(False)
        for region in self.view.get_regions(key):
            self.view.replace(edit, region, content)
        self.view.set_read_only(is_read_only)

def register_listeners(InterfaceClass):
    subclasses.append(InterfaceClass)

def get_interface(view_id):
    return interfaces.get(view_id, None)

class GsInterfaceCloseCommand(TextCommand):

    """
    Clean up references to interfaces for closed views.
    """

    def run(self, edit):
        sublime.set_timeout_async(self.run_async, 0)

    def run_async(self):
        view_id = self.view.id()
        if view_id in interfaces:
            del interfaces[view_id]

class GsInterfaceRefreshCommand(TextCommand):

    """
    Re-render GitSavvy interface view.
    """

    def run(self, edit):
        sublime.set_timeout_async(self.run_async, 0)

    def run_async(self):
        interface_type = self.view.settings().get("git_savvy.interface")
        for InterfaceSubclass in subclasses:
            if InterfaceSubclass.interface_type == interface_type:
                existing_interface = interfaces.get(self.view.id(), None)
                if existing_interface:
                    existing_interface.render(nuke_cursors=False)
                else:
                    interface = InterfaceSubclass(view=self.view)
                    interfaces[interface.view.id()] = interface
<|endoftext|>


<commit_before>'''
Created on Mar 7, 2013

@author: crisr
'''
from __future__ import division, print_function, unicode_literals, absolute_import
import warnings
warnings.simplefilter('default',DeprecationWarning)

import sys
import scipy.stats.distributions  as dist
import numpy as np
from BaseType import BaseType
import scipy.special as polys
from scipy.misc import factorial
import Quadrature

class Distribution(BaseType):
  '''
  a general class containing the distributions
  '''
  def __init__(self):
    BaseType.__init__(self)
    self.upperBoundUsed = False  #True if the distribution is right truncated
    self.lowerBoundUsed = False  #True if the distribution is left truncated
    self.upperBound       = 0.0  #Right bound
    self.lowerBound       = 0.0  #Left bound
    self.adjustmentType   = ''   #this describe how the re-normalization to preserve the probability should be done for truncated distributions
    self.bestQuad         = None #the quadrature that best integrate the distribution

  def readMoreXML(self,xmlNode):
    #this is part of the stochastic collocation sampler not of the distribution!!!!!!
    #FIXME
    try:
      QuadType = xmlNode.attrib['ExpansionQuadrature']
      self.bestQuad = Quadrature.returnInstance(QuadType)
    except:
      pass
    if xmlNode.find('upperBound') !=None:
      self.upperBound = float(xmlNode.find('upperBound').text)
      self.upperBoundUsed = True
    if xmlNode.find('lowerBound')!=None:
      self.lowerBound = float(xmlNode.find('lowerBound').text)
      self.lowerBoundUsed = True
    if xmlNode.find('adjustment') !=None:
      self.adjustment = xmlNode.find('adjustment').text
    else:
      self.adjustment = 'scaling'

  def addInitParams(self,tempDict):
    tempDict['upperBoundUsed'] = self.upperBoundUsed
    tempDict['lowerBoundUsed'] = self.lowerBoundUsed
    tempDict['upperBound'    ] = self.upperBound
    tempDict['lowerBound'    ] = self.lowerBound
    tempDict['adjustmentType'] = self.adjustmentType
    tempDict['bestQuad'      ] = self.bestQuad

  def rvsWithinCDFbounds(self,LowerBound,upperBound):
    point = np.random.rand(1)*(upperBound-LowerBound)+LowerBound
    return self.distribution.ppf(point)

  def rvsWithinbounds(self,LowerBound,upperBound):
    CDFupper = self.distribution.cdf(upperBound)
    CDFlower = self.distribution.cdf(LowerBound)
    return self.rvsWithinCDFbounds(CDFlower,CDFupper)

  def setQuad(self,quad,exp_order):
    self.distQuad=quad
    self.exp_order=exp_order

  def quad(self):
    try: return self.distQuad
    except: raise IOError ('No quadrature has been set for this distr. yet.')

  def polyOrder(self):
    try: return self.exp_order
    except: raise IOError ('Quadrature has not been set for this distr. yet.')

#==============================================================\
#    Distributions convenient for stochastic collocation
#==============================================================\

class Uniform(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.low = 0.0
    self.hi = 0.0
    self.type = 'Uniform'
    self.bestQuad = Quadrature.Legendre

  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self,xmlNode)
    try: self.low = float(xmlNode.find('low').text)
    except: raise Exception('low value needed for uniform distribution')
    try: self.hi = float(xmlNode.find('hi').text)
    except: raise Exception('hi value needed for uniform distribution')
#    self.initializeDistribution() this call is done by the sampler each time a new step start
    self.range=self.hi-self.low
    #assign associated polynomial types
    self.polynomial = polys.legendre
    #define functions locally, then point to them
    def norm(n):
      '''Returns normalization constant for polynomial type, given the poly ordeir'''
      return np.sqrt((2.*n+1.)/2.)

    def standardToActualPoint(x): #standard -> actual
      '''Given a [-1,1] point, converts to parameter value.'''
      return x*self.range/2.+self.distribution.mean()

    def actualToStandardPoint(x): #actual -> standard
      '''Given a parameter value, converts to [-1,1] point.'''
      return (x-self.distribution.mean())/(self.range/2.)

    def standardToActualWeight(x): #standard -> actual
      '''Given normal quadrature weight, returns adjusted weight.'''
      return x/(self.range/2.)

    def probNorm(x): #normalizes probability if total != 1
      '''Returns the poly factor to scale by so that sum(probability)=1.'''
      return self.range

    # point to functions
    self.poly_norm = norm
    self.actual_point = standardToActualPoint
    self.std_point = actualToStandardPoint
    self.actual_weight = standardToActualWeight
    self.probability_norm = probNorm

  def addInitParams(self,tempDict):
    Distribution.addInitParams(self,tempDict)
    tempDict['low'] = self.low
    tempDict['hi'] = self.hi
    # no other additional parameters required

  def initializeDistribution(self):
    self.distribution = dist.uniform(loc=self.low,scale=self.range)

class Normal(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.mean  = 0.0
    self.sigma = 0.0
    self.type = 'Normal'
    self.bestQuad = Quadrature.StatHermite

  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self, xmlNode)
    try: self.mean  = float(xmlNode.find('mean' ).text)
    except: raise Exception('mean value needed for normal distribution')
    try: self.sigma = float(xmlNode.find('sigma').text)
    except: raise Exception('sigma value needed for normal distribution')
    self.initializeDistribution()

  def addInitParams(self,tempDict):
    Distribution.addInitParams(self, tempDict)
    tempDict['mean' ] = self.mean
    tempDict['sigma'] = self.sigma

  def initializeDistribution(self):
    if (not self.upperBoundUsed) and (not self.lowerBoundUsed):
      self.distribution = dist.norm(loc=self.mean,scale=self.sigma)
      self.polynomial = polys.hermitenorm
      def norm(n):
        return (np.sqrt(np.sqrt(2.*np.pi)*factorial(n)))**(-1)

      def standardToActualPoint(x): #standard -> actual
        return x*self.sigma**2/2.+self.distribution.mean()

      def actualToStandardPoint(x): #actual -> standard
        return (x-self.distribution.mean())/(self.sigma**2/2.)

      def standardToActualWeight(x): #standard -> actual
        return x/(self.sigma**2/2.)

      def probNorm(x): #normalizes if total prob. != 1
        return 1.0

      self.poly_norm = norm
      self.actual_point = standardToActualPoint
      self.std_point = actualToStandardPoint
      self.actual_weight = standardToActualWeight
      self.probability_norm = probNorm
    else:
      #FIXME special case distribution for stoch collocation
      if self.lowerBoundUsed == False: a = -sys.float_info[max]
      else:a = (self.lowerBound - self.mean) / self.sigma
      if self.upperBoundUsed == False: b = sys.float_info[max]
      else:b = (self.upperBound - self.mean) / self.sigma
      self.distribution = dist.truncnorm(a,b,loc=self.mean,scale=self.sigma)

class Gamma(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.low = 0.0
    self.alpha = 0.0
    self.beta = 1.0
    self.type = 'Gamma'
    self.bestQuad = Quadrature.Laguerre

  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self,xmlNode)
    try: self.low = float(xmlNode.find('low').text)
    except: raise Exception('low value needed for Gamma distribution')
    try: self.alpha = float(xmlNode.find('alpha').text)
    except: raise Exception('alpha value needed for Gamma distribution')
    try: self.beta = float(xmlNode.find('beta').text)
    except: self.beta=1.0
    self.initializeDistribution()

  def addInitParams(self,tempDict):
    Distribution.addInitParams(self,tempDict)
    tempDict['low'] = self.low
    tempDict['alpha'] = self.alpha
    tempDict['beta'] = self.beta

  def initializeDistribution(self):
    self.distribution = dist.gamma(self.alpha,loc=self.low,scale=self.beta)
    self.polynomial = polys.genlaguerre
    def norm(n):
      return np.sqrt(factorial(n)/polys.gamma(n+self.alpha+1.0))

    def standardToActualPoint(x): #standard -> actual
      return x/self.alpha+self.alpha+self.low #TODO these correct? no beta used

    def actualToStandardPoint(x): #actual -> standard
      return (x-self.low-self.alpha)*self.alpha

    def standardToActualWeight(x): #standard -> actual
      return x

    def probNorm(x): #normalizes probability if total != 1
      return 1.0

    self.poly_norm=norm
    self.actual_point = standardToActualPoint
    self.std_point = actualToStandardPoint
    self.actual_weight = standardToActualWeight
    self.probability_norm = probNorm

class Beta(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.low = 0.0
    self.hi = 0.0
    self.alpha = 0.0
    self.beta = 0.0
    self.type = 'Beta'
    self.bestQuad = Quadrature.Jacobi
    # TODO default to specific Beta distro?

  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self,xmlNode)
    try: self.low = float(xmlNode.find('low').text)
    except: raise Exception('low value needed for Gamma distribution')
    try: self.hi = float(xmlNode.find('hi').text)
    except: raise Exception('hi value needed for Gamma distribution')
    try: self.alpha = float(xmlNode.find('alpha').text)
    except: raise Exception('alpha value needed for Gamma distribution')
    try: self.beta = float(xmlNode.find('beta').text)
    except: raise Exception('beta value needed for Gamma distribution')
    self.initializeDistribution()

  def addInitParams(self,tempDict):
    Distribution.addInitParams(self,tempDict)
    tempDict['low'] = self.low
    tempDict['hi'] = self.hi
    tempDict['alpha'] = self.alpha
    tempDict['beta'] = self.beta

  def initializeDistribution(self):
    self.distribution = dist.beta(self.alpha,self.beta,scale=self.hi-self.low)

#==========================================================\
#    other distributions
#==========================================================\

# Add polynomials, shifting, zero-to-one to these!
class Triangular(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.apex = 0.0
    self.min  = 0.0
    self.max  = 0.0
    self.type = 'Triangular'
    self.bestQuad = None

  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self, xmlNode)
    try: self.apex = float(xmlNode.find('apex').text)
    except: raise Exception('apex value needed for normal distribution')
    try: self.min = float(xmlNode.find('min').text)
    except: raise Exception('min value needed for normal distribution')
    try: self.max = float(xmlNode.find('max').text)
    except: raise Exception('max value needed for normal distribution')
    self.initializeDistribution()

  def addInitParams(self,tempDict):
    Distribution.addInitParams(self, tempDict)
    tempDict['apex' ] = self.apex
    tempDict['min'  ] = self.min
    tempDict['max'  ] = self.max

  def initializeDistribution(self):
    if self.lowerBoundUsed == False and self.upperBoundUsed == False:
      c = (self.apex-self.min)/(self.max-self.min)
      self.distribution = dist.triang(c,loc=self.min,scale=(self.max-self.min))
    else:
      raise IOError ('Truncated triangular not yet implemented')

def returnInstance(Type):
  base = 'Distribution'
  InterfaceDict = {}
  InterfaceDict['Uniform'  ]  = Uniform
  InterfaceDict['Normal'   ]  = Normal
  InterfaceDict['Gamma'    ]  = Gamma
  InterfaceDict['Beta'     ]  = Beta
  InterfaceDict['Triangular'] = Triangular
  try: return InterfaceDict[Type]()
  except: raise NameError('not known '+base+' type '+Type)
<commit_msg>added distributions<commit_after>'''
Created on Mar 7, 2013

@author: crisr
'''
from __future__ import division, print_function, unicode_literals, absolute_import
import warnings
warnings.simplefilter('default',DeprecationWarning)

import sys
import scipy.stats.distributions  as dist
import numpy as np
from BaseType import BaseType
import scipy.special as polys
from scipy.misc import factorial
import Quadrature

class Distribution(BaseType):
  '''
  a general class containing the distributions
  '''
  def __init__(self):
    BaseType.__init__(self)
    self.upperBoundUsed = False  #True if the distribution is right truncated
    self.lowerBoundUsed = False  #True if the distribution is left truncated
    self.upperBound       = 0.0  #Right bound
    self.lowerBound       = 0.0  #Left bound
    self.adjustmentType   = ''   #this describe how the re-normalization to preserve the probability should be done for truncated distributions
    self.bestQuad         = None #the quadrature that best integrate the distribution

  def readMoreXML(self,xmlNode):
    #this is part of the stochastic collocation sampler not of the distribution!!!!!!
    #FIXME
    try:
      QuadType = xmlNode.attrib['ExpansionQuadrature']
      self.bestQuad = Quadrature.returnInstance(QuadType)
    except:
      pass
    if xmlNode.find('upperBound') !=None:
      self.upperBound = float(xmlNode.find('upperBound').text)
      self.upperBoundUsed = True
    if xmlNode.find('lowerBound')!=None:
      self.lowerBound = float(xmlNode.find('lowerBound').text)
      self.lowerBoundUsed = True
    if xmlNode.find('adjustment') !=None:
      self.adjustment = xmlNode.find('adjustment').text
    else:
      self.adjustment = 'scaling'

  def addInitParams(self,tempDict):
    tempDict['upperBoundUsed'] = self.upperBoundUsed
    tempDict['lowerBoundUsed'] = self.lowerBoundUsed
    tempDict['upperBound'    ] = self.upperBound
    tempDict['lowerBound'    ] = self.lowerBound
    tempDict['adjustmentType'] = self.adjustmentType
    tempDict['bestQuad'      ] = self.bestQuad

  def rvsWithinCDFbounds(self,LowerBound,upperBound):
    point = np.random.rand(1)*(upperBound-LowerBound)+LowerBound
    return self.distribution.ppf(point)

  def rvsWithinbounds(self,LowerBound,upperBound):
    CDFupper = self.distribution.cdf(upperBound)
    CDFlower = self.distribution.cdf(LowerBound)
    return self.rvsWithinCDFbounds(CDFlower,CDFupper)

  def setQuad(self,quad,exp_order):
    self.distQuad=quad
    self.exp_order=exp_order

  def quad(self):
    try: return self.distQuad
    except: raise IOError ('No quadrature has been set for this distr. yet.')

  def polyOrder(self):
    try: return self.exp_order
    except: raise IOError ('Quadrature has not been set for this distr. yet.')

#==============================================================\
#    Distributions convenient for stochastic collocation
#==============================================================\

class Uniform(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.low = 0.0
    self.hi = 0.0
    self.type = 'Uniform'
    self.bestQuad = Quadrature.Legendre

  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self,xmlNode)
    try: self.low = float(xmlNode.find('low').text)
    except: raise Exception('low value needed for uniform distribution')
    try: self.hi = float(xmlNode.find('hi').text)
    except: raise Exception('hi value needed for uniform distribution')
#    self.initializeDistribution() this call is done by the sampler each time a new step start
    self.range=self.hi-self.low
    #assign associated polynomial types
    self.polynomial = polys.legendre
    #define functions locally, then point to them
    def norm(n):
      '''Returns normalization constant for polynomial type, given the poly ordeir'''
      return np.sqrt((2.*n+1.)/2.)

    def standardToActualPoint(x): #standard -> actual
      '''Given a [-1,1] point, converts to parameter value.'''
      return x*self.range/2.+self.distribution.mean()

    def actualToStandardPoint(x): #actual -> standard
      '''Given a parameter value, converts to [-1,1] point.'''
      return (x-self.distribution.mean())/(self.range/2.)

    def standardToActualWeight(x): #standard -> actual
      '''Given normal quadrature weight, returns adjusted weight.'''
      return x/(self.range/2.)

    def probNorm(x): #normalizes probability if total != 1
      '''Returns the poly factor to scale by so that sum(probability)=1.'''
      return self.range

    # point to functions
    self.poly_norm = norm
    self.actual_point = standardToActualPoint
    self.std_point = actualToStandardPoint
    self.actual_weight = standardToActualWeight
    self.probability_norm = probNorm

  def addInitParams(self,tempDict):
    Distribution.addInitParams(self,tempDict)
    tempDict['low'] = self.low
    tempDict['hi'] = self.hi
    # no other additional parameters required

  def initializeDistribution(self):
    self.distribution = dist.uniform(loc=self.low,scale=self.range)

class Normal(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.mean  = 0.0
    self.sigma = 0.0
    self.type = 'Normal'
    self.bestQuad = Quadrature.StatHermite

  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self, xmlNode)
    try: self.mean  = float(xmlNode.find('mean' ).text)
    except: raise Exception('mean value needed for normal distribution')
    try: self.sigma = float(xmlNode.find('sigma').text)
    except: raise Exception('sigma value needed for normal distribution')
    self.initializeDistribution()

  def addInitParams(self,tempDict):
    Distribution.addInitParams(self, tempDict)
    tempDict['mean' ] = self.mean
    tempDict['sigma'] = self.sigma

  def initializeDistribution(self):
    if (not self.upperBoundUsed) and (not self.lowerBoundUsed):
      self.distribution = dist.norm(loc=self.mean,scale=self.sigma)
      self.polynomial = polys.hermitenorm
      def norm(n):
        return (np.sqrt(np.sqrt(2.*np.pi)*factorial(n)))**(-1)

      def standardToActualPoint(x): #standard -> actual
        return x*self.sigma**2/2.+self.distribution.mean()

      def actualToStandardPoint(x): #actual -> standard
        return (x-self.distribution.mean())/(self.sigma**2/2.)

      def standardToActualWeight(x): #standard -> actual
        return x/(self.sigma**2/2.)

      def probNorm(x): #normalizes if total prob. != 1
        return 1.0

      self.poly_norm = norm
      self.actual_point = standardToActualPoint
      self.std_point = actualToStandardPoint
      self.actual_weight = standardToActualWeight
      self.probability_norm = probNorm
    else:
      #FIXME special case distribution for stoch collocation
      if self.lowerBoundUsed == False: a = -sys.float_info[max]
      else:a = (self.lowerBound - self.mean) / self.sigma
      if self.upperBoundUsed == False: b = sys.float_info[max]
      else:b = (self.upperBound - self.mean) / self.sigma
      self.distribution = dist.truncnorm(a,b,loc=self.mean,scale=self.sigma)

class Gamma(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.low = 0.0
    self.alpha = 0.0
    self.beta = 1.0
    self.type = 'Gamma'
    self.bestQuad = Quadrature.Laguerre

  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self,xmlNode)
    try: self.low = float(xmlNode.find('low').text)
    except: raise Exception('low value needed for Gamma distribution')
    try: self.alpha = float(xmlNode.find('alpha').text)
    except: raise Exception('alpha value needed for Gamma distribution')
    try: self.beta = float(xmlNode.find('beta').text)
    except: self.beta=1.0
    self.initializeDistribution()

  def addInitParams(self,tempDict):
    Distribution.addInitParams(self,tempDict)
    tempDict['low'] = self.low
    tempDict['alpha'] = self.alpha
    tempDict['beta'] = self.beta

  def initializeDistribution(self):
    self.distribution = dist.gamma(self.alpha,loc=self.low,scale=self.beta)
    self.polynomial = polys.genlaguerre
    def norm(n):
      return np.sqrt(factorial(n)/polys.gamma(n+self.alpha+1.0))

    def standardToActualPoint(x): #standard -> actual
      return x/self.alpha+self.alpha+self.low #TODO these correct? no beta used

    def actualToStandardPoint(x): #actual -> standard
      return (x-self.low-self.alpha)*self.alpha

    def standardToActualWeight(x): #standard -> actual
      return x

    def probNorm(x): #normalizes probability if total != 1
      return 1.0

    self.poly_norm=norm
    self.actual_point = standardToActualPoint
    self.std_point = actualToStandardPoint
    self.actual_weight = standardToActualWeight
    self.probability_norm = probNorm

class Beta(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.low = 0.0
    self.hi = 0.0
    self.alpha = 0.0
    self.beta = 0.0
    self.type = 'Beta'
    self.bestQuad = Quadrature.Jacobi
    # TODO default to specific Beta distro?

  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self,xmlNode)
    try: self.low = float(xmlNode.find('low').text)
    except: raise Exception('low value needed for Gamma distribution')
    try: self.hi = float(xmlNode.find('hi').text)
    except: raise Exception('hi value needed for Gamma distribution')
    try: self.alpha = float(xmlNode.find('alpha').text)
    except: raise Exception('alpha value needed for Gamma distribution')
    try: self.beta = float(xmlNode.find('beta').text)
    except: raise Exception('beta value needed for Gamma distribution')
    self.initializeDistribution()

  def addInitParams(self,tempDict):
    Distribution.addInitParams(self,tempDict)
    tempDict['low'] = self.low
    tempDict['hi'] = self.hi
    tempDict['alpha'] = self.alpha
    tempDict['beta'] = self.beta

  def initializeDistribution(self):
    self.distribution = dist.beta(self.alpha,self.beta,scale=self.hi-self.low)

#==========================================================\
#    other distributions
#==========================================================\

# Add polynomials, shifting, zero-to-one to these!
class Triangular(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.apex = 0.0
    self.min  = 0.0
    self.max  = 0.0
    self.type = 'Triangular'
    self.bestQuad = None

  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self, xmlNode)
    try: self.apex = float(xmlNode.find('apex').text)
    except: raise Exception('apex value needed for normal distribution')
    try: self.min = float(xmlNode.find('min').text)
    except: raise Exception('min value needed for normal distribution')
    try: self.max = float(xmlNode.find('max').text)
    except: raise Exception('max value needed for normal distribution')
    self.initializeDistribution()

  def addInitParams(self,tempDict):
    Distribution.addInitParams(self, tempDict)
    tempDict['apex' ] = self.apex
    tempDict['min'  ] = self.min
    tempDict['max'  ] = self.max

  def initializeDistribution(self):
    if self.lowerBoundUsed == False and self.upperBoundUsed == False:
      c = (self.apex-self.min)/(self.max-self.min)
      self.distribution = dist.triang(c,loc=self.min,scale=(self.max-self.min))
    else:
      raise IOError ('Truncated triangular not yet implemented')

class Poisson(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.mu  = 0.0
    self.type = 'Poisson'

  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self, xmlNode)
    try: self.mu = float(xmlNode.find('mu').text)
    except: raise Exception('mu value needed for poisson distribution')
    self.initializeDistribution()

  def addInitParams(self,tempDict):
    Distribution.addInitParams(self, tempDict)
    tempDict['mu'  ] = self.mu

  def initializeDistribution(self):
    if self.lowerBoundUsed == False and self.upperBoundUsed == False:
      self.distribution = dist.poisson(mu=self.mu)
    else:
      raise IOError ('Truncated poisson not yet implemented')

class Binomial(Distribution):
  def __init__(self):
    Distribution.__init__(self)
    self.n  = 0.0
    self.p  = 0.0
    self.type = 'Binomial'

  def readMoreXML(self,xmlNode):
    Distribution.readMoreXML(self, xmlNode)
    try: self.n = float(xmlNode.find('n').text)
    except: raise Exception('n value needed for Binomial distribution')
    try: self.p = float(xmlNode.find('p').text)
    except: raise Exception('p value needed for Binomial distribution')
    self.initializeDistribution()

  def addInitParams(self,tempDict):
    Distribution.addInitParams(self, tempDict)
    tempDict['n'  ] = self.n
    tempDict['p'  ] = self.p

  def initializeDistribution(self):
    if self.lowerBoundUsed == False and self.upperBoundUsed == False:
      self.distribution = dist.binom(n=self.n,p=self.p)
    else:
      raise IOError ('Truncated Binomial not yet implemented')

def returnInstance(Type):
  base = 'Distribution'
  InterfaceDict = {}
  InterfaceDict['Uniform'  ]  = Uniform
  InterfaceDict['Normal'   ]  = Normal
  InterfaceDict['Gamma'    ]  = Gamma
  InterfaceDict['Beta'     ]  = Beta
  InterfaceDict['Triangular'] = Triangular
  InterfaceDict['Poisson']    = Poisson
  InterfaceDict['Binomial']    = Binomial
  try: return InterfaceDict[Type]()
  except: raise NameError('not known '+base+' type '+Type)

<|endoftext|>


<commit_before>"""
to run it:
sudo pip install oauth2client
python oauth2cli.py --auth_host_port 9000 --client-secret <client-secret> --client-id <client-id>

it creates a json file called "credentials_sdm"
the access_token in the credentials is also printed to stdout
"""

from oauth2client.client import OAuth2WebServerFlow
from oauth2client.file import Storage
import argparse
from oauth2client import tools
import contextlib
import sys
import cStringIO

parser = argparse.ArgumentParser(parents=[tools.argparser])
parser.add_argument('--client-id', type=str, required=True)
parser.add_argument('--client-secret', type=str, required=True)
flags = parser.parse_args()

@contextlib.contextmanager
def nostdout():
    save_stdout = sys.stdout
    sys.stdout = cStringIO.StringIO()
    yield
    sys.stdout = save_stdout

def main(flags):
    flow = OAuth2WebServerFlow(client_id=flags.client_id,
                           client_secret=flags.client_secret,
                           scope='https://www.googleapis.com/auth/userinfo.email')

    storage = Storage('credentials_sdm')

    credentials = tools.run_flow(flow, storage, flags)
    return credentials

if __name__ == '__main__':
    with nostdout():
        c = main(flags)
    print c.get_access_token().access_token
<commit_msg>add option to refresh the token<commit_after>"""
to run it:
sudo pip install oauth2client
python oauth2cli.py --auth_host_port 9000 --client-secret <client-secret> --client-id <client-id>

it creates a json file called "credentials_sdm"
the access_token in the credentials is also printed to stdout
"""

from oauth2client.client import OAuth2WebServerFlow, OAuth2Credentials
from oauth2client.file import Storage
import argparse
from oauth2client import tools
import contextlib
import sys
import cStringIO
import httplib2

FILE_NAME = 'credentials_sdm'

@contextlib.contextmanager
def nostdout():
    save_stdout = sys.stdout
    sys.stdout = cStringIO.StringIO()
    yield
    sys.stdout = save_stdout

def main(args):
    flow = OAuth2WebServerFlow(client_id=args.client_id,
                           client_secret=args.client_secret,
                           scope='https://www.googleapis.com/auth/userinfo.email')
    storage = Storage(FILE_NAME)
    credentials = tools.run_flow(flow, storage, args)
    return credentials

def refresh(args):
    with open(FILE_NAME, 'rU') as f:
        credentials = OAuth2Credentials.from_json(f.read())
    credentials.refresh(httplib2.Http())
    storage = Storage(FILE_NAME)
    storage.put(credentials)
    credentials.set_store(storage)
    return credentials

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers(help='sub-command help')
    subparser_create = subparsers.add_parser('create', help='create help', parents=[tools.argparser])
    subparser_create.add_argument('--client-id', type=str, required=True)
    subparser_create.add_argument('--client-secret', type=str, required=True)
    subparser_create.set_defaults(func=main)
    subparser_refresh = subparsers.add_parser('refresh', help='refresh help')
    subparser_refresh.set_defaults(func=refresh)
    args = parser.parse_args()
    with nostdout():
        c = args.func(args)
    print c.access_token
<|endoftext|>