<commit_before>import copy
import json
import socket

from django.conf import settings
from django.http import Http404

import requests
import six.moves.http_client
from requests import HTTPError
from six.moves.urllib.parse import urlparse

from dimagi.utils.couch.cache.cache_core import get_redis_client

from corehq.apps.formplayer_api.smsforms.exceptions import BadDataError
from corehq.apps.formplayer_api.utils import get_formplayer_url
from corehq.form_processor.utils.general import use_sqlite_backend
from corehq.util.hmac_request import (
    convert_to_bytestring_if_unicode,
    get_hmac_digest,
)

"""
A set of wrappers that return the JSON bodies you use to interact with the formplayer
backend for various sets of tasks.

This API is currently highly beta and could use some hardening.
"""

class TouchformsAuth(object):
    """
    Used to authenticate with touchforms
    """
    def __init__(self, type, key):
        self.type = type
        self.key = key

    def to_dict(self):
        return {'type': self.type, 'key': self.key}

class DjangoAuth(TouchformsAuth):

    def __init__(self, key):
        super(DjangoAuth, self).__init__("django-session", key)

class DigestAuth(TouchformsAuth):

    def __init__(self, username, password):
        self.username = username
        super(DigestAuth, self).__init__("http-digest", password)

    def to_dict(self):
        return {'type': self.type, 'key': self.key, 'username': self.username}

class TouchformsError(ValueError):

    def __init__(self, *args, **kwargs):
        self.response_data = kwargs.pop('response_data', {})
        super(TouchformsError, self).__init__(*args, **kwargs)

class InvalidSessionIdException(TouchformsError):
    pass

class XFormsConfigException(ValueError):
    pass

class XFormsConfig(object):

    def __init__(self, form_path=None, form_content=None, language="",
                 session_data=None, preloader_data={}, instance_content=None,
                 touchforms_url=None, auth=None, domain=None, restore_as=None, restore_as_case_id=None):

        if bool(form_path) == bool(form_content):
            raise XFormsConfigException\
                ("Can specify file path or content but not both!\n"
                 "File Path: %s, Form Content: %s" % (form_path, form_content))

        self.form_path = form_path
        self.form_content = form_content
        self.language = language
        self.session_data = session_data
        self.preloader_data = preloader_data
        self.instance_content = instance_content
        self.auth = auth
        self.restore_as = restore_as
        self.restore_as_case_id = restore_as_case_id
        self.domain = domain

    def get_touchforms_dict(self):
        """
        Translates this config into something touchforms wants to
        work with
        """

        vals = (("action", "new-form"),
                ("form-name", self.form_path),
                ("form-content", self.form_content),
                ("instance-content", self.instance_content),
                ("preloader-data", self.preloader_data),
                ("session-data", self.session_data),
                ("lang", self.language),
                ("form-url", self.form_path))

        # only include anything with a value, or touchforms gets mad
        ret = dict([x for x in vals if x[1]])
        self.add_key_helper('username', ret)
        self.add_key_helper('domain', ret)
        self.add_key_helper('app_id', ret)

        if self.restore_as_case_id:
            # The contact starting the survey is a case who will be
            # filling out the form for itself.
            ret['restoreAsCaseId'] = self.restore_as_case_id
        elif self.restore_as:
            # The contact starting the survey is a user.
            ret['restoreAs'] = self.restore_as
        else:
            raise ValueError("Unable to determine 'restore as' contact for formplayer")

        return ret

    def add_key_helper(self, key, ret):
        if key in self.session_data:
            ret[key] = self.session_data[key]

    def start_session(self):
        """
        Start a new session based on this configuration
        """

        return get_response(json.dumps(self.get_touchforms_dict()), auth=self.auth)

class XformsEvent(object):
    """
    A wrapper for the json event object that comes back from touchforms, which
    looks approximately like this:n

    { "datatype":"select",
      "style":{},
      "choices":["red","green","blue"],
      "caption":"What's your favorite color?",
      "type":"question",
      "answer":null,
      "required":0,
      "ix":"1",
      "help":null
    }
    """
    def __init__(self, datadict):
        self._dict = datadict
        self.type = datadict["type"]
        self.caption = datadict.get("caption", "")
        self.datatype = datadict.get("datatype", "")
        self.output = datadict.get("output", "")
        self.choices = datadict.get("choices", None)

    @property
    def text_prompt(self):
        """
        A text-only prompt for this. Used in pure text (or sms) mode.

        Kept for backwards compatibility. Should use get_text_prompt, below.
        """
        return self.get_text_prompt(None)

    def get_text_prompt(self, select_display_func=None):
        """
        Get a text-only prompt for this. Used in pure text (or sms) mode.

        Allows you to pass in a function to override how selects are displayed.

        The signature of that function should take in the prompt and choice list
        and return a string. The default is select_to_text_compact
        """
        display_func = select_display_func or select_to_text_compact
        if self.datatype == "select" or self.datatype == "multiselect":
            return display_func(self.caption, self._dict["choices"])
        else:
            return self.caption

def select_to_text_compact(caption, choices):
    """
    A function to convert a select item to text in a compact format.
    Format is:

    [question] 1:[choice1], 2:[choice2]...
    """
    return "%s %s." % (caption,
                      ", ".join(["%s:%s" % (i+1, val) for i, val in \
                                 enumerate(choices)]))

def select_to_text_vals_only(caption, choices):
    """
    A function to convert a select item to text in a compact format.
    Format is:

    [question], choices: [choice1], [choice2]...
    """
    return "%s, choices: %s" % (caption, ", ".join(choices))

def select_to_text_readable(caption, choices):
    """
    A function to convert a select item to text in a more verbose, readable
    format.
    Format is:

    [question] Send 1 for [choice1], 2 for [choice2]...
    """
    return "%s Send %s" % (caption,
                      ", ".join(["%s for %s" % (i+1, val) for i, val in \
                                 enumerate(choices)]))

def select_to_text_caption_only(caption, choices):
    """
    A select choices => text function that ignores choice captions entirely.
    All presentation of choices must be included in the main question caption.
    A DRY violation, for sure, but gives the maximum flexibility
    """
    return caption

class XformsResponse(object):
    """
    A wrapper for the json that comes back from touchforms, which
    looks approximately like this:
    {"event":
        { "datatype":"select",
          "style":{},
          ... (see above)
         },
     "session_id": 'd0addaa40dbcefefc6a687472a4e65d2',
     "status":"accepted",
     "seq_id":1}

    Although errors come back more like this:
    {'status': 'validation-error',
     'seq_id': 2,
     'reason': 'some message about constraint',
     'type': 'constraint'}

    """

    def __init__(self, datadict):
        self._dict = datadict
        self.is_error = False
        self.error = None
        if "event" in datadict and datadict["event"] is not None:
            self.event = XformsEvent(datadict["event"])
            self.text_prompt = self.event.text_prompt
        else:
            self.event = None

        self.seq_id = datadict.get("seq_id","")
        self.session_id = datadict.get("session_id", "")
        self.status = datadict.get("status", "")

        # custom logic to handle errors
        if self.status == "validation-error":
            assert self.event is None, "There should be no touchforms event for errors"
            self.is_error = True
            self.text_prompt = datadict.get("reason", "that is not a legal answer")

        # custom logic to handle http related errors
        elif self.status == "http-error":
            assert self.event is None, "There should be no touchforms event for errors"
            self.error = datadict.get("error")
            self.is_error = True
            self.status_code = datadict.get("status_code")
            self.args = datadict.get("args")
            self.url = datadict.get("url")
        elif self.event is None:
            raise TouchformsError("unhandleable response: %s" % json.dumps(datadict),
                response_data=datadict)

    @classmethod
    def server_down(cls):
        # TODO: this should probably be configurable
        return XformsResponse({"status": "http-error",
                               "error": "No response from server. Please "
                                        "contact your administrator for help."})

def formplayer_post_data_helper(d, content_type, url):
    data = json.dumps(d).encode('utf-8')
    headers = {}
    headers["Content-Type"] = content_type
    headers["content-length"] = str(len(data))
    headers["X-MAC-DIGEST"] = get_hmac_digest(settings.FORMPLAYER_INTERNAL_AUTH_KEY, data)
    response = requests.post(
        url,
        data=data,
        headers=headers
    )
    if response.status_code == 404:
        raise Http404(response.reason)
    if 500 <= response.status_code < 600:
        http_error_msg = '%s Server Error: %s for url: %s' % (response.status_code, response.reason, response.url)
        raise HTTPError(http_error_msg, response=response)
    return response.json()

def post_data(data, auth=None, content_type="application/json"):
    try:
        d = json.loads(data)
    except TypeError:
        raise BadDataError('unhandleable touchforms query: %s' % data)

    domain = d.get("domain")

    if not domain:
        raise ValueError("Expected domain")

    d = get_formplayer_session_data(d)
    return formplayer_post_data_helper(d, content_type, get_formplayer_url() + "/" + d["action"])

def get_formplayer_session_data(data):
    data['oneQuestionPerScreen'] = True
    data['nav_mode'] = 'prompt'
    if "session_id" in data:
        session_id = data["session_id"]
    elif "session-id" in data:
        session_id = data["session-id"]
    else:
        return data

    data["session_id"] = session_id
    data["session-id"] = session_id
    return data

def get_response(data, auth=None):
    try:
        response_json = post_data(data, auth=auth)
    except socket.error as e:
        return XformsResponse.server_down()
    try:
        return XformsResponse(response_json)
    except Exception as e:
        raise e

def get_raw_instance(session_id, domain=None, auth=None):
    """
    Gets the raw xml instance of the current session regardless of the state that we're in (used for logging partially complete
    forms to couch when errors happen).
    """

    data = {
        "action":"get-instance",
        "session-id": session_id,
        "domain": domain
    }

    response = post_data(json.dumps(data), auth)
    if "error" in response:
        error = response["error"]
        if error == "Form session not found":
            raise InvalidSessionIdException("Invalid Session Id")
        else:
            raise TouchformsError(error)
    return response

def start_form_session(form_path, content=None, language="", session_data={}):
    """
    Start a new form session
    """
    # TODO: this method has been deprecated and the config object
    # should just be used directly. Temporarily left to support legacy code.
    return XFormsConfig(form_path=form_path,
                        instance_content=content,
                        session_data=session_data,
                        language=language).start_session()

def answer_question(session_id, answer, domain, auth=None):
    """
    Answer a question.
    """
    data = {"action": "answer",
            "session-id": session_id,
            "answer": answer,
            "domain": domain}
    return get_response(json.dumps(data), auth)

def current_question(session_id, domain, auth=None):
    """
    Retrieves information about the current question.
    """
    data = {"action": "current",
            "session-id": session_id,
            "domain": domain}
    return get_response(json.dumps(data), auth)

def next(session_id, domain, auth=None):
    """
    Moves to the next question.
    """
    data = {"action": "next",
            "session-id": session_id,
            "domain": domain}
    return get_response(json.dumps(data), auth)
<commit_msg>Add X-FORMPLAYER-SESSION header for sticky sms form sessions<commit_after>import copy
import json
import socket

from django.conf import settings
from django.http import Http404

import requests
import six.moves.http_client
from requests import HTTPError
from six.moves.urllib.parse import urlparse

from dimagi.utils.couch.cache.cache_core import get_redis_client

from corehq.apps.formplayer_api.smsforms.exceptions import BadDataError
from corehq.apps.formplayer_api.utils import get_formplayer_url
from corehq.form_processor.utils.general import use_sqlite_backend
from corehq.util.hmac_request import (
    convert_to_bytestring_if_unicode,
    get_hmac_digest,
)

"""
A set of wrappers that return the JSON bodies you use to interact with the formplayer
backend for various sets of tasks.

This API is currently highly beta and could use some hardening.
"""

class TouchformsAuth(object):
    """
    Used to authenticate with touchforms
    """
    def __init__(self, type, key):
        self.type = type
        self.key = key

    def to_dict(self):
        return {'type': self.type, 'key': self.key}

class DjangoAuth(TouchformsAuth):

    def __init__(self, key):
        super(DjangoAuth, self).__init__("django-session", key)

class DigestAuth(TouchformsAuth):

    def __init__(self, username, password):
        self.username = username
        super(DigestAuth, self).__init__("http-digest", password)

    def to_dict(self):
        return {'type': self.type, 'key': self.key, 'username': self.username}

class TouchformsError(ValueError):

    def __init__(self, *args, **kwargs):
        self.response_data = kwargs.pop('response_data', {})
        super(TouchformsError, self).__init__(*args, **kwargs)

class InvalidSessionIdException(TouchformsError):
    pass

class XFormsConfigException(ValueError):
    pass

class XFormsConfig(object):

    def __init__(self, form_path=None, form_content=None, language="",
                 session_data=None, preloader_data={}, instance_content=None,
                 touchforms_url=None, auth=None, domain=None, restore_as=None, restore_as_case_id=None):

        if bool(form_path) == bool(form_content):
            raise XFormsConfigException\
                ("Can specify file path or content but not both!\n"
                 "File Path: %s, Form Content: %s" % (form_path, form_content))

        self.form_path = form_path
        self.form_content = form_content
        self.language = language
        self.session_data = session_data
        self.preloader_data = preloader_data
        self.instance_content = instance_content
        self.auth = auth
        self.restore_as = restore_as
        self.restore_as_case_id = restore_as_case_id
        self.domain = domain

    def get_touchforms_dict(self):
        """
        Translates this config into something touchforms wants to
        work with
        """

        vals = (("action", "new-form"),
                ("form-name", self.form_path),
                ("form-content", self.form_content),
                ("instance-content", self.instance_content),
                ("preloader-data", self.preloader_data),
                ("session-data", self.session_data),
                ("lang", self.language),
                ("form-url", self.form_path))

        # only include anything with a value, or touchforms gets mad
        ret = dict([x for x in vals if x[1]])
        self.add_key_helper('username', ret)
        self.add_key_helper('domain', ret)
        self.add_key_helper('app_id', ret)

        if self.restore_as_case_id:
            # The contact starting the survey is a case who will be
            # filling out the form for itself.
            ret['restoreAsCaseId'] = self.restore_as_case_id
        elif self.restore_as:
            # The contact starting the survey is a user.
            ret['restoreAs'] = self.restore_as
        else:
            raise ValueError("Unable to determine 'restore as' contact for formplayer")

        return ret

    def add_key_helper(self, key, ret):
        if key in self.session_data:
            ret[key] = self.session_data[key]

    def start_session(self):
        """
        Start a new session based on this configuration
        """

        return get_response(json.dumps(self.get_touchforms_dict()), auth=self.auth)

class XformsEvent(object):
    """
    A wrapper for the json event object that comes back from touchforms, which
    looks approximately like this:n

    { "datatype":"select",
      "style":{},
      "choices":["red","green","blue"],
      "caption":"What's your favorite color?",
      "type":"question",
      "answer":null,
      "required":0,
      "ix":"1",
      "help":null
    }
    """
    def __init__(self, datadict):
        self._dict = datadict
        self.type = datadict["type"]
        self.caption = datadict.get("caption", "")
        self.datatype = datadict.get("datatype", "")
        self.output = datadict.get("output", "")
        self.choices = datadict.get("choices", None)

    @property
    def text_prompt(self):
        """
        A text-only prompt for this. Used in pure text (or sms) mode.

        Kept for backwards compatibility. Should use get_text_prompt, below.
        """
        return self.get_text_prompt(None)

    def get_text_prompt(self, select_display_func=None):
        """
        Get a text-only prompt for this. Used in pure text (or sms) mode.

        Allows you to pass in a function to override how selects are displayed.

        The signature of that function should take in the prompt and choice list
        and return a string. The default is select_to_text_compact
        """
        display_func = select_display_func or select_to_text_compact
        if self.datatype == "select" or self.datatype == "multiselect":
            return display_func(self.caption, self._dict["choices"])
        else:
            return self.caption

def select_to_text_compact(caption, choices):
    """
    A function to convert a select item to text in a compact format.
    Format is:

    [question] 1:[choice1], 2:[choice2]...
    """
    return "%s %s." % (caption,
                      ", ".join(["%s:%s" % (i+1, val) for i, val in \
                                 enumerate(choices)]))

def select_to_text_vals_only(caption, choices):
    """
    A function to convert a select item to text in a compact format.
    Format is:

    [question], choices: [choice1], [choice2]...
    """
    return "%s, choices: %s" % (caption, ", ".join(choices))

def select_to_text_readable(caption, choices):
    """
    A function to convert a select item to text in a more verbose, readable
    format.
    Format is:

    [question] Send 1 for [choice1], 2 for [choice2]...
    """
    return "%s Send %s" % (caption,
                      ", ".join(["%s for %s" % (i+1, val) for i, val in \
                                 enumerate(choices)]))

def select_to_text_caption_only(caption, choices):
    """
    A select choices => text function that ignores choice captions entirely.
    All presentation of choices must be included in the main question caption.
    A DRY violation, for sure, but gives the maximum flexibility
    """
    return caption

class XformsResponse(object):
    """
    A wrapper for the json that comes back from touchforms, which
    looks approximately like this:
    {"event":
        { "datatype":"select",
          "style":{},
          ... (see above)
         },
     "session_id": 'd0addaa40dbcefefc6a687472a4e65d2',
     "status":"accepted",
     "seq_id":1}

    Although errors come back more like this:
    {'status': 'validation-error',
     'seq_id': 2,
     'reason': 'some message about constraint',
     'type': 'constraint'}

    """

    def __init__(self, datadict):
        self._dict = datadict
        self.is_error = False
        self.error = None
        if "event" in datadict and datadict["event"] is not None:
            self.event = XformsEvent(datadict["event"])
            self.text_prompt = self.event.text_prompt
        else:
            self.event = None

        self.seq_id = datadict.get("seq_id","")
        self.session_id = datadict.get("session_id", "")
        self.status = datadict.get("status", "")

        # custom logic to handle errors
        if self.status == "validation-error":
            assert self.event is None, "There should be no touchforms event for errors"
            self.is_error = True
            self.text_prompt = datadict.get("reason", "that is not a legal answer")

        # custom logic to handle http related errors
        elif self.status == "http-error":
            assert self.event is None, "There should be no touchforms event for errors"
            self.error = datadict.get("error")
            self.is_error = True
            self.status_code = datadict.get("status_code")
            self.args = datadict.get("args")
            self.url = datadict.get("url")
        elif self.event is None:
            raise TouchformsError("unhandleable response: %s" % json.dumps(datadict),
                response_data=datadict)

    @classmethod
    def server_down(cls):
        # TODO: this should probably be configurable
        return XformsResponse({"status": "http-error",
                               "error": "No response from server. Please "
                                        "contact your administrator for help."})

def formplayer_post_data_helper(d, content_type, url):
    data = json.dumps(d).encode('utf-8')
    session_id = data['session-id']
    headers = {}
    headers["Content-Type"] = content_type
    headers["content-length"] = str(len(data))
    headers["X-MAC-DIGEST"] = get_hmac_digest(settings.FORMPLAYER_INTERNAL_AUTH_KEY, data)
    headers["X-FORMPLAYER-SESSION"] = session_id
    response = requests.post(
        url,
        data=data,
        headers=headers
    )
    if response.status_code == 404:
        raise Http404(response.reason)
    if 500 <= response.status_code < 600:
        http_error_msg = '%s Server Error: %s for url: %s' % (response.status_code, response.reason, response.url)
        raise HTTPError(http_error_msg, response=response)
    return response.json()

def post_data(data, auth=None, content_type="application/json"):
    try:
        d = json.loads(data)
    except TypeError:
        raise BadDataError('unhandleable touchforms query: %s' % data)

    domain = d.get("domain")

    if not domain:
        raise ValueError("Expected domain")

    d = get_formplayer_session_data(d)
    return formplayer_post_data_helper(d, content_type, get_formplayer_url() + "/" + d["action"])

def get_formplayer_session_data(data):
    data['oneQuestionPerScreen'] = True
    data['nav_mode'] = 'prompt'
    if "session_id" in data:
        session_id = data["session_id"]
    elif "session-id" in data:
        session_id = data["session-id"]
    else:
        return data

    data["session_id"] = session_id
    data["session-id"] = session_id
    return data

def get_response(data, auth=None):
    try:
        response_json = post_data(data, auth=auth)
    except socket.error as e:
        return XformsResponse.server_down()
    try:
        return XformsResponse(response_json)
    except Exception as e:
        raise e

def get_raw_instance(session_id, domain=None, auth=None):
    """
    Gets the raw xml instance of the current session regardless of the state that we're in (used for logging partially complete
    forms to couch when errors happen).
    """

    data = {
        "action":"get-instance",
        "session-id": session_id,
        "domain": domain
    }

    response = post_data(json.dumps(data), auth)
    if "error" in response:
        error = response["error"]
        if error == "Form session not found":
            raise InvalidSessionIdException("Invalid Session Id")
        else:
            raise TouchformsError(error)
    return response

def start_form_session(form_path, content=None, language="", session_data={}):
    """
    Start a new form session
    """
    # TODO: this method has been deprecated and the config object
    # should just be used directly. Temporarily left to support legacy code.
    return XFormsConfig(form_path=form_path,
                        instance_content=content,
                        session_data=session_data,
                        language=language).start_session()

def answer_question(session_id, answer, domain, auth=None):
    """
    Answer a question.
    """
    data = {"action": "answer",
            "session-id": session_id,
            "answer": answer,
            "domain": domain}
    return get_response(json.dumps(data), auth)

def current_question(session_id, domain, auth=None):
    """
    Retrieves information about the current question.
    """
    data = {"action": "current",
            "session-id": session_id,
            "domain": domain}
    return get_response(json.dumps(data), auth)

def next(session_id, domain, auth=None):
    """
    Moves to the next question.
    """
    data = {"action": "next",
            "session-id": session_id,
            "domain": domain}
    return get_response(json.dumps(data), auth)
<|endoftext|>


<commit_before>from contextlib import closing
from nose.tools import eq_, ok_
from datetime import datetime
from pytz import timezone

from minerva.test import with_conn

from minerva_db import clear_database

from minerva import directory
from minerva.directory import existence

def prepare_datebase(conn):
    with closing(conn.cursor()) as cursor:
        cursor.execute("DELETE FROM directory.entity")
        cursor.execute("DELETE FROM directory.existence")

    clear_database(conn)

def get_existence_curr(conn):
    with closing(conn.cursor()) as cursor:
        cursor.execute("""
SELECT entity.dn, e.timestamp, e.exists
FROM directory.existence_curr e
JOIN directory.entity on entity.id = e.entity_id
ORDER BY entity.id, e.timestamp""")
        return cursor.fetchall()

def check_existence(conn, rows):
    with closing(conn.cursor()) as cursor:
        cursor.execute("""
SELECT entity.dn, e.timestamp, e.exists
FROM directory.existence e
JOIN directory.entity on entity.id = e.entity_id
ORDER BY e.timestamp, entity.id""")

        data_rows = cursor.fetchall()

    if len(rows) != len(data_rows):
        return False

    for index, row in enumerate(data_rows):
        if row != rows[index]:
            return False

    return True

TIMEZONE = timezone("Europe/Amsterdam")

@with_conn(prepare_datebase)
def test_existence_simple(conn):
    dn = "network=1,cell=1"

    dt = TIMEZONE.localize(datetime(2014, 01, 01, 14, 0, 0))

    directory.create_entity(conn, dn)

    ex = existence.Existence(conn)
    ex.mark_existing([dn])
    ex.flush(dt)

    ok_(check_existence(conn, [(dn,dt,True)]))

@with_conn(prepare_datebase)
def test_existence_onoff(conn):
    with closing(conn.cursor()) as cursor:
        dn1 = "network=1,cell=1"
        dn2 = "network=1,cell=2"

        dt1 = TIMEZONE.localize(datetime(2014, 01, 01, 14, 0, 0))
        dt2 = TIMEZONE.localize(datetime(2014, 02, 01, 14, 0, 0))

        ex = existence.Existence(conn)

        directory.create_entity(conn, dn1)
        directory.create_entity(conn, dn2)

        ex.mark_existing([dn1])
        ex.flush(dt1)

        ex.mark_existing([dn2])
        ex.flush(dt2)

        ok_(check_existence(conn, [(dn1, dt1, True), (dn1, dt2, False), (dn2, dt2, True)]))

@with_conn(prepare_datebase)
def test_existence_history_on(conn):
    with closing(conn.cursor()) as cursor:
        dn1 = "network=1,cell=1"
        dn2 = "network=1,cell=2"

        dt1 = TIMEZONE.localize(datetime(2014, 01, 01, 14, 0, 0))
        dt2 = TIMEZONE.localize(datetime(2014, 02, 01, 14, 0, 0))

        ex = existence.Existence(conn)

        directory.create_entity(conn, dn1)
        directory.create_entity(conn, dn2)

        ex.mark_existing([dn1])
        ex.flush(dt2)

        ex.mark_existing([dn2])
        ex.flush(dt1)

        ok_(check_existence(conn, [(dn2, dt1, True), (dn1, dt2, True)]))

@with_conn(prepare_datebase)
def test_existence_history_off(conn):
    with closing(conn.cursor()) as cursor:
        dn1 = "network=1,cell=1"
        dn2 = "network=1,cell=2"

        dt1 = TIMEZONE.localize(datetime(2014, 01, 01, 14, 0, 0))
        dt2 = TIMEZONE.localize(datetime(2014, 02, 01, 14, 0, 0))
        dt3 = TIMEZONE.localize(datetime(2014, 03, 01, 14, 0, 0))

        ex = existence.Existence(conn)

        directory.create_entity(conn, dn1)
        directory.create_entity(conn, dn2)

        ex.mark_existing([dn1])
        ex.flush(dt1)

        ex.mark_existing([dn2])
        ex.flush(dt2)

        ex.mark_existing([dn1, dn2])
        ex.flush(dt3)

        ok_(check_existence(conn, [(dn1, dt1, True), (dn1, dt2, False), (dn2, dt2, True), (dn1, dt3, True)]))

@with_conn(prepare_datebase)
def test_existence_duplicate(conn):
    dn = "network=1,cell=1"
    dt = TIMEZONE.localize(datetime(2014, 01, 01, 14, 0, 0))

    directory.create_entity(conn, dn)
    ex = existence.Existence(conn)

    ex.mark_existing([dn])
    ex.flush(dt)

    ex.mark_existing([dn])
    ex.flush(dt)

    ok_(check_existence(conn, [(dn,dt,True)]))

@with_conn(prepare_datebase)
def test_existence_still_on(conn):
    dn = "network=1,cell=1"
    dt1 = TIMEZONE.localize(datetime(2014, 01, 01, 14, 0, 0))
    dt2 = TIMEZONE.localize(datetime(2014, 02, 01, 14, 0, 0))

    directory.create_entity(conn, dn)
    ex = existence.Existence(conn)

    ex.mark_existing([dn])
    ex.flush(dt1)

    ex.mark_existing([dn])
    ex.flush(dt2)

    ok_(check_existence(conn, [(dn,dt1,True)]))
<commit_msg>Updated existence test<commit_after>from contextlib import closing
from nose.tools import eq_, ok_
from datetime import datetime
from pytz import timezone

from minerva.test import with_conn

from minerva_db import clear_database

from minerva import directory
from minerva.directory import existence

def prepare_datebase(conn):
    with closing(conn.cursor()) as cursor:
        cursor.execute("DELETE FROM directory.entity")
        cursor.execute("DELETE FROM directory.existence")

    clear_database(conn)

def check_existence(conn, rows):
    with closing(conn.cursor()) as cursor:
        cursor.execute("""
SELECT entity.dn, e.timestamp, e.exists
FROM directory.existence e
JOIN directory.entity on entity.id = e.entity_id
ORDER BY e.timestamp, entity.id""")

        data_rows = cursor.fetchall()

    if len(rows) != len(data_rows):
        return False

    for index, row in enumerate(data_rows):
        if row != rows[index]:
            return False

    return True

TIMEZONE = timezone("Europe/Amsterdam")

@with_conn(prepare_datebase)
def test_existence_simple(conn):
    dn = "network=1,cell=1"

    dt = TIMEZONE.localize(datetime(2014, 01, 01, 14, 0, 0))

    directory.create_entity(conn, dn)

    ex = existence.Existence(conn)
    ex.mark_existing([dn])
    ex.flush(dt)

    ok_(check_existence(conn, [(dn,dt,True)]))

@with_conn(prepare_datebase)
def test_existence_onoff(conn):
    with closing(conn.cursor()) as cursor:
        dn1 = "network=1,cell=1"
        dn2 = "network=1,cell=2"

        dt1 = TIMEZONE.localize(datetime(2014, 01, 01, 14, 0, 0))
        dt2 = TIMEZONE.localize(datetime(2014, 02, 01, 14, 0, 0))

        ex = existence.Existence(conn)

        directory.create_entity(conn, dn1)
        directory.create_entity(conn, dn2)

        ex.mark_existing([dn1])
        ex.flush(dt1)

        ex.mark_existing([dn2])
        ex.flush(dt2)

        ok_(check_existence(conn, [(dn1, dt1, True), (dn1, dt2, False), (dn2, dt2, True)]))

@with_conn(prepare_datebase)
def test_existence_history_on(conn):
    with closing(conn.cursor()) as cursor:
        dn1 = "network=1,cell=1"
        dn2 = "network=1,cell=2"

        dt1 = TIMEZONE.localize(datetime(2014, 01, 01, 14, 0, 0))
        dt2 = TIMEZONE.localize(datetime(2014, 02, 01, 14, 0, 0))

        ex = existence.Existence(conn)

        directory.create_entity(conn, dn1)
        directory.create_entity(conn, dn2)

        ex.mark_existing([dn1])
        ex.flush(dt2)

        ex.mark_existing([dn2])
        ex.flush(dt1)

        ok_(check_existence(conn, [(dn2, dt1, True), (dn1, dt2, True)]))

@with_conn(prepare_datebase)
def test_existence_history_off(conn):
    with closing(conn.cursor()) as cursor:
        dn1 = "network=1,cell=1"
        dn2 = "network=1,cell=2"

        dt1 = TIMEZONE.localize(datetime(2014, 01, 01, 14, 0, 0))
        dt2 = TIMEZONE.localize(datetime(2014, 02, 01, 14, 0, 0))
        dt3 = TIMEZONE.localize(datetime(2014, 03, 01, 14, 0, 0))

        ex = existence.Existence(conn)

        directory.create_entity(conn, dn1)
        directory.create_entity(conn, dn2)

        ex.mark_existing([dn1])
        ex.flush(dt1)

        ex.mark_existing([dn2])
        ex.flush(dt2)

        ex.mark_existing([dn1, dn2])
        ex.flush(dt3)

        ok_(check_existence(conn, [(dn1, dt1, True), (dn1, dt2, False), (dn2, dt2, True), (dn1, dt3, True)]))

@with_conn(prepare_datebase)
def test_existence_duplicate(conn):
    dn = "network=1,cell=1"
    dt = TIMEZONE.localize(datetime(2014, 01, 01, 14, 0, 0))

    directory.create_entity(conn, dn)
    ex = existence.Existence(conn)

    ex.mark_existing([dn])
    ex.flush(dt)

    ex.mark_existing([dn])
    ex.flush(dt)

    ok_(check_existence(conn, [(dn,dt,True)]))

@with_conn(prepare_datebase)
def test_existence_still_on(conn):
    dn = "network=1,cell=1"
    dt1 = TIMEZONE.localize(datetime(2014, 01, 01, 14, 0, 0))
    dt2 = TIMEZONE.localize(datetime(2014, 02, 01, 14, 0, 0))

    directory.create_entity(conn, dn)
    ex = existence.Existence(conn)

    ex.mark_existing([dn])
    ex.flush(dt1)

    ex.mark_existing([dn])
    ex.flush(dt2)

    ok_(check_existence(conn, [(dn,dt1,True)]))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import imaplib
import re

try:
    import imbox
except ImportError:
    imbox = None

from intelmq.lib.bot import CollectorBot

class MailCollectorBot(CollectorBot):

    def init(self):
        if imbox is None:
            raise ValueError('Could not import imbox. Please install it.')

    def connect_mailbox(self):
        self.logger.debug("Connecting to %s.", self.parameters.mail_host)
        mailbox = imbox.Imbox(self.parameters.mail_host,
                              self.parameters.mail_user,
                              self.parameters.mail_password,
                              self.parameters.mail_ssl)
        return mailbox

    def process(self):
        mailbox = self.connect_mailbox()
        emails = mailbox.messages(folder=self.parameters.folder, unread=True,
                                  sent_to=getattr(self.parameters, "sent_to", None),
                                  sent_from=getattr(self.parameters, "sent_from", None))

        if emails:
            for uid, message in emails:

                if (self.parameters.subject_regex and
                        not re.search(self.parameters.subject_regex,
                                      re.sub(r"\r\n\s", " ", message.subject))):
                    self.logger.debug("Message with date %s skipped because subject %r does not match.",
                                      message.date, message.subject)
                    continue

                if self.process_message(uid, message):
                    try:
                        mailbox.mark_seen(uid)
                    except imaplib.abort:
                        # Disconnect, see https://github.com/certtools/intelmq/issues/852
                        mailbox = self.connect_mailbox()
                        mailbox.mark_seen(uid)
        else:
            self.logger.debug("No unread mails to check.")
        mailbox.logout()

    def process_message(self, uid, message) -> bool:
        """
        Returns:
            seen: Mark the message as seen or not
        """
        raise NotImplementedError
<commit_msg>Add CA support in Mail URL fetcher<commit_after># -*- coding: utf-8 -*-
import imaplib
import re
import ssl

try:
    import imbox
except ImportError:
    imbox = None

from intelmq.lib.bot import CollectorBot

class MailCollectorBot(CollectorBot):

    def init(self):
        if imbox is None:
            raise ValueError('Could not import imbox. Please install it.')

    def connect_mailbox(self):
        self.logger.debug("Connecting to %s.", self.parameters.mail_host)
        ca_file = getattr(self.parameters, 'ca_file', None)
        ssl_custom_context = ssl.create_default_context(cafile=ca_file)
        mailbox = imbox.Imbox(self.parameters.mail_host,
                              self.parameters.mail_user,
                              self.parameters.mail_password,
                              self.parameters.mail_ssl,
                              ssl_context=ssl_custom_context)
        return mailbox

    def process(self):
        mailbox = self.connect_mailbox()
        emails = mailbox.messages(folder=self.parameters.folder, unread=True,
                                  sent_to=getattr(self.parameters, "sent_to", None),
                                  sent_from=getattr(self.parameters, "sent_from", None))

        if emails:
            for uid, message in emails:

                if (self.parameters.subject_regex and
                        not re.search(self.parameters.subject_regex,
                                      re.sub(r"\r\n\s", " ", message.subject))):
                    self.logger.debug("Message with date %s skipped because subject %r does not match.",
                                      message.date, message.subject)
                    continue

                if self.process_message(uid, message):
                    try:
                        mailbox.mark_seen(uid)
                    except imaplib.abort:
                        # Disconnect, see https://github.com/certtools/intelmq/issues/852
                        mailbox = self.connect_mailbox()
                        mailbox.mark_seen(uid)
        else:
            self.logger.debug("No unread mails to check.")
        mailbox.logout()

    def process_message(self, uid, message) -> bool:
        """
        Returns:
            seen: Mark the message as seen or not
        """
        raise NotImplementedError
<|endoftext|>


<commit_before>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

from cosmo_tester.framework.testenv import TestCase
import json
from urllib2 import urlopen
from fabric.api import env, reboot

class RebootManagerTest(TestCase):
    def _reboot_server(self):
        env.update({
            'user': self.env.managment_user_name,
            'key_filename': self.env.management_key_path,
            'host_string': self.env.management_ip,
        })
        reboot()

    def _get_undefined_services(self):
        return [key for key, val in self.status.iteritems() if val is None]

    def _get_stopped_services(self):
        return [key for key, val in self.status.iteritems()
                if val and not val['instances']]

    def _get_status(self):
        out = urlopen("http://{0}:8100/status".format(self.env.management_ip))
        return json.load(out)

    def setUp(self, *args, **kwargs):
        super(RebootManagerTest, self).setUp(*args, **kwargs)
        out = urlopen("http://{0}:8100/status".format(self.env.management_ip))
        self.status = json.load(out)

    def test_00_pre_reboot(self):
        undefined = self._get_undefined_services()
        self.assertEqual(undefined, [], 'undefined services: {0}'
                         .format(','.join(undefined)))
        stopped = self._get_stopped_services()
        self.assertEqual(stopped, [], 'stopped services: {0}'
                         .format(','.join(stopped)))

    def test_01_during_reboot(self):
            pre_reboot_status = self.status
            self._reboot_server()
            self.status = self._get_status()
            self.assertEqual(pre_reboot_status, self.status,
                             'pre and post reboot status is not equal')

    def test_01_post_reboot(self):
        undefined = self._get_undefined_services()
        self.assertEqual(undefined, [], 'undefined services: {0}'
                         .format(','.join(undefined)))
        stopped = self._get_stopped_services()
        self.assertEqual(stopped, [], 'stopped services: {0}'
                         .format(','.join(stopped)))
<commit_msg>replaces urllib with cosmo-rest-client<commit_after>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

from cosmo_tester.framework.testenv import TestCase
from cosmo_manager_rest_client.cosmo_manager_rest_client import (
    CosmoManagerRestClient)
from fabric.api import env, reboot

class RebootManagerTest(TestCase):
    def _reboot_server(self):
        env.update({
            'user': self.env.managment_user_name,
            'key_filename': self.env.management_key_path,
            'host_string': self.env.management_ip,
        })
        reboot()

    def _get_undefined_services(self):
        return [each.display_name for each in self.status if each.name is None]

    def _get_stopped_services(self):
        return [each.display_name for each in self.status
                if each and not each.instances]

    def _get_status(self):
        rest = CosmoManagerRestClient(self.env.management_ip)
        return rest.list_services()

    def setUp(self, *args, **kwargs):
        super(RebootManagerTest, self).setUp(*args, **kwargs)
        self.status = self._get_status()

    def test_00_pre_reboot(self):
        undefined = self._get_undefined_services()
        self.assertEqual(undefined, [], 'undefined services: {0}'
                         .format(','.join(undefined)))
        stopped = self._get_stopped_services()
        self.assertEqual(stopped, [], 'stopped services: {0}'
                         .format(','.join(stopped)))

    def test_01_during_reboot(self):
            pre_reboot_status = self.status
            self._reboot_server()
            self.status = self._get_status()
            self.assertEqual(pre_reboot_status, self.status,
                             'pre and post reboot status is not equal')

    def test_01_post_reboot(self):
        undefined = self._get_undefined_services()
        self.assertEqual(undefined, [], 'undefined services: {0}'
                         .format(','.join(undefined)))
        stopped = self._get_stopped_services()
        self.assertEqual(stopped, [], 'stopped services: {0}'
                         .format(','.join(stopped)))
<|endoftext|>


<commit_before>from intelmq.lib.bot import Bot, sys
from intelmq.lib.utils import force_decode
from intelmq.lib.event import Event
#from intelmq.lib.cache import *

class ArborParserBot(Bot):

    def process(self):
        report = self.receive_message()

        if report:
            for row in report.split('\n'):
                row = row.strip()

                if len(row) == 0 or row.startswith('other'):
                    continue

                row = force_decode(row).split()
                event = Event()

                columns = ["ip"]
                for key, value in zip(columns, row):
                    event.add(key, value)

                self.send_message(event)

        self.acknowledge_message()

if __name__ == "__main__":
    bot = ArborParserBot(sys.argv[1])
    bot.start()
<commit_msg>remove unnecessary imports<commit_after>from intelmq.lib.bot import Bot, sys
from intelmq.lib.utils import force_decode
from intelmq.lib.event import Event

class ArborParserBot(Bot):

    def process(self):
        report = self.receive_message()

        if report:
            for row in report.split('\n'):
                row = row.strip()

                if len(row) == 0 or row.startswith('other'):
                    continue

                row = force_decode(row).split()
                event = Event()

                columns = ["ip"]
                for key, value in zip(columns, row):
                    event.add(key, value)

                self.send_message(event)

        self.acknowledge_message()

if __name__ == "__main__":
    bot = ArborParserBot(sys.argv[1])
    bot.start()
<|endoftext|>


<commit_before>from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
from airflow.hooks.base_hook import BaseHook
from airflow.plugins_manager import AirflowPlugin
import cpgintegrate
import requests

class XComDatasetToCkan(BaseOperator):

    @apply_defaults
    def __init__(self, ckan_connection_id, ckan_package_id, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.ckan_connection_id = ckan_connection_id
        self.ckan_package_id = ckan_package_id

    def execute(self, context):

        conn = BaseHook.get_connection(self.ckan_connection_id)

        source_task_id = self.upstream_task_ids[0]

        push_frame = context['ti'].xcom_pull(source_task_id)
        old_frame = context['ti'].xcom_pull(include_prior_dates=True)

        existing_resource_list = requests.get(
            url=conn.host + '/api/3/action/package_show',
            headers={"Authorization": conn.get_password()},
            params={"id": self.ckan_package_id},
        ).json()['result']['resources']

        res_create = False
        try:
            request_data = {"id": next(res['id'] for res in existing_resource_list if res['name'] == source_task_id)}
            url_ending = '/api/3/action/resource_update'
        except StopIteration:
            request_data = {"package_id": self.ckan_package_id, "name": source_task_id}
            url_ending = '/api/3/action/resource_create'
            self.log.info("Creating resource %s", source_task_id)
            res_create = True

        if res_create or not(push_frame.equals(old_frame)):
            res = requests.post(
                url=conn.host + url_ending,
                data=request_data,
                headers={"Authorization": conn.get_password()},
                files={"upload": (source_task_id+".csv", push_frame.to_csv())},
            )
            self.log.info("HTTP Status Code: %s", res.status_code)
            assert res.status_code == 200

            # Push metadata if exists'
            if hasattr(push_frame, 'get_json_column_info'):
                datadict_res = requests.post(
                    url=conn.host + '/api/3/action/datastore_create',
                    data='{"resource_id":"%s", "force":"true","fields":%s}' %
                         (res.json()['result']['id'], push_frame.get_json_column_info()),
                    headers={"Authorization": conn.get_password(), "Content-Type": "application/json"},
                )
                self.log.info("Data Dictionary Push Status Code: %s", datadict_res.status_code)
                assert datadict_res.status_code == 200

            return push_frame

class CPGDatasetToXCom(BaseOperator):
    ui_color = '#7DF9FF'

    @apply_defaults
    def __init__(self, connector_class, connection_id, dataset_args=None, dataset_kwargs=None, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.connector_class = connector_class
        self.connection_id = connection_id
        self.dataset_args = dataset_args or []
        self.dataset_kwargs = dataset_kwargs or {}

    def _get_connector(self):
        conn = BaseHook.get_connection(self.connection_id)
        return self.connector_class(auth=(conn.login, conn.get_password()), **vars(conn), **conn.extra_dejson)

    def execute(self, context):
        return self._get_connector().get_dataset(*self.dataset_args, **self.dataset_kwargs)

class CPGProcessorToXCom(CPGDatasetToXCom):
    ui_color = '#E7FEFF'

    @apply_defaults
    def __init__(self, processor, iter_files_args=None, iter_files_kwargs=None,
                 processor_args=None, processor_kwargs=None, file_subject_id=False, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.iter_files_args = iter_files_args or []
        self.iter_files_kwargs = iter_files_kwargs or {}
        self.processor = processor
        self.processor_args = processor_args or []
        self.processor_kwargs = processor_kwargs or {}
        self.file_subject_id = file_subject_id

    def execute(self, context):
        connector_instance = self._get_connector()
        processor_instance = self.processor(*self.processor_args, **self.processor_kwargs).to_frame \
            if isinstance(self.processor, type) else self.processor
        return (cpgintegrate
                .process_files(connector_instance.iter_files(*self.iter_files_args, **self.iter_files_kwargs),
                               processor_instance)
                .drop([] if self.file_subject_id else ["FileSubjectID"], axis=1))

class XComDatasetProcess(BaseOperator):
    cols_always_present = [cpgintegrate.SOURCE_FIELD_NAME]

    @apply_defaults
    def __init__(self, post_processor=None, filter_cols=None, drop_na_cols=True,
                 row_filter=lambda row: True, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.post_processor = post_processor or (lambda x: x)
        if type(filter_cols) == list:
            self.column_filter = {"items": filter_cols+self.cols_always_present}
        elif type(filter_cols) == str:
            self.column_filter = {"regex": str}
        else:
            self.column_filter = {"regex": ".*"}
        self.row_filter = row_filter
        self.drop_na_cols = drop_na_cols

    def execute(self, context):
        out_frame = self.post_processor(context['ti'].xcom_pull(self.upstream_task_ids[0])
                                        .filter(**self.column_filter)
                                        .loc[lambda df: df.apply(self.row_filter, axis=1)])
        if self.drop_na_cols:
            out_frame.dropna(axis=1, how='all', inplace=True)
        return out_frame

class AirflowCPGPlugin(AirflowPlugin):
    name = "cpg_plugin"
    operators = [CPGDatasetToXCom, CPGProcessorToXCom, XComDatasetProcess, XComDatasetToCkan]
<commit_msg>Adds frame unchanged log<commit_after>from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
from airflow.hooks.base_hook import BaseHook
from airflow.plugins_manager import AirflowPlugin
import cpgintegrate
import requests

class XComDatasetToCkan(BaseOperator):

    @apply_defaults
    def __init__(self, ckan_connection_id, ckan_package_id, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.ckan_connection_id = ckan_connection_id
        self.ckan_package_id = ckan_package_id

    def execute(self, context):

        conn = BaseHook.get_connection(self.ckan_connection_id)

        source_task_id = self.upstream_task_ids[0]

        push_frame = context['ti'].xcom_pull(source_task_id)
        old_frame = context['ti'].xcom_pull(include_prior_dates=True)

        existing_resource_list = requests.get(
            url=conn.host + '/api/3/action/package_show',
            headers={"Authorization": conn.get_password()},
            params={"id": self.ckan_package_id},
        ).json()['result']['resources']

        res_create = False
        try:
            request_data = {"id": next(res['id'] for res in existing_resource_list if res['name'] == source_task_id)}
            url_ending = '/api/3/action/resource_update'
        except StopIteration:
            request_data = {"package_id": self.ckan_package_id, "name": source_task_id}
            url_ending = '/api/3/action/resource_create'
            self.log.info("Creating resource %s", source_task_id)
            res_create = True

        if res_create or not(push_frame.equals(old_frame)):
            res = requests.post(
                url=conn.host + url_ending,
                data=request_data,
                headers={"Authorization": conn.get_password()},
                files={"upload": (source_task_id+".csv", push_frame.to_csv())},
            )
            self.log.info("HTTP Status Code: %s", res.status_code)
            assert res.status_code == 200

            # Push metadata if exists'
            if hasattr(push_frame, 'get_json_column_info'):
                datadict_res = requests.post(
                    url=conn.host + '/api/3/action/datastore_create',
                    data='{"resource_id":"%s", "force":"true","fields":%s}' %
                         (res.json()['result']['id'], push_frame.get_json_column_info()),
                    headers={"Authorization": conn.get_password(), "Content-Type": "application/json"},
                )
                self.log.info("Data Dictionary Push Status Code: %s", datadict_res.status_code)
                assert datadict_res.status_code == 200

            return push_frame
        else:
            self.log.info("Frame unchanged, skipping push")

class CPGDatasetToXCom(BaseOperator):
    ui_color = '#7DF9FF'

    @apply_defaults
    def __init__(self, connector_class, connection_id, dataset_args=None, dataset_kwargs=None, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.connector_class = connector_class
        self.connection_id = connection_id
        self.dataset_args = dataset_args or []
        self.dataset_kwargs = dataset_kwargs or {}

    def _get_connector(self):
        conn = BaseHook.get_connection(self.connection_id)
        return self.connector_class(auth=(conn.login, conn.get_password()), **vars(conn), **conn.extra_dejson)

    def execute(self, context):
        return self._get_connector().get_dataset(*self.dataset_args, **self.dataset_kwargs)

class CPGProcessorToXCom(CPGDatasetToXCom):
    ui_color = '#E7FEFF'

    @apply_defaults
    def __init__(self, processor, iter_files_args=None, iter_files_kwargs=None,
                 processor_args=None, processor_kwargs=None, file_subject_id=False, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.iter_files_args = iter_files_args or []
        self.iter_files_kwargs = iter_files_kwargs or {}
        self.processor = processor
        self.processor_args = processor_args or []
        self.processor_kwargs = processor_kwargs or {}
        self.file_subject_id = file_subject_id

    def execute(self, context):
        connector_instance = self._get_connector()
        processor_instance = self.processor(*self.processor_args, **self.processor_kwargs).to_frame \
            if isinstance(self.processor, type) else self.processor
        return (cpgintegrate
                .process_files(connector_instance.iter_files(*self.iter_files_args, **self.iter_files_kwargs),
                               processor_instance)
                .drop([] if self.file_subject_id else ["FileSubjectID"], axis=1))

class XComDatasetProcess(BaseOperator):
    cols_always_present = [cpgintegrate.SOURCE_FIELD_NAME]

    @apply_defaults
    def __init__(self, post_processor=None, filter_cols=None, drop_na_cols=True,
                 row_filter=lambda row: True, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.post_processor = post_processor or (lambda x: x)
        if type(filter_cols) == list:
            self.column_filter = {"items": filter_cols+self.cols_always_present}
        elif type(filter_cols) == str:
            self.column_filter = {"regex": str}
        else:
            self.column_filter = {"regex": ".*"}
        self.row_filter = row_filter
        self.drop_na_cols = drop_na_cols

    def execute(self, context):
        out_frame = self.post_processor(context['ti'].xcom_pull(self.upstream_task_ids[0])
                                        .filter(**self.column_filter)
                                        .loc[lambda df: df.apply(self.row_filter, axis=1)])
        if self.drop_na_cols:
            out_frame.dropna(axis=1, how='all', inplace=True)
        return out_frame

class AirflowCPGPlugin(AirflowPlugin):
    name = "cpg_plugin"
    operators = [CPGDatasetToXCom, CPGProcessorToXCom, XComDatasetProcess, XComDatasetToCkan]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""some measures for evaluation of prediction, tests and model selection

Created on Tue Nov 08 15:23:20 2011

Author: Josef Perktold
License: BSD-3

"""
import numpy as np

from statsmodels.tools.validation import array_like

def mse(x1, x2, axis=0):
    """mean squared error

    Parameters
    ----------
    x1, x2 : array_like
       The performance measure depends on the difference between these two
       arrays.
    axis : int
       axis along which the summary statistic is calculated

    Returns
    -------
    mse : ndarray or float
       mean squared error along given axis.

    Notes
    -----
    If ``x1`` and ``x2`` have different shapes, then they need to broadcast.
    This uses ``numpy.asanyarray`` to convert the input. Whether this is the
    desired result or not depends on the array subclass, for example
    numpy matrices will silently produce an incorrect result.
    """
    x1 = np.asanyarray(x1)
    x2 = np.asanyarray(x2)
    return np.mean((x1-x2)**2, axis=axis)

def rmse(x1, x2, axis=0):
    """root mean squared error

    Parameters
    ----------
    x1, x2 : array_like
       The performance measure depends on the difference between these two
       arrays.
    axis : int
       axis along which the summary statistic is calculated

    Returns
    -------
    rmse : ndarray or float
       root mean squared error along given axis.

    Notes
    -----
    If ``x1`` and ``x2`` have different shapes, then they need to broadcast.
    This uses ``numpy.asanyarray`` to convert the input. Whether this is the
    desired result or not depends on the array subclass, for example
    numpy matrices will silently produce an incorrect result.
    """
    x1 = np.asanyarray(x1)
    x2 = np.asanyarray(x2)
    return np.sqrt(mse(x1, x2, axis=axis))

def maxabs(x1, x2, axis=0):
    """maximum absolute error

    Parameters
    ----------
    x1, x2 : array_like
       The performance measure depends on the difference between these two
       arrays.
    axis : int
       axis along which the summary statistic is calculated

    Returns
    -------
    maxabs : ndarray or float
       maximum absolute difference along given axis.

    Notes
    -----
    If ``x1`` and ``x2`` have different shapes, then they need to broadcast.
    This uses ``numpy.asanyarray`` to convert the input. Whether this is the
    desired result or not depends on the array subclass.
    """
    x1 = np.asanyarray(x1)
    x2 = np.asanyarray(x2)
    return np.max(np.abs(x1-x2), axis=axis)

def meanabs(x1, x2, axis=0):
    """mean absolute error

    Parameters
    ----------
    x1, x2 : array_like
       The performance measure depends on the difference between these two
       arrays.
    axis : int
       axis along which the summary statistic is calculated

    Returns
    -------
    meanabs : ndarray or float
       mean absolute difference along given axis.

    Notes
    -----
    If ``x1`` and ``x2`` have different shapes, then they need to broadcast.
    This uses ``numpy.asanyarray`` to convert the input. Whether this is the
    desired result or not depends on the array subclass.
    """
    x1 = np.asanyarray(x1)
    x2 = np.asanyarray(x2)
    return np.mean(np.abs(x1-x2), axis=axis)

def medianabs(x1, x2, axis=0):
    """median absolute error

    Parameters
    ----------
    x1, x2 : array_like
       The performance measure depends on the difference between these two
       arrays.
    axis : int
       axis along which the summary statistic is calculated

    Returns
    -------
    medianabs : ndarray or float
       median absolute difference along given axis.

    Notes
    -----
    If ``x1`` and ``x2`` have different shapes, then they need to broadcast.
    This uses ``numpy.asanyarray`` to convert the input. Whether this is the
    desired result or not depends on the array subclass.
    """
    x1 = np.asanyarray(x1)
    x2 = np.asanyarray(x2)
    return np.median(np.abs(x1-x2), axis=axis)

def bias(x1, x2, axis=0):
    """bias, mean error

    Parameters
    ----------
    x1, x2 : array_like
       The performance measure depends on the difference between these two
       arrays.
    axis : int
       axis along which the summary statistic is calculated

    Returns
    -------
    bias : ndarray or float
       bias, or mean difference along given axis.

    Notes
    -----
    If ``x1`` and ``x2`` have different shapes, then they need to broadcast.
    This uses ``numpy.asanyarray`` to convert the input. Whether this is the
    desired result or not depends on the array subclass.
    """
    x1 = np.asanyarray(x1)
    x2 = np.asanyarray(x2)
    return np.mean(x1-x2, axis=axis)

def medianbias(x1, x2, axis=0):
    """median bias, median error

    Parameters
    ----------
    x1, x2 : array_like
       The performance measure depends on the difference between these two
       arrays.
    axis : int
       axis along which the summary statistic is calculated

    Returns
    -------
    medianbias : ndarray or float
       median bias, or median difference along given axis.

    Notes
    -----
    If ``x1`` and ``x2`` have different shapes, then they need to broadcast.
    This uses ``numpy.asanyarray`` to convert the input. Whether this is the
    desired result or not depends on the array subclass.
    """
    x1 = np.asanyarray(x1)
    x2 = np.asanyarray(x2)
    return np.median(x1-x2, axis=axis)

def vare(x1, x2, ddof=0, axis=0):
    """variance of error

    Parameters
    ----------
    x1, x2 : array_like
       The performance measure depends on the difference between these two
       arrays.
    axis : int
       axis along which the summary statistic is calculated

    Returns
    -------
    vare : ndarray or float
       variance of difference along given axis.

    Notes
    -----
    If ``x1`` and ``x2`` have different shapes, then they need to broadcast.
    This uses ``numpy.asanyarray`` to convert the input. Whether this is the
    desired result or not depends on the array subclass.
    """
    x1 = np.asanyarray(x1)
    x2 = np.asanyarray(x2)
    return np.var(x1-x2, ddof=ddof, axis=axis)

def stde(x1, x2, ddof=0, axis=0):
    """standard deviation of error

    Parameters
    ----------
    x1, x2 : array_like
       The performance measure depends on the difference between these two
       arrays.
    axis : int
       axis along which the summary statistic is calculated

    Returns
    -------
    stde : ndarray or float
       standard deviation of difference along given axis.

    Notes
    -----
    If ``x1`` and ``x2`` have different shapes, then they need to broadcast.
    This uses ``numpy.asanyarray`` to convert the input. Whether this is the
    desired result or not depends on the array subclass.
    """
    x1 = np.asanyarray(x1)
    x2 = np.asanyarray(x2)
    return np.std(x1-x2, ddof=ddof, axis=axis)

def iqr(x1, x2, axis=0):
    """
    Interquartile range of error

    Parameters
    ----------
    x1 : array_like
       One of the inputs into the IQR calculation.
    x2 : array_like
       The other input into the IQR calculation.
    axis : {None, int}
       axis along which the summary statistic is calculated

    Returns
    -------
    irq : {float, ndarray}
       Interquartile range along given axis.

    Notes
    -----
    If ``x1`` and ``x2`` have different shapes, then they must broadcast.
    """
    x1 = array_like(x1, 'x1', dtype=None, ndim=None)
    x2 = array_like(x2, 'x1', dtype=None, ndim=None)
    if axis is None:
        x1 = x1.ravel()
        x2 = x2.ravel()
        axis = 0
    xdiff = np.sort(x1 - x2, axis=axis)
    nobs = x1.shape[axis]
    idx = np.round((nobs-1) * np.array([0.25, 0.75])).astype(int)
    sl = [slice(None)] * xdiff.ndim
    sl[axis] = idx
    iqr = np.diff(xdiff[tuple(sl)], axis=axis)
    iqr = np.squeeze(iqr)  # drop reduced dimension
    return iqr

# Information Criteria
# ---------------------

def aic(llf, nobs, df_modelwc):
    """
    Akaike information criterion

    Parameters
    ----------
    llf : {float, array_like}
        value of the loglikelihood
    nobs : int
        number of observations
    df_modelwc : int
        number of parameters including constant

    Returns
    -------
    aic : float
        information criterion

    References
    ----------
    https://en.wikipedia.org/wiki/Akaike_information_criterion
    """
    return -2. * llf + 2. * df_modelwc

def aicc(llf, nobs, df_modelwc):
    """
    Akaike information criterion (AIC) with small sample correction

    Parameters
    ----------
    llf : {float, array_like}
        value of the loglikelihood
    nobs : int
        number of observations
    df_modelwc : int
        number of parameters including constant

    Returns
    -------
    aicc : float
        information criterion

    References
    ----------
    https://en.wikipedia.org/wiki/Akaike_information_criterion#AICc
    """
    return -2. * llf + 2. * df_modelwc * nobs / (nobs - df_modelwc - 1.)

def bic(llf, nobs, df_modelwc):
    """
    Bayesian information criterion (BIC) or Schwarz criterion

    Parameters
    ----------
    llf : {float, array_like}
        value of the loglikelihood
    nobs : int
        number of observations
    df_modelwc : int
        number of parameters including constant

    Returns
    -------
    bic : float
        information criterion

    References
    ----------
    https://en.wikipedia.org/wiki/Bayesian_information_criterion
    """
    return -2. * llf + np.log(nobs) * df_modelwc

def hqic(llf, nobs, df_modelwc):
    """
    Hannan-Quinn information criterion (HQC)

    Parameters
    ----------
    llf : {float, array_like}
        value of the loglikelihood
    nobs : int
        number of observations
    df_modelwc : int
        number of parameters including constant

    Returns
    -------
    hqic : float
        information criterion

    References
    ----------
    Wikipedia does not say much
    """
    return -2. * llf + 2 * np.log(np.log(nobs)) * df_modelwc

# IC based on residual sigma

def aic_sigma(sigma2, nobs, df_modelwc, islog=False):
    r"""
    Akaike information criterion

    Parameters
    ----------
    sigma2 : float
        estimate of the residual variance or determinant of Sigma_hat in the
        multivariate case. If islog is true, then it is assumed that sigma
        is already log-ed, for example logdetSigma.
    nobs : int
        number of observations
    df_modelwc : int
        number of parameters including constant

    Returns
    -------
    aic : float
        information criterion

    Notes
    -----
    A constant has been dropped in comparison to the loglikelihood base
    information criteria. The information criteria should be used to compare
    only comparable models.

    For example, AIC is defined in terms of the loglikelihood as

    :math:`-2 llf + 2 k`

    in terms of :math:`\hat{\sigma}^2`

    :math:`log(\hat{\sigma}^2) + 2 k / n`

    in terms of the determinant of :math:`\hat{\Sigma}`

    :math:`log(\|\hat{\Sigma}\|) + 2 k / n`

    Note: In our definition we do not divide by n in the log-likelihood
    version.

    TODO: Latex math

    reference for example lecture notes by Herman Bierens

    See Also
    --------

    References
    ----------
    https://en.wikipedia.org/wiki/Akaike_information_criterion
    """
    if not islog:
        sigma2 = np.log(sigma2)
    return sigma2 + aic(0, nobs, df_modelwc) / nobs

def aicc_sigma(sigma2, nobs, df_modelwc, islog=False):
    """Akaike information criterion (AIC) with small sample correction

    Parameters
    ----------
    sigma2 : float
        estimate of the residual variance or determinant of Sigma_hat in the
        multivariate case. If islog is true, then it is assumed that sigma
        is already log-ed, for example logdetSigma.
    nobs : int
        number of observations
    df_modelwc : int
        number of parameters including constant

    Returns
    -------
    aicc : float
        information criterion

    Notes
    -----
    A constant has been dropped in comparison to the loglikelihood base
    information criteria. These should be used to compare for comparable
    models.

    References
    ----------
    https://en.wikipedia.org/wiki/Akaike_information_criterion#AICc
    """
    if not islog:
        sigma2 = np.log(sigma2)
    return sigma2 + aicc(0, nobs, df_modelwc) / nobs

def bic_sigma(sigma2, nobs, df_modelwc, islog=False):
    """Bayesian information criterion (BIC) or Schwarz criterion

    Parameters
    ----------
    sigma2 : float
        estimate of the residual variance or determinant of Sigma_hat in the
        multivariate case. If islog is true, then it is assumed that sigma
        is already log-ed, for example logdetSigma.
    nobs : int
        number of observations
    df_modelwc : int
        number of parameters including constant

    Returns
    -------
    bic : float
        information criterion

    Notes
    -----
    A constant has been dropped in comparison to the loglikelihood base
    information criteria. These should be used to compare for comparable
    models.

    References
    ----------
    https://en.wikipedia.org/wiki/Bayesian_information_criterion
    """
    if not islog:
        sigma2 = np.log(sigma2)
    return sigma2 + bic(0, nobs, df_modelwc) / nobs

def hqic_sigma(sigma2, nobs, df_modelwc, islog=False):
    """Hannan-Quinn information criterion (HQC)

    Parameters
    ----------
    sigma2 : float
        estimate of the residual variance or determinant of Sigma_hat in the
        multivariate case. If islog is true, then it is assumed that sigma
        is already log-ed, for example logdetSigma.
    nobs : int
        number of observations
    df_modelwc : int
        number of parameters including constant

    Returns
    -------
    hqic : float
        information criterion

    Notes
    -----
    A constant has been dropped in comparison to the loglikelihood base
    information criteria. These should be used to compare for comparable
    models.

    References
    ----------
    xxx
    """
    if not islog:
        sigma2 = np.log(sigma2)
    return sigma2 + hqic(0, nobs, df_modelwc) / nobs

# from var_model.py, VAR only? separates neqs and k_vars per equation
# def fpe_sigma():
#     ((nobs + self.df_model) / self.df_resid) ** neqs * np.exp(ld)

__all__ = [maxabs, meanabs, medianabs, medianbias, mse, rmse, stde, vare,
           aic, aic_sigma, aicc, aicc_sigma, bias, bic, bic_sigma,
           hqic, hqic_sigma, iqr]
<commit_msg>ENH: Add Root Mean Square Percentage Error as rmspe()<commit_after># -*- coding: utf-8 -*-
"""some measures for evaluation of prediction, tests and model selection

Created on Tue Nov 08 15:23:20 2011
Updated on Wed Jun 03 10:42:20 2020

Authors: Josef Perktold & Peter Prescott
License: BSD-3

"""
import numpy as np

from statsmodels.tools.validation import array_like

def mse(x1, x2, axis=0):
    """mean squared error

    Parameters
    ----------
    x1, x2 : array_like
       The performance measure depends on the difference between these two
       arrays.
    axis : int
       axis along which the summary statistic is calculated

    Returns
    -------
    mse : ndarray or float
       mean squared error along given axis.

    Notes
    -----
    If ``x1`` and ``x2`` have different shapes, then they need to broadcast.
    This uses ``numpy.asanyarray`` to convert the input. Whether this is the
    desired result or not depends on the array subclass, for example
    numpy matrices will silently produce an incorrect result.
    """
    x1 = np.asanyarray(x1)
    x2 = np.asanyarray(x2)
    return np.mean((x1-x2)**2, axis=axis)

def rmse(x1, x2, axis=0):
    """root mean squared error

    Parameters
    ----------
    x1, x2 : array_like
       The performance measure depends on the difference between these two
       arrays.
    axis : int
       axis along which the summary statistic is calculated

    Returns
    -------
    rmse : ndarray or float
       root mean squared error along given axis.

    Notes
    -----
    If ``x1`` and ``x2`` have different shapes, then they need to broadcast.
    This uses ``numpy.asanyarray`` to convert the input. Whether this is the
    desired result or not depends on the array subclass, for example
    numpy matrices will silently produce an incorrect result.
    """
    x1 = np.asanyarray(x1)
    x2 = np.asanyarray(x2)
    return np.sqrt(mse(x1, x2, axis=axis))

def rmspe(y, y_hat, axis=0, zeros=np.nan):
    """
    Root Mean Squared Percentage Error

    Parameters
    ----------
    y : array_like
      The actual value.
    y_hat : array_like
       The predicted value.
    axis : int
       Axis along which the summary statistic is calculated
    zeros : float
       Value to assign to error where actual value is zero

    Returns
    -------
    rmspe : ndarray or float
       Root Mean Squared Percentage Error along given axis.
    """
    y_hat = np.asarray(y_hat)
    y = np.asarray(y)
    error = y - y_hat
    percentage_error = np.divide(error, y, out=np.full_like(error, zeros), where=y!=0)
    return np.nanmean(percentage_error**2, axis=axis) * 100

def maxabs(x1, x2, axis=0):
    """maximum absolute error

    Parameters
    ----------
    x1, x2 : array_like
       The performance measure depends on the difference between these two
       arrays.
    axis : int
       axis along which the summary statistic is calculated

    Returns
    -------
    maxabs : ndarray or float
       maximum absolute difference along given axis.

    Notes
    -----
    If ``x1`` and ``x2`` have different shapes, then they need to broadcast.
    This uses ``numpy.asanyarray`` to convert the input. Whether this is the
    desired result or not depends on the array subclass.
    """
    x1 = np.asanyarray(x1)
    x2 = np.asanyarray(x2)
    return np.max(np.abs(x1-x2), axis=axis)

def meanabs(x1, x2, axis=0):
    """mean absolute error

    Parameters
    ----------
    x1, x2 : array_like
       The performance measure depends on the difference between these two
       arrays.
    axis : int
       axis along which the summary statistic is calculated

    Returns
    -------
    meanabs : ndarray or float
       mean absolute difference along given axis.

    Notes
    -----
    If ``x1`` and ``x2`` have different shapes, then they need to broadcast.
    This uses ``numpy.asanyarray`` to convert the input. Whether this is the
    desired result or not depends on the array subclass.
    """
    x1 = np.asanyarray(x1)
    x2 = np.asanyarray(x2)
    return np.mean(np.abs(x1-x2), axis=axis)

def medianabs(x1, x2, axis=0):
    """median absolute error

    Parameters
    ----------
    x1, x2 : array_like
       The performance measure depends on the difference between these two
       arrays.
    axis : int
       axis along which the summary statistic is calculated

    Returns
    -------
    medianabs : ndarray or float
       median absolute difference along given axis.

    Notes
    -----
    If ``x1`` and ``x2`` have different shapes, then they need to broadcast.
    This uses ``numpy.asanyarray`` to convert the input. Whether this is the
    desired result or not depends on the array subclass.
    """
    x1 = np.asanyarray(x1)
    x2 = np.asanyarray(x2)
    return np.median(np.abs(x1-x2), axis=axis)

def bias(x1, x2, axis=0):
    """bias, mean error

    Parameters
    ----------
    x1, x2 : array_like
       The performance measure depends on the difference between these two
       arrays.
    axis : int
       axis along which the summary statistic is calculated

    Returns
    -------
    bias : ndarray or float
       bias, or mean difference along given axis.

    Notes
    -----
    If ``x1`` and ``x2`` have different shapes, then they need to broadcast.
    This uses ``numpy.asanyarray`` to convert the input. Whether this is the
    desired result or not depends on the array subclass.
    """
    x1 = np.asanyarray(x1)
    x2 = np.asanyarray(x2)
    return np.mean(x1-x2, axis=axis)

def medianbias(x1, x2, axis=0):
    """median bias, median error

    Parameters
    ----------
    x1, x2 : array_like
       The performance measure depends on the difference between these two
       arrays.
    axis : int
       axis along which the summary statistic is calculated

    Returns
    -------
    medianbias : ndarray or float
       median bias, or median difference along given axis.

    Notes
    -----
    If ``x1`` and ``x2`` have different shapes, then they need to broadcast.
    This uses ``numpy.asanyarray`` to convert the input. Whether this is the
    desired result or not depends on the array subclass.
    """
    x1 = np.asanyarray(x1)
    x2 = np.asanyarray(x2)
    return np.median(x1-x2, axis=axis)

def vare(x1, x2, ddof=0, axis=0):
    """variance of error

    Parameters
    ----------
    x1, x2 : array_like
       The performance measure depends on the difference between these two
       arrays.
    axis : int
       axis along which the summary statistic is calculated

    Returns
    -------
    vare : ndarray or float
       variance of difference along given axis.

    Notes
    -----
    If ``x1`` and ``x2`` have different shapes, then they need to broadcast.
    This uses ``numpy.asanyarray`` to convert the input. Whether this is the
    desired result or not depends on the array subclass.
    """
    x1 = np.asanyarray(x1)
    x2 = np.asanyarray(x2)
    return np.var(x1-x2, ddof=ddof, axis=axis)

def stde(x1, x2, ddof=0, axis=0):
    """standard deviation of error

    Parameters
    ----------
    x1, x2 : array_like
       The performance measure depends on the difference between these two
       arrays.
    axis : int
       axis along which the summary statistic is calculated

    Returns
    -------
    stde : ndarray or float
       standard deviation of difference along given axis.

    Notes
    -----
    If ``x1`` and ``x2`` have different shapes, then they need to broadcast.
    This uses ``numpy.asanyarray`` to convert the input. Whether this is the
    desired result or not depends on the array subclass.
    """
    x1 = np.asanyarray(x1)
    x2 = np.asanyarray(x2)
    return np.std(x1-x2, ddof=ddof, axis=axis)

def iqr(x1, x2, axis=0):
    """
    Interquartile range of error

    Parameters
    ----------
    x1 : array_like
       One of the inputs into the IQR calculation.
    x2 : array_like
       The other input into the IQR calculation.
    axis : {None, int}
       axis along which the summary statistic is calculated

    Returns
    -------
    irq : {float, ndarray}
       Interquartile range along given axis.

    Notes
    -----
    If ``x1`` and ``x2`` have different shapes, then they must broadcast.
    """
    x1 = array_like(x1, 'x1', dtype=None, ndim=None)
    x2 = array_like(x2, 'x1', dtype=None, ndim=None)
    if axis is None:
        x1 = x1.ravel()
        x2 = x2.ravel()
        axis = 0
    xdiff = np.sort(x1 - x2, axis=axis)
    nobs = x1.shape[axis]
    idx = np.round((nobs-1) * np.array([0.25, 0.75])).astype(int)
    sl = [slice(None)] * xdiff.ndim
    sl[axis] = idx
    iqr = np.diff(xdiff[tuple(sl)], axis=axis)
    iqr = np.squeeze(iqr)  # drop reduced dimension
    return iqr

# Information Criteria
# ---------------------

def aic(llf, nobs, df_modelwc):
    """
    Akaike information criterion

    Parameters
    ----------
    llf : {float, array_like}
        value of the loglikelihood
    nobs : int
        number of observations
    df_modelwc : int
        number of parameters including constant

    Returns
    -------
    aic : float
        information criterion

    References
    ----------
    https://en.wikipedia.org/wiki/Akaike_information_criterion
    """
    return -2. * llf + 2. * df_modelwc

def aicc(llf, nobs, df_modelwc):
    """
    Akaike information criterion (AIC) with small sample correction

    Parameters
    ----------
    llf : {float, array_like}
        value of the loglikelihood
    nobs : int
        number of observations
    df_modelwc : int
        number of parameters including constant

    Returns
    -------
    aicc : float
        information criterion

    References
    ----------
    https://en.wikipedia.org/wiki/Akaike_information_criterion#AICc
    """
    return -2. * llf + 2. * df_modelwc * nobs / (nobs - df_modelwc - 1.)

def bic(llf, nobs, df_modelwc):
    """
    Bayesian information criterion (BIC) or Schwarz criterion

    Parameters
    ----------
    llf : {float, array_like}
        value of the loglikelihood
    nobs : int
        number of observations
    df_modelwc : int
        number of parameters including constant

    Returns
    -------
    bic : float
        information criterion

    References
    ----------
    https://en.wikipedia.org/wiki/Bayesian_information_criterion
    """
    return -2. * llf + np.log(nobs) * df_modelwc

def hqic(llf, nobs, df_modelwc):
    """
    Hannan-Quinn information criterion (HQC)

    Parameters
    ----------
    llf : {float, array_like}
        value of the loglikelihood
    nobs : int
        number of observations
    df_modelwc : int
        number of parameters including constant

    Returns
    -------
    hqic : float
        information criterion

    References
    ----------
    Wikipedia does not say much
    """
    return -2. * llf + 2 * np.log(np.log(nobs)) * df_modelwc

# IC based on residual sigma

def aic_sigma(sigma2, nobs, df_modelwc, islog=False):
    r"""
    Akaike information criterion

    Parameters
    ----------
    sigma2 : float
        estimate of the residual variance or determinant of Sigma_hat in the
        multivariate case. If islog is true, then it is assumed that sigma
        is already log-ed, for example logdetSigma.
    nobs : int
        number of observations
    df_modelwc : int
        number of parameters including constant

    Returns
    -------
    aic : float
        information criterion

    Notes
    -----
    A constant has been dropped in comparison to the loglikelihood base
    information criteria. The information criteria should be used to compare
    only comparable models.

    For example, AIC is defined in terms of the loglikelihood as

    :math:`-2 llf + 2 k`

    in terms of :math:`\hat{\sigma}^2`

    :math:`log(\hat{\sigma}^2) + 2 k / n`

    in terms of the determinant of :math:`\hat{\Sigma}`

    :math:`log(\|\hat{\Sigma}\|) + 2 k / n`

    Note: In our definition we do not divide by n in the log-likelihood
    version.

    TODO: Latex math

    reference for example lecture notes by Herman Bierens

    See Also
    --------

    References
    ----------
    https://en.wikipedia.org/wiki/Akaike_information_criterion
    """
    if not islog:
        sigma2 = np.log(sigma2)
    return sigma2 + aic(0, nobs, df_modelwc) / nobs

def aicc_sigma(sigma2, nobs, df_modelwc, islog=False):
    """Akaike information criterion (AIC) with small sample correction

    Parameters
    ----------
    sigma2 : float
        estimate of the residual variance or determinant of Sigma_hat in the
        multivariate case. If islog is true, then it is assumed that sigma
        is already log-ed, for example logdetSigma.
    nobs : int
        number of observations
    df_modelwc : int
        number of parameters including constant

    Returns
    -------
    aicc : float
        information criterion

    Notes
    -----
    A constant has been dropped in comparison to the loglikelihood base
    information criteria. These should be used to compare for comparable
    models.

    References
    ----------
    https://en.wikipedia.org/wiki/Akaike_information_criterion#AICc
    """
    if not islog:
        sigma2 = np.log(sigma2)
    return sigma2 + aicc(0, nobs, df_modelwc) / nobs

def bic_sigma(sigma2, nobs, df_modelwc, islog=False):
    """Bayesian information criterion (BIC) or Schwarz criterion

    Parameters
    ----------
    sigma2 : float
        estimate of the residual variance or determinant of Sigma_hat in the
        multivariate case. If islog is true, then it is assumed that sigma
        is already log-ed, for example logdetSigma.
    nobs : int
        number of observations
    df_modelwc : int
        number of parameters including constant

    Returns
    -------
    bic : float
        information criterion

    Notes
    -----
    A constant has been dropped in comparison to the loglikelihood base
    information criteria. These should be used to compare for comparable
    models.

    References
    ----------
    https://en.wikipedia.org/wiki/Bayesian_information_criterion
    """
    if not islog:
        sigma2 = np.log(sigma2)
    return sigma2 + bic(0, nobs, df_modelwc) / nobs

def hqic_sigma(sigma2, nobs, df_modelwc, islog=False):
    """Hannan-Quinn information criterion (HQC)

    Parameters
    ----------
    sigma2 : float
        estimate of the residual variance or determinant of Sigma_hat in the
        multivariate case. If islog is true, then it is assumed that sigma
        is already log-ed, for example logdetSigma.
    nobs : int
        number of observations
    df_modelwc : int
        number of parameters including constant

    Returns
    -------
    hqic : float
        information criterion

    Notes
    -----
    A constant has been dropped in comparison to the loglikelihood base
    information criteria. These should be used to compare for comparable
    models.

    References
    ----------
    xxx
    """
    if not islog:
        sigma2 = np.log(sigma2)
    return sigma2 + hqic(0, nobs, df_modelwc) / nobs

# from var_model.py, VAR only? separates neqs and k_vars per equation
# def fpe_sigma():
#     ((nobs + self.df_model) / self.df_resid) ** neqs * np.exp(ld)

__all__ = [maxabs, meanabs, medianabs, medianbias, mse, rmse, rmspe, stde, vare,
           aic, aic_sigma, aicc, aicc_sigma, bias, bic, bic_sigma,
           hqic, hqic_sigma, iqr]
<|endoftext|>


<commit_before>"""Class for generator for a resource."""

from django.http import Http404
from abc import ABC, abstractmethod
from utils.str_to_bool import str_to_bool

class BaseResourceGenerator(ABC):
    """Class for generator for a resource."""

    default_valid_options = {
        "paper_size": ["a4", "letter"]
    }
    additional_valid_options = dict()

    def __init__(self, requested_options=None):
        """Construct BaseResourceGenerator instance.

        Args:
            requested_options: QueryDict of requested_options (QueryDict).
        """
        self.valid_options = BaseResourceGenerator.default_valid_options
        self.valid_options.update(self.additional_valid_options)
        if requested_options:
            self.requested_options = self.process_requested_options(requested_options)

    @abstractmethod
    def data(self):
        """Abstract method to be implemented by subclasses.

        Raise:
            NotImplementedError: When data() method of the ResourceGenerator
            class is called.
        """
        raise NotImplementedError("Subclass does not implement the data method.")

    @property
    def subtitle(self):
        """Return the subtitle string of the resource.

        Used after the resource name in the filename, and
        also on the resource image.

        Returns:
            Text for subtitle (str).
        """
        return self.requested_options["paper_size"]

    def process_requested_options(self, requested_options):
        """Convert requested options to usable types.

        Method does the following:
        - Update all values through str_to_bool utility function.
        - Raises 404 error is requested option cannot be found.
        - Raises 404 is option given with invalid value.

        Returns:
            QueryDict of converted requested options (QueryDict).
        """
        requested_options = requested_options.copy()
        for option in self.valid_options.keys():
            values = requested_options.getlist(option)
            if not values:
                raise Http404("{} parameter not specified.".format(option))
            for (i, value) in enumerate(values):
                update_value = str_to_bool(value)
                if update_value not in self.valid_options[option]:
                    raise Http404("{} for parameter {} is not valid.".format(update_value, option))
                values[i] = update_value
            requested_options.setlist(option, values)
        return requested_options
<commit_msg>Fix bug in BaseResourceGenerator where valid options are shared across all generators<commit_after>"""Class for generator for a resource."""

from django.http import Http404
from abc import ABC, abstractmethod
from utils.str_to_bool import str_to_bool
from copy import deepcopy

class BaseResourceGenerator(ABC):
    """Class for generator for a resource."""

    default_valid_options = {
        "paper_size": ["a4", "letter"]
    }
    additional_valid_options = dict()

    def __init__(self, requested_options=None):
        """Construct BaseResourceGenerator instance.

        Args:
            requested_options: QueryDict of requested_options (QueryDict).
        """
        # Use deepcopy to avoid successive generators from sharing the same
        # valid_options dictionary.
        self.valid_options = deepcopy(BaseResourceGenerator.default_valid_options)
        self.valid_options.update(self.additional_valid_options)
        if requested_options:
            self.requested_options = self.process_requested_options(requested_options)

    @abstractmethod
    def data(self):
        """Abstract method to be implemented by subclasses.

        Raise:
            NotImplementedError: When data() method of the ResourceGenerator
            class is called.
        """
        raise NotImplementedError("Subclass does not implement the data method.")

    @property
    def subtitle(self):
        """Return the subtitle string of the resource.

        Used after the resource name in the filename, and
        also on the resource image.

        Returns:
            Text for subtitle (str).
        """
        return self.requested_options["paper_size"]

    def process_requested_options(self, requested_options):
        """Convert requested options to usable types.

        Method does the following:
        - Update all values through str_to_bool utility function.
        - Raises 404 error is requested option cannot be found.
        - Raises 404 is option given with invalid value.

        Returns:
            QueryDict of converted requested options (QueryDict).
        """
        requested_options = requested_options.copy()
        for option in self.valid_options.keys():
            values = requested_options.getlist(option)
            if not values:
                raise Http404("{} parameter not specified.".format(option))
            for (i, value) in enumerate(values):
                update_value = str_to_bool(value)
                if update_value not in self.valid_options[option]:
                    raise Http404("{} for parameter {} is not valid.".format(update_value, option))
                values[i] = update_value
            requested_options.setlist(option, values)
        return requested_options
<|endoftext|>