<commit_before>from Cookie import SimpleCookie
import cookielib
import copy
import requests
import time
from saml2 import class_name
from saml2.pack import http_form_post_message
from saml2.pack import make_soap_enveloped_saml_thingy
from saml2.pack import http_redirect_message

import logging
from saml2.soap import parse_soap_enveloped_saml_response

logger = logging.getLogger(__name__)

__author__ = 'rolandh'

ATTRS = {"version":None,
         "name":"",
         "value": None,
         "port": None,
         "port_specified": False,
         "domain": "",
         "domain_specified": False,
         "domain_initial_dot": False,
         "path": "",
         "path_specified": False,
         "secure": False,
         "expires": None,
         "discard": True,
         "comment": None,
         "comment_url": None,
         "rest": "",
         "rfc2109": True}

PAIRS = {
    "port": "port_specified",
    "domain": "domain_specified",
    "path": "path_specified"
}

class ConnectionError(Exception):
    pass

def _since_epoch(cdate):
    # date format 'Wed, 06-Jun-2012 01:34:34 GMT'
    cdate = cdate[5:-4]
    try:
        t = time.strptime(cdate, "%d-%b-%Y %H:%M:%S")
    except ValueError:
        t = time.strptime(cdate, "%d-%b-%y %H:%M:%S")
    return int(time.mktime(t))

class HTTPBase(object):
    def __init__(self, verify=True, ca_bundle=None, key_file=None,
                 cert_file=None):
        self.request_args = {"allow_redirects": False,}
        self.cookies = {}
        self.cookiejar = cookielib.CookieJar()

        self.request_args["verify"] = verify
        if ca_bundle:
            self.request_args["verify"] = ca_bundle
        if key_file:
            self.request_args["cert"] = (cert_file, key_file)

        self.sec = None

    def _cookies(self):
        cookie_dict = {}

        for _, a in list(self.cookiejar._cookies.items()):
            for _, b in list(a.items()):
                for cookie in list(b.values()):
                    # print cookie
                    cookie_dict[cookie.name] = cookie.value

        return cookie_dict

    def set_cookie(self, kaka, request):
        """Returns a cookielib.Cookie based on a set-cookie header line"""

        # default rfc2109=False
        # max-age, httponly
        for cookie_name, morsel in kaka.items():
            std_attr = ATTRS.copy()
            std_attr["name"] = cookie_name
            _tmp = morsel.coded_value
            if _tmp.startswith('"') and _tmp.endswith('"'):
                std_attr["value"] = _tmp[1:-1]
            else:
                std_attr["value"] = _tmp

            std_attr["version"] = 0
            # copy attributes that have values
            for attr in morsel.keys():
                if attr in ATTRS:
                    if morsel[attr]:
                        if attr == "expires":
                            std_attr[attr]=_since_epoch(morsel[attr])
                        else:
                            std_attr[attr]=morsel[attr]
                elif attr == "max-age":
                    if morsel["max-age"]:
                        std_attr["expires"] = _since_epoch(morsel["max-age"])

            for att, set in PAIRS.items():
                if std_attr[att]:
                    std_attr[set] = True

            if std_attr["domain"] and std_attr["domain"].startswith("."):
                std_attr["domain_initial_dot"] = True

            if morsel["max-age"] is 0:
                try:
                    self.cookiejar.clear(domain=std_attr["domain"],
                                         path=std_attr["path"],
                                         name=std_attr["name"])
                except ValueError:
                    pass
            else:
                new_cookie = cookielib.Cookie(**std_attr)

                self.cookiejar.set_cookie(new_cookie)

    def send(self, url, method="GET", **kwargs):
        _kwargs = copy.copy(self.request_args)
        if kwargs:
            _kwargs.update(kwargs)

        if self.cookiejar:
            _kwargs["cookies"] = self._cookies()
            #logger.info("SENT COOKIEs: %s" % (_kwargs["cookies"],))
        try:
            r = requests.request(method, url, **_kwargs)
        except requests.ConnectionError, exc:
            raise ConnectionError("%s" % exc)

        try:
            #logger.info("RECEIVED COOKIEs: %s" % (r.headers["set-cookie"],))
            self.set_cookie(SimpleCookie(r.headers["set-cookie"]), r)
        except AttributeError, err:
            pass

        return r

    def use_http_form_post(self, message, destination, relay_state):
        """
        Return a form that will automagically execute and POST the message
        to the recipient.

        :param message:
        :param destination:
        :param relay_state:
        :return: tuple (header, message)
        """
        if not isinstance(message, basestring):
            request = "%s" % (message,)

        return http_form_post_message(message, destination, relay_state)

    def use_http_get(self, message, destination, relay_state):
        """
        Send a message using GET, this is the HTTP-Redirect case so
        no direct response is expected to this request.

        :param request:
        :param destination:
        :param relay_state:
        :return: tuple (header, None)
        """
        if not isinstance(message, basestring):
            request = "%s" % (message,)

        return http_redirect_message(message, destination, relay_state)

    def send_using_soap(self, request, destination, headers=None, sign=False):
        """
        Send a message using SOAP+POST

        :param request:
        :param destination:
        :param headers:
        :param sign:
        :return:
        """
        if headers is None:
            headers = {"content-type": "application/soap+xml"}
        else:
            headers.update({"content-type": "application/soap+xml"})

        soap_message = make_soap_enveloped_saml_thingy(request)

        if sign and self.sec:
            _signed = self.sec.sign_statement_using_xmlsec(soap_message,
                                                           class_name(request),
                                                           nodeid=request.id)
            soap_message = _signed

        #_response = self.server.post(soap_message, headers, path=path)
        try:
            response = self.send(destination, "POST", data=soap_message,
                                 headers=headers)
        except Exception, exc:
            logger.info("HTTPClient exception: %s" % (exc,))
            return None

        if response:
            logger.info("SOAP response: %s" % response)
            return parse_soap_enveloped_saml_response(response)
        else:
            return False

<commit_msg>Editorial<commit_after>from Cookie import SimpleCookie
import cookielib
import copy
import requests
import time
from saml2 import class_name
from saml2.pack import http_form_post_message
from saml2.pack import make_soap_enveloped_saml_thingy
from saml2.pack import http_redirect_message

import logging
from saml2.soap import parse_soap_enveloped_saml_response

logger = logging.getLogger(__name__)

__author__ = 'rolandh'

ATTRS = {"version":None,
         "name":"",
         "value": None,
         "port": None,
         "port_specified": False,
         "domain": "",
         "domain_specified": False,
         "domain_initial_dot": False,
         "path": "",
         "path_specified": False,
         "secure": False,
         "expires": None,
         "discard": True,
         "comment": None,
         "comment_url": None,
         "rest": "",
         "rfc2109": True}

PAIRS = {
    "port": "port_specified",
    "domain": "domain_specified",
    "path": "path_specified"
}

class ConnectionError(Exception):
    pass

def _since_epoch(cdate):
    # date format 'Wed, 06-Jun-2012 01:34:34 GMT'
    cdate = cdate[5:-4]
    try:
        t = time.strptime(cdate, "%d-%b-%Y %H:%M:%S")
    except ValueError:
        t = time.strptime(cdate, "%d-%b-%y %H:%M:%S")
    return int(time.mktime(t))

class HTTPBase(object):
    def __init__(self, verify=True, ca_bundle=None, key_file=None,
                 cert_file=None):
        self.request_args = {"allow_redirects": False,}
        self.cookies = {}
        self.cookiejar = cookielib.CookieJar()

        self.request_args["verify"] = verify
        if ca_bundle:
            self.request_args["verify"] = ca_bundle
        if key_file:
            self.request_args["cert"] = (cert_file, key_file)

        self.sec = None

    def _cookies(self):
        cookie_dict = {}

        for _, a in list(self.cookiejar._cookies.items()):
            for _, b in list(a.items()):
                for cookie in list(b.values()):
                    # print cookie
                    cookie_dict[cookie.name] = cookie.value

        return cookie_dict

    def set_cookie(self, kaka, request):
        """Returns a cookielib.Cookie based on a set-cookie header line"""

        # default rfc2109=False
        # max-age, httponly
        for cookie_name, morsel in kaka.items():
            std_attr = ATTRS.copy()
            std_attr["name"] = cookie_name
            _tmp = morsel.coded_value
            if _tmp.startswith('"') and _tmp.endswith('"'):
                std_attr["value"] = _tmp[1:-1]
            else:
                std_attr["value"] = _tmp

            std_attr["version"] = 0
            # copy attributes that have values
            for attr in morsel.keys():
                if attr in ATTRS:
                    if morsel[attr]:
                        if attr == "expires":
                            std_attr[attr]=_since_epoch(morsel[attr])
                        else:
                            std_attr[attr]=morsel[attr]
                elif attr == "max-age":
                    if morsel["max-age"]:
                        std_attr["expires"] = _since_epoch(morsel["max-age"])

            for att, set in PAIRS.items():
                if std_attr[att]:
                    std_attr[set] = True

            if std_attr["domain"] and std_attr["domain"].startswith("."):
                std_attr["domain_initial_dot"] = True

            if morsel["max-age"] is 0:
                try:
                    self.cookiejar.clear(domain=std_attr["domain"],
                                         path=std_attr["path"],
                                         name=std_attr["name"])
                except ValueError:
                    pass
            else:
                new_cookie = cookielib.Cookie(**std_attr)

                self.cookiejar.set_cookie(new_cookie)

    def send(self, url, method="GET", **kwargs):
        _kwargs = copy.copy(self.request_args)
        if kwargs:
            _kwargs.update(kwargs)

        if self.cookiejar:
            _kwargs["cookies"] = self._cookies()
            #logger.info("SENT COOKIEs: %s" % (_kwargs["cookies"],))
        try:
            r = requests.request(method, url, **_kwargs)
        except requests.ConnectionError, exc:
            raise ConnectionError("%s" % exc)

        try:
            #logger.info("RECEIVED COOKIEs: %s" % (r.headers["set-cookie"],))
            self.set_cookie(SimpleCookie(r.headers["set-cookie"]), r)
        except AttributeError, err:
            pass

        return r

    def use_http_form_post(self, message, destination, relay_state):
        """
        Return a form that will automagically execute and POST the message
        to the recipient.

        :param message:
        :param destination:
        :param relay_state:
        :return: tuple (header, message)
        """
        if not isinstance(message, basestring):
            request = "%s" % (message,)

        return http_form_post_message(message, destination, relay_state)

    def use_http_get(self, message, destination, relay_state):
        """
        Send a message using GET, this is the HTTP-Redirect case so
        no direct response is expected to this request.

        :param message:
        :param destination:
        :param relay_state:
        :return: tuple (header, None)
        """
        if not isinstance(message, basestring):
            request = "%s" % (message,)

        return http_redirect_message(message, destination, relay_state)

    def send_using_soap(self, request, destination, headers=None, sign=False):
        """
        Send a message using SOAP+POST

        :param request:
        :param destination:
        :param headers:
        :param sign:
        :return:
        """
        if headers is None:
            headers = {"content-type": "application/soap+xml"}
        else:
            headers.update({"content-type": "application/soap+xml"})

        soap_message = make_soap_enveloped_saml_thingy(request)

        if sign and self.sec:
            _signed = self.sec.sign_statement_using_xmlsec(soap_message,
                                                           class_name(request),
                                                           nodeid=request.id)
            soap_message = _signed

        #_response = self.server.post(soap_message, headers, path=path)
        try:
            response = self.send(destination, "POST", data=soap_message,
                                 headers=headers)
        except Exception, exc:
            logger.info("HTTPClient exception: %s" % (exc,))
            return None

        if response:
            logger.info("SOAP response: %s" % response)
            return parse_soap_enveloped_saml_response(response)
        else:
            return False

<|endoftext|>


<commit_before>"""
sentry.coreapi
~~~~~~~~~~~~~~

:copyright: (c) 2010-2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
# TODO: We should make the API a class, and UDP/HTTP just inherit from it
#       This will make it so we can more easily control logging with various
#       metadata (rather than generic log messages which arent useful).

from datetime import datetime, timedelta
import base64
import logging
import uuid
import zlib

from django.contrib.auth.models import User
from django.utils.encoding import smart_str

from sentry.conf import settings
from sentry.exceptions import InvalidInterface, InvalidData, InvalidTimestamp
from sentry.models import Project, ProjectKey, TeamMember, Team
from sentry.plugins import plugins
from sentry.tasks.store import store_event
from sentry.utils import is_float, json
from sentry.utils.auth import parse_auth_header
from sentry.utils.imports import import_string
from sentry.utils.queue import maybe_delay

logger = logging.getLogger('sentry.coreapi.errors')

MAX_CULPRIT_LENGTH = 200
MAX_MESSAGE_LENGTH = 1000

RESERVED_FIELDS = (
    'project',
    'event_id',
    'message',
    'checksum',
    'culprit',
    'level',
    'time_spent',
    'logger',
    'server_name',
    'site',
    'timestamp',
    'extra',
    'modules',
    'tags',
    'platform',
)

class APIError(Exception):
    http_status = 400
    msg = 'Invalid request'

    def __init__(self, msg=None):
        if msg:
            self.msg = msg

    def __str__(self):
        return self.msg or ''

class APIUnauthorized(APIError):
    http_status = 401
    msg = 'Unauthorized'

class APIForbidden(APIError):
    http_status = 403

class APITimestampExpired(APIError):
    http_status = 410

def extract_auth_vars(request):
    if request.META.get('HTTP_X_SENTRY_AUTH', '').startswith('Sentry'):
        return parse_auth_header(request.META['HTTP_X_SENTRY_AUTH'])
    elif request.META.get('HTTP_AUTHORIZATION', '').startswith('Sentry'):
        return parse_auth_header(request.META['HTTP_AUTHORIZATION'])
    else:
        return request.GET

def project_from_auth_vars(auth_vars):
    api_key = auth_vars.get('sentry_key')
    if not api_key:
        raise APIForbidden('Invalid api key')
    try:
        pk = ProjectKey.objects.get_from_cache(public_key=api_key)
    except ProjectKey.DoesNotExist:
        raise APIForbidden('Invalid api key')

    if pk.secret_key != auth_vars.get('sentry_secret', pk.secret_key):
        raise APIForbidden('Invalid api key')

    project = Project.objects.get_from_cache(pk=pk.project_id)

    if pk.user:
        try:
            team = Team.objects.get_from_cache(pk=project.team_id)
        except Team.DoesNotExist:
            raise APIUnauthorized('Member does not have access to project')

        try:
            TeamMember.objects.get(team=team, user=pk.user, is_active=True)
        except TeamMember.DoesNotExist:
            raise APIUnauthorized('Member does not have access to project')

        # We have to refetch this as it may have been catched
        pk.user = User.objects.get(id=pk.user_id)
        if not pk.user.is_active:
            raise APIUnauthorized('Account is not active')

    return project, pk.user

def project_from_api_key_and_id(api_key, project_id):
    """
    Given a public api key and a project id returns
    a project instance or throws APIUnauthorized.
    """
    try:
        pk = ProjectKey.objects.get_from_cache(public_key=api_key)
    except ProjectKey.DoesNotExist:
        raise APIUnauthorized('Invalid api key')

    if str(project_id).isdigit():
        if str(pk.project_id) != str(project_id):
            raise APIUnauthorized('Invalid project id')
    else:
        if str(pk.project.slug) != str(project_id):
            raise APIUnauthorized('Invalid project id')

    project = Project.objects.get_from_cache(pk=pk.project_id)

    if pk.user:
        team = Team.objects.get_from_cache(pk=project.team_id)

        try:
            tm = TeamMember.objects.get(team=team, user=pk.user, is_active=True)
        except TeamMember.DoesNotExist:
            raise APIUnauthorized('Member does not have access to project')

        # We have to refetch this as it may have been catched
        pk.user = User.objects.get(id=pk.user_id)
        if not pk.user.is_active:
            raise APIUnauthorized('Account is not active')

        tm.project = project

        result = plugins.first('has_perm', tm.user, 'create_event', project)
        if result is False:
            raise APIForbidden('Creation of this event was blocked')

    return project

def project_from_id(request):
    """
    Given a request returns a project instance or throws
    APIUnauthorized.
    """
    if not request.user.is_active:
        raise APIUnauthorized('Account is not active')

    try:
        project = Project.objects.get_from_cache(pk=request.GET['project_id'])
    except Project.DoesNotExist:
        raise APIUnauthorized('Invalid project')

    try:
        team = Team.objects.get_from_cache(pk=project.team_id)
    except Project.DoesNotExist:
        raise APIUnauthorized('Member does not have access to project')

    try:
        TeamMember.objects.get(
            user=request.user,
            team=team,
            is_active=True,
        )
    except TeamMember.DoesNotExist:
        raise APIUnauthorized('Member does not have access to project')

    result = plugins.first('has_perm', request.user, 'create_event', project)
    if result is False:
        raise APIForbidden('Creation of this event was blocked')

    return project

def decode_and_decompress_data(encoded_data):
    try:
        try:
            return base64.b64decode(encoded_data).decode('zlib')
        except zlib.error:
            return base64.b64decode(encoded_data)
    except Exception, e:
        # This error should be caught as it suggests that there's a
        # bug somewhere in the client's code.
        logger.exception('Bad data received')
        raise APIForbidden('Bad data decoding request (%s, %s)' % (
            e.__class__.__name__, e))

def safely_load_json_string(json_string):
    try:
        obj = json.loads(json_string)
    except Exception, e:
        # This error should be caught as it suggests that there's a
        # bug somewhere in the client's code.
        logger.exception('Bad data received')
        raise APIForbidden('Bad data reconstructing object (%s, %s)' % (
            e.__class__.__name__, e))

    # XXX: ensure keys are coerced to strings
    return dict((smart_str(k), v) for k, v in obj.iteritems())

def ensure_valid_project_id(desired_project, data):
    # Confirm they're using either the master key, or their specified project
    # matches with the signed project.
    if desired_project and data.get('project'):
        if str(data.get('project')) not in [str(desired_project.id), desired_project.slug]:
            raise APIForbidden('Invalid credentials')
        data['project'] = desired_project.id
    elif not desired_project:
        data['project'] = 1
    elif not data.get('project'):
        data['project'] = desired_project.id

def process_data_timestamp(data):
    if is_float(data['timestamp']):
        try:
            data['timestamp'] = datetime.fromtimestamp(float(data['timestamp']))
        except Exception:
            raise InvalidTimestamp('Invalid value for timestamp: %r' % data['timestamp'])
    elif not isinstance(data['timestamp'], datetime):
        if '.' in data['timestamp']:
            format = '%Y-%m-%dT%H:%M:%S.%f'
        else:
            format = '%Y-%m-%dT%H:%M:%S'
        if 'Z' in data['timestamp']:
            # support UTC market, but not other timestamps
            format += 'Z'
        try:
            data['timestamp'] = datetime.strptime(data['timestamp'], format)
        except Exception:
            raise InvalidTimestamp('Invalid value for timestamp: %r' % data['timestamp'])

    if data['timestamp'] > datetime.now() + timedelta(minutes=1):
        raise InvalidTimestamp('Invalid value for timestamp (in future): %r' % data['timestamp'])

    return data

def validate_data(project, data, client=None):
    ensure_valid_project_id(project, data)

    if not data.get('message'):
        data['message'] = '<no message value>'
    elif len(data['message']) > MAX_MESSAGE_LENGTH:
        raise InvalidData('Value \'message\' is too long. Input is %d chars, max is %s.' % (
            len(data['message']), MAX_MESSAGE_LENGTH))

    if data.get('culprit') and len(data['culprit']) > 200:
        raise InvalidData('Value \'culprit\' is too long. Input is %d chars, max is %s.' % (
            len(data['culprit']), MAX_CULPRIT_LENGTH))

    if not data.get('event_id'):
        data['event_id'] = uuid.uuid4().hex
    elif len(data['event_id']) > 32:
        raise InvalidData('Invalid value for \'event_id\': must be a 32 character identifier')

    if 'timestamp' in data:
        try:
            process_data_timestamp(data)
        except InvalidTimestamp:
            # Log the error, remove the timestamp, and continue
            logger.error('Client %r passed an invalid value for timestamp %r',
                client or '<unknown client>',
                data['timestamp'],
            )
            del data['timestamp']

    if data.get('modules') and type(data['modules']) != dict:
        raise InvalidData('Invalid type for \'modules\': must be a mapping')

    for k, v in data.items():
        if k in RESERVED_FIELDS:
            continue

        if '.' not in k:
            logger.error('Ignoring unknown attribute %r passed by client %r',
                k,
                client or '<unknown client>',
            )
            del data[k]
            continue

        try:
            interface = import_string(k)
        except (ImportError, AttributeError), e:
            raise InvalidInterface('%r is not a valid interface name: %s' % (k, e))

        if not v:
            logger.error('Ignoring empty interface %r passed by client %r',
                k,
                client or '<unknown client>',
            )
            del data[k]
            continue

        try:
            data[k] = interface(**v).serialize()
        except Exception, e:
            logger.error('Client %r passed an invalid value for interface %r',
                client or '<unknown client>',
                interface, exc_info=True)
            raise InvalidData('Unable to validate interface, %r: %s' % (k, e))

    level = data.get('level') or settings.DEFAULT_LOG_LEVEL
    if isinstance(level, basestring) and not level.isdigit():
        # assume it's something like 'warning'
        try:
            data['level'] = settings.LOG_LEVEL_REVERSE_MAP[level]
        except KeyError:
            raise InvalidData('Invalid logging level specified: %r' % level)

    return data

def insert_data_to_database(data):
    maybe_delay(store_event, data=data)
<commit_msg>Log values<commit_after>"""
sentry.coreapi
~~~~~~~~~~~~~~

:copyright: (c) 2010-2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
# TODO: We should make the API a class, and UDP/HTTP just inherit from it
#       This will make it so we can more easily control logging with various
#       metadata (rather than generic log messages which arent useful).

from datetime import datetime, timedelta
import base64
import logging
import uuid
import zlib

from django.contrib.auth.models import User
from django.utils.encoding import smart_str

from sentry.conf import settings
from sentry.exceptions import InvalidInterface, InvalidData, InvalidTimestamp
from sentry.models import Project, ProjectKey, TeamMember, Team
from sentry.plugins import plugins
from sentry.tasks.store import store_event
from sentry.utils import is_float, json
from sentry.utils.auth import parse_auth_header
from sentry.utils.imports import import_string
from sentry.utils.queue import maybe_delay

logger = logging.getLogger('sentry.coreapi.errors')

MAX_CULPRIT_LENGTH = 200
MAX_MESSAGE_LENGTH = 1000

RESERVED_FIELDS = (
    'project',
    'event_id',
    'message',
    'checksum',
    'culprit',
    'level',
    'time_spent',
    'logger',
    'server_name',
    'site',
    'timestamp',
    'extra',
    'modules',
    'tags',
    'platform',
)

class APIError(Exception):
    http_status = 400
    msg = 'Invalid request'

    def __init__(self, msg=None):
        if msg:
            self.msg = msg

    def __str__(self):
        return self.msg or ''

class APIUnauthorized(APIError):
    http_status = 401
    msg = 'Unauthorized'

class APIForbidden(APIError):
    http_status = 403

class APITimestampExpired(APIError):
    http_status = 410

def extract_auth_vars(request):
    if request.META.get('HTTP_X_SENTRY_AUTH', '').startswith('Sentry'):
        return parse_auth_header(request.META['HTTP_X_SENTRY_AUTH'])
    elif request.META.get('HTTP_AUTHORIZATION', '').startswith('Sentry'):
        return parse_auth_header(request.META['HTTP_AUTHORIZATION'])
    else:
        return request.GET

def project_from_auth_vars(auth_vars):
    api_key = auth_vars.get('sentry_key')
    if not api_key:
        raise APIForbidden('Invalid api key')
    try:
        pk = ProjectKey.objects.get_from_cache(public_key=api_key)
    except ProjectKey.DoesNotExist:
        raise APIForbidden('Invalid api key')

    if pk.secret_key != auth_vars.get('sentry_secret', pk.secret_key):
        raise APIForbidden('Invalid api key')

    project = Project.objects.get_from_cache(pk=pk.project_id)

    if pk.user:
        try:
            team = Team.objects.get_from_cache(pk=project.team_id)
        except Team.DoesNotExist:
            raise APIUnauthorized('Member does not have access to project')

        try:
            TeamMember.objects.get(team=team, user=pk.user, is_active=True)
        except TeamMember.DoesNotExist:
            raise APIUnauthorized('Member does not have access to project')

        # We have to refetch this as it may have been catched
        pk.user = User.objects.get(id=pk.user_id)
        if not pk.user.is_active:
            raise APIUnauthorized('Account is not active')

    return project, pk.user

def project_from_api_key_and_id(api_key, project_id):
    """
    Given a public api key and a project id returns
    a project instance or throws APIUnauthorized.
    """
    try:
        pk = ProjectKey.objects.get_from_cache(public_key=api_key)
    except ProjectKey.DoesNotExist:
        raise APIUnauthorized('Invalid api key')

    if str(project_id).isdigit():
        if str(pk.project_id) != str(project_id):
            raise APIUnauthorized('Invalid project id')
    else:
        if str(pk.project.slug) != str(project_id):
            raise APIUnauthorized('Invalid project id')

    project = Project.objects.get_from_cache(pk=pk.project_id)

    if pk.user:
        team = Team.objects.get_from_cache(pk=project.team_id)

        try:
            tm = TeamMember.objects.get(team=team, user=pk.user, is_active=True)
        except TeamMember.DoesNotExist:
            raise APIUnauthorized('Member does not have access to project')

        # We have to refetch this as it may have been catched
        pk.user = User.objects.get(id=pk.user_id)
        if not pk.user.is_active:
            raise APIUnauthorized('Account is not active')

        tm.project = project

        result = plugins.first('has_perm', tm.user, 'create_event', project)
        if result is False:
            raise APIForbidden('Creation of this event was blocked')

    return project

def project_from_id(request):
    """
    Given a request returns a project instance or throws
    APIUnauthorized.
    """
    if not request.user.is_active:
        raise APIUnauthorized('Account is not active')

    try:
        project = Project.objects.get_from_cache(pk=request.GET['project_id'])
    except Project.DoesNotExist:
        raise APIUnauthorized('Invalid project')

    try:
        team = Team.objects.get_from_cache(pk=project.team_id)
    except Project.DoesNotExist:
        raise APIUnauthorized('Member does not have access to project')

    try:
        TeamMember.objects.get(
            user=request.user,
            team=team,
            is_active=True,
        )
    except TeamMember.DoesNotExist:
        raise APIUnauthorized('Member does not have access to project')

    result = plugins.first('has_perm', request.user, 'create_event', project)
    if result is False:
        raise APIForbidden('Creation of this event was blocked')

    return project

def decode_and_decompress_data(encoded_data):
    try:
        try:
            return base64.b64decode(encoded_data).decode('zlib')
        except zlib.error:
            return base64.b64decode(encoded_data)
    except Exception, e:
        # This error should be caught as it suggests that there's a
        # bug somewhere in the client's code.
        logger.exception('Bad data received')
        raise APIForbidden('Bad data decoding request (%s, %s)' % (
            e.__class__.__name__, e))

def safely_load_json_string(json_string):
    try:
        obj = json.loads(json_string)
    except Exception, e:
        # This error should be caught as it suggests that there's a
        # bug somewhere in the client's code.
        logger.exception('Bad data received')
        raise APIForbidden('Bad data reconstructing object (%s, %s)' % (
            e.__class__.__name__, e))

    # XXX: ensure keys are coerced to strings
    return dict((smart_str(k), v) for k, v in obj.iteritems())

def ensure_valid_project_id(desired_project, data):
    # Confirm they're using either the master key, or their specified project
    # matches with the signed project.
    if desired_project and data.get('project'):
        if str(data.get('project')) not in [str(desired_project.id), desired_project.slug]:
            raise APIForbidden('Invalid credentials')
        data['project'] = desired_project.id
    elif not desired_project:
        data['project'] = 1
    elif not data.get('project'):
        data['project'] = desired_project.id

def process_data_timestamp(data):
    if is_float(data['timestamp']):
        try:
            data['timestamp'] = datetime.fromtimestamp(float(data['timestamp']))
        except Exception:
            raise InvalidTimestamp('Invalid value for timestamp: %r' % data['timestamp'])
    elif not isinstance(data['timestamp'], datetime):
        if '.' in data['timestamp']:
            format = '%Y-%m-%dT%H:%M:%S.%f'
        else:
            format = '%Y-%m-%dT%H:%M:%S'
        if 'Z' in data['timestamp']:
            # support UTC market, but not other timestamps
            format += 'Z'
        try:
            data['timestamp'] = datetime.strptime(data['timestamp'], format)
        except Exception:
            raise InvalidTimestamp('Invalid value for timestamp: %r' % data['timestamp'])

    if data['timestamp'] > datetime.now() + timedelta(minutes=1):
        raise InvalidTimestamp('Invalid value for timestamp (in future): %r' % data['timestamp'])

    return data

def validate_data(project, data, client=None):
    ensure_valid_project_id(project, data)

    if not data.get('message'):
        data['message'] = '<no message value>'
    elif len(data['message']) > MAX_MESSAGE_LENGTH:
        raise InvalidData('Value \'message\' is too long. Input is %d chars, max is %s.' % (
            len(data['message']), MAX_MESSAGE_LENGTH))

    if data.get('culprit') and len(data['culprit']) > 200:
        raise InvalidData('Value \'culprit\' is too long. Input is %d chars, max is %s.' % (
            len(data['culprit']), MAX_CULPRIT_LENGTH))

    if not data.get('event_id'):
        data['event_id'] = uuid.uuid4().hex
    elif len(data['event_id']) > 32:
        raise InvalidData('Invalid value for \'event_id\': must be a 32 character identifier')

    if 'timestamp' in data:
        try:
            process_data_timestamp(data)
        except InvalidTimestamp:
            # Log the error, remove the timestamp, and continue
            logger.error('Client %r passed an invalid value for timestamp %r',
                client or '<unknown client>',
                data['timestamp'],
            )
            del data['timestamp']

    if data.get('modules') and type(data['modules']) != dict:
        raise InvalidData('Invalid type for \'modules\': must be a mapping')

    for k, v in data.items():
        if k in RESERVED_FIELDS:
            continue

        if '.' not in k:
            logger.error('Ignoring unknown attribute %r passed by client %r',
                k,
                client or '<unknown client>',
            )
            del data[k]
            continue

        try:
            interface = import_string(k)
        except (ImportError, AttributeError), e:
            raise InvalidInterface('%r is not a valid interface name: %s' % (k, e))

        if not v:
            logger.error('Ignoring empty interface %r passed by client %r',
                k,
                client or '<unknown client>',
            )
            del data[k]
            continue

        try:
            data[k] = interface(**v).serialize()
        except Exception, e:
            logger.error('Client %r passed an invalid value for interface %r',
                client or '<unknown client>',
                interface, exc_info=True, extra={'values': v})
            raise InvalidData('Unable to validate interface, %r: %s' % (k, e))

    level = data.get('level') or settings.DEFAULT_LOG_LEVEL
    if isinstance(level, basestring) and not level.isdigit():
        # assume it's something like 'warning'
        try:
            data['level'] = settings.LOG_LEVEL_REVERSE_MAP[level]
        except KeyError:
            raise InvalidData('Invalid logging level specified: %r' % level)

    return data

def insert_data_to_database(data):
    maybe_delay(store_event, data=data)
<|endoftext|>


<commit_before>from Entity import Entity

class Station(Entity):
  def __init__(self, config):
    super.__init__(config)
    self.weaponStocks = config['weaponStocks']
    self.maxDock = config['maxDock']
    self.docked = config['docked']
    self.producing = config['producing']
    self.rechargeRate = config['rechargeRate']

  def dock(self, ship):
    if len(self.docked) < self.maxDock:
      self.docked.append(ship)
      return True
    return False

  def unDock(self, ship):
    if (ship in self.docked):
      self.docked.remove(ship)
      return True
    return False

  def tick(self, duration):
    self.shields += self.shieldMultiplier * duration
    self.energy += self.energyMultiplier * duration

    if self.energy > 0:
      for ship in self.docked:
        if ship.energy < ship.maxEnergy:
          recharged = min(ship.maxEnergy - ship.energy, self.rechargeRate * duration, self.energy)
          ship.energy += recharged
          self.energy -= recharged

    return super.tick(duration).extend([{[self.id, "weaponStocks"]:self.weaponStocks},
                                   {[self.id, "maxDock"]:self.maxDock},
                                   {[self.id, "docked"]:self.docked}])
<commit_msg>Add production to stations<commit_after>from Entity import Entity

class Station(Entity):
  def __init__(self, config):
    super.__init__(config)
    self.weaponStocks = config['weaponStocks']
    self.maxDock = config['maxDock']
    self.docked = config['docked']
    self.producing = config['producing']
    self.rechargeRate = config['rechargeRate']

  def dock(self, ship):
    if len(self.docked) < self.maxDock:
      self.docked.append(ship)
      return True
    return False

  def unDock(self, ship):
    if (ship in self.docked):
      self.docked.remove(ship)
      return True
    return False

  def setProduction(self, thing):
    # thing should be a commodity of some sort
    self.building = thing
    self.buildProgress = 0

  def tick(self, duration):
    self.shields += self.shieldMultiplier * duration
    self.energy += self.energyMultiplier * duration

    if self.building:
      self.buildProgress += duration
      if self.buildProgress > self.building.buildCost:
        # We have built something!

        if self.building.name in self.weaponStocks:
          self.weaponStocks[self.building.name] += 1
        else:
          self.weaponStocks[self.building.name] = 1

        self.buildProgress -= self.building.buildCost

    if self.energy > 0:
      for ship in self.docked:
        if ship.energy < ship.maxEnergy:
          recharged = min(ship.maxEnergy - ship.energy, self.rechargeRate * duration, self.energy)
          ship.energy += recharged
          self.energy -= recharged

    return super.tick(duration).extend([{[self.id, "weaponStocks"]:self.weaponStocks},
                                   {[self.id, "maxDock"]:self.maxDock},
                                   {[self.id, "docked"]:self.docked}])
<|endoftext|>


<commit_before>import smtplib
from os.path import basename
from email.mime.base import MIMEBase
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.utils import COMMASPACE, formatdate
from email import Encoders

from flask import current_app

def send_message(recipients, subject, attachments, body):
    send_from = current_app.rz_config.mail_default_sender
    smtp_hostname = current_app.rz_config.mail_hostname
    send_message_helper(smtp_hostname=smtp_hostname,
                        send_from=send_from, recipients=recipients,
                        subject=subject, attachments=attachments, body=body)

def send_message_helper(smtp_hostname, send_from, recipients, subject, attachments, body):
    """
    attachments is a list of tuples (filename, mimetype, data)

    note: helper exists for ease of testing, since it doesn't use flask only
    python batteries-included packages
    """
    assert isinstance(recipients, list)
    msg = MIMEMultipart()
    msg['From'] = send_from
    msg['To'] = COMMASPACE.join(recipients)
    msg['Date'] = formatdate(localtime=True)
    msg['Subject'] = subject
    msg.attach(MIMEText(body))

    for filename, mimetype, data in attachments:
        maintype, subtype = mimetype.split('/')
        part = MIMEBase(maintype, subtype)
        part.set_payload(data)
        Encoders.encode_base64(part)
        part.add_header('Content-Disposition', 'attachment', filename=filename)
        #Content_Disposition: attachment; filename="feedback_page.html"
        msg.attach(part)

    smtp = smtplib.SMTP(smtp_hostname)
    smtp.sendmail(send_from, recipients, msg.as_string())
    smtp.close()

if __name__ == '__main__':
    # FIXME - move to actual test suite
    send_message_helper('localhost', 'alon@localhost', ['alon@localhost'], 'test subject', [
        ('diary.txt', 'text/plain', "bla bla bla yeah that's right bla")], 'this is it pal')
<commit_msg>rz_mail: send_message logging<commit_after>import smtplib
from os.path import basename
from email.mime.base import MIMEBase
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.utils import COMMASPACE, formatdate
from email import Encoders

from flask import current_app
import logging
log = logging.getLogger('rhizi')

def send_message(recipients, subject, attachments, body):
    send_from = current_app.rz_config.mail_default_sender
    smtp_hostname = current_app.rz_config.mail_hostname
    send_message_helper(smtp_hostname=smtp_hostname,
                        send_from=send_from, recipients=recipients,
                        subject=subject, attachments=attachments, body=body)

    log.info('email sent: recipients: %s: subject: %s, attachment-count: %d' % (recipients, subject, len(attachments)))

def send_message_helper(smtp_hostname, send_from, recipients, subject, attachments, body):
    """
    attachments is a list of tuples (filename, mimetype, data)

    note: helper exists for ease of testing, since it doesn't use flask only
    python batteries-included packages
    """
    assert isinstance(recipients, list)
    msg = MIMEMultipart()
    msg['From'] = send_from
    msg['To'] = COMMASPACE.join(recipients)
    msg['Date'] = formatdate(localtime=True)
    msg['Subject'] = subject
    msg.attach(MIMEText(body))

    for filename, mimetype, data in attachments:
        maintype, subtype = mimetype.split('/')
        part = MIMEBase(maintype, subtype)
        part.set_payload(data)
        Encoders.encode_base64(part)
        part.add_header('Content-Disposition', 'attachment', filename=filename)
        #Content_Disposition: attachment; filename="feedback_page.html"
        msg.attach(part)

    smtp = smtplib.SMTP(smtp_hostname)
    smtp.sendmail(send_from, recipients, msg.as_string())
    smtp.close()

if __name__ == '__main__':
    # FIXME - move to actual test suite
    send_message_helper('localhost', 'alon@localhost', ['alon@localhost'], 'test subject', [
        ('diary.txt', 'text/plain', "bla bla bla yeah that's right bla")], 'this is it pal')
<|endoftext|>


<commit_before>from datetime import datetime
from datetime import timedelta
from flask import current_app
from flask import redirect
from flask import render_template
from flask import request
from flask import session
from flask import url_for
import flask
import json
import logging
import re
import uuid

from crypt_util import hash_pw
import crypt_util
from rz_req_handling import make_response__json, make_response__json__html
from rz_user_db import User_Account

log = logging.getLogger('rhizi')

class User_Signup_Request(dict):

    def __init__(self, email_address,
                       first_name,
                       last_name,
                       pw_plaintxt,
                       rz_username):

        self['rz_username'] = rz_username
        self['email_address'] = email_address
        self['last_name'] = last_name
        self['first_name'] = first_name
        self['pw_plaintxt'] = pw_plaintxt

        self['submission_date'] = None
        self['validation_key'] = None

    def __str__(self):
        return 'email_address: %s' % (self['email_address'])

    def has_expired(self):
        date_now = datetime.now()
        dt_6h = timedelta(hours=6)
        dt = self['submission_date'] - date_now
        return dt > dt_6h

def filter_expired_requests(us_req_map):

    for su_req_email, su_req in us_req_map.items():
        if su_req.has_expired():
            log.info('user sign-up request expired: %s' % (su_req))
            del us_req_map[su_req_email]

def activate_user_account(us_req):

    user_db = current_app.user_db

    existing_account = None
    try:
        existing_account = user_db.lookup_user__by_email_address(us_req['email_address'])
    except Exception as _:
        pass  # thrown if user was not found

    if None != existing_account:
        raise Exception('account activation code reused: existing-account: %s' % (existing_account))

    # calc pw hash
    pw_plaintxt = us_req['pw_plaintxt']
    salt = current_app.rz_config.secret_key
    pw_hash = hash_pw(str(pw_plaintxt), salt)

    u_account = User_Account(first_name=us_req['first_name'],
                              last_name=us_req['last_name'],
                              rz_username=us_req['rz_username'],
                              email_address=us_req['email_address'],
                              pw_hash=pw_hash,
                              role_set=['user'])

    user_db.user_add(u_account)

    log.info('user account activated: email: %s, rz_username: %s' % (us_req['email_address'], us_req['rz_username']))

def rest__login():

    def sanitize_input(req):
        req_json = request.get_json()
        u = req_json['username']
        p = req_json['password']
        return u, p

    if request.method == 'POST':
        try:
            u, p = sanitize_input(request)
        except:
            log.warn('failed to sanitize inputs. request: %s' % request)
            return make_response__json(status=401)  # return empty response
        try:
            salt = current_app.rz_config.secret_key
            pw_hash = hash_pw(p, salt)
            current_app.user_db.validate_login(u, pw_hash)
        except Exception as e:
            # login failed
            log.warn('login: unauthorized: user: %s' % (u))
            return make_response__json(status=401)  # return empty response

        # login successful
        session['username'] = u
        log.debug('login: success: user: %s' % (u))
        return make_response__json(status=200)  # return empty response

    if request.method == 'GET':
        return render_template('login.html')

def rest__logout():
    """
    REST API endpoint: logout
    """
    # remove the username from the session if it's there
    u = session.pop('username', None)
    log.debug('logout: success: user: %s' % (u))
    return redirect(url_for('login'))

def rest__user_signup():
    """
    REST API endpoint: user sign up
    """
    def add_user_signup_req(us_req_map, us_req):
        key = us_req['email_address']
        us_req_map[key] = us_req  # finally, add request

    def rm_user_signup_req(us_req_map, us_req):
        key = us_req['email_address']
        del us_req_map[key]

    def sanitize_and_validate_input(req):
        req_json = request.get_json()

        # TODO: sanitize
        field_to_regex_map = {'first_name': r'\w{3,16}',
                              'last_name': r'\w{3,16}',
                              'rz_username': r'\w{3,16}',
                              'email_address': r'[^@]+@[^@]+\.[^@]+',
                              'pw_plaintxt': r'[A-Za-z0-9]{8,32}',  # avoid symbols
                              }

        for f_name, regex in field_to_regex_map.items():
            f_val = req_json.get(f_name)
            if None == f_val:
                raise Exception('malformed signup request: missing field: %s' % (f_name))
            if None == re.match(regex, f_val):
                raise Exception('malformed signup request: regex match failure: regex: %s, input: %s' % (regex, f_val))

        first_name = req_json['first_name']
        last_name = req_json['last_name']
        rz_username = req_json['rz_username']
        email_address = req_json['email_address']
        pw_plaintxt = req_json['pw_plaintxt']

        ret = User_Signup_Request(email_address=email_address,
                                  first_name=first_name,
                                  last_name=last_name,
                                  pw_plaintxt=pw_plaintxt,
                                  rz_username=rz_username)

        return ret

    def generate_email_confirmation_key():
        """
        generate a random UUID based string ID
        """
        return str(uuid.uuid4()).replace('-', '')

    def get_or_init_usreq_map():
        """
        lazy user signup request map getter
        """

        if not hasattr(current_app, 'usreq_email_to_req_map'):
            setattr(current_app, 'usreq_email_to_req_map', {})
        us_req_map = current_app.usreq_email_to_req_map

        return us_req_map

    html_ok__submitted = '
Your request has been successfully submitted.
Please check your email to activate your account.
'
    html_ok__already_pending = '
Your request has already been submitted.
please check your email to activate your account.
'
    html_err__tech_difficulty = '
We are experiencing technical difficulty processing your request,
please try again later.
'
    html_err__activation_failure = '
Activation failure,
please try again later.
'

    # use incoming request as house keeping trigger
    us_req_map = get_or_init_usreq_map()
    filter_expired_requests(us_req_map)

    if request.method == 'POST':
        # FIXME: add captcha
        # if req_json['captcha_solution'] = ...

        try:
            us_req = sanitize_and_validate_input(request)
        except Exception as e:
            log.exception(e)
            return make_response__json__html(status=400, html_str=html_err__tech_difficulty)

        # FIXME: implement form validation

        existing_req = us_req_map.get(us_req['email_address'])
        if None != existing_req:
            # already pending
            log.warning('user signup: request already pending: %s' % (existing_req))
            return make_response__json__html(status=200, html_str=html_ok__already_pending)

        us_req['submission_date'] = datetime.now()
        us_req['validation_key'] = generate_email_confirmation_key()

        # send activation email
        try:
            send_user_activation_link__email(us_req)
            add_user_signup_req(us_req_map, us_req)
            return make_response__json__html(html_str=html_ok__submitted)
        except Exception as e:
            log.exception('user sign-up: failed to send validation email', e)
            return make_response__json__html(status=500, html_str=html_err__tech_difficulty)

    if request.method == 'GET':

        v_tok = request.args.get('v_tok')
        if None != v_tok:  # validation token passed
            for _, us_req in us_req_map.items():
                other_req_v_tok = us_req['validation_key']
                if None == other_req_v_tok:
                    continue
                if other_req_v_tok == v_tok:

                    if us_req.has_expired():  # check again whether request has expired
                        return render_template('user_signup.html', state='activation_failure', html_str__err=html_err__activation_failure)

                    try:
                        activate_user_account(us_req)
                    except Exception as e:
                        return render_template('user_signup.html', state='activation_failure', html_str__err=html_err__activation_failure)

                    # activation success
                    rm_user_signup_req(us_req_map, us_req)
                    return render_template('user_signup.html', state='activation_success')

            # request expired & already removed OR bad token
            log.warning('user sign-up: request not found or bad token: remote-address: %s' % (request.remote_addr))
            return render_template('user_signup.html', state='activation_failure', html_str__err=html_err__activation_failure)

        else:  # no validation token: new user
            return render_template('user_signup.html')

def send_user_activation_link__email(us_req):
    """
    Send user feedback by email along with screen capture attachments
    """

    # FIXME: use HTTPS
    confirmation_link = 'http://%s/signup?v_tok=%s' % (current_app.rz_config.SERVER_NAME,
                                                       us_req['validation_key'])

    msg_body = ['Hello %s %s,' % (us_req['first_name'],
                                  us_req['last_name']),
                '',
                'Thank you for your joining the Rhizi community!',
                '',
                'Below is a sign up confirmation link - open it in your browser to activate your account:',
                '',
                confirmation_link,
                '',
                'Happy knowledge editing!',
                'The Rhizi team.'
                ]
    msg_body = '\n'.join(msg_body)

    try:
        send_email_message(recipients=[us_req['email_address']],
                           subject="Rhizi sign up request",
                           body=msg_body)
        return json.dumps({})  # expects json encoded, contents discarded

    except Exception:
        log.exception('send_user_feedback__email: exception while sending email')
        return make_response__json(status=500)
<commit_msg>send_user_activation_link__email(): return activation_link, log<commit_after>from datetime import datetime
from datetime import timedelta
from flask import current_app
from flask import redirect
from flask import render_template
from flask import request
from flask import session
from flask import url_for
import flask
import json
import logging
import re
import uuid

from crypt_util import hash_pw
import crypt_util
from rz_req_handling import make_response__json, make_response__json__html
from rz_user_db import User_Account

log = logging.getLogger('rhizi')

class User_Signup_Request(dict):

    def __init__(self, email_address,
                       first_name,
                       last_name,
                       pw_plaintxt,
                       rz_username):

        self['rz_username'] = rz_username
        self['email_address'] = email_address
        self['last_name'] = last_name
        self['first_name'] = first_name
        self['pw_plaintxt'] = pw_plaintxt

        self['submission_date'] = None
        self['validation_key'] = None

    def __str__(self):
        return 'email_address: %s' % (self['email_address'])

    def has_expired(self):
        date_now = datetime.now()
        dt_6h = timedelta(hours=6)
        dt = self['submission_date'] - date_now
        return dt > dt_6h

def filter_expired_requests(us_req_map):

    for su_req_email, su_req in us_req_map.items():
        if su_req.has_expired():
            log.info('user sign-up request expired: %s' % (su_req))
            del us_req_map[su_req_email]

def activate_user_account(us_req):

    user_db = current_app.user_db

    existing_account = None
    try:
        existing_account = user_db.lookup_user__by_email_address(us_req['email_address'])
    except Exception as _:
        pass  # thrown if user was not found

    if None != existing_account:
        raise Exception('account activation code reused: existing-account: %s' % (existing_account))

    # calc pw hash
    pw_plaintxt = us_req['pw_plaintxt']
    salt = current_app.rz_config.secret_key
    pw_hash = hash_pw(str(pw_plaintxt), salt)

    u_account = User_Account(first_name=us_req['first_name'],
                              last_name=us_req['last_name'],
                              rz_username=us_req['rz_username'],
                              email_address=us_req['email_address'],
                              pw_hash=pw_hash,
                              role_set=['user'])

    user_db.user_add(u_account)

    log.info('user account activated: email: %s, rz_username: %s' % (us_req['email_address'], us_req['rz_username']))

def rest__login():

    def sanitize_input(req):
        req_json = request.get_json()
        u = req_json['username']
        p = req_json['password']
        return u, p

    if request.method == 'POST':
        try:
            u, p = sanitize_input(request)
        except:
            log.warn('failed to sanitize inputs. request: %s' % request)
            return make_response__json(status=401)  # return empty response
        try:
            salt = current_app.rz_config.secret_key
            pw_hash = hash_pw(p, salt)
            current_app.user_db.validate_login(u, pw_hash)
        except Exception as e:
            # login failed
            log.warn('login: unauthorized: user: %s' % (u))
            return make_response__json(status=401)  # return empty response

        # login successful
        session['username'] = u
        log.debug('login: success: user: %s' % (u))
        return make_response__json(status=200)  # return empty response

    if request.method == 'GET':
        return render_template('login.html')

def rest__logout():
    """
    REST API endpoint: logout
    """
    # remove the username from the session if it's there
    u = session.pop('username', None)
    log.debug('logout: success: user: %s' % (u))
    return redirect(url_for('login'))

def rest__user_signup():
    """
    REST API endpoint: user sign up
    """
    def add_user_signup_req(us_req_map, us_req):
        key = us_req['email_address']
        us_req_map[key] = us_req  # finally, add request

    def rm_user_signup_req(us_req_map, us_req):
        key = us_req['email_address']
        del us_req_map[key]

    def sanitize_and_validate_input(req):
        req_json = request.get_json()

        # TODO: sanitize
        field_to_regex_map = {'first_name': r'\w{3,16}',
                              'last_name': r'\w{3,16}',
                              'rz_username': r'\w{3,16}',
                              'email_address': r'[^@]+@[^@]+\.[^@]+',
                              'pw_plaintxt': r'[A-Za-z0-9]{8,32}',  # avoid symbols
                              }

        for f_name, regex in field_to_regex_map.items():
            f_val = req_json.get(f_name)
            if None == f_val:
                raise Exception('malformed signup request: missing field: %s' % (f_name))
            if None == re.match(regex, f_val):
                raise Exception('malformed signup request: regex match failure: regex: %s, input: %s' % (regex, f_val))

        first_name = req_json['first_name']
        last_name = req_json['last_name']
        rz_username = req_json['rz_username']
        email_address = req_json['email_address']
        pw_plaintxt = req_json['pw_plaintxt']

        ret = User_Signup_Request(email_address=email_address,
                                  first_name=first_name,
                                  last_name=last_name,
                                  pw_plaintxt=pw_plaintxt,
                                  rz_username=rz_username)

        return ret

    def generate_email_confirmation_key():
        """
        generate a random UUID based string ID
        """
        return str(uuid.uuid4()).replace('-', '')

    def get_or_init_usreq_map():
        """
        lazy user signup request map getter
        """

        if not hasattr(current_app, 'usreq_email_to_req_map'):
            setattr(current_app, 'usreq_email_to_req_map', {})
        us_req_map = current_app.usreq_email_to_req_map

        return us_req_map

    html_ok__submitted = '
Your request has been successfully submitted.
Please check your email to activate your account.
'
    html_ok__already_pending = '
Your request has already been submitted.
please check your email to activate your account.
'
    html_err__tech_difficulty = '
We are experiencing technical difficulty processing your request,
please try again later.
'
    html_err__activation_failure = '
Activation failure,
please try again later.
'

    # use incoming request as house keeping trigger
    us_req_map = get_or_init_usreq_map()
    filter_expired_requests(us_req_map)

    if request.method == 'POST':
        # FIXME: add captcha
        # if req_json['captcha_solution'] = ...

        try:
            us_req = sanitize_and_validate_input(request)
        except Exception as e:
            log.exception(e)
            return make_response__json__html(status=400, html_str=html_err__tech_difficulty)

        # FIXME: implement form validation

        existing_req = us_req_map.get(us_req['email_address'])
        if None != existing_req:
            # already pending
            log.warning('user signup: request already pending: %s' % (existing_req))
            return make_response__json__html(status=200, html_str=html_ok__already_pending)

        us_req['submission_date'] = datetime.now()
        us_req['validation_key'] = generate_email_confirmation_key()

        # send activation email
        try:
            send_user_activation_link__email(us_req)
            add_user_signup_req(us_req_map, us_req)
            return make_response__json__html(html_str=html_ok__submitted)
        except Exception as e:
            log.exception('user sign-up: failed to send validation email', e)
            return make_response__json__html(status=500, html_str=html_err__tech_difficulty)

    if request.method == 'GET':

        v_tok = request.args.get('v_tok')
        if None != v_tok:  # validation token passed
            for _, us_req in us_req_map.items():
                other_req_v_tok = us_req['validation_key']
                if None == other_req_v_tok:
                    continue
                if other_req_v_tok == v_tok:

                    if us_req.has_expired():  # check again whether request has expired
                        return render_template('user_signup.html', state='activation_failure', html_str__err=html_err__activation_failure)

                    try:
                        activate_user_account(us_req)
                    except Exception as e:
                        return render_template('user_signup.html', state='activation_failure', html_str__err=html_err__activation_failure)

                    # activation success
                    rm_user_signup_req(us_req_map, us_req)
                    return render_template('user_signup.html', state='activation_success')

            # request expired & already removed OR bad token
            log.warning('user sign-up: request not found or bad token: remote-address: %s' % (request.remote_addr))
            return render_template('user_signup.html', state='activation_failure', html_str__err=html_err__activation_failure)

        else:  # no validation token: new user
            return render_template('user_signup.html')

def send_user_activation_link__email(us_req):
    """
    Send user feedback by email along with screen capture attachments

    @return: activation_link
    """

    # FIXME: use HTTPS
    activation_link = 'http://%s/signup?v_tok=%s' % (current_app.rz_config.SERVER_NAME,
                                                       us_req['validation_key'])

    msg_body = ['Hello %s %s,' % (us_req['first_name'],
                                  us_req['last_name']),
                '',
                'Thank you for your joining the Rhizi community!',
                '',
                'Below is a sign up confirmation link - open it in your browser to activate your account:',
                '',
                activation_link,
                '',
                'Happy knowledge editing!',
                'The Rhizi team.'
                ]
    msg_body = '\n'.join(msg_body)

    try:
        send_email_message(recipients=[us_req['email_address']],
                           subject="Rhizi sign up request",
                           body=msg_body)
        return activation_link

    except Exception:
        log.exception('send_user_feedback__email: exception while sending email')
<|endoftext|>


<commit_before>import logging

import numpy
from .psd import ParametricSpectrum

__all__ = ["arcovar", "arcovar_marple", "pcovar", 'arcovar_marple']

def arcovar_marple(x, order):
    r"""Estimate AR model parameters using covariance method

    This implementation is based on [Marple]_. This code is far more
    complicated and slower than :func:`arcovar` function, which is now the official version.
    See :func:`arcovar` for a detailed description of Covariance method.

    This function should be used in place of arcovar only if order<=4, for
    which :func:`arcovar` does not work.

    Fast algorithm for the solution of the covariance least squares normal
    equations from Marple.

    :param array X:  Array of complex data samples
    :param int oder: Order of linear prediction model

    :return:
        * AF   - Array of complex forward linear prediction coefficients
        * PF   - Real forward linear prediction variance at order IP
        * AB   - Array of complex backward linear prediction coefficients
        * PB   - Real backward linear prediction variance at order IP
        * PV   - store linear prediction coefficients

    .. note:: this code and the original code in Marple diverge for ip>10.
        it seems that this is related to single precision used with
        complex type in fortran whereas numpy uses double precision for
        complex type.

    :validation: the AR parameters are the same as those returned by
        a completely different function :func:`arcovar`.

    :References: [Marple]_
    """
    assert len(x) >= order, "X must be dimensioned >=N"

    #   ----------------------------------------------------- Initialization
    x = numpy.array(x)
    N = len(x)

    # Equations 8.C.42
    r0 = sum(abs(x)**2.)
    r1 = abs(x[0])**2
    rN = abs(x[N-1])**2

    pf = r0 - r1
    pb = r0 - rN

    delta = 1. - r1 / r0
    gamma = 1. - rN / r0

    c = numpy.zeros(N, dtype=complex)
    d = numpy.zeros(N, dtype=complex)
    r = numpy.zeros(N, dtype=complex)
    af = numpy.zeros(N, dtype=complex)
    ab = numpy.zeros(N, dtype=complex)

    c[0] = x[N-1].conjugate() / r0
    d[0] = x[0].conjugate() / r0

    # special case
    if order == 0:
        pf = r0 / float(N)
        pb = pf
        return af, pf, ab, pb, 0

    # ----------------------------------------------------------  MAIN LOOP

    #ip +1 because we want to enter in the loop to run the first part of the code.
    pbv = []
    for m in range(0, order+1):
        logging.debug('----------------------------m=', m)
        logging.debug(c[0:2])
        logging.debug(d[0:2])
        r1 = 1./pf
        r2 = 1./pb
        r3 = 1./delta
        r4 = 1./gamma

        #logging.debug('starting r1r2r3r4=', r1, r2, r3, r4, pf, pb, delta, gamma)

        #Order update: AF and AB vectors ; time update: C and D vectors
        temp = 0.+0.j
        for k in range(m+1, N):
            temp = temp + x[k]*x[k-m-1].conjugate()

        r[m] =  temp.conjugate()
        theta = x[0] * c[m]
        #print(('theta', theta))
        #    print(('c=', c[0:2]))
        #    print(('d=', d[0:2]))
        if m == 0:
            pass
        else:
            for k in range(0, m):
                theta = theta + x[m-k] * c[k]                   # Eq. (8.C.39)
                r[k] = r[k] - x[N-m-1] * x[N-m+k].conjugate() # Eq. (8.C.32)
                temp = temp + af[m-k-1] * r[k].conjugate()
                #print 'loop1 k=', k
                #print '         theta=',theta, 'r[k]=',r[k], 'temp=', temp
                #print '         c=',c[k], 'af=',af[m-k-1]
        """if m > 0:
            if debug:
                print((m, N-m))
                print(('Xk=0',x[m-0],x[N-m-1], x[N-m+0]))
            if m > 1:
                if debug:
                    print('Xk=1',x[m-1],x[N-m-1], x[N-m+1])
        """
        c1 = -temp * r2
        c2 = -r1 * temp.conjugate()
        c3 = theta * r3
        c4 = r4 *theta.conjugate()
        #if debug:
        #    print('c1 c2 c3 c4 before af=',c1 ,c2 ,c3 ,c4)
        af[m] = c1                    #             ! Eq. (8.C.19)
        ab[m] = c2                    #             ! Eq. (8.C.22)
        save = c[m]
        c[m] = save + c3*d[m]
        d[m] = d[m] + c4*save
        #if debug:
        #    print('res',m,'af[m]=',af[m], ab[m], save, 'temp=',temp)

        if m == 0:
            pass
        else:
            #if debug:print('af before', af[0:2])
            for k in range(0, m):
                save = af[k]
                af[k] = save + c1 * ab[m-k-1] # Eq. (8.C.18)
                ab[m-k-1] = ab[m-k-1] + c2 * save   # Eq. (8.C.21)

                save = c[k]
                c[k] = save + c3*d[k]       # Eq. (8.C.37)
                d[k] = d[k] + c4*save       # Eq. (8.C.38)
                #if debug:
                #    print('loop2 k=', k)
                #    print('      af[k]=', af[k])
                #    print('      ab[m-k-1]=', ab[m-k-1])
                #    print('      c[k]=', c[k])
                #    print('      d[k]=', d[k])
            #if debug:
            #    print('af after=', af[0:2])
            #    print('ab=', ab[0:2])

        r5 = temp.real**2 + temp.imag**2
        pf = pf - r5*r2         # Eq. (8.C.20)
        pb = pb - r5*r1         # Eq. (8.C.23)
        r5 = theta.real**2 + theta.imag**2
        delta = delta - r5*r4               # Eq. (8.C.39)
        gamma = gamma - r5*r3               # Eq. (8.C.40)
        #if debug:
        #    print('r5r2r1deltagamma', r5, r2, r1 , delta, gamma)
        #    print('pf before norm', pf, pb, N-m-1)
        if m != order-1:
            pass
        else:
            pf = pf / float(N-m-1)
            pb = pb / float(N-m-1)
            #if debug:
            #    print('ENDING', N-m-1)
            break

        #if debug:
        #    print('pf and pb', pf, pb)
        if pf > 0 and pb > 0:
            pass
        else:
            ValueError("Negative PF or PB value")
        if (delta > 0. and delta <=1 and gamma > 0. and gamma <=1):
            pass
        else:
            ValueError("Invalid delta or gamma value")

        #C   Time update:  AF and AB vectors; order update:  C and D vectors

        r1 = 1./pf
        r2 = 1./pb
        r3 = 1./delta
        r4 = 1./gamma
        #if debug:
        #    print('--------time update', r1, r2, r3, r4, m+1, N-m-1, x[m+1], x[N-m-2])
        ef = x[m+1]
        eb = x[(N-1)-m-1]
        #if debug:
        #    print('delta, gamma=', delta, gamma)
        #if debug:
        #    print('before eb=', eb, ' ef=', ef)

        for k in range(0,m+1):
            #print 'k=', k, 'ef=', ef, ' eb=',eb,' af=',af[k], ab[k]
            #print x[m-k],x[N-m+k-1]
            ef = ef + af[k] * x[m-k]             # Eq. (8.C.1)
            eb = eb + ab[k] * x[N-m+k-1]                   # Eq. (8.C.2)

        #ef = sum(af)

        #if debug:
        #    print('efweb', ef , eb)
        c1 = ef*r3
        c2 = eb*r4
        c3 = eb.conjugate() * r2
        c4 = ef.conjugate() * r1
        #if debug:
        #    print('c1c2c3c4', c1, c2, c3, c4)
        #    print('af before', af[0:2])
        for k in range(m, -1, -1):
            save = af[k]
            af[k] = save + c1 * d[k]                    #  Eq. (8.C.33)
            d[k+1] = d[k] + c4 * save                    # Eq. (8.C.25)
            save = ab[k]
            ab[k] = save + c2 * c[m-k]                 # Eq. (8.C.35)
            c[m-k] = c[m-k] + c3 * save              # Eq. (8.C.24)
        #if debug:
        #    print('af after', af[0:2])
        #    print('d', d[0:2])
        #    print('ab', ab[0:2])
        #    print('c', c[0:2])
        #if debug:print('Pb before', pf, pb)
        c[m+1] = c3
        d[0] = c4
        #r5 = abs(ef)**2
        r5 = ef.real**2 + ef.imag**2
        pf = pf - r5 * r3                              # Eq. (8.C.34)

        delta = delta-r5 * r1                        # Eq. (8.C.30)
        #r5 = abs(eb)**2
        r5 = eb.real**2 + eb.imag**2
        pb = pb - r5 * r4                              # Eq. (8.C.36)
        #if debug:
        #    print('Pb---------------------', m, pb, r5, r4)
        gamma = gamma-r5*r2                        # Eq. (8.C.31)
        #if debug:
        #    print('Gamma', gamma)
        #    print('r5 r3,r1,and delta', r5, r3, r1, delta)
        #    print('eb=', eb)
        #    print('ef=', ef)
        #    print('pf=', pf)
        #    print('pb=', pb)
        pbv.append(pb)

        if (pf > 0. and pb > 0.):
            pass
        else:
            ValueError("Negative PF or PB value")
        #if debug:
        #    print(delta, gamma)
        if (delta > 0. and delta <= 1.) and (gamma > 0. and gamma <= 1.):
            pass
        else:
            ValueError("Invalid delta or gamma value")

    #af=array of forward coeff
    #ab=array of barward coeff
    #pb=backward variance
    #pf=forward variance
    return af, pf, ab, pb, pbv

def arcovar(x, order):
    r"""Simple and fast implementation of the covariance AR estimate

    This code is 10 times faster than :func:`arcovar_marple` and more importantly
    only 10 lines of code, compared to a 200 loc for :func:`arcovar_marple`

    :param array X:  Array of complex data samples
    :param int oder: Order of linear prediction model

    :return:
        * a - Array of complex forward linear prediction coefficients
        * e - error

    The covariance method fits a Pth order autoregressive (AR) model to the
    input signal, which is assumed to be the output of
    an AR system driven by white noise. This method minimizes the forward
    prediction error in the least-squares sense. The output vector
    contains the normalized estimate of the AR system parameters

    The white noise input variance estimate is also returned.

    If is the power spectral density of y(n), then:

    .. math:: \frac{e}{\left| A(e^{jw}) \right|^2} = \frac{e}{\left| 1+\sum_{k-1}^P a(k)e^{-jwk}\right|^2}

    Because the method characterizes the input data using an all-pole model,
    the correct choice of the model order p is important.

    .. plot::
        :width: 80%
        :include-source:

        from spectrum import *
        from pylab import plot, log10, linspace, axis
        ar_values, error = arcovar(marple_data, 15)
        psd = arma2psd(ar_values, sides='centerdc')
        plot(linspace(-0.5, 0.5, len(psd)), 10*log10(psd/max(psd)))
        axis([-0.5, 0.5, -60, 0])

    .. seealso:: :class:`pcovar`

    :validation: the AR parameters are the same as those returned by
        a completely different function :func:`arcovar_marple`.

    :References: [Mathworks]_
    """

    from spectrum import corrmtx
    import scipy.linalg

    X = corrmtx(x, order, 'covariance')
    Xc = numpy.matrix(X[:, 1:])
    X1 = numpy.array(X[:, 0])

    # Coefficients estimated via the covariance method
    # Here we use lstsq rathre than solve function because Xc is not square
    # matrix

    a, _residues, _rank, _singular_values = scipy.linalg.lstsq(-Xc, X1)

    # Estimate the input white noise variance
    Cz = numpy.dot(X1.conj().transpose(), Xc)
    e = numpy.dot(X1.conj().transpose(), X1) + numpy.dot(Cz, a)
    assert e.imag < 1e-4, 'wierd behaviour'
    e = float(e.real) # ignore imag part that should be small

    return a, e

class pcovar(ParametricSpectrum):
    """Class to create PSD based on covariance algorithm

    See :func:`arcovar` for description.

    .. plot::
        :width: 80%
        :include-source:

        from spectrum import *
        p = pcovar(marple_data, 15, NFFT=4096)
        p.plot(sides='centerdc')

    .. seealso:: :class:`arcovar`
    """
    def __init__(self, data, order, NFFT=None, sampling=1.,
                 scale_by_freq=False):
        """**Constructor**

        For a detailled description of the parameters, see :func:`arcovar`.

        :param array data:     input data (list or numpy.array)
        :param int order:
        :param int NFFT:       total length of the final data sets (padded with zero if needed; default is 4096)
        :param float sampling: sampling frequency of the input :attr:`data`.

        """
        super(pcovar, self).__init__(data, ar_order=order,
                                            NFFT=NFFT, sampling=sampling,
                                            scale_by_freq=scale_by_freq)

    def __call__(self):
        from spectrum import arma2psd
        ar, _e = arcovar(self.data, self.ar_order)
        self.ar = ar
        psd = arma2psd(A=ar, T=self.sampling, NFFT=self.NFFT)

        if self.datatype == 'real':
            from .tools import twosided_2_onesided
            newpsd  = twosided_2_onesided(psd)
            #\psd[0:self.NFFT/2]*2
            #newpsd[0] /= 2.
            #newpsd = numpy.append(newpsd, psd[-1])
            self.psd = newpsd
        else:
            self.psd = psd
        if self.scale_by_freq is True:
            self.scale()

    def _str_title(self):
        return "Covariance PSD estimate\n"

    def __str__(self):
        return super(pcovar, self).__str__()

<commit_msg>Fix Fs=0 and Fs/2 real case for pcovar<commit_after>import logging

import numpy as np
from .psd import ParametricSpectrum

__all__ = ["arcovar", "arcovar_marple", "pcovar", 'arcovar_marple']

def arcovar_marple(x, order):
    r"""Estimate AR model parameters using covariance method

    This implementation is based on [Marple]_. This code is far more
    complicated and slower than :func:`arcovar` function, which is now the official version.
    See :func:`arcovar` for a detailed description of Covariance method.

    This function should be used in place of arcovar only if order<=4, for
    which :func:`arcovar` does not work.

    Fast algorithm for the solution of the covariance least squares normal
    equations from Marple.

    :param array X:  Array of complex data samples
    :param int oder: Order of linear prediction model

    :return:
        * AF   - Array of complex forward linear prediction coefficients
        * PF   - Real forward linear prediction variance at order IP
        * AB   - Array of complex backward linear prediction coefficients
        * PB   - Real backward linear prediction variance at order IP
        * PV   - store linear prediction coefficients

    .. note:: this code and the original code in Marple diverge for ip>10.
        it seems that this is related to single precision used with
        complex type in fortran whereas numpy uses double precision for
        complex type.

    :validation: the AR parameters are the same as those returned by
        a completely different function :func:`arcovar`.

    :References: [Marple]_
    """
    assert len(x) >= order, "X must be dimensioned >=N"

    #   ----------------------------------------------------- Initialization
    x = np.array(x)
    N = len(x)

    # Equations 8.C.42
    r0 = sum(abs(x)**2.)
    r1 = abs(x[0])**2
    rN = abs(x[N-1])**2

    pf = r0 - r1
    pb = r0 - rN

    delta = 1. - r1 / r0
    gamma = 1. - rN / r0

    c = np.zeros(N, dtype=complex)
    d = np.zeros(N, dtype=complex)
    r = np.zeros(N, dtype=complex)
    af = np.zeros(N, dtype=complex)
    ab = np.zeros(N, dtype=complex)

    c[0] = x[N-1].conjugate() / r0
    d[0] = x[0].conjugate() / r0

    # special case
    if order == 0:
        pf = r0 / float(N)
        pb = pf
        return af, pf, ab, pb, 0

    # ----------------------------------------------------------  MAIN LOOP

    #ip +1 because we want to enter in the loop to run the first part of the code.
    pbv = []
    for m in range(0, order+1):
        logging.debug('----------------------------m=', m)
        logging.debug(c[0:2])
        logging.debug(d[0:2])
        r1 = 1./pf
        r2 = 1./pb
        r3 = 1./delta
        r4 = 1./gamma

        #logging.debug('starting r1r2r3r4=', r1, r2, r3, r4, pf, pb, delta, gamma)

        #Order update: AF and AB vectors ; time update: C and D vectors
        temp = 0.+0.j
        for k in range(m+1, N):
            temp = temp + x[k]*x[k-m-1].conjugate()

        r[m] =  temp.conjugate()
        theta = x[0] * c[m]
        #print(('theta', theta))
        #    print(('c=', c[0:2]))
        #    print(('d=', d[0:2]))
        if m == 0:
            pass
        else:
            for k in range(0, m):
                theta = theta + x[m-k] * c[k]                   # Eq. (8.C.39)
                r[k] = r[k] - x[N-m-1] * x[N-m+k].conjugate() # Eq. (8.C.32)
                temp = temp + af[m-k-1] * r[k].conjugate()
                #print 'loop1 k=', k
                #print '         theta=',theta, 'r[k]=',r[k], 'temp=', temp
                #print '         c=',c[k], 'af=',af[m-k-1]
        """if m > 0:
            if debug:
                print((m, N-m))
                print(('Xk=0',x[m-0],x[N-m-1], x[N-m+0]))
            if m > 1:
                if debug:
                    print('Xk=1',x[m-1],x[N-m-1], x[N-m+1])
        """
        c1 = -temp * r2
        c2 = -r1 * temp.conjugate()
        c3 = theta * r3
        c4 = r4 *theta.conjugate()
        #if debug:
        #    print('c1 c2 c3 c4 before af=',c1 ,c2 ,c3 ,c4)
        af[m] = c1                    #             ! Eq. (8.C.19)
        ab[m] = c2                    #             ! Eq. (8.C.22)
        save = c[m]
        c[m] = save + c3*d[m]
        d[m] = d[m] + c4*save
        #if debug:
        #    print('res',m,'af[m]=',af[m], ab[m], save, 'temp=',temp)

        if m == 0:
            pass
        else:
            #if debug:print('af before', af[0:2])
            for k in range(0, m):
                save = af[k]
                af[k] = save + c1 * ab[m-k-1] # Eq. (8.C.18)
                ab[m-k-1] = ab[m-k-1] + c2 * save   # Eq. (8.C.21)

                save = c[k]
                c[k] = save + c3*d[k]       # Eq. (8.C.37)
                d[k] = d[k] + c4*save       # Eq. (8.C.38)
                #if debug:
                #    print('loop2 k=', k)
                #    print('      af[k]=', af[k])
                #    print('      ab[m-k-1]=', ab[m-k-1])
                #    print('      c[k]=', c[k])
                #    print('      d[k]=', d[k])
            #if debug:
            #    print('af after=', af[0:2])
            #    print('ab=', ab[0:2])

        r5 = temp.real**2 + temp.imag**2
        pf = pf - r5*r2         # Eq. (8.C.20)
        pb = pb - r5*r1         # Eq. (8.C.23)
        r5 = theta.real**2 + theta.imag**2
        delta = delta - r5*r4               # Eq. (8.C.39)
        gamma = gamma - r5*r3               # Eq. (8.C.40)
        #if debug:
        #    print('r5r2r1deltagamma', r5, r2, r1 , delta, gamma)
        #    print('pf before norm', pf, pb, N-m-1)
        if m != order-1:
            pass
        else:
            pf = pf / float(N-m-1)
            pb = pb / float(N-m-1)
            #if debug:
            #    print('ENDING', N-m-1)
            break

        #if debug:
        #    print('pf and pb', pf, pb)
        if pf > 0 and pb > 0:
            pass
        else:
            ValueError("Negative PF or PB value")
        if (delta > 0. and delta <=1 and gamma > 0. and gamma <=1):
            pass
        else:
            ValueError("Invalid delta or gamma value")

        #C   Time update:  AF and AB vectors; order update:  C and D vectors

        r1 = 1./pf
        r2 = 1./pb
        r3 = 1./delta
        r4 = 1./gamma
        #if debug:
        #    print('--------time update', r1, r2, r3, r4, m+1, N-m-1, x[m+1], x[N-m-2])
        ef = x[m+1]
        eb = x[(N-1)-m-1]

        for k in range(0,m+1):
            #print 'k=', k, 'ef=', ef, ' eb=',eb,' af=',af[k], ab[k]
            #print x[m-k],x[N-m+k-1]
            ef = ef + af[k] * x[m-k]             # Eq. (8.C.1)
            eb = eb + ab[k] * x[N-m+k-1]                   # Eq. (8.C.2)

        #ef = sum(af)

        #if debug:
        #    print('efweb', ef , eb)
        c1 = ef*r3
        c2 = eb*r4
        c3 = eb.conjugate() * r2
        c4 = ef.conjugate() * r1
        #if debug:
        #    print('c1c2c3c4', c1, c2, c3, c4)
        #    print('af before', af[0:2])
        for k in range(m, -1, -1):
            save = af[k]
            af[k] = save + c1 * d[k]                    #  Eq. (8.C.33)
            d[k+1] = d[k] + c4 * save                    # Eq. (8.C.25)
            save = ab[k]
            ab[k] = save + c2 * c[m-k]                 # Eq. (8.C.35)
            c[m-k] = c[m-k] + c3 * save              # Eq. (8.C.24)
        #if debug:
        #    print('af after', af[0:2])
        #    print('d', d[0:2])
        #    print('ab', ab[0:2])
        #    print('c', c[0:2])
        #if debug:print('Pb before', pf, pb)
        c[m+1] = c3
        d[0] = c4
        #r5 = abs(ef)**2
        r5 = ef.real**2 + ef.imag**2
        pf = pf - r5 * r3                              # Eq. (8.C.34)

        delta = delta-r5 * r1                        # Eq. (8.C.30)
        #r5 = abs(eb)**2
        r5 = eb.real**2 + eb.imag**2
        pb = pb - r5 * r4                              # Eq. (8.C.36)
        #if debug:
        #    print('Pb---------------------', m, pb, r5, r4)
        gamma = gamma-r5*r2                        # Eq. (8.C.31)
        pbv.append(pb)

        if (pf > 0. and pb > 0.):
            pass
        else:
            ValueError("Negative PF or PB value")
        #if debug:
        #    print(delta, gamma)
        if (delta > 0. and delta <= 1.) and (gamma > 0. and gamma <= 1.):
            pass
        else:
            ValueError("Invalid delta or gamma value")

    #af=array of forward coeff
    #ab=array of barward coeff
    #pb=backward variance
    #pf=forward variance
    return af, pf, ab, pb, pbv

def arcovar(x, order):
    r"""Simple and fast implementation of the covariance AR estimate

    This code is 10 times faster than :func:`arcovar_marple` and more importantly
    only 10 lines of code, compared to a 200 loc for :func:`arcovar_marple`

    :param array X:  Array of complex data samples
    :param int oder: Order of linear prediction model

    :return:
        * a - Array of complex forward linear prediction coefficients
        * e - error

    The covariance method fits a Pth order autoregressive (AR) model to the
    input signal, which is assumed to be the output of
    an AR system driven by white noise. This method minimizes the forward
    prediction error in the least-squares sense. The output vector
    contains the normalized estimate of the AR system parameters

    The white noise input variance estimate is also returned.

    If is the power spectral density of y(n), then:

    .. math:: \frac{e}{\left| A(e^{jw}) \right|^2} = \frac{e}{\left| 1+\sum_{k-1}^P a(k)e^{-jwk}\right|^2}

    Because the method characterizes the input data using an all-pole model,
    the correct choice of the model order p is important.

    .. plot::
        :width: 80%
        :include-source:

        from spectrum import arcovar, marple_data, arma2psd
        from pylab import plot, log10, linspace, axis

        ar_values, error = arcovar(marple_data, 15)
        psd = arma2psd(ar_values, sides='centerdc')
        plot(linspace(-0.5, 0.5, len(psd)), 10*log10(psd/max(psd)))
        axis([-0.5, 0.5, -60, 0])

    .. seealso:: :class:`pcovar`

    :validation: the AR parameters are the same as those returned by
        a completely different function :func:`arcovar_marple`.

    :References: [Mathworks]_
    """

    from spectrum import corrmtx
    import scipy.linalg

    X = corrmtx(x, order, 'covariance')
    Xc = np.matrix(X[:, 1:])
    X1 = np.array(X[:, 0])

    # Coefficients estimated via the covariance method
    # Here we use lstsq rathre than solve function because Xc is not square
    # matrix

    a, _residues, _rank, _singular_values = scipy.linalg.lstsq(-Xc, X1)

    # Estimate the input white noise variance
    Cz = np.dot(X1.conj().transpose(), Xc)
    e = np.dot(X1.conj().transpose(), X1) + np.dot(Cz, a)
    assert e.imag < 1e-4, 'wierd behaviour'
    e = float(e.real) # ignore imag part that should be small

    return a, e

class pcovar(ParametricSpectrum):
    """Class to create PSD based on covariance algorithm

    See :func:`arcovar` for description.

    .. plot::
        :width: 80%
        :include-source:

        from spectrum import pcovar, marple_data
        p = pcovar(marple_data, 15, NFFT=4096)
        p.plot(sides='centerdc')

    .. seealso:: :class:`arcovar`
    """
    def __init__(self, data, order, NFFT=None, sampling=1.,
                 scale_by_freq=False):
        """**Constructor**

        For a detailled description of the parameters, see :func:`arcovar`.

        :param array data: input data (list or numpy.array)
        :param int order:
        :param int NFFT: total length of the final data sets (padded
            with zero if needed; default is 4096)
        :param float sampling: sampling frequency of the input :attr:`data`.

        """
        super(pcovar, self).__init__(data, ar_order=order,
                                            NFFT=NFFT, sampling=sampling,
                                            scale_by_freq=scale_by_freq)

    def __call__(self):
        from spectrum import arma2psd
        ar, _e = arcovar(self.data, self.ar_order)
        self.ar = ar
        psd = arma2psd(A=ar, T=self.sampling, NFFT=self.NFFT)

        if self.datatype == 'real':
            #from .tools import twosided_2_onesided
            #newpsd  = twosided_2_onesided(psd)
            #\psd[0:self.NFFT/2]*2
            #newpsd[0] /= 2.
            #newpsd = numpy.append(newpsd, psd[-1])
            newpsd  = psd[0:int(self.NFFT//2)] * 2
            # TODO: check the last value is correct or required ?
            newpsd = np.append(newpsd, psd[int(self.NFFT//2)]*2)
            self.psd = newpsd
        else:
            self.psd = psd
        if self.scale_by_freq is True:
            self.scale()

    def _str_title(self):
        return "Covariance PSD estimate\n"

    def __str__(self):
        return super(pcovar, self).__str__()

<|endoftext|>