<commit_before># -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from ._base import SQLBaseStore
from synapse.util.caches.descriptors import cached

from twisted.internet import defer, reactor

from canonicaljson import encode_canonical_json

from collections import namedtuple

import itertools
import logging
import ujson as json

logger = logging.getLogger(__name__)

_TransactionRow = namedtuple(
    "_TransactionRow", (
        "id", "transaction_id", "destination", "ts", "response_code",
        "response_json",
    )
)

_UpdateTransactionRow = namedtuple(
    "_TransactionRow", (
        "response_code", "response_json",
    )
)

class TransactionStore(SQLBaseStore):
    """A collection of queries for handling PDUs.
    """

    def __init__(self, hs):
        super(TransactionStore, self).__init__(hs)

        # New transactions that are currently in flights
        self.inflight_transactions = {}

        # Newly delievered transactions that *weren't* persisted while in flight
        self.new_delivered_transactions = {}

        # Newly delivered transactions that *were* persisted while in flight
        self.update_delivered_transactions = {}

        self.last_transaction = {}

        reactor.addSystemEventTrigger("before", "shutdown", self._persist_in_mem_txns)
        self._clock.looping_call(self._persist_in_mem_txns, 1000)

        self._clock.looping_call(self._cleanup_transactions, 30 * 60 * 1000)

    def get_received_txn_response(self, transaction_id, origin):
        """For an incoming transaction from a given origin, check if we have
        already responded to it. If so, return the response code and response
        body (as a dict).

        Args:
            transaction_id (str)
            origin(str)

        Returns:
            tuple: None if we have not previously responded to
            this transaction or a 2-tuple of (int, dict)
        """

        return self.runInteraction(
            "get_received_txn_response",
            self._get_received_txn_response, transaction_id, origin
        )

    def _get_received_txn_response(self, txn, transaction_id, origin):
        result = self._simple_select_one_txn(
            txn,
            table="received_transactions",
            keyvalues={
                "transaction_id": transaction_id,
                "origin": origin,
            },
            retcols=(
                "transaction_id", "origin", "ts", "response_code", "response_json",
                "has_been_referenced",
            ),
            allow_none=True,
        )

        if result and result["response_code"]:
            return result["response_code"], json.loads(str(result["response_json"]))
        else:
            return None

    def set_received_txn_response(self, transaction_id, origin, code,
                                  response_dict):
        """Persist the response we returened for an incoming transaction, and
        should return for subsequent transactions with the same transaction_id
        and origin.

        Args:
            txn
            transaction_id (str)
            origin (str)
            code (int)
            response_json (str)
        """

        return self._simple_insert(
            table="received_transactions",
            values={
                "transaction_id": transaction_id,
                "origin": origin,
                "response_code": code,
                "response_json": buffer(encode_canonical_json(response_dict)),
                "ts": self._clock.time_msec(),
            },
            or_ignore=True,
            desc="set_received_txn_response",
        )

    def prep_send_transaction(self, transaction_id, destination,
                              origin_server_ts):
        """Persists an outgoing transaction and calculates the values for the
        previous transaction id list.

        This should be called before sending the transaction so that it has the
        correct value for the `prev_ids` key.

        Args:
            transaction_id (str)
            destination (str)
            origin_server_ts (int)

        Returns:
            list: A list of previous transaction ids.
        """

        auto_id = self._transaction_id_gen.get_next()

        txn_row = _TransactionRow(
            id=auto_id,
            transaction_id=transaction_id,
            destination=destination,
            ts=origin_server_ts,
            response_code=0,
            response_json=None,
        )

        self.inflight_transactions.setdefault(destination, {})[transaction_id] = txn_row

        prev_txn = self.last_transaction.get(destination)
        if prev_txn:
            return defer.succeed(prev_txn)
        else:
            return self.runInteraction(
                "_get_prevs_txn",
                self._get_prevs_txn,
                destination,
            )

    def _get_prevs_txn(self, txn, destination):
        # First we find out what the prev_txns should be.
        # Since we know that we are only sending one transaction at a time,
        # we can simply take the last one.
        query = (
            "SELECT * FROM sent_transactions"
            " WHERE destination = ?"
            " ORDER BY id DESC LIMIT 1"
        )

        txn.execute(query, (destination,))
        results = self.cursor_to_dict(txn)

        prev_txns = [r["transaction_id"] for r in results]

        return prev_txns

    def delivered_txn(self, transaction_id, destination, code, response_dict):
        """Persists the response for an outgoing transaction.

        Args:
            transaction_id (str)
            destination (str)
            code (int)
            response_json (str)
        """

        txn_row = self.inflight_transactions.get(
            destination, {}
        ).pop(transaction_id, None)

        self.last_transaction[destination] = transaction_id

        if txn_row:
            d = self.new_delivered_transactions.setdefault(destination, {})
            d[transaction_id] = txn_row._replace(
                response_code=code,
                response_json=None,  # For now, don't persist response
            )
        else:
            d = self.update_delivered_transactions.setdefault(destination, {})
            # For now, don't persist response
            d[transaction_id] = _UpdateTransactionRow(code, None)

    def get_transactions_after(self, transaction_id, destination):
        """Get all transactions after a given local transaction_id.

        Args:
            transaction_id (str)
            destination (str)

        Returns:
            list: A list of dicts
        """
        return self.runInteraction(
            "get_transactions_after",
            self._get_transactions_after, transaction_id, destination
        )

    def _get_transactions_after(self, txn, transaction_id, destination):
        query = (
            "SELECT * FROM sent_transactions"
            " WHERE destination = ? AND id >"
            " ("
            " SELECT id FROM sent_transactions"
            " WHERE transaction_id = ? AND destination = ?"
            " )"
        )

        txn.execute(query, (destination, transaction_id, destination))

        return self.cursor_to_dict(txn)

    @cached(max_entries=10000)
    def get_destination_retry_timings(self, destination):
        """Gets the current retry timings (if any) for a given destination.

        Args:
            destination (str)

        Returns:
            None if not retrying
            Otherwise a dict for the retry scheme
        """
        return self.runInteraction(
            "get_destination_retry_timings",
            self._get_destination_retry_timings, destination)

    def _get_destination_retry_timings(self, txn, destination):
        result = self._simple_select_one_txn(
            txn,
            table="destinations",
            keyvalues={
                "destination": destination,
            },
            retcols=("destination", "retry_last_ts", "retry_interval"),
            allow_none=True,
        )

        if result and result["retry_last_ts"] > 0:
            return result
        else:
            return None

    def set_destination_retry_timings(self, destination,
                                      retry_last_ts, retry_interval):
        """Sets the current retry timings for a given destination.
        Both timings should be zero if retrying is no longer occuring.

        Args:
            destination (str)
            retry_last_ts (int) - time of last retry attempt in unix epoch ms
            retry_interval (int) - how long until next retry in ms
        """

        # XXX: we could chose to not bother persisting this if our cache thinks
        # this is a NOOP
        return self.runInteraction(
            "set_destination_retry_timings",
            self._set_destination_retry_timings,
            destination,
            retry_last_ts,
            retry_interval,
        )

    def _set_destination_retry_timings(self, txn, destination,
                                       retry_last_ts, retry_interval):
        txn.call_after(self.get_destination_retry_timings.invalidate, (destination,))

        self._simple_upsert_txn(
            txn,
            "destinations",
            keyvalues={
                "destination": destination,
            },
            values={
                "retry_last_ts": retry_last_ts,
                "retry_interval": retry_interval,
            },
            insertion_values={
                "destination": destination,
                "retry_last_ts": retry_last_ts,
                "retry_interval": retry_interval,
            }
        )

    def get_destinations_needing_retry(self):
        """Get all destinations which are due a retry for sending a transaction.

        Returns:
            list: A list of dicts
        """

        return self.runInteraction(
            "get_destinations_needing_retry",
            self._get_destinations_needing_retry
        )

    def _get_destinations_needing_retry(self, txn):
        query = (
            "SELECT * FROM destinations"
            " WHERE retry_last_ts > 0 and retry_next_ts < ?"
        )

        txn.execute(query, (self._clock.time_msec(),))
        return self.cursor_to_dict(txn)

    @defer.inlineCallbacks
    def _persist_in_mem_txns(self):
        try:
            inflight = self.inflight_transactions
            new_delivered = self.new_delivered_transactions
            update_delivered = self.update_delivered_transactions

            self.inflight_transactions = {}
            self.new_delivered_transactions = {}
            self.update_delivered_transactions = {}

            full_rows = [
                row._asdict()
                for txn_map in itertools.chain(inflight.values(), new_delivered.values())
                for row in txn_map.values()
            ]

            def f(txn):
                if full_rows:
                    self._simple_insert_many_txn(
                        txn=txn,
                        table="sent_transactions",
                        values=full_rows
                    )

                for dest, txn_map in update_delivered.items():
                    for txn_id, update_row in txn_map.items():
                        self._simple_update_one_txn(
                            txn,
                            table="sent_transactions",
                            keyvalues={
                                "transaction_id": txn_id,
                                "destination": dest,
                            },
                            updatevalues={
                                "response_code": update_row.response_code,
                                "response_json": None,  # For now, don't persist response
                            }
                        )

            if full_rows or update_delivered:
                yield self.runInteraction("_persist_in_mem_txns", f)
        except:
            logger.exception("Failed to persist transactions!")

    def _cleanup_transactions(self):
        now = self._clock.time_msec()
        month_ago = now - 30 * 24 * 60 * 60 * 1000
        six_hours_ago = now - 6 * 60 * 60 * 1000

        def _cleanup_transactions_txn(txn):
            txn.execute("DELETE FROM received_transactions WHERE ts < ?", (month_ago,))
            txn.execute("DELETE FROM sent_transactions WHERE ts < ?", (six_hours_ago,))

        return self.runInteraction("_persist_in_mem_txns", _cleanup_transactions_txn)
<commit_msg>Increase batching of sent transaction inserts<commit_after># -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from ._base import SQLBaseStore
from synapse.util.caches.descriptors import cached

from twisted.internet import defer, reactor

from canonicaljson import encode_canonical_json

from collections import namedtuple

import itertools
import logging
import ujson as json

logger = logging.getLogger(__name__)

_TransactionRow = namedtuple(
    "_TransactionRow", (
        "id", "transaction_id", "destination", "ts", "response_code",
        "response_json",
    )
)

_UpdateTransactionRow = namedtuple(
    "_TransactionRow", (
        "response_code", "response_json",
    )
)

class TransactionStore(SQLBaseStore):
    """A collection of queries for handling PDUs.
    """

    def __init__(self, hs):
        super(TransactionStore, self).__init__(hs)

        # New transactions that are currently in flights
        self.inflight_transactions = {}

        # Newly delievered transactions that *weren't* persisted while in flight
        self.new_delivered_transactions = {}

        # Newly delivered transactions that *were* persisted while in flight
        self.update_delivered_transactions = {}

        self.last_transaction = {}

        reactor.addSystemEventTrigger("before", "shutdown", self._persist_in_mem_txns)
        self._clock.looping_call(self._persist_in_mem_txns, 10 * 1000)

        self._clock.looping_call(self._cleanup_transactions, 30 * 60 * 1000)

    def get_received_txn_response(self, transaction_id, origin):
        """For an incoming transaction from a given origin, check if we have
        already responded to it. If so, return the response code and response
        body (as a dict).

        Args:
            transaction_id (str)
            origin(str)

        Returns:
            tuple: None if we have not previously responded to
            this transaction or a 2-tuple of (int, dict)
        """

        return self.runInteraction(
            "get_received_txn_response",
            self._get_received_txn_response, transaction_id, origin
        )

    def _get_received_txn_response(self, txn, transaction_id, origin):
        result = self._simple_select_one_txn(
            txn,
            table="received_transactions",
            keyvalues={
                "transaction_id": transaction_id,
                "origin": origin,
            },
            retcols=(
                "transaction_id", "origin", "ts", "response_code", "response_json",
                "has_been_referenced",
            ),
            allow_none=True,
        )

        if result and result["response_code"]:
            return result["response_code"], json.loads(str(result["response_json"]))
        else:
            return None

    def set_received_txn_response(self, transaction_id, origin, code,
                                  response_dict):
        """Persist the response we returened for an incoming transaction, and
        should return for subsequent transactions with the same transaction_id
        and origin.

        Args:
            txn
            transaction_id (str)
            origin (str)
            code (int)
            response_json (str)
        """

        return self._simple_insert(
            table="received_transactions",
            values={
                "transaction_id": transaction_id,
                "origin": origin,
                "response_code": code,
                "response_json": buffer(encode_canonical_json(response_dict)),
                "ts": self._clock.time_msec(),
            },
            or_ignore=True,
            desc="set_received_txn_response",
        )

    def prep_send_transaction(self, transaction_id, destination,
                              origin_server_ts):
        """Persists an outgoing transaction and calculates the values for the
        previous transaction id list.

        This should be called before sending the transaction so that it has the
        correct value for the `prev_ids` key.

        Args:
            transaction_id (str)
            destination (str)
            origin_server_ts (int)

        Returns:
            list: A list of previous transaction ids.
        """

        auto_id = self._transaction_id_gen.get_next()

        txn_row = _TransactionRow(
            id=auto_id,
            transaction_id=transaction_id,
            destination=destination,
            ts=origin_server_ts,
            response_code=0,
            response_json=None,
        )

        self.inflight_transactions.setdefault(destination, {})[transaction_id] = txn_row

        prev_txn = self.last_transaction.get(destination)
        if prev_txn:
            return defer.succeed(prev_txn)
        else:
            return self.runInteraction(
                "_get_prevs_txn",
                self._get_prevs_txn,
                destination,
            )

    def _get_prevs_txn(self, txn, destination):
        # First we find out what the prev_txns should be.
        # Since we know that we are only sending one transaction at a time,
        # we can simply take the last one.
        query = (
            "SELECT * FROM sent_transactions"
            " WHERE destination = ?"
            " ORDER BY id DESC LIMIT 1"
        )

        txn.execute(query, (destination,))
        results = self.cursor_to_dict(txn)

        prev_txns = [r["transaction_id"] for r in results]

        return prev_txns

    def delivered_txn(self, transaction_id, destination, code, response_dict):
        """Persists the response for an outgoing transaction.

        Args:
            transaction_id (str)
            destination (str)
            code (int)
            response_json (str)
        """

        txn_row = self.inflight_transactions.get(
            destination, {}
        ).pop(transaction_id, None)

        self.last_transaction[destination] = transaction_id

        if txn_row:
            d = self.new_delivered_transactions.setdefault(destination, {})
            d[transaction_id] = txn_row._replace(
                response_code=code,
                response_json=None,  # For now, don't persist response
            )
        else:
            d = self.update_delivered_transactions.setdefault(destination, {})
            # For now, don't persist response
            d[transaction_id] = _UpdateTransactionRow(code, None)

    def get_transactions_after(self, transaction_id, destination):
        """Get all transactions after a given local transaction_id.

        Args:
            transaction_id (str)
            destination (str)

        Returns:
            list: A list of dicts
        """
        return self.runInteraction(
            "get_transactions_after",
            self._get_transactions_after, transaction_id, destination
        )

    def _get_transactions_after(self, txn, transaction_id, destination):
        query = (
            "SELECT * FROM sent_transactions"
            " WHERE destination = ? AND id >"
            " ("
            " SELECT id FROM sent_transactions"
            " WHERE transaction_id = ? AND destination = ?"
            " )"
        )

        txn.execute(query, (destination, transaction_id, destination))

        return self.cursor_to_dict(txn)

    @cached(max_entries=10000)
    def get_destination_retry_timings(self, destination):
        """Gets the current retry timings (if any) for a given destination.

        Args:
            destination (str)

        Returns:
            None if not retrying
            Otherwise a dict for the retry scheme
        """
        return self.runInteraction(
            "get_destination_retry_timings",
            self._get_destination_retry_timings, destination)

    def _get_destination_retry_timings(self, txn, destination):
        result = self._simple_select_one_txn(
            txn,
            table="destinations",
            keyvalues={
                "destination": destination,
            },
            retcols=("destination", "retry_last_ts", "retry_interval"),
            allow_none=True,
        )

        if result and result["retry_last_ts"] > 0:
            return result
        else:
            return None

    def set_destination_retry_timings(self, destination,
                                      retry_last_ts, retry_interval):
        """Sets the current retry timings for a given destination.
        Both timings should be zero if retrying is no longer occuring.

        Args:
            destination (str)
            retry_last_ts (int) - time of last retry attempt in unix epoch ms
            retry_interval (int) - how long until next retry in ms
        """

        # XXX: we could chose to not bother persisting this if our cache thinks
        # this is a NOOP
        return self.runInteraction(
            "set_destination_retry_timings",
            self._set_destination_retry_timings,
            destination,
            retry_last_ts,
            retry_interval,
        )

    def _set_destination_retry_timings(self, txn, destination,
                                       retry_last_ts, retry_interval):
        txn.call_after(self.get_destination_retry_timings.invalidate, (destination,))

        self._simple_upsert_txn(
            txn,
            "destinations",
            keyvalues={
                "destination": destination,
            },
            values={
                "retry_last_ts": retry_last_ts,
                "retry_interval": retry_interval,
            },
            insertion_values={
                "destination": destination,
                "retry_last_ts": retry_last_ts,
                "retry_interval": retry_interval,
            }
        )

    def get_destinations_needing_retry(self):
        """Get all destinations which are due a retry for sending a transaction.

        Returns:
            list: A list of dicts
        """

        return self.runInteraction(
            "get_destinations_needing_retry",
            self._get_destinations_needing_retry
        )

    def _get_destinations_needing_retry(self, txn):
        query = (
            "SELECT * FROM destinations"
            " WHERE retry_last_ts > 0 and retry_next_ts < ?"
        )

        txn.execute(query, (self._clock.time_msec(),))
        return self.cursor_to_dict(txn)

    @defer.inlineCallbacks
    def _persist_in_mem_txns(self):
        try:
            inflight = self.inflight_transactions
            new_delivered = self.new_delivered_transactions
            update_delivered = self.update_delivered_transactions

            self.inflight_transactions = {}
            self.new_delivered_transactions = {}
            self.update_delivered_transactions = {}

            full_rows = [
                row._asdict()
                for txn_map in itertools.chain(inflight.values(), new_delivered.values())
                for row in txn_map.values()
            ]

            def f(txn):
                if full_rows:
                    self._simple_insert_many_txn(
                        txn=txn,
                        table="sent_transactions",
                        values=full_rows
                    )

                for dest, txn_map in update_delivered.items():
                    for txn_id, update_row in txn_map.items():
                        self._simple_update_one_txn(
                            txn,
                            table="sent_transactions",
                            keyvalues={
                                "transaction_id": txn_id,
                                "destination": dest,
                            },
                            updatevalues={
                                "response_code": update_row.response_code,
                                "response_json": None,  # For now, don't persist response
                            }
                        )

            if full_rows or update_delivered:
                yield self.runInteraction("_persist_in_mem_txns", f)
        except:
            logger.exception("Failed to persist transactions!")

    def _cleanup_transactions(self):
        now = self._clock.time_msec()
        month_ago = now - 30 * 24 * 60 * 60 * 1000
        six_hours_ago = now - 6 * 60 * 60 * 1000

        def _cleanup_transactions_txn(txn):
            txn.execute("DELETE FROM received_transactions WHERE ts < ?", (month_ago,))
            txn.execute("DELETE FROM sent_transactions WHERE ts < ?", (six_hours_ago,))

        return self.runInteraction("_persist_in_mem_txns", _cleanup_transactions_txn)
<|endoftext|>


<commit_before>import urllib
import urlparse

from django.template import (Library, loader, TemplateSyntaxError, Node,
                             Variable, VariableDoesNotExist)
from django.template.defaultfilters import escapejs
from django.utils.encoding import smart_str
from django.utils.safestring import mark_safe
from django.utils.html import escape

import jinja2
from jingo import register, env

from utils.helpers import get_random_string

from django.template.defaultfilters import \
     (addslashes, escapejs, fix_ampersands,
      floatformat, iriencode, linenumbers,
      slugify, truncatewords, truncatewords_html,
      urlencode, cut, rjust, ljust, linebreaks,
      linebreaksbr, removetags, unordered_list,
      date, time, timesince, timeuntil, yesno,
      pluralize, phone2numeric, capfirst)

for func in (addslashes, escapejs, fix_ampersands,
             floatformat, iriencode, linenumbers,
             slugify, truncatewords, truncatewords_html,
             urlencode, cut, rjust, ljust, linebreaks,
             linebreaksbr, removetags, unordered_list,
             date, time, timesince, timeuntil, yesno,
             pluralize, phone2numeric,capfirst):
        register.filter(func)

@jinja2.contextfunction
@register.function
def querystring(context, query, **params):
    """
    Add paramaters to a query string. New params will be appended, duplicates will
    be overwritten.
    """

    # separate querystring from route
    qs = []
    if context != None :
        url_parts = context['request'].get_full_path().split('?')
        qs = url_parts[1].split('&') if len(url_parts) == 2 else []

    #collect all querystring params
    for i in query.split('&'):
        qs.append(i)

    #mash them together into a dictionary
    query_dictionary = {}
    for (i,v) in enumerate( qs ):
        parts = v.split('=')
        if( len(parts) == 2 ):
            query_dictionary[parts[0]] = parts[1]

    query_dictionary.update(**params)

    #convert dictionary to querystring with all params that have values
    qs = []
    for (k,v) in query_dictionary.items():
        qs.append( k+'='+urllib.quote(str(v)) )

    return '&'.join(sorted(qs))

@jinja2.contextfunction
@register.function
def select_selected(context, value):
    bits = value.split('=');
    if context['request'].GET.get(bits[0]) == bits[1]:
        return 'selected=selected'
    else:
        return ''

@register.function
def replace(item, value):
    """Replaces first part of ``value`` with the second one

    :param: value (string) list of 2 items separated by comma
    :result: (string) ``item`` with the first string replaced by the other
    """
    items = value.split(',')
    if len(items) != 2:
        raise TemplateSyntaxError(
                "Replace filter argument is a comma separated list of 2 items")
    return item.replace(items[0], items[1])

@jinja2.contextfunction
@register.function
def escape_template(context, template):
    t = env.get_template(template).render(context)
    return jinja2.Markup(escapejs(t))

@jinja2.contextfunction
@register.function
def safe_csrf_token(context):
    csrf_token = context.get('csrf_token', None)
    if csrf_token:
        if csrf_token == 'NOTPROVIDED':
            return mark_safe(u"")
        else:
            return mark_safe(u"
"
                    "<input type='hidden' name='csrfmiddlewaretoken' "
                    "value='%s' />
" % escape(csrf_token))
    else:
        # It's very probable that the token is missing because of
        # misconfiguration, so we raise a warning
        from django.conf import settings
        if settings.DEBUG:
            import warnings
            warnings.warn("A {% csrf_token %} was used in a template, "
                    "but the context did not provide the value.  "
                    "This is usually caused by not using RequestContext.")
        return u''

@register.function
def hashtag(length=10):
    """ return random string """
    return get_random_string(length)

def urlparams(url_, **query):
    """
    Update the parameters of a url and return it.
    """
    url = urlparse.urlparse(url_)
    query_string = querystring(None, url.query, **query)
    new = urlparse.ParseResult(url.scheme, url.netloc, url.path, url.params,
                               query_string, url.fragment)
    return new.geturl()

def urlencode(items):
    """A Unicode-safe URLencoder."""
    try:
        return urllib.urlencode(items)
    except UnicodeEncodeError:
        return urllib.urlencode([(k, smart_str(v)) for k, v in items])<commit_msg>adding a shipyard() jingo helper<commit_after>import urllib
import urlparse

from django.conf import settings
from django.template import (Library, loader, TemplateSyntaxError, Node,
                             Variable, VariableDoesNotExist)
from django.template.defaultfilters import escapejs
from django.utils.encoding import smart_str
from django.utils.safestring import mark_safe
from django.utils.html import escape

import jinja2
from jingo import register, env

from utils.helpers import get_random_string

from django.template.defaultfilters import \
     (addslashes, escapejs, fix_ampersands,
      floatformat, iriencode, linenumbers,
      slugify, truncatewords, truncatewords_html,
      urlencode, cut, rjust, ljust, linebreaks,
      linebreaksbr, removetags, unordered_list,
      date, time, timesince, timeuntil, yesno,
      pluralize, phone2numeric, capfirst)

for func in (addslashes, escapejs, fix_ampersands,
             floatformat, iriencode, linenumbers,
             slugify, truncatewords, truncatewords_html,
             urlencode, cut, rjust, ljust, linebreaks,
             linebreaksbr, removetags, unordered_list,
             date, time, timesince, timeuntil, yesno,
             pluralize, phone2numeric,capfirst):
        register.filter(func)

@jinja2.contextfunction
@register.function
def querystring(context, query, **params):
    """
    Add paramaters to a query string. New params will be appended, duplicates will
    be overwritten.
    """

    # separate querystring from route
    qs = []
    if context != None :
        url_parts = context['request'].get_full_path().split('?')
        qs = url_parts[1].split('&') if len(url_parts) == 2 else []

    #collect all querystring params
    for i in query.split('&'):
        qs.append(i)

    #mash them together into a dictionary
    query_dictionary = {}
    for (i,v) in enumerate( qs ):
        parts = v.split('=')
        if( len(parts) == 2 ):
            query_dictionary[parts[0]] = parts[1]

    query_dictionary.update(**params)

    #convert dictionary to querystring with all params that have values
    qs = []
    for (k,v) in query_dictionary.items():
        qs.append( k+'='+urllib.quote(str(v)) )
    return '&'.join(sorted(qs))

@jinja2.contextfunction
@register.function
def select_selected(context, value):
    bits = value.split('=');
    if context['request'].GET.get(bits[0]) == bits[1]:
        return 'selected=selected'
    else:
        return ''

@register.function
def replace(item, value):
    """Replaces first part of ``value`` with the second one

    :param: value (string) list of 2 items separated by comma
    :result: (string) ``item`` with the first string replaced by the other
    """
    items = value.split(',')
    if len(items) != 2:
        raise TemplateSyntaxError(
                "Replace filter argument is a comma separated list of 2 items")
    return item.replace(items[0], items[1])

@jinja2.contextfunction
@register.function
def escape_template(context, template):
    t = env.get_template(template).render(context)
    return jinja2.Markup(escapejs(t))

@jinja2.contextfunction
@register.function
def safe_csrf_token(context):
    csrf_token = context.get('csrf_token', None)
    if csrf_token:
        if csrf_token == 'NOTPROVIDED':
            return mark_safe(u"")
        else:
            return mark_safe(u"
"
                    "<input type='hidden' name='csrfmiddlewaretoken' "
                    "value='%s' />
" % escape(csrf_token))
    else:
        # It's very probable that the token is missing because of
        # misconfiguration, so we raise a warning
        if settings.DEBUG:
            import warnings
            warnings.warn("A {% csrf_token %} was used in a template, "
                    "but the context did not provide the value.  "
                    "This is usually caused by not using RequestContext.")
        return u''

@register.function
def hashtag(length=10):
    """ return random string """
    return get_random_string(length)

_SHIPYARD_REQUIRE = u'/media/lib/shipyard/scripts/require.js'
_SHIPYARD_SUFFIX = '-min.js'

@register.function
def shipyard(path, compiled_path=None):
    " return script tag for Shipyard apps "
    if settings.DEBUG:
        out = (u'' %
                (_SHIPYARD_REQUIRE, path))
    else:
        if not compiled_path:
            compiled_path = path + _SHIPYARD_SUFFIX
        out = mark_safe(u'' %
                (compiled_path, settings.BUILD_ID))
    return jinja2.Markup(out)

def urlparams(url_, **query):
    """
    Update the parameters of a url and return it.
    """
    url = urlparse.urlparse(url_)
    query_string = querystring(None, url.query, **query)
    new = urlparse.ParseResult(url.scheme, url.netloc, url.path, url.params,
                               query_string, url.fragment)
    return new.geturl()

def urlencode(items):
    """A Unicode-safe URLencoder."""
    try:
        return urllib.urlencode(items)
    except UnicodeEncodeError:
        return urllib.urlencode([(k, smart_str(v)) for k, v in items])
<|endoftext|>


<commit_before># coding: utf-8
from .exact_sampling import *
from .approximate_sampling import *
import matplotlib.pyplot as plt

class Discrete_DPP:

	def __init__(self, kernel, ensemble_type, projection_kernel=False):

		self.nb_items = kernel.shape[0]

		self.ensemble_type = ensemble_type
		self.__check_ensemble_type_param_validity()

		self.projection_kernel = projection_kernel
		self.__check_projection_kernel_param_validity()

		self.K = None
		self.L = None
		# If valid kernel, diagonalization only for non projection kernel.
		self.eig_vals_K = None
		self.eig_vals_L = None
		self.eig_vecs = None
		self.__check_kernel_for_dpp_validity(kernel)

		self.sampling_mode = None # Default 'GS'
		### Exact sampling
		# 'GS' for Gram-Schmidt,
		# 'Schur' for Schur complement
		# 'KuTa12' for Kulesza (Algo 1).
		### Approx sampling
		# 'AED' 'AD' 'E' A=Add E=Exchange D=Delete
		self.list_of_samples = []

	def __str__(self):
		str_info = ["Discrete DPP defined by:",
								"- {}-ensemble on {} items",
								"- Projection kernel: {}",
								"- sampling mode = {}",
								"- number of samples = {}"]

		return "\n".join(str_info).format(self.ensemble_type, self.nb_items,
																		"Yes" if self.projection_kernel else "No",
																		self.sampling_mode,
																		len(self.list_of_samples))

	def __check_ensemble_type_param_validity(self):

		if self.ensemble_type not in ('K', 'L'):
			str_list = ["Invalid ensemble_type parameter, use:",
									"- 'K' for inclusion probability kernel",
									"- 'L' for marginal kernel.",
									"Given {}".format(self.ensemble_type)]

			raise ValueError("\n".join(str_list))

	def __check_projection_kernel_param_validity(self):
		# Recall we first checked the kernel to be symmetric (K=K.T), here we check the reproducing property of the (orthogonal) projection kernel.
		# For this we perform a cheap test to check K^2 = K K.T = K
		if not isinstance(self.projection_kernel, bool):
			str_list = ["Invalid projection_kernel argument: must be True/False",
									"Given projection_kernel={}".format(self.projection_kernel)]
			raise ValueError("\n".join(str_list))

	def __check_kernel_for_dpp_validity(self, kernel):
		"""Check symmetry, projection, and validity:
		- For K-ensemble 0<=K<=I
		- For L-ensemble L>=0"""

		# Check symmetry: kernel.T = kernel
		if not np.allclose(kernel, kernel.T):
			raise ValueError("Invalid kernel: not symmetric")

		# Check if 'K' kernel is orthogonal projection K^2 = K K.T = K
		# Only case for which eigendecomposition is not necessary
		if (self.ensemble_type == 'K') & (self.projection_kernel):
			# Cheap test checking reproducing property
			nb_tmp = 5
			items_to_check = np.arange(nb_tmp)
			K_i_ = kernel[items_to_check, :]
			K_ii = kernel[items_to_check, items_to_check]

			if np.allclose(np_inner1d(K_i_, K_i_), K_ii):
					self.K = kernel
			else:
				raise ValueError("Invalid kernel: doesn't seem to be a projection")

		else:
			# Eigendecomposition necessary for non projection kernels
			eig_vals, eig_vecs = la.eigh(kernel)
			tol = 1e-8 # tolerance on the eigenvalues

			# If K-ensemble
			if self.ensemble_type == 'K':
				# Check 0 <= K <= I
				if np.all((-tol <= eig_vals) & (eig_vals <= 1.0+tol)):
					self.K = kernel
					self.eig_vals_K = eig_vals
					self.eig_vecs = eig_vecs

				else:
					raise ValueError("Invalid kernel for K-ensemble. Eigen values are not in [0,1]")

			# If L-ensemble
			elif self.ensemble_type == 'L':
				# Check L >= 0
				if np.all(eig_vals >= -tol):
					self.L = kernel
					self.eig_vals_L = eig_vals
					self.eig_vecs = eig_vecs

				else:
					raise ValueError("Invalid kernel for L-ensemble. Eigen values !>= 0")

	def info(self):
		print(self.__str__())

	def compute_K_kernel(self):
		"""K = L(I+L)^-1 = I - (I+L)^-1"""
		if self.ensemble_type == 'K':
			raise ValueError("'K' kernel is already available")

		else:
			self.eig_vals_K = self.eig_vals_L/(1.0 + self.eig_vals_L)
			self.K = (self.eig_vecs * self.eig_vals_K).dot(self.eig_vecs.T)

	def compute_L_kernel(self):
		"""L = K(I-K)^-1 = (I-K)^-1 - I"""
		if self.ensemble_type == 'L':
			raise ValueError("'L' kernel is already available")

		else:
			try: # to compute eigenvalues of kernel L = K(I-K)^-1
				np.seterr(divide='raise')
				self.eig_vals_L = self.eig_vals_K/(1.0 - self.eig_vals_K)
				self.L = (self.eig_vecs * self.eig_vals_L).dot(self.eig_vecs.T)

			except FloatingPointError as e:
				str_list = ["WARNING: {}.".format(e),
										"Eigenvalues of 'L' kernel (L=K(I-K)^-1) cannot be computed.",
										"'K' kernel has some eigenvalues equal are very close to 1.",
										"Hint: 'K' kernel might be a projection."]
				print("\n".join(str_list))

	def flush_samples(self):
		self.list_of_samples = []

	### Exact sampling
	def sample_exact(self, sampling_mode="GS"):

		self.sampling_mode = sampling_mode

		if (self.ensemble_type == 'K') & (self.projection_kernel):
			# If K is orthogonal projection no need for eigendecomposition, update conditional via Gram-Schmidt on columns (equiv on rows) of K
			sampl = proj_dpp_sampler_kernel(self.K,
																			self.sampling_mode)

		else:
			if self.eig_vals_K is None: # Compute eig_vals_K from eig_vals_L
				self.eig_vals_K = self.eig_vals_L/(1.0 + self.eig_vals_L)

			sampl = dpp_sampler_eig(self.eig_vals_K,
															self.eig_vecs,
															self.sampling_mode)

		self.list_of_samples.append(sampl)

	### Approximate sampling
	def sample_approx(self, sampling_mode="AED", nb_iter=10, T_max=None):

		self.list_of_samples.append(sampl)

	def plot(self):
		"""Display a heatmap of the kernel provided, either K- or L-ensemble"""

		print("Heat map of '{}'-kernel".format('K' if self.ensemble_type == 'K'\
																							else 'L'))
		fig, ax = plt.subplots(1,1)

		heatmap = ax.pcolor(self.K if self.ensemble_type == 'K' else self.L,
												cmap='jet')

		ax.set_aspect('equal')

		ticks = np.arange(self.nb_items)
		ticks_label = [r'${}$'.format(tic) for tic in ticks]

		ax.xaxis.tick_top()
		ax.set_xticks(ticks+0.5, minor=False)

		ax.invert_yaxis()
		ax.set_yticks(ticks+0.5, minor=False)

		ax.set_xticklabels(ticks_label, minor=False)
		ax.set_yticklabels(ticks_label, minor=False)

		plt.colorbar(heatmap)
		plt.show()<commit_msg>Redesign of (k-)dpp classes to use approximate sampling without eigendecomposition at construction<commit_after># coding: utf-8
from .exact_sampling import *
from .approximate_sampling import *
import matplotlib.pyplot as plt

class Discrete_DPP:

	def __init__(self, kernel, ensemble_type, projection_kernel=False):

		self.nb_items = kernel.shape[0]
		self.projection_kernel = projection_kernel
		self.__check_projection_kernel(kernel)

		self.K = None
		self.L = None
		self.ensemble_type = ensemble_type
		self.__check_ensemble_type(kernel)

		self.list_of_samples = []
		self.sampling_mode = None # Default 'GS'
		### Exact sampling: eigendecomposition may be required
		# 'GS' for Gram-Schmidt,
		# 'Schur' for Schur complement
		# 'KuTa12' for Kulesza (Algo 1).
		self.eigen_decomposition_available = False
		self.eig_vals_K = None
		self.eig_vals_L = None
		self.eig_vecs = None
		self.__check_kernel_for_dpp_validity(kernel)

		### Approx sampling: eigendecomposition not required
		# 'AED' Add-Exchange-Delete
		# 'AD' Add-Delete
		# 'E' Exchange

	def __str__(self):
		str_info = ["Discrete DPP defined by:",
								"- {}-ensemble on {} items",
								"- Projection kernel: {}",
								"- sampling mode = {}",
								"- number of samples = {}"]

		return "\n".join(str_info).format(self.ensemble_type, self.nb_items,
																		"Yes" if self.projection_kernel else "No",
																		self.sampling_mode,
																		len(self.list_of_samples))

	def __check_projection_kernel(self):
		if not isinstance(self.projection_kernel, bool):
			str_list = ["Invalid projection_kernel argument: must be True/False",
									"Given projection_kernel={}".format(self.projection_kernel)]
			raise ValueError("\n".join(str_list))

		elif self.projection_kernel:
			# Cheap test checking reproducing property
			nb_tmp = 5
			items_to_check = np.arange(nb_tmp)
			K_i_ = kernel[items_to_check, :]
			K_ii = kernel[items_to_check, items_to_check]

			if not np.allclose(np_inner1d(K_i_, K_i_), K_ii):
				raise ValueError("Invalid kernel: doesn't seem to be a projection")

	def __check_ensemble_type(self, kernel):

		if self.ensemble_type == 'K':
			self.K = kernel
		elif self.ensemble_type == 'L':
			self.L = kernel
		else:
			str_list = ["Invalid ensemble_type parameter, use:",
									"- 'K' for inclusion probability kernel",
									"- 'L' for marginal kernel.",
									"Given {}".format(self.ensemble_type)]

			raise ValueError("\n".join(str_list))

	def __eigendecompose(self):
		"""Check symmetry, projection, and validity:
		- For K-ensemble 0<=K<=I
		- For L-ensemble L>=0"""
		if self.eigen_decomposition_available:
			pass
		else:
			# Eigendecomposition necessary for non projection kernel
			tol = 1e-8 # tolerance on the eigenvalues
			self.eigendecomposition = True

			# If K-ensemble
			if self.ensemble_type == 'K':
				self.eig_vals_K, self.eig_vecs = la.eigh(self.K)
				# Check 0 <= K <= I
				if not np.all((-tol<=self.eig_vals_K) & (self.eig_vals_K<=1.0+tol)):
					raise ValueError("Invalid kernel for K-ensemble. Eigen values are not in [0,1]")

			# If L-ensemble
			elif self.ensemble_type == 'L':
				self.eig_vals_L, self.eig_vecs = la.eigh(self.L)
				# Check L >= 0
				if not np.all(eig_vals >= -tol):
					raise ValueError("Invalid kernel for L-ensemble. Eigen values !>= 0")

	def info(self):
		print(self.__str__())

	def compute_K_kernel(self):
		"""K = L(I+L)^-1 = I - (I+L)^-1"""
		if self.K is not None:
			raise ValueError("'K' kernel is already available")

		elif self.eigen_decomposition_available:
			self.eig_vals_K = self.eig_vals_L/(1.0 + self.eig_vals_L)
			self.K = (self.eig_vecs * self.eig_vals_K).dot(self.eig_vecs.T)

		else:
			print("Eigendecomposition performed")
			self.__eigendecompose()
			self.compute_K_kernel()

	def compute_L_kernel(self):
		"""L = K(I-K)^-1 = (I-K)^-1 - I"""
		if self.L is not None:
			raise ValueError("'L' kernel is already available")

		elif self.eigen_decomposition_available:
			try: # to compute eigenvalues of kernel L = K(I-K)^-1
				np.seterr(divide='raise')
				self.eig_vals_L = self.eig_vals_K/(1.0 - self.eig_vals_K)
				self.L = (self.eig_vecs * self.eig_vals_L).dot(self.eig_vecs.T)

			except FloatingPointError as e:
				str_list = ["WARNING: {}.".format(e),
										"Eigenvalues of 'L' kernel (L=K(I-K)^-1) cannot be computed.",
										"'K' kernel has some eigenvalues equal are very close to 1.",
										"Hint: 'K' kernel might be a projection."]
				print("\n".join(str_list))

		else:
			print("Eigendecomposition performed")
			self.__eigendecompose()
			self.compute_L_kernel()

	def flush_samples(self):
		self.list_of_samples = []

	### Exact sampling
	def sample_exact(self, sampling_mode="GS"):

		self.sampling_mode = sampling_mode

		if (self.ensemble_type == 'K') & (self.projection_kernel):
			# If K is orthogonal projection no need for eigendecomposition, update conditional via Gram-Schmidt on columns (equiv on rows) of K
			sampl = proj_dpp_sampler_kernel(self.K,
																			self.sampling_mode)

		else:
			if self.eig_vals_K is None: # Compute eig_vals_K from eig_vals_L
				self.eig_vals_K = self.eig_vals_L/(1.0 + self.eig_vals_L)

			sampl = dpp_sampler_eig(self.eig_vals_K,
															self.eig_vecs,
															self.sampling_mode)

		self.list_of_samples.append(sampl)

	### Approximate sampling
	def sample_approx(self, sampling_mode="AED", nb_iter=10, T_max=None):

		self.list_of_samples.append(sampl)

	def plot(self):
		"""Display a heatmap of the kernel provided, either K- or L-ensemble"""

		print("Heat map of '{}'-kernel".format('K' if self.ensemble_type == 'K'\
																							else 'L'))
		fig, ax = plt.subplots(1,1)

		heatmap = ax.pcolor(self.K if self.ensemble_type == 'K' else self.L,
												cmap='jet')

		ax.set_aspect('equal')

		ticks = np.arange(self.nb_items)
		ticks_label = [r'${}$'.format(tic) for tic in ticks]

		ax.xaxis.tick_top()
		ax.set_xticks(ticks+0.5, minor=False)

		ax.invert_yaxis()
		ax.set_yticks(ticks+0.5, minor=False)

		ax.set_xticklabels(ticks_label, minor=False)
		ax.set_yticklabels(ticks_label, minor=False)

		plt.colorbar(heatmap)
		plt.show()<|endoftext|>


<commit_before># Copyright (C) 2010-2012 Yaco Sistemas (http://www.yaco.es)
# Copyright (C) 2009 Lorenzo Gil Sanchez <lorenzo.gil.sanchez@gmail.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#            http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging

from django.conf import settings
from django.contrib.auth.backends import ModelBackend
from django.contrib.auth.models import User, SiteProfileNotAvailable
from django.core.exceptions import ObjectDoesNotExist

logger = logging.getLogger('djangosaml2')

class Saml2Backend(ModelBackend):

    def authenticate(self, session_info=None, attribute_mapping=None,
                     create_unknown_user=True):
        if session_info is None or attribute_mapping is None:
            logger.error('Session info or attribute mapping are None')
            return None

        if not 'ava' in session_info:
            logger.error('"ava" key not found in session_info')
            return None

        attributes = session_info['ava']
        if not attributes:
            logger.error('The attributes dictionary is empty')

        try:
            django_user_main_attribute = settings.SAML_MAIN_USER_ATTRIBUTE
        except AttributeError:
            django_user_main_attribute = 'username'

        logger.debug('attributes: %s' % attributes)
        logger.debug('attribute_mapping: %s' % attribute_mapping)
        saml_user = None
        for saml_attr, django_fields in attribute_mapping.items():
            if (django_user_main_attribute in django_fields
                and saml_attr in attributes):
                saml_user = attributes[saml_attr][0]

        if saml_user is None:
            logger.error('Could not find saml_user value')
            return None

        user = None
        main_attribute = self.clean_user_main_attribute(saml_user)

        user_query_args = {django_user_main_attribute: main_attribute}

        # Note that this could be accomplished in one try-except clause, but
        # instead we use get_or_create when creating unknown users since it has
        # built-in safeguards for multiple threads.
        if create_unknown_user:
            logger.debug('Check if the user "%s" exists or create otherwise'
                         % main_attribute)
            user, created = User.objects.get_or_create(**user_query_args)
            if created:
                logger.debug('New user created')
                user = self.configure_user(user, attributes, attribute_mapping)
            else:
                logger.debug('User updated')
                user = self.update_user(user, attributes, attribute_mapping)
        else:
            logger.debug('Retrieving existing user "%s"' % main_attribute)
            try:
                user = User.objects.get(**user_query_args)
                user = self.update_user(user, attributes, attribute_mapping)
            except User.DoesNotExist:
                logger.error('The user "%s" does not exist' % main_attribute)
                pass

        return user

    def clean_user_main_attribute(self, main_attribute):
        """Performs any cleaning on the user main attribute (which
        usually is "username") prior to using it to get or
        create the user object.  Returns the cleaned attribute.

        By default, returns the attribute unchanged.
        """
        return main_attribute

    def configure_user(self, user, attributes, attribute_mapping):
        """Configures a user after creation and returns the updated user.

        By default, returns the user with his attributes updated.
        """
        user.set_unusable_password()
        return self.update_user(user, attributes, attribute_mapping,
                                force_save=True)

    def update_user(self, user, attributes, attribute_mapping,
                    force_save=False):
        """Update a user with a set of attributes and returns the updated user.

        By default it uses a mapping defined in the settings constant
        SAML_ATTRIBUTE_MAPPING. For each attribute, if the user object has
        that field defined it will be set, otherwise it will try to set
        it in the profile object.
        """
        if not attribute_mapping:
            return user

        try:
            profile = user.get_profile()
        except ObjectDoesNotExist:
            profile = None
        except SiteProfileNotAvailable:
            profile = None

        user_modified = False
        profile_modified = False
        for saml_attr, django_attrs in attribute_mapping.items():
            try:
                for attr in django_attrs:
                    if hasattr(user, attr):
                        setattr(user, attr, attributes[saml_attr][0])
                        user_modified = True

                    elif profile is not None and hasattr(profile, attr):
                        setattr(profile, attr, attributes[saml_attr][0])
                        profile_modified = True

            except KeyError:
                # the saml attribute is missing
                pass

        if user_modified or force_save:
            user.save()

        if profile is not None and (profile_modified or force_save):
            profile.save()

        return user
<commit_msg>check if there are more than one user when authenticating<commit_after># Copyright (C) 2010-2012 Yaco Sistemas (http://www.yaco.es)
# Copyright (C) 2009 Lorenzo Gil Sanchez <lorenzo.gil.sanchez@gmail.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#            http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging

from django.conf import settings
from django.contrib.auth.backends import ModelBackend
from django.contrib.auth.models import User, SiteProfileNotAvailable
from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned

logger = logging.getLogger('djangosaml2')

class Saml2Backend(ModelBackend):

    def authenticate(self, session_info=None, attribute_mapping=None,
                     create_unknown_user=True):
        if session_info is None or attribute_mapping is None:
            logger.error('Session info or attribute mapping are None')
            return None

        if not 'ava' in session_info:
            logger.error('"ava" key not found in session_info')
            return None

        attributes = session_info['ava']
        if not attributes:
            logger.error('The attributes dictionary is empty')

        django_user_main_attribute = getattr(
            settings, 'SAML_DJANGO_USER_MAIN_ATTRIBUTE', 'username')

        logger.debug('attributes: %s' % attributes)
        logger.debug('attribute_mapping: %s' % attribute_mapping)
        saml_user = None
        for saml_attr, django_fields in attribute_mapping.items():
            if (django_user_main_attribute in django_fields
                and saml_attr in attributes):
                saml_user = attributes[saml_attr][0]

        if saml_user is None:
            logger.error('Could not find saml_user value')
            return None

        user = None
        main_attribute = self.clean_user_main_attribute(saml_user)

        user_query_args = {django_user_main_attribute: main_attribute}

        # Note that this could be accomplished in one try-except clause, but
        # instead we use get_or_create when creating unknown users since it has
        # built-in safeguards for multiple threads.
        if create_unknown_user:
            logger.debug('Check if the user "%s" exists or create otherwise'
                         % main_attribute)
            try:
                user, created = User.objects.get_or_create(**user_query_args)
            except MultipleObjectsReturned:
                logger.error("There are more than one user with %s = %s" %
                             (django_user_main_attribute, main_attribute))
                return None

            if created:
                logger.debug('New user created')
                user = self.configure_user(user, attributes, attribute_mapping)
            else:
                logger.debug('User updated')
                user = self.update_user(user, attributes, attribute_mapping)
        else:
            logger.debug('Retrieving existing user "%s"' % main_attribute)
            try:
                user = User.objects.get(**user_query_args)
                user = self.update_user(user, attributes, attribute_mapping)
            except User.DoesNotExist:
                logger.error('The user "%s" does not exist' % main_attribute)
                return None
            except MultipleObjectsReturned:
                logger.error("There are more than one user with %s = %s" %
                             (django_user_main_attribute, main_attribute))
                return None

        return user

    def clean_user_main_attribute(self, main_attribute):
        """Performs any cleaning on the user main attribute (which
        usually is "username") prior to using it to get or
        create the user object.  Returns the cleaned attribute.

        By default, returns the attribute unchanged.
        """
        return main_attribute

    def configure_user(self, user, attributes, attribute_mapping):
        """Configures a user after creation and returns the updated user.

        By default, returns the user with his attributes updated.
        """
        user.set_unusable_password()
        return self.update_user(user, attributes, attribute_mapping,
                                force_save=True)

    def update_user(self, user, attributes, attribute_mapping,
                    force_save=False):
        """Update a user with a set of attributes and returns the updated user.

        By default it uses a mapping defined in the settings constant
        SAML_ATTRIBUTE_MAPPING. For each attribute, if the user object has
        that field defined it will be set, otherwise it will try to set
        it in the profile object.
        """
        if not attribute_mapping:
            return user

        try:
            profile = user.get_profile()
        except ObjectDoesNotExist:
            profile = None
        except SiteProfileNotAvailable:
            profile = None

        user_modified = False
        profile_modified = False
        for saml_attr, django_attrs in attribute_mapping.items():
            try:
                for attr in django_attrs:
                    if hasattr(user, attr):
                        setattr(user, attr, attributes[saml_attr][0])
                        user_modified = True

                    elif profile is not None and hasattr(profile, attr):
                        setattr(profile, attr, attributes[saml_attr][0])
                        profile_modified = True

            except KeyError:
                # the saml attribute is missing
                pass

        if user_modified or force_save:
            user.save()

        if profile is not None and (profile_modified or force_save):
            profile.save()

        return user
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from ._base import SQLBaseStore, Table

from collections import namedtuple

import logging

logger = logging.getLogger(__name__)

class TransactionStore(SQLBaseStore):
    """A collection of queries for handling PDUs.
    """

    def get_received_txn_response(self, transaction_id, origin):
        """For an incoming transaction from a given origin, check if we have
        already responded to it. If so, return the response code and response
        body (as a dict).

        Args:
            transaction_id (str)
            origin(str)

        Returns:
            tuple: None if we have not previously responded to
            this transaction or a 2-tuple of (int, dict)
        """

        return self.runInteraction(
            "get_received_txn_response",
            self._get_received_txn_response, transaction_id, origin
        )

    def _get_received_txn_response(self, txn, transaction_id, origin):
        where_clause = "transaction_id = ? AND origin = ?"
        query = ReceivedTransactionsTable.select_statement(where_clause)

        txn.execute(query, (transaction_id, origin))

        results = ReceivedTransactionsTable.decode_results(txn.fetchall())

        if results and results[0].response_code:
            return (results[0].response_code, results[0].response_json)
        else:
            return None

    def set_received_txn_response(self, transaction_id, origin, code,
                                  response_dict):
        """Persist the response we returened for an incoming transaction, and
        should return for subsequent transactions with the same transaction_id
        and origin.

        Args:
            txn
            transaction_id (str)
            origin (str)
            code (int)
            response_json (str)
        """

        return self.runInteraction(
            "set_received_txn_response",
            self._set_received_txn_response,
            transaction_id, origin, code, response_dict
        )

    def _set_received_txn_response(self, txn, transaction_id, origin, code,
                                   response_json):
        query = (
            "UPDATE %s "
            "SET response_code = ?, response_json = ? "
            "WHERE transaction_id = ? AND origin = ?"
        ) % ReceivedTransactionsTable.table_name

        txn.execute(query, (code, response_json, transaction_id, origin))

    def prep_send_transaction(self, transaction_id, destination,
                              origin_server_ts):
        """Persists an outgoing transaction and calculates the values for the
        previous transaction id list.

        This should be called before sending the transaction so that it has the
        correct value for the `prev_ids` key.

        Args:
            transaction_id (str)
            destination (str)
            origin_server_ts (int)

        Returns:
            list: A list of previous transaction ids.
        """

        return self.runInteraction(
            "prep_send_transaction",
            self._prep_send_transaction,
            transaction_id, destination, origin_server_ts
        )

    def _prep_send_transaction(self, txn, transaction_id, destination,
                               origin_server_ts):

        # First we find out what the prev_txns should be.
        # Since we know that we are only sending one transaction at a time,
        # we can simply take the last one.
        query = "%s ORDER BY id DESC LIMIT 1" % (
                SentTransactions.select_statement("destination = ?"),
            )

        results = txn.execute(query, (destination,))
        results = SentTransactions.decode_results(results)

        prev_txns = [r.transaction_id for r in results]

        # Actually add the new transaction to the sent_transactions table.

        query = SentTransactions.insert_statement()
        txn.execute(query, SentTransactions.EntryType(
            None,
            transaction_id=transaction_id,
            destination=destination,
            ts=origin_server_ts,
            response_code=0,
            response_json=None
        ))

        # Update the tx id -> pdu id mapping

        # values = [
        #     (transaction_id, destination, pdu[0], pdu[1])
        #     for pdu in pdu_list
        # ]
        #
        # logger.debug("Inserting: %s", repr(values))
        #
        # query = TransactionsToPduTable.insert_statement()
        # txn.executemany(query, values)

        return prev_txns

    def delivered_txn(self, transaction_id, destination, code, response_dict):
        """Persists the response for an outgoing transaction.

        Args:
            transaction_id (str)
            destination (str)
            code (int)
            response_json (str)
        """
        return self.runInteraction(
            "delivered_txn",
            self._delivered_txn,
            transaction_id, destination, code, response_dict
        )

    def _delivered_txn(cls, txn, transaction_id, destination,
                       code, response_json):
        query = (
            "UPDATE %s "
            "SET response_code = ?, response_json = ? "
            "WHERE transaction_id = ? AND destination = ?"
        ) % SentTransactions.table_name

        txn.execute(query, (code, response_json, transaction_id, destination))

    def get_transactions_after(self, transaction_id, destination):
        """Get all transactions after a given local transaction_id.

        Args:
            transaction_id (str)
            destination (str)

        Returns:
            list: A list of `ReceivedTransactionsTable.EntryType`
        """
        return self.runInteraction(
            "get_transactions_after",
            self._get_transactions_after, transaction_id, destination
        )

    def _get_transactions_after(cls, txn, transaction_id, destination):
        where = (
            "destination = ? AND id > (select id FROM %s WHERE "
            "transaction_id = ? AND destination = ?)"
        ) % (
            SentTransactions.table_name
        )
        query = SentTransactions.select_statement(where)

        txn.execute(query, (destination, transaction_id, destination))

        return ReceivedTransactionsTable.decode_results(txn.fetchall())

    def get_destination_retry_timings(self, destination):
        """Gets the current retry timings (if any) for a given destination.

        Args:
            destination (str)

        Returns:
            None if not retrying
            tuple: (retry_last_ts, retry_interval)
                retry_ts: time of last retry attempt in unix epoch ms
                retry_interval: how long until next retry in ms
        """
        return self.runInteraction(
            "get_destination_retry_timings",
            self._get_destination_retry_timings, destination)

    def _get_destination_retry_timings(cls, txn, destination):
        query = DestinationsTable.select_statement("destination = ?")
        txn.execute(query, (destination,))
        result = DestinationsTable.decode_single_result(txn.fetchone())
        if result and result[0] > 0:
            return result
        else:
            return None

    def set_destination_retry_timings(self, destination, retry_last_ts, retry_interval):
        """Sets the current retry timings for a given destination.
        Both timings should be zero if retrying is no longer occuring.

        Args:
            destination (str)
            retry_last_ts (int) - time of last retry attempt in unix epoch ms
            retry_interval (int) - how long until next retry in ms
        """
        return self.runInteraction(
            "set_destination_retry_timings",
            self._set_destination_retry_timings, destination, retry_last_ts, retry_interval)

    def _set_destination_retry_timings(cls, txn, destination, retry_last_ts, retry_interval):

        query = (
            "INSERT OR REPLACE INTO %s "
            "(retry_last_ts, retry_interval) "
            "VALUES (?, ?) "
            "WHERE destination = ?"
        ) % DestinationsTable.table_name

        txn.execute(query, (retry_last_ts, retry_interval, destination))

    def get_destinations_needing_retry(self):
        """Get all destinations which are due a retry for sending a transaction.

        Returns:
            list: A list of `DestinationsTable.EntryType`
        """
        return self.runInteraction(
            "get_destinations_needing_retry",
            self._get_destinations_needing_retry
        )

    def _get_destinations_needing_retry(cls, txn):
        where = "retry_last_ts > 0 and retry_next_ts < now()"
        query = DestinationsTable.select_statement(where)
        txn.execute(query)
        return DestinationsTable.decode_results(txn.fetchall())

class ReceivedTransactionsTable(Table):
    table_name = "received_transactions"

    fields = [
        "transaction_id",
        "origin",
        "ts",
        "response_code",
        "response_json",
        "has_been_referenced",
    ]

    EntryType = namedtuple("ReceivedTransactionsEntry", fields)

class SentTransactions(Table):
    table_name = "sent_transactions"

    fields = [
        "id",
        "transaction_id",
        "destination",
        "ts",
        "response_code",
        "response_json",
    ]

    EntryType = namedtuple("SentTransactionsEntry", fields)

class TransactionsToPduTable(Table):
    table_name = "transaction_id_to_pdu"

    fields = [
        "transaction_id",
        "destination",
        "pdu_id",
        "pdu_origin",
    ]

    EntryType = namedtuple("TransactionsToPduEntry", fields)

class DestinationsTable(Table):
    table_name = "destinations"

    fields = [
        "destination",
        "retry_last_ts",
        "retry_interval",
    ]

    EntryType = namedtuple("DestinationsEntry", fields)<commit_msg>add a write-through cache on the retry schedule<commit_after># -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from ._base import SQLBaseStore, Table

from collections import namedtuple

import logging

logger = logging.getLogger(__name__)

class TransactionStore(SQLBaseStore):
    """A collection of queries for handling PDUs.
    """

    # a write-through cache of DestinationsTable.EntryType indexed by destination string
    destination_retry_cache = {}

    def get_received_txn_response(self, transaction_id, origin):
        """For an incoming transaction from a given origin, check if we have
        already responded to it. If so, return the response code and response
        body (as a dict).

        Args:
            transaction_id (str)
            origin(str)

        Returns:
            tuple: None if we have not previously responded to
            this transaction or a 2-tuple of (int, dict)
        """

        return self.runInteraction(
            "get_received_txn_response",
            self._get_received_txn_response, transaction_id, origin
        )

    def _get_received_txn_response(self, txn, transaction_id, origin):
        where_clause = "transaction_id = ? AND origin = ?"
        query = ReceivedTransactionsTable.select_statement(where_clause)

        txn.execute(query, (transaction_id, origin))

        results = ReceivedTransactionsTable.decode_results(txn.fetchall())

        if results and results[0].response_code:
            return (results[0].response_code, results[0].response_json)
        else:
            return None

    def set_received_txn_response(self, transaction_id, origin, code,
                                  response_dict):
        """Persist the response we returened for an incoming transaction, and
        should return for subsequent transactions with the same transaction_id
        and origin.

        Args:
            txn
            transaction_id (str)
            origin (str)
            code (int)
            response_json (str)
        """

        return self.runInteraction(
            "set_received_txn_response",
            self._set_received_txn_response,
            transaction_id, origin, code, response_dict
        )

    def _set_received_txn_response(self, txn, transaction_id, origin, code,
                                   response_json):
        query = (
            "UPDATE %s "
            "SET response_code = ?, response_json = ? "
            "WHERE transaction_id = ? AND origin = ?"
        ) % ReceivedTransactionsTable.table_name

        txn.execute(query, (code, response_json, transaction_id, origin))

    def prep_send_transaction(self, transaction_id, destination,
                              origin_server_ts):
        """Persists an outgoing transaction and calculates the values for the
        previous transaction id list.

        This should be called before sending the transaction so that it has the
        correct value for the `prev_ids` key.

        Args:
            transaction_id (str)
            destination (str)
            origin_server_ts (int)

        Returns:
            list: A list of previous transaction ids.
        """

        return self.runInteraction(
            "prep_send_transaction",
            self._prep_send_transaction,
            transaction_id, destination, origin_server_ts
        )

    def _prep_send_transaction(self, txn, transaction_id, destination,
                               origin_server_ts):

        # First we find out what the prev_txns should be.
        # Since we know that we are only sending one transaction at a time,
        # we can simply take the last one.
        query = "%s ORDER BY id DESC LIMIT 1" % (
                SentTransactions.select_statement("destination = ?"),
            )

        results = txn.execute(query, (destination,))
        results = SentTransactions.decode_results(results)

        prev_txns = [r.transaction_id for r in results]

        # Actually add the new transaction to the sent_transactions table.

        query = SentTransactions.insert_statement()
        txn.execute(query, SentTransactions.EntryType(
            None,
            transaction_id=transaction_id,
            destination=destination,
            ts=origin_server_ts,
            response_code=0,
            response_json=None
        ))

        # Update the tx id -> pdu id mapping

        # values = [
        #     (transaction_id, destination, pdu[0], pdu[1])
        #     for pdu in pdu_list
        # ]
        #
        # logger.debug("Inserting: %s", repr(values))
        #
        # query = TransactionsToPduTable.insert_statement()
        # txn.executemany(query, values)

        return prev_txns

    def delivered_txn(self, transaction_id, destination, code, response_dict):
        """Persists the response for an outgoing transaction.

        Args:
            transaction_id (str)
            destination (str)
            code (int)
            response_json (str)
        """
        return self.runInteraction(
            "delivered_txn",
            self._delivered_txn,
            transaction_id, destination, code, response_dict
        )

    def _delivered_txn(cls, txn, transaction_id, destination,
                       code, response_json):
        query = (
            "UPDATE %s "
            "SET response_code = ?, response_json = ? "
            "WHERE transaction_id = ? AND destination = ?"
        ) % SentTransactions.table_name

        txn.execute(query, (code, response_json, transaction_id, destination))

    def get_transactions_after(self, transaction_id, destination):
        """Get all transactions after a given local transaction_id.

        Args:
            transaction_id (str)
            destination (str)

        Returns:
            list: A list of `ReceivedTransactionsTable.EntryType`
        """
        return self.runInteraction(
            "get_transactions_after",
            self._get_transactions_after, transaction_id, destination
        )

    def _get_transactions_after(cls, txn, transaction_id, destination):
        where = (
            "destination = ? AND id > (select id FROM %s WHERE "
            "transaction_id = ? AND destination = ?)"
        ) % (
            SentTransactions.table_name
        )
        query = SentTransactions.select_statement(where)

        txn.execute(query, (destination, transaction_id, destination))

        return ReceivedTransactionsTable.decode_results(txn.fetchall())

    def get_destination_retry_timings(self, destination):
        """Gets the current retry timings (if any) for a given destination.

        Args:
            destination (str)

        Returns:
            None if not retrying
            Otherwise a DestinationsTable.EntryType for the retry scheme
        """
        if self.destination_retry_cache[destination]:
            return self.destination_retry_cache[destination]

        return self.runInteraction(
            "get_destination_retry_timings",
            self._get_destination_retry_timings, destination)

    def _get_destination_retry_timings(cls, txn, destination):
        query = DestinationsTable.select_statement("destination = ?")
        txn.execute(query, (destination,))
        result = DestinationsTable.decode_single_result(txn.fetchone())
        if result and result.retry_last_ts > 0:
            return result
        else:
            return None

    def set_destination_retry_timings(self, destination, retry_last_ts, retry_interval):
        """Sets the current retry timings for a given destination.
        Both timings should be zero if retrying is no longer occuring.

        Args:
            destination (str)
            retry_last_ts (int) - time of last retry attempt in unix epoch ms
            retry_interval (int) - how long until next retry in ms
        """

        self.destination_retry_cache[destination] = (
            DestinationsTable.EntryType(destination, retry_last_ts, retry_interval)
        )

        # xxx: we could chose to not bother persisting this if our cache things this is a NOOP
        return self.runInteraction(
            "set_destination_retry_timings",
            self._set_destination_retry_timings, destination, retry_last_ts, retry_interval)

    def _set_destination_retry_timings(cls, txn, destination, retry_last_ts, retry_interval):

        query = (
            "INSERT OR REPLACE INTO %s "
            "(retry_last_ts, retry_interval) "
            "VALUES (?, ?) "
            "WHERE destination = ?"
        ) % DestinationsTable.table_name

        txn.execute(query, (retry_last_ts, retry_interval, destination))

    def get_destinations_needing_retry(self):
        """Get all destinations which are due a retry for sending a transaction.

        Returns:
            list: A list of `DestinationsTable.EntryType`
        """

        return self.runInteraction(
            "get_destinations_needing_retry",
            self._get_destinations_needing_retry
        )

    def _get_destinations_needing_retry(cls, txn):
        where = "retry_last_ts > 0 and retry_next_ts < now()"
        query = DestinationsTable.select_statement(where)
        txn.execute(query)
        return DestinationsTable.decode_results(txn.fetchall())

class ReceivedTransactionsTable(Table):
    table_name = "received_transactions"

    fields = [
        "transaction_id",
        "origin",
        "ts",
        "response_code",
        "response_json",
        "has_been_referenced",
    ]

    EntryType = namedtuple("ReceivedTransactionsEntry", fields)

class SentTransactions(Table):
    table_name = "sent_transactions"

    fields = [
        "id",
        "transaction_id",
        "destination",
        "ts",
        "response_code",
        "response_json",
    ]

    EntryType = namedtuple("SentTransactionsEntry", fields)

class TransactionsToPduTable(Table):
    table_name = "transaction_id_to_pdu"

    fields = [
        "transaction_id",
        "destination",
        "pdu_id",
        "pdu_origin",
    ]

    EntryType = namedtuple("TransactionsToPduEntry", fields)

class DestinationsTable(Table):
    table_name = "destinations"

    fields = [
        "destination",
        "retry_last_ts",
        "retry_interval",
    ]

    EntryType = namedtuple("DestinationsEntry", fields)<|endoftext|>