<commit_before># Copyright 2015 Spotify AB. All rights reserved.
#
# The contents of this file are licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
import napalm.exceptions
import sys

class NetworkDriver:

    def __init__(self, hostname, username, password, timeout):
        """
        This is the base class you have to inherit from when writing your own Network Driver to manage any device. You
        will, in addition, have to override all the methods specified on this class. Make sure you follow the guidelines
        for every method and that you return the correct data.

        :param hostname: (str) IP or FQDN of the device you want to connect to.
        :param username: (str) Username you want to use
        :param password: (str) Password
        :param timeout: (int) Time in seconds to wait for the device to respond.
        :return:
        """
        raise NotImplementedError

    def __enter__(self):
        try:
            self.open()
        except:
            exc_info = sys.exc_info()
            self.__raise_clean_exception(exc_info[0], exc_info[1], exc_info[2])
        return self

    def __exit__(self, exc_type, exc_value, exc_traceback):
        self.close()
        if exc_type is not None:
            self.__raise_clean_exception(exc_type, exc_value, exc_traceback)

    @staticmethod
    def __raise_clean_exception(exc_type, exc_value, exc_traceback):
        """
        This method is going to check if the exception exc_type is part of the builtins exceptions or part of the
        napalm exceptions. If it is not, it will print a message on the screen giving instructions to fill a bug.

        :param exc_type: Exception class.
        :param exc_value: Exception object.
        :param exc_traceback: Traceback.
        :return:
        """
        if exc_type.__name__ not in dir(napalm.exceptions) and \
                        exc_type.__name__ not in __builtins__.keys():
            epilog = ("NAPALM didn't catch this exception. Please, fill a bugfix on "
                      "https://github.com/napalm-automation/napalm/issues\n"
                      "Don't forget to include this traceback.")
            print(epilog)
        raise exc_type, exc_value, exc_traceback

    def open(self):
        """
        Opens a connection to the device.
        """
        raise NotImplementedError

    def close(self):
        """
        Closes the connection to the device.
        """
        raise NotImplementedError

    def load_replace_candidate(self, filename=None, config=None):
        """
        Populates the candidate configuration. You can populate it from a file or from a string. If you send both a
        filename and a string containing the configuration, the file takes precedence.

        If you use this method the existing configuration will be replaced entirely by the candidate configuration once
        you commit the changes. This method will not change the configuration by itself.

        :param filename: Path to the file containing the desired configuration. By default is None.
        :param config: String containing the desired configuration.
        :raise ReplaceConfigException: If there is an error on the configuration sent.
        """
        raise NotImplementedError

    def load_merge_candidate(self, filename=None, config=None):
        """
        Populates the candidate configuration. You can populate it from a file or from a string. If you send both a
        filename and a string containing the configuration, the file takes precedence.

        If you use this method the existing configuration will be merged with the candidate configuration once
        you commit the changes. This method will not change the configuration by itself.

        :param filename: Path to the file containing the desired configuration. By default is None.
        :param config: String containing the desired configuration.
        :raise MergeConfigException: If there is an error on the configuration sent.
        """
        raise NotImplementedError

    def compare_config(self):
        """
        :return: A string showing the difference between the running configuration and the candidate configuration. The\
        running_config is loaded automatically just before doing the comparison so there is no need for you to do it.
        """
        raise NotImplementedError

    def commit_config(self):
        """
        Commits the changes requested by the method load_replace_candidate or load_merge_candidate.
        """
        raise NotImplementedError

    def discard_config(self):
        """
        Discards the configuration loaded into the candidate.
        """
        raise NotImplementedError

    def rollback(self):
        """
        If changes were made, revert changes to the original state.
        """
        raise NotImplementedError

    def get_facts(self):
        """
        Returns a dictionary containing the following information:
         * uptime - Uptime of the device in seconds.
         * vendor - Manufacturer of the device.
         * model - Device model.
         * hostname - Hostname of the device
         * fqdn - Fqdn of the device
         * os_version - String with the OS version running on the device.
         * serial_number - Serial number of the device
         * interface_list - List of the interfaces of the device

        For example::

            {
            'uptime': 151005.57332897186,
            'vendor': u'Arista',
            'os_version': u'4.14.3-2329074.gaatlantarel',
            'serial_number': u'SN0123A34AS',
            'model': u'vEOS',
            'hostname': u'eos-router',
            'fqdn': u'eos-router',
            'interface_list': [u'Ethernet2', u'Management1', u'Ethernet1', u'Ethernet3']
            }

        """
        raise NotImplementedError

    def get_interfaces(self):
        """
        Returns a dictionary of dictionaries. The keys for the first dictionary will be the interfaces in the devices.\
        The inner dictionary will containing the following data for each interface:
         * is_up (True/False)
         * is_enabled (True/False)
         * description (string)
         * last_flapped (int in seconds)
         * speed (int in Mbit)
         * mac_address (string)

        For example::

            {
            u'Management1':
                {
                'is_up': False,
                'is_enabled': False,
                'description': u'',
                'last_flapped': -1,
                'speed': 1000,
                'mac_address': u'dead:beef:dead',
                },
            u'Ethernet1':
                {
                'is_up': True,
                'is_enabled': True,
                'description': u'foo',
                'last_flapped': 1429978575.1554043,
                'speed': 1000,
                'mac_address': u'beef:dead:beef',
                },
            u'Ethernet2':
                {
                'is_up': True,
                'is_enabled': True,
                'description': u'bla',
                'last_flapped': 1429978575.1555667,
                'speed': 1000,
                'mac_address': u'beef:beef:beef',
                },
            u'Ethernet3':
                {
                'is_up': False,
                'is_enabled': True,
                'description': u'bar',
                'last_flapped': -1,
                'speed': 1000,
                'mac_address': u'dead:dead:dead',
                }
            }
        """
        raise NotImplementedError

    def get_lldp_neighbors(self):
        """
        Returns a dictionary where the keys are local ports and the value is a list of dictionaries with the following \
        information:
            * hostname
            * port

        For example::

            {
            u'Ethernet2':
                [
                    {
                    'hostname': u'junos-unittest',
                    'port': u'520',
                    }
                ],
            u'Ethernet3':
                [
                    {
                    'hostname': u'junos-unittest',
                    'port': u'522',
                    }
                ],
            u'Ethernet1':
                [
                    {
                    'hostname': u'junos-unittest',
                    'port': u'519',
                    },
                    {
                    'hostname': u'ios-xrv-unittest',
                    'port': u'Gi0/0/0/0',
                    }
                ],
            u'Management1':
                [
                    {
                    'hostname': u'junos-unittest',
                    'port': u'508',
                    }
                ]
            }
        """
        raise NotImplementedError

    def get_bgp_neighbors(self):
        """
        Returns a dictionary of dictionaries. The keys for the first dictionary will be the vrf (global if no vrf).
        The inner dictionary will contain the following data for each vrf:
         * router_id
         * peers - another dictionary of dictionaries. Outer keys are the IPs of the neighbors. The inner keys are:
           * local_as (int)
           * remote_as (int)
           * remote_id - peer router id
           * is_up (True/False)
           * is_enabled (True/False)
           * description (string)
           * uptime (int in seconds)
           * address_family (dictionary) - A dictionary of address families available for the neighbor. So far it can
             be 'ipv4' or 'ipv6'
               * received_prefixes (int)
               * accepted_prefixes (int)
               * sent_prefixes (int)
        """
        raise NotImplementedError

    def get_environment(self):
        """
        Returns a dictionary where:
            * fans is a dictionary of dictionaries where the key is the location and the values:
                * status (boolean) - True if it's ok, false if it's broken
            * temperature is a dictionary of dictionaries where the key is the location and the values:
                * temperature (float) - Temperature in celsius the sensor is reporting.
                * is_alert (boolean) - True if the temperature is above the alert threshold
                * is_critical (boolean) - True if the temperature is above the critical threshold
            * power is a dictionary of dictionaries where the key is the PSU id and the values:
                * status (boolean) - True if it's ok, false if it's broken
                * capacity (float) - Capacity in W that the power supply can support
                * output (float) - Watts drawn by the system
            * cpu is a dictionary of dictionaries where the key is the ID and the values
                * %usage
            * memory is a dictionary with:
                * available_ram (int) - Total amount of RAM installed in the device
                * used_ram (int) - RAM in use in the device
        """
        raise NotImplementedError

    def get_interfaces_counters(self):
        """
        Returns a dictionary of dictionaries where the first key is an interface name and the inner dictionary contains
        the following keys:
            * tx_errors (int)
            * rx_errors (int)
            * tx_discards (int)
            * rx_discards (int)
            * tx_octets (int)
            * rx_octets (int)
            * tx_unicast_packets (int)
            * rx_unicast_packets (int)
            * tx_multicast_packets (int)
            * rx_multicast_packets (int)
            * tx_broadcast_packets (int)
            * rx_broadcast_packets (int)

        Example:

            {
                u'Ethernet2': {
                    'tx_multicast_packets': 699,
                    'tx_discards': 0,
                    'tx_octets': 88577,
                    'tx_errors': 0,
                    'rx_octets': 0,
                    'tx_unicast_packets': 0,
                    'rx_errors': 0,
                    'tx_broadcast_packets': 0,
                    'rx_multicast_packets': 0,
                    'rx_broadcast_packets': 0,
                    'rx_discards': 0,
                    'rx_unicast_packets': 0
                },
                u'Management1': {
                     'tx_multicast_packets': 0,
                     'tx_discards': 0,
                     'tx_octets': 159159,
                     'tx_errors': 0,
                     'rx_octets': 167644,
                     'tx_unicast_packets': 1241,
                     'rx_errors': 0,
                     'tx_broadcast_packets': 0,
                     'rx_multicast_packets': 0,
                     'rx_broadcast_packets': 80,
                     'rx_discards': 0,
                     'rx_unicast_packets': 0
                },
                u'Ethernet1': {
                     'tx_multicast_packets': 293,
                     'tx_discards': 0,
                     'tx_octets': 38639,
                     'tx_errors': 0,
                     'rx_octets': 0,
                     'tx_unicast_packets': 0,
                     'rx_errors': 0,
                     'tx_broadcast_packets': 0,
                     'rx_multicast_packets': 0,
                     'rx_broadcast_packets': 0,
                     'rx_discards': 0,
                     'rx_unicast_packets': 0
                }
            }
        """
        raise NotImplementedError
<commit_msg>Added context manager support<commit_after># Copyright 2015 Spotify AB. All rights reserved.
#
# The contents of this file are licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
import napalm.exceptions
import sys

class NetworkDriver:

    def __init__(self, hostname, username, password, timeout):
        """
        This is the base class you have to inherit from when writing your own Network Driver to manage any device. You
        will, in addition, have to override all the methods specified on this class. Make sure you follow the guidelines
        for every method and that you return the correct data.

        :param hostname: (str) IP or FQDN of the device you want to connect to.
        :param username: (str) Username you want to use
        :param password: (str) Password
        :param timeout: (int) Time in seconds to wait for the device to respond.
        :return:
        """
        raise NotImplementedError

    def __enter__(self):
        try:
            self.open()
        except:
            exc_info = sys.exc_info()
            self.__raise_clean_exception(exc_info[0], exc_info[1], exc_info[2])
        return self

    def __exit__(self, exc_type, exc_value, exc_traceback):
        self.close()
        if exc_type is not None:
            self.__raise_clean_exception(exc_type, exc_value, exc_traceback)

    @staticmethod
    def __raise_clean_exception(exc_type, exc_value, exc_traceback):
        """
        This method is going to check if the exception exc_type is part of the builtins exceptions or part of the
        napalm exceptions. If it is not, it will print a message on the screen giving instructions to fill a bug.
        Finally it will raise the original exception.

        :param exc_type: Exception class.
        :param exc_value: Exception object.
        :param exc_traceback: Traceback.
        """
        if exc_type.__name__ not in dir(napalm.exceptions) and \
                        exc_type.__name__ not in __builtins__.keys():
            epilog = ("NAPALM didn't catch this exception. Please, fill a bugfix on "
                      "https://github.com/napalm-automation/napalm/issues\n"
                      "Don't forget to include this traceback.")
            print(epilog)
        raise exc_type, exc_value, exc_traceback

    def open(self):
        """
        Opens a connection to the device.
        """
        raise NotImplementedError

    def close(self):
        """
        Closes the connection to the device.
        """
        raise NotImplementedError

    def load_replace_candidate(self, filename=None, config=None):
        """
        Populates the candidate configuration. You can populate it from a file or from a string. If you send both a
        filename and a string containing the configuration, the file takes precedence.

        If you use this method the existing configuration will be replaced entirely by the candidate configuration once
        you commit the changes. This method will not change the configuration by itself.

        :param filename: Path to the file containing the desired configuration. By default is None.
        :param config: String containing the desired configuration.
        :raise ReplaceConfigException: If there is an error on the configuration sent.
        """
        raise NotImplementedError

    def load_merge_candidate(self, filename=None, config=None):
        """
        Populates the candidate configuration. You can populate it from a file or from a string. If you send both a
        filename and a string containing the configuration, the file takes precedence.

        If you use this method the existing configuration will be merged with the candidate configuration once
        you commit the changes. This method will not change the configuration by itself.

        :param filename: Path to the file containing the desired configuration. By default is None.
        :param config: String containing the desired configuration.
        :raise MergeConfigException: If there is an error on the configuration sent.
        """
        raise NotImplementedError

    def compare_config(self):
        """
        :return: A string showing the difference between the running configuration and the candidate configuration. The\
        running_config is loaded automatically just before doing the comparison so there is no need for you to do it.
        """
        raise NotImplementedError

    def commit_config(self):
        """
        Commits the changes requested by the method load_replace_candidate or load_merge_candidate.
        """
        raise NotImplementedError

    def discard_config(self):
        """
        Discards the configuration loaded into the candidate.
        """
        raise NotImplementedError

    def rollback(self):
        """
        If changes were made, revert changes to the original state.
        """
        raise NotImplementedError

    def get_facts(self):
        """
        Returns a dictionary containing the following information:
         * uptime - Uptime of the device in seconds.
         * vendor - Manufacturer of the device.
         * model - Device model.
         * hostname - Hostname of the device
         * fqdn - Fqdn of the device
         * os_version - String with the OS version running on the device.
         * serial_number - Serial number of the device
         * interface_list - List of the interfaces of the device

        For example::

            {
            'uptime': 151005.57332897186,
            'vendor': u'Arista',
            'os_version': u'4.14.3-2329074.gaatlantarel',
            'serial_number': u'SN0123A34AS',
            'model': u'vEOS',
            'hostname': u'eos-router',
            'fqdn': u'eos-router',
            'interface_list': [u'Ethernet2', u'Management1', u'Ethernet1', u'Ethernet3']
            }

        """
        raise NotImplementedError

    def get_interfaces(self):
        """
        Returns a dictionary of dictionaries. The keys for the first dictionary will be the interfaces in the devices.\
        The inner dictionary will containing the following data for each interface:
         * is_up (True/False)
         * is_enabled (True/False)
         * description (string)
         * last_flapped (int in seconds)
         * speed (int in Mbit)
         * mac_address (string)

        For example::

            {
            u'Management1':
                {
                'is_up': False,
                'is_enabled': False,
                'description': u'',
                'last_flapped': -1,
                'speed': 1000,
                'mac_address': u'dead:beef:dead',
                },
            u'Ethernet1':
                {
                'is_up': True,
                'is_enabled': True,
                'description': u'foo',
                'last_flapped': 1429978575.1554043,
                'speed': 1000,
                'mac_address': u'beef:dead:beef',
                },
            u'Ethernet2':
                {
                'is_up': True,
                'is_enabled': True,
                'description': u'bla',
                'last_flapped': 1429978575.1555667,
                'speed': 1000,
                'mac_address': u'beef:beef:beef',
                },
            u'Ethernet3':
                {
                'is_up': False,
                'is_enabled': True,
                'description': u'bar',
                'last_flapped': -1,
                'speed': 1000,
                'mac_address': u'dead:dead:dead',
                }
            }
        """
        raise NotImplementedError

    def get_lldp_neighbors(self):
        """
        Returns a dictionary where the keys are local ports and the value is a list of dictionaries with the following \
        information:
            * hostname
            * port

        For example::

            {
            u'Ethernet2':
                [
                    {
                    'hostname': u'junos-unittest',
                    'port': u'520',
                    }
                ],
            u'Ethernet3':
                [
                    {
                    'hostname': u'junos-unittest',
                    'port': u'522',
                    }
                ],
            u'Ethernet1':
                [
                    {
                    'hostname': u'junos-unittest',
                    'port': u'519',
                    },
                    {
                    'hostname': u'ios-xrv-unittest',
                    'port': u'Gi0/0/0/0',
                    }
                ],
            u'Management1':
                [
                    {
                    'hostname': u'junos-unittest',
                    'port': u'508',
                    }
                ]
            }
        """
        raise NotImplementedError

    def get_bgp_neighbors(self):
        """
        Returns a dictionary of dictionaries. The keys for the first dictionary will be the vrf (global if no vrf).
        The inner dictionary will contain the following data for each vrf:
         * router_id
         * peers - another dictionary of dictionaries. Outer keys are the IPs of the neighbors. The inner keys are:
           * local_as (int)
           * remote_as (int)
           * remote_id - peer router id
           * is_up (True/False)
           * is_enabled (True/False)
           * description (string)
           * uptime (int in seconds)
           * address_family (dictionary) - A dictionary of address families available for the neighbor. So far it can
             be 'ipv4' or 'ipv6'
               * received_prefixes (int)
               * accepted_prefixes (int)
               * sent_prefixes (int)
        """
        raise NotImplementedError

    def get_environment(self):
        """
        Returns a dictionary where:
            * fans is a dictionary of dictionaries where the key is the location and the values:
                * status (boolean) - True if it's ok, false if it's broken
            * temperature is a dictionary of dictionaries where the key is the location and the values:
                * temperature (float) - Temperature in celsius the sensor is reporting.
                * is_alert (boolean) - True if the temperature is above the alert threshold
                * is_critical (boolean) - True if the temperature is above the critical threshold
            * power is a dictionary of dictionaries where the key is the PSU id and the values:
                * status (boolean) - True if it's ok, false if it's broken
                * capacity (float) - Capacity in W that the power supply can support
                * output (float) - Watts drawn by the system
            * cpu is a dictionary of dictionaries where the key is the ID and the values
                * %usage
            * memory is a dictionary with:
                * available_ram (int) - Total amount of RAM installed in the device
                * used_ram (int) - RAM in use in the device
        """
        raise NotImplementedError

    def get_interfaces_counters(self):
        """
        Returns a dictionary of dictionaries where the first key is an interface name and the inner dictionary contains
        the following keys:
            * tx_errors (int)
            * rx_errors (int)
            * tx_discards (int)
            * rx_discards (int)
            * tx_octets (int)
            * rx_octets (int)
            * tx_unicast_packets (int)
            * rx_unicast_packets (int)
            * tx_multicast_packets (int)
            * rx_multicast_packets (int)
            * tx_broadcast_packets (int)
            * rx_broadcast_packets (int)

        Example:

            {
                u'Ethernet2': {
                    'tx_multicast_packets': 699,
                    'tx_discards': 0,
                    'tx_octets': 88577,
                    'tx_errors': 0,
                    'rx_octets': 0,
                    'tx_unicast_packets': 0,
                    'rx_errors': 0,
                    'tx_broadcast_packets': 0,
                    'rx_multicast_packets': 0,
                    'rx_broadcast_packets': 0,
                    'rx_discards': 0,
                    'rx_unicast_packets': 0
                },
                u'Management1': {
                     'tx_multicast_packets': 0,
                     'tx_discards': 0,
                     'tx_octets': 159159,
                     'tx_errors': 0,
                     'rx_octets': 167644,
                     'tx_unicast_packets': 1241,
                     'rx_errors': 0,
                     'tx_broadcast_packets': 0,
                     'rx_multicast_packets': 0,
                     'rx_broadcast_packets': 80,
                     'rx_discards': 0,
                     'rx_unicast_packets': 0
                },
                u'Ethernet1': {
                     'tx_multicast_packets': 293,
                     'tx_discards': 0,
                     'tx_octets': 38639,
                     'tx_errors': 0,
                     'rx_octets': 0,
                     'tx_unicast_packets': 0,
                     'rx_errors': 0,
                     'tx_broadcast_packets': 0,
                     'rx_multicast_packets': 0,
                     'rx_broadcast_packets': 0,
                     'rx_discards': 0,
                     'rx_unicast_packets': 0
                }
            }
        """
        raise NotImplementedError
<|endoftext|>


<commit_before>from django.conf import settings
from django.core.mail import EmailMultiAlternatives
from django.core.urlresolvers import reverse
from django.db import models
from django.utils.html import escape
import logging
import os
import re

EMAIL_RE = re.compile(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)")
log = logging.getLogger(__name__)

def secure_random_string():
    return os.urandom(24).encode('hex')

class Party(models.Model):
    pa_id = models.CharField(max_length=100, null=False, blank=False, help_text="Peoples Assembly ID", unique=True)
    name = models.CharField(max_length=200, null=False, blank=False)
    slug = models.CharField(max_length=70, null=False, blank=False)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    @property
    def icon_url(self):
        return '/static/images/parties/%s.png' % self.slug

    def as_dict(self):
        return {
            'id': self.id,
            'pa_id': self.pa_id,
            'name': self.name,
            'slug': self.slug,
            'icon_url': self.icon_url,
        }

class ConstituencyBranch(models.Model):
    pa_id = models.CharField(max_length=100, null=False, blank=False, help_text="Peoples Assembly ID", unique=True)
    name = models.CharField(max_length=200, null=False, blank=False)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def as_dict(self):
        return {
            'id': self.id,
            'pa_id': self.pa_id,
            'name': self.name,
        }

class Person(models.Model):
    pa_id = models.CharField(max_length=100, null=False, blank=False, help_text="Peoples Assembly ID", unique=True)
    name = models.CharField(max_length=70, null=False, blank=False)
    pa_url = models.TextField(null=False, blank=False)
    in_national_assembly = models.BooleanField(default=True)
    portrait_url = models.CharField(max_length=300, null=True, blank=True)
    party = models.ForeignKey(Party, null=True, blank=False)
    constituency_branches = models.ManyToManyField(ConstituencyBranch)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def as_dict(self):
        return {
            'id': self.id,
            'pa_id': self.pa_id,
            'name': self.name,
            'contactdetails': [d.as_dict() for d in self.contactdetails.all()],
            'pa_url': self.pa_url,
            'portrait_url': self.portrait_url,
            'party': self.party.as_dict() if self.party else None,
        }

    def __unicode__(self):
        return self.name

class ContactDetail(models.Model):
    person = models.ForeignKey(Person, related_name="contactdetails")
    type = models.CharField(max_length=40, help_text="Type of contact detail")
    value = models.CharField(max_length=255, help_text="The actual detail")
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def as_dict(self):
        return {
            'id': self.id,
            'type': self.type,
            'value': self.value,
        }

    def __unicode__(self):
        return "%s (%s) %s" % (self.person.name, self.type, self.value,)

class Email(models.Model):
    to_person = models.ForeignKey(Person)
    to_addresses = models.TextField(null=True, blank=True, help_text="The actuall address(es) we sent to belonging to the recipient at the time, regardless of the addresses that we currently have for them.")
    remote_ip = models.CharField(max_length=20, help_text="User's remote IP")
    user_agent = models.CharField(max_length=200, help_text="User's user agent string")
    subject = models.CharField(max_length=200, null=False, blank=False)
    body_txt = models.TextField(null=False, blank=False)
    from_name = models.CharField(max_length=70, null=False, blank=False)
    from_email = models.EmailField(null=False, blank=False)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    # This is what we use make it viewable online, but only to anyone who knows the URL
    secure_id = models.CharField(max_length=100, blank=True, unique=True, default=secure_random_string)
    # This is what we only present to the sender so that they can answer questions
    # that we can tie to an (eventually anonymous) individual.
    # THIS SHOULD NOT BE PART OF THE STANDARD as_dict dict.
    sender_secret = models.CharField(max_length=100, blank=True, unique=True, default=secure_random_string)

    @property
    def body_html(self):
        # VERY IMPORTANT: Escape any HTML someone might have entered
        html_escaped_text = escape(self.body_txt)
        return re.sub("(\r\n|\n|\r)", "
", html_escaped_text)

    def as_dict(self):
        return {
            'id': self.id,
            'to_person': self.to_person.as_dict(),
            'to_addresses': self.to_addresses,
            'subject': self.subject,
            'body_html': self.body_html,
            'body_txt': self.body_txt,
            'from_name': self.from_name,
            'from_email': self.from_email,
            'secure_id': self.secure_id,
        }

    def send(self):
        if not EMAIL_RE.match(self.from_email):
            self.from_email = "noreply@openup.org.za"
        sender = "%s <%s>" % (self.from_name, self.from_email)
        recipients = ["%s <jd@openup.org.za>" % self.to_person.name.strip() for c in self.to_person.contactdetails.filter(type='email').all()]
        if settings.SEND_EMAILS:
            log.info("Sending email to %s from %s" % (recipients, self.from_email))

            url = settings.BASE_URL + reverse('email-detail', kwargs={'secure_id': self.secure_id})
            body_txt = "%s\n\nThis message can also be viewed at %s" % (self.body_txt, url)
            body_html = "%s

This message can also be viewed at %s" % (
                self.body_html, url, url)

            email = EmailMultiAlternatives(
                self.subject,
                body_txt,
                sender,
                recipients,
                cc=[sender],
            )
            email.attach_alternative(body_html, "text/html")
            email.send()
        self.to_addresses = ", ".join(recipients)
        self.save()

    def __unicode__(self):
        return u"From: %s To: %s" % (self.from_email, self.to_addresses)

class SenderQA(models.Model):
    email = models.ForeignKey(Email)
    question = models.TextField(null=False, blank=False)
    answer = models.TextField(null=False, blank=False)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def __unicode__(self):
        return u"%d Q: %s...A: %s" % (self.email.id, self.question[:100], self.answer)
<commit_msg>Remove email-detail URL from email<commit_after>from django.conf import settings
from django.core.mail import EmailMultiAlternatives
from django.core.urlresolvers import reverse
from django.db import models
from django.utils.html import escape
import logging
import os
import re

EMAIL_RE = re.compile(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)")
log = logging.getLogger(__name__)

def secure_random_string():
    return os.urandom(24).encode('hex')

class Party(models.Model):
    pa_id = models.CharField(max_length=100, null=False, blank=False, help_text="Peoples Assembly ID", unique=True)
    name = models.CharField(max_length=200, null=False, blank=False)
    slug = models.CharField(max_length=70, null=False, blank=False)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    @property
    def icon_url(self):
        return '/static/images/parties/%s.png' % self.slug

    def as_dict(self):
        return {
            'id': self.id,
            'pa_id': self.pa_id,
            'name': self.name,
            'slug': self.slug,
            'icon_url': self.icon_url,
        }

class ConstituencyBranch(models.Model):
    pa_id = models.CharField(max_length=100, null=False, blank=False, help_text="Peoples Assembly ID", unique=True)
    name = models.CharField(max_length=200, null=False, blank=False)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def as_dict(self):
        return {
            'id': self.id,
            'pa_id': self.pa_id,
            'name': self.name,
        }

class Person(models.Model):
    pa_id = models.CharField(max_length=100, null=False, blank=False, help_text="Peoples Assembly ID", unique=True)
    name = models.CharField(max_length=70, null=False, blank=False)
    pa_url = models.TextField(null=False, blank=False)
    in_national_assembly = models.BooleanField(default=True)
    portrait_url = models.CharField(max_length=300, null=True, blank=True)
    party = models.ForeignKey(Party, null=True, blank=False)
    constituency_branches = models.ManyToManyField(ConstituencyBranch)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def as_dict(self):
        return {
            'id': self.id,
            'pa_id': self.pa_id,
            'name': self.name,
            'contactdetails': [d.as_dict() for d in self.contactdetails.all()],
            'pa_url': self.pa_url,
            'portrait_url': self.portrait_url,
            'party': self.party.as_dict() if self.party else None,
        }

    def __unicode__(self):
        return self.name

class ContactDetail(models.Model):
    person = models.ForeignKey(Person, related_name="contactdetails")
    type = models.CharField(max_length=40, help_text="Type of contact detail")
    value = models.CharField(max_length=255, help_text="The actual detail")
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def as_dict(self):
        return {
            'id': self.id,
            'type': self.type,
            'value': self.value,
        }

    def __unicode__(self):
        return "%s (%s) %s" % (self.person.name, self.type, self.value,)

class Email(models.Model):
    to_person = models.ForeignKey(Person)
    to_addresses = models.TextField(null=True, blank=True, help_text="The actuall address(es) we sent to belonging to the recipient at the time, regardless of the addresses that we currently have for them.")
    remote_ip = models.CharField(max_length=20, help_text="User's remote IP")
    user_agent = models.CharField(max_length=200, help_text="User's user agent string")
    subject = models.CharField(max_length=200, null=False, blank=False)
    body_txt = models.TextField(null=False, blank=False)
    from_name = models.CharField(max_length=70, null=False, blank=False)
    from_email = models.EmailField(null=False, blank=False)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)
    # This is what we use make it viewable online, but only to anyone who knows the URL
    secure_id = models.CharField(max_length=100, blank=True, unique=True, default=secure_random_string)
    # This is what we only present to the sender so that they can answer questions
    # that we can tie to an (eventually anonymous) individual.
    # THIS SHOULD NOT BE PART OF THE STANDARD as_dict dict.
    sender_secret = models.CharField(max_length=100, blank=True, unique=True, default=secure_random_string)

    @property
    def body_html(self):
        # VERY IMPORTANT: Escape any HTML someone might have entered
        html_escaped_text = escape(self.body_txt)
        return re.sub("(\r\n|\n|\r)", "
", html_escaped_text)

    def as_dict(self):
        return {
            'id': self.id,
            'to_person': self.to_person.as_dict(),
            'to_addresses': self.to_addresses,
            'subject': self.subject,
            'body_html': self.body_html,
            'body_txt': self.body_txt,
            'from_name': self.from_name,
            'from_email': self.from_email,
            'secure_id': self.secure_id,
        }

    def send(self):
        if not EMAIL_RE.match(self.from_email):
            self.from_email = "noreply@openup.org.za"
        sender = "%s <%s>" % (self.from_name, self.from_email)
        recipients = ["%s <jd@openup.org.za>" % self.to_person.name.strip() for c in self.to_person.contactdetails.filter(type='email').all()]
        if settings.SEND_EMAILS:
            log.info("Sending email to %s from %s" % (recipients, self.from_email))

            url = settings.BASE_URL + reverse('email-detail', kwargs={'secure_id': self.secure_id})

            email = EmailMultiAlternatives(
                self.subject,
                self.body_txt,
                sender,
                recipients,
                cc=[sender],
            )
            email.attach_alternative(self.body_html, "text/html")
            email.send()
        self.to_addresses = ", ".join(recipients)
        self.save()

    def __unicode__(self):
        return u"From: %s To: %s" % (self.from_email, self.to_addresses)

class SenderQA(models.Model):
    email = models.ForeignKey(Email)
    question = models.TextField(null=False, blank=False)
    answer = models.TextField(null=False, blank=False)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    def __unicode__(self):
        return u"%d Q: %s...A: %s" % (self.email.id, self.question[:100], self.answer)
<|endoftext|>


<commit_before>import flask
import nbserve
import os

flask_app = flask.Flask(nbserve.__progname__)
flask_app.config['DEBUG'] = True

#############
# Initialize some IPython and RunIPy services.
from IPython.html.services.notebooks.filenbmanager import FileNotebookManager
nbmanager = FileNotebookManager(notebook_dir='.')
##
# This thread initializes a notebook runner, so that it's
# ready to go on first page access.
runner = None
import threading
def make_notebook_runner():
    global runner
    from runipy.notebook_runner import NotebookRunner
    runner = NotebookRunner(None)
    print "Runner is ready."
make_notebook_runner_thread = threading.Thread(target=make_notebook_runner)
make_notebook_runner_thread.start()

def set_working_directory(path):
    if not os.path.exists(path):
        raise IOError('Path not found: %s' % os.path.abspath(path))
    nbmanager.notebook_dir = path

@flask_app.route('/')
def render_index():
    template = """

Notebooks

        {% for notebook in notebooks %}

{{notebook.name}}

        {% endfor %}

    """
    return flask.render_template_string(template, notebooks=nbmanager.list_notebooks('.'))

@flask_app.route('/<nbname>/')
def render_page(nbname):
    global runner
    from IPython.nbconvert.exporters.html import HTMLExporter

    if not nbmanager.notebook_exists(nbname):
        print "Notebook %s does not exist." % nbname
        flask.abort(404)

    print "Loading notebook %s" % nbname
    nbmanager.trust_notebook(nbname)
    nb = nbmanager.get_notebook(nbname)
    print "Making runner..."''

    # This is an ugly little bit to deal with a sporadic
    #  'queue empty' bug in iPython that only seems to
    #  happen on the integration servers...
    #  see https://github.com/paulgb/runipy/issues/36
    N_RUN_RETRIES = 4
    from Queue import Empty

    for i in range(N_RUN_RETRIES):
        try:
            if runner is None:
                make_notebook_runner_thread.join()

            # Do as complete of a reset of the kernel as we can.
            # Unfortunately, this doesn't really do a 'hard' reset
            # of any modules...
            class ResetCell(dict):
                """Simulates just enough of a notebook cell to get this
                '%reset -f' cell executed using the existing runipy
                 machinery."""
                input = "%reset -f"
            runner.run_cell(ResetCell())
            runner.nb = nb['content']
            print "Running notebook"
            runner.run_notebook(skip_exceptions=True)
            break
        except Empty as e:
            print "WARNING: Empty bug happened."
            if i >= (N_RUN_RETRIES - 1):
                raise

    print "Exporting notebook"
    from IPython.config import Config
    exporter = HTMLExporter(
        #config=Config({'HTMLExporter':{'default_template':args.template}})
        #config=Config({'HTMLExporter':{'default_template':'nbserve'}})
        #template='nbserve'
    )
    exporter.environment.loader.loaders[0].searchpath += [os.path.join(os.path.split(__file__)[0],'templates')]
    exporter.template_file = 'nbserve.tpl'
    from clearinputpreprocessor import ClearInputPreprocessor
    clear_input_preprocessor = ClearInputPreprocessor()
    clear_input_preprocessor.enabled = True
    output, resources = runner.nb, None
    #output, resources = clear_input_preprocessor(output, resources)
    output, resources = exporter.from_notebook_node(runner.nb)
    print "Returning."
    resources['metadata']['name'] = 'Test title.'
    return output

if __name__ == "__main__":
    flask_app.run()
<commit_msg>Simplified code, but its not properly seeing the template...<commit_after>import flask
import nbserve
import os

flask_app = flask.Flask(nbserve.__progname__)
flask_app.config['DEBUG'] = True

#############
# Initialize some IPython and RunIPy services.
from IPython.html.services.notebooks.filenbmanager import FileNotebookManager
nbmanager = FileNotebookManager(notebook_dir='.')
##
# This thread initializes a notebook runner, so that it's
# ready to go on first page access.
runner = None
import threading
def make_notebook_runner():
    global runner
    from runipy.notebook_runner import NotebookRunner
    runner = NotebookRunner(None)
    print "Runner is ready."
make_notebook_runner_thread = threading.Thread(target=make_notebook_runner)
make_notebook_runner_thread.start()

def set_working_directory(path):
    if not os.path.exists(path):
        raise IOError('Path not found: %s' % os.path.abspath(path))
    nbmanager.notebook_dir = path

@flask_app.route('/')
def render_index():
    template = """

Notebooks

        {% for notebook in notebooks %}

{{notebook.name}}

        {% endfor %}

    """
    return flask.render_template_string(template, notebooks=nbmanager.list_notebooks('.'))

@flask_app.route('/<nbname>/')
def render_page(nbname):
    global runner
    from IPython.nbconvert.exporters.html import HTMLExporter

    if not nbmanager.notebook_exists(nbname):
        print "Notebook %s does not exist." % nbname
        flask.abort(404)

    print "Loading notebook %s" % nbname
    nbmanager.trust_notebook(nbname)
    nb = nbmanager.get_notebook(nbname)
    print "Making runner..."''

    # This is an ugly little bit to deal with a sporadic
    #  'queue empty' bug in iPython that only seems to
    #  happen on the integration servers...
    #  see https://github.com/paulgb/runipy/issues/36
    N_RUN_RETRIES = 4
    from Queue import Empty

    for i in range(N_RUN_RETRIES):
        try:
            if runner is None:
                make_notebook_runner_thread.join()

            # Do as complete of a reset of the kernel as we can.
            # Unfortunately, this doesn't really do a 'hard' reset
            # of any modules...
            class ResetCell(dict):
                """Simulates just enough of a notebook cell to get this
                '%reset -f' cell executed using the existing runipy
                 machinery."""
                input = "%reset -f"
            runner.run_cell(ResetCell())
            runner.nb = nb['content']
            print "Running notebook"
            runner.run_notebook(skip_exceptions=True)
            break
        except Empty as e:
            print "WARNING: Empty bug happened."
            if i >= (N_RUN_RETRIES - 1):
                raise

    print "Exporting notebook"
    import jinja2
    template = jinja2.Template("""{%- extends 'full.tpl' -%}

{% block input_group -%}
{% endblock input_group %}

{% block in_prompt -%}
{%- endblock in_prompt %}

{% block empty_in_prompt -%}
{%- endblock empty_in_prompt %}

{% block output %}
{{ super.super() }}
{% endblock output %}""")
    #templateLoader = jinja2.FileSystemLoader( searchpath="/" )
    #templateEnv = jinja2.Environment( loader=templateLoader )

    #TEMPLATE_FILE = "/home/user/site/example2.jinja"
    #template = templateEnv.get_template( TEMPLATE_FILE )

    #from IPython.config import Config
    exporter = HTMLExporter(
        #config=Config({'HTMLExporter':{'default_template':args.template}})
        #config=Config({'HTMLExporter':{'default_template':'nbserve'}})

    )
    exporter.template = template
    #exporter.environment.loader.loaders[0].searchpath += [os.path.join(os.path.split(__file__)[0],'templates')]
    #exporter.template_file = 'nbserve.tpl'
    #from clearinputpreprocessor import ClearInputPreprocessor
    #clear_input_preprocessor = ClearInputPreprocessor()
    #clear_input_preprocessor.enabled = True
    #output, resources = runner.nb, None
    #output, resources = clear_input_preprocessor(output, resources)
    output, resources = exporter.from_notebook_node(runner.nb)
    print "Returning."
    #resources['metadata']['name'] = 'Test title.'
    return output

if __name__ == "__main__":
    flask_app.run()
<|endoftext|>


<commit_before># coding: utf-8
from django.contrib.auth.models import User
from tastypie.resources import ModelResource
from tastypie import fields
from tastypie.contrib.contenttypes.fields import GenericForeignKeyField
from tastypie.authentication import ApiKeyAuthentication
from tastypie.authorization import DjangoAuthorization

from journalmanager.models import (
    Journal,
    UseLicense,
    Sponsor,
    Collection,
    Issue,
    Section,
    DataChangeEvent,
    RegularPressRelease,
    AheadPressRelease,
    PressReleaseTranslation,
    PressReleaseArticle,
)

class ApiKeyAuthMeta:
    authentication = ApiKeyAuthentication()
    authorization = DjangoAuthorization()

class SectionResource(ModelResource):
    journal = fields.ForeignKey('api.resources.JournalResource',
        'journal')
    issues = fields.OneToManyField('api.resources.IssueResource',
        'issue_set')
    titles = fields.CharField(readonly=True)

    class Meta(ApiKeyAuthMeta):
        queryset = Section.objects.all()
        resource_name = 'sections'
        allowed_methods = ['get']
        excludes = ['legacy_code']
        filtering = {
            "journal": ('exact'),
        }

    def dehydrate_titles(self, bundle):
        return [(title.language.iso_code, title.title)
            for title in bundle.obj.titles.all()]

class IssueResource(ModelResource):
    journal = fields.ForeignKey('api.resources.JournalResource',
        'journal')
    sections = fields.ManyToManyField(SectionResource, 'section')
    thematic_titles = fields.CharField(readonly=True)
    is_press_release = fields.BooleanField(readonly=True)

    class Meta(ApiKeyAuthMeta):
        queryset = Issue.objects.all()
        resource_name = 'issues'
        allowed_methods = ['get', ]
        filtering = {
            "journal": ('exact'),
            "is_marked_up": ('exact'),
        }

    def build_filters(self, filters=None):
        """
        Custom filter that retrieves data by the collection's name_slug.
        """
        if filters is None:
            filters = {}

        orm_filters = super(IssueResource, self).build_filters(filters)

        if 'collection' in filters:
            issues = Issue.objects.filter(
                journal__collection__name_slug=filters['collection'])
            orm_filters['pk__in'] = issues

        return orm_filters

    def dehydrate_thematic_titles(self, bundle):
        return dict([title.language.iso_code, title.title]
            for title in bundle.obj.issuetitle_set.all())

    def dehydrate_is_press_release(self, bundle):
        return False

class CollectionResource(ModelResource):

    class Meta(ApiKeyAuthMeta):
        queryset = Collection.objects.all()
        resource_name = 'collections'
        allowed_methods = ['get', ]

class SponsorResource(ModelResource):
    class Meta(ApiKeyAuthMeta):
        queryset = Sponsor.objects.all()
        resource_name = 'sponsors'
        allowed_methods = ['get', ]

class UseLicenseResource(ModelResource):
    class Meta(ApiKeyAuthMeta):
        queryset = UseLicense.objects.all()
        resource_name = 'uselicenses'
        allowed_methods = ['get', ]

class UserResource(ModelResource):
    class Meta(ApiKeyAuthMeta):
        queryset = User.objects.all()
        resource_name = 'users'
        allowed_methods = ['get', ]
        excludes = [
            'email',
            'password',
            'is_active',
            'is_staff',
            'is_superuser',
        ]

class JournalResource(ModelResource):
    missions = fields.CharField(readonly=True)
    other_titles = fields.CharField(readonly=True)
    creator = fields.ForeignKey(UserResource, 'creator')
    abstract_keyword_languages = fields.CharField(readonly=True)
    languages = fields.CharField(readonly=True)
    use_license = fields.ForeignKey(UseLicenseResource, 'use_license', full=True)
    sponsors = fields.ManyToManyField(SponsorResource, 'sponsor')
    collections = fields.ForeignKey(CollectionResource, 'collection')
    issues = fields.OneToManyField(IssueResource, 'issue_set')
    sections = fields.OneToManyField(SectionResource, 'section_set')
    pub_status_history = fields.ListField(readonly=True)
    contact = fields.DictField(readonly=True)
    study_areas = fields.ListField(readonly=True)

    #recursive field
    previous_title = fields.ForeignKey('self', 'previous_title', null=True)

    class Meta(ApiKeyAuthMeta):
        queryset = Journal.objects.all().filter()
        resource_name = 'journals'
        allowed_methods = ['get', ]
        filtering = {
            'is_trashed': ('exact',),
        }

    def build_filters(self, filters=None):
        """
        Custom filter that retrieves data by the collection's name_slug.
        """
        if filters is None:
            filters = {}

        orm_filters = super(JournalResource, self).build_filters(filters)

        if 'collection' in filters:
            journals = Journal.objects.filter(
                collection__name_slug=filters['collection'])
            orm_filters['pk__in'] = journals

        if 'pubstatus' in filters:
            # keep the previous filtering
            try:
                j = orm_filters['pk__in']
            except KeyError:
                j = Journal.objects

            statuses = filters.getlist('pubstatus')
            journals = j.filter(
                pub_status__in=statuses)
            orm_filters['pk__in'] = journals

        return orm_filters

    def dehydrate_missions(self, bundle):
        return [(mission.language.iso_code, mission.description)
            for mission in bundle.obj.missions.all()]

    def dehydrate_other_titles(self, bundle):
        return [(title.category, title.title)
            for title in bundle.obj.other_titles.all()]

    def dehydrate_languages(self, bundle):
        return [language.iso_code
            for language in bundle.obj.languages.all()]

    def dehydrate_pub_status_history(self, bundle):
        return [{'date': event.created_at,
                'status': event.status}
            for event in bundle.obj.status_history.order_by('-created_at').all()]

    def dehydrate_study_areas(self, bundle):
        return [area.study_area
            for area in bundle.obj.study_areas.all()]

class DataChangeEventResource(ModelResource):
    collection_uri = fields.ForeignKey(CollectionResource, 'collection')
    seq = fields.IntegerField(attribute='pk', readonly=True)
    object_uri = GenericForeignKeyField({
        Journal: JournalResource,
        Issue: IssueResource,
    }, 'content_object')

    class Meta(ApiKeyAuthMeta):
        resource_name = 'changes'
        queryset = DataChangeEvent.objects.all()
        excludes = [
            'object_id',
            'id',
        ]
        allowed_methods = ['get', ]

    def build_filters(self, filters=None):
        """
        Custom filter that retrieves data by the collection's name_slug.
        """
        if filters is None:
            filters = {}

        orm_filters = super(DataChangeEventResource, self).build_filters(filters)

        if 'since' in filters:
            events = DataChangeEvent.objects.filter(
                pk__gte=int(filters['since']))
            orm_filters['pk__in'] = events

        return orm_filters

class PressReleaseTranslationResource(ModelResource):
    language = fields.CharField(readonly=True)

    class Meta(ApiKeyAuthMeta):
        resource_name = 'prtranslations'
        queryset = PressReleaseTranslation.objects.all()
        allowed_methods = ['get', ]

    def dehydrate_language(self, bundle):
        return bundle.obj.language.iso_code

class PressReleaseResource(ModelResource):
    issue_uri = fields.ForeignKey(IssueResource, 'issue')
    translations = fields.OneToManyField(PressReleaseTranslationResource,
                                         'translations',
                                         full=True)
    articles = fields.CharField(readonly=True)
    issue_meta = fields.CharField(readonly=True)

    class Meta(ApiKeyAuthMeta):
        resource_name = 'pressreleases'
        queryset = RegularPressRelease.objects.all()
        allowed_methods = ['get', ]

    def build_filters(self, filters=None):
        """
        Custom filter that retrieves data by the article PID.
        """
        if filters is None:
            filters = {}

        orm_filters = super(PressReleaseResource, self).build_filters(filters)

        if 'article_pid' in filters:
            preleases = RegularPressRelease.objects.filter(
                articles__article_pid=filters['article_pid'])
            orm_filters['pk__in'] = preleases

        elif 'journal_pid' in filters:
            preleases = RegularPressRelease.objects.by_journal_pid(
                filters['journal_pid'])
            orm_filters['pk__in'] = preleases
        elif 'issue_pid' in filters:
            preleases = RegularPressRelease.objects.by_issue_pid(
                filters['issue_pid'])
            orm_filters['pk__in'] = preleases

        return orm_filters

    def dehydrate_articles(self, bundle):
        return [art.article_pid for art in bundle.obj.articles.all()]

    def dehydrate_issue_meta(self, bundle):
        issue = bundle.obj.issue

        meta_data = {
            'scielo_pid': issue.scielo_pid,
            'short_title': issue.journal.short_title,
            'volume': issue.volume,
            'number': issue.number,
            'suppl_volume': issue.suppl_volume,
            'suppl_number': issue.suppl_number,
            'publication_start_month': issue.publication_start_month,
            'publication_end_month': issue.publication_end_month,
            'publication_city': issue.journal.publication_city,
            'publication_year': issue.publication_year,
        }

        return meta_data

class AheadPressReleaseResource(ModelResource):
    journal_uri = fields.ForeignKey(JournalResource, 'journal')
    translations = fields.OneToManyField(PressReleaseTranslationResource,
                                         'translations',
                                         full=True)
    articles = fields.CharField(readonly=True)

    class Meta(ApiKeyAuthMeta):
        resource_name = 'apressreleases'
        queryset = AheadPressRelease.objects.all()
        allowed_methods = ['get', ]

    def dehydrate_articles(self, bundle):
        return [art.article_pid for art in bundle.obj.articles.all()]

    def build_filters(self, filters=None):
        """
        Custom filter that retrieves data by the article PID.
        """
        if filters is None:
            filters = {}

        orm_filters = super(AheadPressReleaseResource, self).build_filters(filters)

        if 'article_pid' in filters:
            preleases = AheadPressRelease.objects.filter(
                articles__article_pid=filters['article_pid'])
            orm_filters['pk__in'] = preleases

        elif 'journal_pid' in filters:
            preleases = AheadPressRelease.objects.by_journal_pid(
                filters['journal_pid'])
            orm_filters['pk__in'] = preleases

        return orm_filters
<commit_msg>Adicionado o id do press release no ordering da API<commit_after># coding: utf-8
from django.contrib.auth.models import User
from tastypie.resources import ModelResource
from tastypie import fields
from tastypie.contrib.contenttypes.fields import GenericForeignKeyField
from tastypie.authentication import ApiKeyAuthentication
from tastypie.authorization import DjangoAuthorization

from journalmanager.models import (
    Journal,
    UseLicense,
    Sponsor,
    Collection,
    Issue,
    Section,
    DataChangeEvent,
    RegularPressRelease,
    AheadPressRelease,
    PressReleaseTranslation,
    PressReleaseArticle,
)

class ApiKeyAuthMeta:
    authentication = ApiKeyAuthentication()
    authorization = DjangoAuthorization()

class SectionResource(ModelResource):
    journal = fields.ForeignKey('api.resources.JournalResource',
        'journal')
    issues = fields.OneToManyField('api.resources.IssueResource',
        'issue_set')
    titles = fields.CharField(readonly=True)

    class Meta(ApiKeyAuthMeta):
        queryset = Section.objects.all()
        resource_name = 'sections'
        allowed_methods = ['get']
        excludes = ['legacy_code']
        filtering = {
            "journal": ('exact'),
        }

    def dehydrate_titles(self, bundle):
        return [(title.language.iso_code, title.title)
            for title in bundle.obj.titles.all()]

class IssueResource(ModelResource):
    journal = fields.ForeignKey('api.resources.JournalResource',
        'journal')
    sections = fields.ManyToManyField(SectionResource, 'section')
    thematic_titles = fields.CharField(readonly=True)
    is_press_release = fields.BooleanField(readonly=True)

    class Meta(ApiKeyAuthMeta):
        queryset = Issue.objects.all()
        resource_name = 'issues'
        allowed_methods = ['get', ]
        filtering = {
            "journal": ('exact'),
            "is_marked_up": ('exact'),
        }

    def build_filters(self, filters=None):
        """
        Custom filter that retrieves data by the collection's name_slug.
        """
        if filters is None:
            filters = {}

        orm_filters = super(IssueResource, self).build_filters(filters)

        if 'collection' in filters:
            issues = Issue.objects.filter(
                journal__collection__name_slug=filters['collection'])
            orm_filters['pk__in'] = issues

        return orm_filters

    def dehydrate_thematic_titles(self, bundle):
        return dict([title.language.iso_code, title.title]
            for title in bundle.obj.issuetitle_set.all())

    def dehydrate_is_press_release(self, bundle):
        return False

class CollectionResource(ModelResource):

    class Meta(ApiKeyAuthMeta):
        queryset = Collection.objects.all()
        resource_name = 'collections'
        allowed_methods = ['get', ]

class SponsorResource(ModelResource):
    class Meta(ApiKeyAuthMeta):
        queryset = Sponsor.objects.all()
        resource_name = 'sponsors'
        allowed_methods = ['get', ]

class UseLicenseResource(ModelResource):
    class Meta(ApiKeyAuthMeta):
        queryset = UseLicense.objects.all()
        resource_name = 'uselicenses'
        allowed_methods = ['get', ]

class UserResource(ModelResource):
    class Meta(ApiKeyAuthMeta):
        queryset = User.objects.all()
        resource_name = 'users'
        allowed_methods = ['get', ]
        excludes = [
            'email',
            'password',
            'is_active',
            'is_staff',
            'is_superuser',
        ]

class JournalResource(ModelResource):
    missions = fields.CharField(readonly=True)
    other_titles = fields.CharField(readonly=True)
    creator = fields.ForeignKey(UserResource, 'creator')
    abstract_keyword_languages = fields.CharField(readonly=True)
    languages = fields.CharField(readonly=True)
    use_license = fields.ForeignKey(UseLicenseResource, 'use_license', full=True)
    sponsors = fields.ManyToManyField(SponsorResource, 'sponsor')
    collections = fields.ForeignKey(CollectionResource, 'collection')
    issues = fields.OneToManyField(IssueResource, 'issue_set')
    sections = fields.OneToManyField(SectionResource, 'section_set')
    pub_status_history = fields.ListField(readonly=True)
    contact = fields.DictField(readonly=True)
    study_areas = fields.ListField(readonly=True)

    #recursive field
    previous_title = fields.ForeignKey('self', 'previous_title', null=True)

    class Meta(ApiKeyAuthMeta):
        queryset = Journal.objects.all().filter()
        resource_name = 'journals'
        allowed_methods = ['get', ]
        filtering = {
            'is_trashed': ('exact',),
        }

    def build_filters(self, filters=None):
        """
        Custom filter that retrieves data by the collection's name_slug.
        """
        if filters is None:
            filters = {}

        orm_filters = super(JournalResource, self).build_filters(filters)

        if 'collection' in filters:
            journals = Journal.objects.filter(
                collection__name_slug=filters['collection'])
            orm_filters['pk__in'] = journals

        if 'pubstatus' in filters:
            # keep the previous filtering
            try:
                j = orm_filters['pk__in']
            except KeyError:
                j = Journal.objects

            statuses = filters.getlist('pubstatus')
            journals = j.filter(
                pub_status__in=statuses)
            orm_filters['pk__in'] = journals

        return orm_filters

    def dehydrate_missions(self, bundle):
        return [(mission.language.iso_code, mission.description)
            for mission in bundle.obj.missions.all()]

    def dehydrate_other_titles(self, bundle):
        return [(title.category, title.title)
            for title in bundle.obj.other_titles.all()]

    def dehydrate_languages(self, bundle):
        return [language.iso_code
            for language in bundle.obj.languages.all()]

    def dehydrate_pub_status_history(self, bundle):
        return [{'date': event.created_at,
                'status': event.status}
            for event in bundle.obj.status_history.order_by('-created_at').all()]

    def dehydrate_study_areas(self, bundle):
        return [area.study_area
            for area in bundle.obj.study_areas.all()]

class DataChangeEventResource(ModelResource):
    collection_uri = fields.ForeignKey(CollectionResource, 'collection')
    seq = fields.IntegerField(attribute='pk', readonly=True)
    object_uri = GenericForeignKeyField({
        Journal: JournalResource,
        Issue: IssueResource,
    }, 'content_object')

    class Meta(ApiKeyAuthMeta):
        resource_name = 'changes'
        queryset = DataChangeEvent.objects.all()
        excludes = [
            'object_id',
            'id',
        ]
        allowed_methods = ['get', ]

    def build_filters(self, filters=None):
        """
        Custom filter that retrieves data by the collection's name_slug.
        """
        if filters is None:
            filters = {}

        orm_filters = super(DataChangeEventResource, self).build_filters(filters)

        if 'since' in filters:
            events = DataChangeEvent.objects.filter(
                pk__gte=int(filters['since']))
            orm_filters['pk__in'] = events

        return orm_filters

class PressReleaseTranslationResource(ModelResource):
    language = fields.CharField(readonly=True)

    class Meta(ApiKeyAuthMeta):
        resource_name = 'prtranslations'
        queryset = PressReleaseTranslation.objects.all()
        allowed_methods = ['get', ]

    def dehydrate_language(self, bundle):
        return bundle.obj.language.iso_code

class PressReleaseResource(ModelResource):
    issue_uri = fields.ForeignKey(IssueResource, 'issue')
    translations = fields.OneToManyField(PressReleaseTranslationResource,
                                         'translations',
                                         full=True)
    articles = fields.CharField(readonly=True)
    issue_meta = fields.CharField(readonly=True)

    class Meta(ApiKeyAuthMeta):
        resource_name = 'pressreleases'
        queryset = RegularPressRelease.objects.all()
        allowed_methods = ['get', ]
        ordering = ['id']

    def build_filters(self, filters=None):
        """
        Custom filter that retrieves data by the article PID.
        """
        if filters is None:
            filters = {}

        orm_filters = super(PressReleaseResource, self).build_filters(filters)

        if 'article_pid' in filters:
            preleases = RegularPressRelease.objects.filter(
                articles__article_pid=filters['article_pid'])
            orm_filters['pk__in'] = preleases

        elif 'journal_pid' in filters:
            preleases = RegularPressRelease.objects.by_journal_pid(
                filters['journal_pid'])
            orm_filters['pk__in'] = preleases
        elif 'issue_pid' in filters:
            preleases = RegularPressRelease.objects.by_issue_pid(
                filters['issue_pid'])
            orm_filters['pk__in'] = preleases

        return orm_filters

    def dehydrate_articles(self, bundle):
        return [art.article_pid for art in bundle.obj.articles.all()]

    def dehydrate_issue_meta(self, bundle):
        issue = bundle.obj.issue

        meta_data = {
            'scielo_pid': issue.scielo_pid,
            'short_title': issue.journal.short_title,
            'volume': issue.volume,
            'number': issue.number,
            'suppl_volume': issue.suppl_volume,
            'suppl_number': issue.suppl_number,
            'publication_start_month': issue.publication_start_month,
            'publication_end_month': issue.publication_end_month,
            'publication_city': issue.journal.publication_city,
            'publication_year': issue.publication_year,
        }

        return meta_data

class AheadPressReleaseResource(ModelResource):
    journal_uri = fields.ForeignKey(JournalResource, 'journal')
    translations = fields.OneToManyField(PressReleaseTranslationResource,
                                         'translations',
                                         full=True)
    articles = fields.CharField(readonly=True)

    class Meta(ApiKeyAuthMeta):
        resource_name = 'apressreleases'
        queryset = AheadPressRelease.objects.all()
        allowed_methods = ['get', ]

    def dehydrate_articles(self, bundle):
        return [art.article_pid for art in bundle.obj.articles.all()]

    def build_filters(self, filters=None):
        """
        Custom filter that retrieves data by the article PID.
        """
        if filters is None:
            filters = {}

        orm_filters = super(AheadPressReleaseResource, self).build_filters(filters)

        if 'article_pid' in filters:
            preleases = AheadPressRelease.objects.filter(
                articles__article_pid=filters['article_pid'])
            orm_filters['pk__in'] = preleases

        elif 'journal_pid' in filters:
            preleases = AheadPressRelease.objects.by_journal_pid(
                filters['journal_pid'])
            orm_filters['pk__in'] = preleases

        return orm_filters
<|endoftext|>


<commit_before>"""Context class."""

import logging
import sys

from google.appengine.api import datastore_errors
from google.appengine.api import memcache

from core import datastore_rpc

from ndb import model, tasks, eventloop, utils

class AutoBatcher(object):

  def __init__(self, todo_task):
    self._todo_task = todo_task  # Task called with list of (future, arg) pairs
    self._todo = []  # List of (future, arg) pairs
    self._running = None  # Currently running task, if any

  def __repr__(self):
    return '%s(%s)' % (self.__class__.__name__, self._todo_task.__name__)

  def add(self, arg):
    fut = tasks.Future('%s.add(%s)' % (self, arg))
    if not self._todo:  # Schedule the callback
      # We use the fact that regular tasks are queued at time None,
      # which puts them at absolute time 0 (i.e. ASAP -- still on a
      # FIFO basis).  Callbacks explicitly scheduled with a delay of 0
      # are only run after all immediately runnable tasks have run.
      eventloop.queue_task(0, self._autobatcher_callback)
    self._todo.append((fut, arg))
    return fut

  def _autobatcher_callback(self):
    if not self._todo:
      return
    if self._running is not None:
      # Another callback may still be running.
      if not self._running.done():
        # Wait for it to complete first, then try again.
        self._running.add_callback(self._autobatcher_callback)
        return
      self._running = None
    # We cannot postpone the inevitable any longer.
    todo = self._todo
    self._todo = []  # Get ready for the next batch
    logging.info('AutoBatcher(%s): %d items',
                 self._todo_task.__name__, len(todo))
    self._running = self._todo_task(todo)
    # Add a callback to the Future to propagate exceptions,
    # since this Future is not normally checked otherwise.
    self._running.add_callback(self._running.check_success)

  @tasks.task
  def flush(self):
    while self._running or self._todo:
      if self._running:
        if self._running.done():
          self._running.check_success()
          self._running = None
        else:
          yield self._running
      else:
        self._autobatcher_callback()

# TODO: Rename?  To what?  Session???
class Context(object):

  def __init__(self, conn=None, auto_batcher_class=AutoBatcher):
    if conn is None:
      conn = model.conn  # TODO: Get rid of this?
    self._conn = conn
    self._auto_batcher_class = auto_batcher_class
    self._get_batcher = auto_batcher_class(self._get_task)
    self._put_batcher = auto_batcher_class(self._put_task)
    self._delete_batcher = auto_batcher_class(self._delete_task)
    self._cache = {}
    self._cache_policy = None
    self._memcache_policy = None
    # TODO: Also add a way to compute the memcache expiration time.

  @tasks.task
  def flush(self):
    yield (self._get_batcher.flush(),
           self._put_batcher.flush(),
           self._delete_batcher.flush())

  @tasks.task
  def _get_task(self, todo):
    assert todo
    # First check memcache.
    keys = set(key for _, key in todo)
    memkeymap = dict((key, key.urlsafe())
                     for key in keys if self.should_memcache(key))
    if memkeymap:
      results = memcache.get_multi(memkeymap.values())
      leftover = []
##      del todo[1:]  # Uncommenting this creates an interesting bug.
      for fut, key in todo:
        mkey = memkeymap[key]
        if mkey in results:
          pb = results[mkey]
          ent = self._conn.adapter.pb_to_entity(pb)
          fut.set_result(ent)
        else:
          leftover.append((fut, key))
      todo = leftover
    if todo:
      keys = [key for (_, key) in todo]
      results = yield self._conn.async_get(None, keys)
      for ent, (fut, _) in zip(results, todo):
        fut.set_result(ent)

  @tasks.task
  def _put_task(self, todo):
    assert todo
    # TODO: What if the same entity is being put twice?
    # TODO: What if two entities with the same key are being put?
    ents = [ent for (_, ent) in todo]
    results = yield self._conn.async_put(None, ents)
    for key, (fut, ent) in zip(results, todo):
      if key != ent.key:
        assert ent.key is None or not list(ent.key.flat())[-1]
        ent.key = key
      fut.set_result(key)
    # Now update memcache.
    # TODO: Could we update memcache *before* calling async_put()?
    # (Hm, not for new entities but possibly for updated ones.)
    mapping = {}
    for _, ent in todo:
      if self.should_memcache(ent.key):
        pb = self._conn.adapter.entity_to_pb(ent)
        mapping[ent.key.urlsafe()] = pb
    if mapping:
      # TODO: Optionally set the memcache expiration time;
      # maybe configurable based on key (or even entity).
      failures = memcache.set_multi(mapping)
      if failures:
        badkeys = []
        for failure in failures:
          badkeys.append(mapping[failure].key)
        logging.info('memcache failed to set %d out of %d keys: %s',
                     len(failures), len(mapping), badkeys)

  @tasks.task
  def _delete_task(self, todo):
    assert todo
    keys = set(key for (_, key) in todo)
    yield self._conn.async_delete(None, keys)
    for fut, _ in todo:
      fut.set_result(None)
    # Now update memcache.
    memkeys = [key.urlsafe() for key in keys if self.should_memcache(key)]
    if memkeys:
      memcache.delete_multi(memkeys)
      # The value returned by delete_multi() is pretty much useless, it
      # could be the keys were never cached in the first place.

  def set_cache_policy(self, func):
    self._cache_policy = func

  def should_cache(self, key):
    if self._cache_policy is None:
      return True
    return self._cache_policy(key)

  def set_memcache_policy(self, func):
    self._memcache_policy = func

  def should_memcache(self, key):
    if self._memcache_policy is None:
      return True
    return self._memcache_policy(key)

  # TODO: What about conflicting requests to different autobatchers,
  # e.g. task A calls get() on a given key while task B calls
  # delete()?  The outcome is nondeterministic, depending on which
  # autobatcher gets run first.  Maybe we should just flag such
  # conflicts as errors, with an overridable policy to resolve them
  # differently?

  @tasks.task
  def get(self, key):
    if key in self._cache:
      entity = self._cache[key]  # May be None, meaning "doesn't exist".
    else:
      entity = yield self._get_batcher.add(key)
      if self.should_cache(key):
        self._cache[key] = entity
    raise tasks.Return(entity)

  @tasks.task
  def put(self, entity):
    key = yield self._put_batcher.add(entity)
    if entity.key != key:
      logging.info('replacing key %s with %s', entity.key, key)
      entity.key = key
    # TODO: For updated entities, could we update the cache first?
    if self.should_cache(key):
      self._cache[key] = entity
    raise tasks.Return(key)

  @tasks.task
  def delete(self, key):
    yield self._delete_batcher.add(key)
    if key in self._cache:
      self._cache[key] = None

  @tasks.task
  def allocate_ids(self, key, size=None, max=None):
    lo_hi = yield self._conn.async_allocate_ids(None, key, size, max)
    raise tasks.Return(lo_hi)

  def map_query(self, query, callback,
                options=None, reducer=None, initial=None):
    mfut = tasks.MultiFuture('map_query', reducer, initial)

    @tasks.task
    def helper():
      rpc = query.run_async(self._conn, options)
      count = 0
      while rpc is not None:
        batch = yield rpc
        rpc = batch.next_batch_async(options)
        for ent in batch.results:
          key = ent.key
          if key in self._cache:
            if self._cache[key] is None:
              # This is a weird case.  Apparently this entity was
              # deleted concurrently with the query.  Let's just
              # pretend he delete happened first.
              logging.info('Conflict: entity %s was deleted', key)
              continue
            # Replace the entity the callback will see with the one
            # from the cache.
            if ent != self._cache[key]:
              logging.info('Conflict: entity %s was modified', key)
            ent = self._cache[key]
          else:
            if self.should_cache(key):
              self._cache[key] = ent
          count += 1
          val = callback(ent)  # TODO: If this raises something, log and ignore
          if isinstance(val, tasks.Future):
            mfut.add_dependent(val)
          else:
            mfut.process_value(val)
      mfut.complete()
      raise tasks.Return(count)

    return mfut, helper()

  @tasks.task
  def transaction(self, callback, retry=3, entity_group=None):
    # Will invoke callback(ctx) one or more times with ctx set to a new,
    # transactional Context.  Returns a Future.  Callback must be a task.
    if entity_group is not None:
      app = entity_group._Key__reference.app()
    else:
      app = model._DefaultAppId()
    yield self.flush()
    for i in range(1 + max(0, retry)):
      transaction = yield self._conn.async_begin_transaction(None, app)
      tconn = datastore_rpc.TransactionalConnection(
        adapter=self._conn.adapter,
        config=self._conn.config,
        transaction=transaction,
        entity_group=entity_group)
      tctx = self.__class__(conn=tconn,
                            auto_batcher_class=self._auto_batcher_class)
      tctx.set_memcache_policy(lambda key: False)
      fut = callback(tctx)
      assert isinstance(fut, tasks.Future)
      try:
        try:
          result = yield fut
        finally:
          yield tctx.flush()
      except Exception, err:
        t, e, tb = sys.exc_info()
        yield tconn.async_rollback(None)  # TODO: Don't block???
        raise t, e, tb
      else:
        ok = yield tconn.async_commit(None)
        if ok:
          self._cache.update(tctx._cache)
          self._flush_memcache(tctx._cache)
          raise tasks.Return(result)
    # Out of retries
    raise datastore_errors.TransactionFailedError(
      'The transaction could not be committed. Please try again.')

  def _flush_memcache(self, keys):
    keys = set(key for key in keys if self.should_memcache(key))
    if keys:
      memkeys = [key.urlsafe() for key in keys]
      memcache.delete_multi(memkeys)

  @tasks.task
  def get_or_insert(self, model_class, name, parent=None, **kwds):
    assert isinstance(name, basestring) and name
    if parent is None:
      pairs = []
    else:
      pairs = list(parent.pairs())
    pairs.append((model_class.GetKind(), name))
    key = model.Key(pairs=pairs)
    ent = yield self.get(key)
    if ent is None:
      @tasks.task
      def txn(ctx):
        ent = yield ctx.get(key)
        if ent is None:
          ent = model_class(**kwds)  # TODO: Check for forbidden keys
          ent.key = key
          yield ctx.put(ent)
        raise tasks.Return(ent)
      ent = yield self.transaction(txn)
    raise tasks.Return(ent)

def toplevel(func):
  """Decorator that adds a fresh Context as self.ctx *and* taskifies it."""
  @utils.wrapping(func)
  def add_context_wrapper(self, *args):
    __ndb_debug__ = utils.func_info(func)
    tasks.Future.clear_all_pending()
    self.ctx = Context()
    return tasks.taskify(func)(self, *args)
  return add_context_wrapper
<commit_msg>A few TODOs.<commit_after>"""Context class."""

import logging
import sys

from google.appengine.api import datastore_errors
from google.appengine.api import memcache

from core import datastore_rpc

from ndb import model, tasks, eventloop, utils

class AutoBatcher(object):

  def __init__(self, todo_task):
    self._todo_task = todo_task  # Task called with list of (future, arg) pairs
    self._todo = []  # List of (future, arg) pairs
    self._running = None  # Currently running task, if any

  def __repr__(self):
    return '%s(%s)' % (self.__class__.__name__, self._todo_task.__name__)

  def add(self, arg):
    fut = tasks.Future('%s.add(%s)' % (self, arg))
    if not self._todo:  # Schedule the callback
      # We use the fact that regular tasks are queued at time None,
      # which puts them at absolute time 0 (i.e. ASAP -- still on a
      # FIFO basis).  Callbacks explicitly scheduled with a delay of 0
      # are only run after all immediately runnable tasks have run.
      eventloop.queue_task(0, self._autobatcher_callback)
    self._todo.append((fut, arg))
    return fut

  def _autobatcher_callback(self):
    if not self._todo:
      return
    if self._running is not None:
      # Another callback may still be running.
      if not self._running.done():
        # Wait for it to complete first, then try again.
        self._running.add_callback(self._autobatcher_callback)
        return
      self._running = None
    # We cannot postpone the inevitable any longer.
    todo = self._todo
    self._todo = []  # Get ready for the next batch
    logging.info('AutoBatcher(%s): %d items',
                 self._todo_task.__name__, len(todo))
    self._running = self._todo_task(todo)
    # Add a callback to the Future to propagate exceptions,
    # since this Future is not normally checked otherwise.
    self._running.add_callback(self._running.check_success)

  @tasks.task
  def flush(self):
    while self._running or self._todo:
      if self._running:
        if self._running.done():
          self._running.check_success()
          self._running = None
        else:
          yield self._running
      else:
        self._autobatcher_callback()

# TODO: Rename?  To what?  Session???
class Context(object):

  def __init__(self, conn=None, auto_batcher_class=AutoBatcher):
    if conn is None:
      conn = model.conn  # TODO: Get rid of this?
    self._conn = conn
    self._auto_batcher_class = auto_batcher_class
    self._get_batcher = auto_batcher_class(self._get_task)
    self._put_batcher = auto_batcher_class(self._put_task)
    self._delete_batcher = auto_batcher_class(self._delete_task)
    self._cache = {}
    self._cache_policy = None
    self._memcache_policy = None
    # TODO: Also add a way to compute the memcache expiration time.

  @tasks.task
  def flush(self):
    yield (self._get_batcher.flush(),
           self._put_batcher.flush(),
           self._delete_batcher.flush())

  @tasks.task
  def _get_task(self, todo):
    assert todo
    # First check memcache.
    keys = set(key for _, key in todo)
    memkeymap = dict((key, key.urlsafe())
                     for key in keys if self.should_memcache(key))
    if memkeymap:
      results = memcache.get_multi(memkeymap.values())
      leftover = []
##      del todo[1:]  # Uncommenting this creates an interesting bug.
      for fut, key in todo:
        mkey = memkeymap[key]
        if mkey in results:
          pb = results[mkey]
          ent = self._conn.adapter.pb_to_entity(pb)
          fut.set_result(ent)
        else:
          leftover.append((fut, key))
      todo = leftover
    if todo:
      keys = [key for (_, key) in todo]
      # TODO: What if async_get() created a non-trivial MultiRpc?
      results = yield self._conn.async_get(None, keys)
      for ent, (fut, _) in zip(results, todo):
        fut.set_result(ent)

  @tasks.task
  def _put_task(self, todo):
    assert todo
    # TODO: What if the same entity is being put twice?
    # TODO: What if two entities with the same key are being put?
    ents = [ent for (_, ent) in todo]
    results = yield self._conn.async_put(None, ents)
    for key, (fut, ent) in zip(results, todo):
      if key != ent.key:
        assert ent.key is None or not list(ent.key.flat())[-1]
        ent.key = key
      fut.set_result(key)
    # Now update memcache.
    # TODO: Could we update memcache *before* calling async_put()?
    # (Hm, not for new entities but possibly for updated ones.)
    mapping = {}
    for _, ent in todo:
      if self.should_memcache(ent.key):
        pb = self._conn.adapter.entity_to_pb(ent)
        mapping[ent.key.urlsafe()] = pb
    if mapping:
      # TODO: Optionally set the memcache expiration time;
      # maybe configurable based on key (or even entity).
      failures = memcache.set_multi(mapping)
      if failures:
        badkeys = []
        for failure in failures:
          badkeys.append(mapping[failure].key)
        logging.info('memcache failed to set %d out of %d keys: %s',
                     len(failures), len(mapping), badkeys)

  @tasks.task
  def _delete_task(self, todo):
    assert todo
    keys = set(key for (_, key) in todo)
    yield self._conn.async_delete(None, keys)
    for fut, _ in todo:
      fut.set_result(None)
    # Now update memcache.
    memkeys = [key.urlsafe() for key in keys if self.should_memcache(key)]
    if memkeys:
      memcache.delete_multi(memkeys)
      # The value returned by delete_multi() is pretty much useless, it
      # could be the keys were never cached in the first place.

  def set_cache_policy(self, func):
    self._cache_policy = func

  def should_cache(self, key):
    if self._cache_policy is None:
      return True
    return self._cache_policy(key)

  def set_memcache_policy(self, func):
    self._memcache_policy = func

  def should_memcache(self, key):
    if self._memcache_policy is None:
      return True
    return self._memcache_policy(key)

  # TODO: What about conflicting requests to different autobatchers,
  # e.g. task A calls get() on a given key while task B calls
  # delete()?  The outcome is nondeterministic, depending on which
  # autobatcher gets run first.  Maybe we should just flag such
  # conflicts as errors, with an overridable policy to resolve them
  # differently?

  @tasks.task
  def get(self, key):
    if key in self._cache:
      entity = self._cache[key]  # May be None, meaning "doesn't exist".
    else:
      entity = yield self._get_batcher.add(key)
      if self.should_cache(key):
        self._cache[key] = entity
    raise tasks.Return(entity)

  @tasks.task
  def put(self, entity):
    key = yield self._put_batcher.add(entity)
    if entity.key != key:
      logging.info('replacing key %s with %s', entity.key, key)
      entity.key = key
    # TODO: For updated entities, could we update the cache first?
    if self.should_cache(key):
      self._cache[key] = entity
    raise tasks.Return(key)

  @tasks.task
  def delete(self, key):
    yield self._delete_batcher.add(key)
    if key in self._cache:
      self._cache[key] = None

  @tasks.task
  def allocate_ids(self, key, size=None, max=None):
    lo_hi = yield self._conn.async_allocate_ids(None, key, size, max)
    raise tasks.Return(lo_hi)

  def map_query(self, query, callback,
                options=None, reducer=None, initial=None):
    mfut = tasks.MultiFuture('map_query', reducer, initial)

    @tasks.task
    def helper():
      rpc = query.run_async(self._conn, options)
      count = 0
      while rpc is not None:
        batch = yield rpc
        rpc = batch.next_batch_async(options)
        for ent in batch.results:
          key = ent.key
          if key in self._cache:
            if self._cache[key] is None:
              # This is a weird case.  Apparently this entity was
              # deleted concurrently with the query.  Let's just
              # pretend he delete happened first.
              logging.info('Conflict: entity %s was deleted', key)
              continue
            # Replace the entity the callback will see with the one
            # from the cache.
            if ent != self._cache[key]:
              logging.info('Conflict: entity %s was modified', key)
            ent = self._cache[key]
          else:
            if self.should_cache(key):
              self._cache[key] = ent
          count += 1
          val = callback(ent)  # TODO: If this raises something, log and ignore
          if isinstance(val, tasks.Future):
            mfut.add_dependent(val)
          else:
            mfut.process_value(val)
      mfut.complete()
      raise tasks.Return(count)

    return mfut, helper()

  @tasks.task
  def transaction(self, callback, retry=3, entity_group=None):
    # Will invoke callback(ctx) one or more times with ctx set to a new,
    # transactional Context.  Returns a Future.  Callback must be a task.
    if entity_group is not None:
      app = entity_group._Key__reference.app()
    else:
      app = model._DefaultAppId()
    yield self.flush()
    for i in range(1 + max(0, retry)):
      transaction = yield self._conn.async_begin_transaction(None, app)
      tconn = datastore_rpc.TransactionalConnection(
        adapter=self._conn.adapter,
        config=self._conn.config,
        transaction=transaction,
        entity_group=entity_group)
      tctx = self.__class__(conn=tconn,
                            auto_batcher_class=self._auto_batcher_class)
      tctx.set_memcache_policy(lambda key: False)
      fut = callback(tctx)
      assert isinstance(fut, tasks.Future)
      try:
        try:
          result = yield fut
        finally:
          yield tctx.flush()
      except Exception, err:
        t, e, tb = sys.exc_info()
        yield tconn.async_rollback(None)  # TODO: Don't block???
        raise t, e, tb
      else:
        ok = yield tconn.async_commit(None)
        if ok:
          # TODO: This is questionable when self is transactional.
          self._cache.update(tctx._cache)
          self._flush_memcache(tctx._cache)
          raise tasks.Return(result)
    # Out of retries
    raise datastore_errors.TransactionFailedError(
      'The transaction could not be committed. Please try again.')

  def _flush_memcache(self, keys):
    keys = set(key for key in keys if self.should_memcache(key))
    if keys:
      memkeys = [key.urlsafe() for key in keys]
      memcache.delete_multi(memkeys)

  @tasks.task
  def get_or_insert(self, model_class, name, parent=None, **kwds):
    assert isinstance(name, basestring) and name
    if parent is None:
      pairs = []
    else:
      pairs = list(parent.pairs())
    pairs.append((model_class.GetKind(), name))
    key = model.Key(pairs=pairs)
    # TODO: Can the cache be trusted here?
    ent = yield self.get(key)
    if ent is None:
      @tasks.task
      def txn(ctx):
        ent = yield ctx.get(key)
        if ent is None:
          ent = model_class(**kwds)  # TODO: Check for forbidden keys
          ent.key = key
          yield ctx.put(ent)
        raise tasks.Return(ent)
      ent = yield self.transaction(txn)
    raise tasks.Return(ent)

def toplevel(func):
  """Decorator that adds a fresh Context as self.ctx *and* taskifies it."""
  @utils.wrapping(func)
  def add_context_wrapper(self, *args):
    __ndb_debug__ = utils.func_info(func)
    tasks.Future.clear_all_pending()
    self.ctx = Context()
    return tasks.taskify(func)(self, *args)
  return add_context_wrapper
<|endoftext|>