<commit_before>import logging

from django.template import Library, Node

from .. import utils
from .. import models

register = Library()
log = logging.getLogger(__name__)

@register.inclusion_tag('schedule/tags/event.html', takes_context=True)
def eventinfo(context, evt):
    event = None
    if isinstance(evt, basestring):
        log.debug("Got event as string: " + evt)
        if '_evt_cache' in context:
            log.debug("Checking local cache")
            event = context['_evt_cache'].get(evt)
        if event is None:
            event = utils.prepare_event(evt)
    else:
        event = evt
    return {
        'event': event
    }

class PreloadEventsNode(Node):
    def __init__(self, nodelist, event_names=None):
        self.nodelist = nodelist
        self.event_names = event_names or []

    def __repr__(self):
        return "<PreloadEventNode>"

    def render(self, context):
        cache = {}
        for evt in utils.load_event_models(self.event_names):
            if isinstance(evt, models.Session):
                evtkey = 'session:{0}'.format(evt.pk)
            else:
                evtkey = 'side:{0}'.format(evt.pk)
            cache[evtkey] = utils.prepare_event(evt)
        print cache
        context.update({
            '_evt_cache': cache
            })
        output = self.nodelist.render(context)
        context.pop()
        return output

@register.tag('preloadevents')
def do_preloadevents(parser, token):
    bits = token.split_contents()
    event_names = bits[1:]
    if not event_names:
        raise TemplateSyntaxError("You have to specify at least one event")
    nodelist = parser.parse(('endpreloadevents',))
    parser.delete_first_token()
    return PreloadEventsNode(nodelist, event_names=event_names)<commit_msg>Removes unnecessary logging<commit_after>import logging

from django.template import Library, Node

from .. import utils
from .. import models

register = Library()
log = logging.getLogger(__name__)

@register.inclusion_tag('schedule/tags/event.html', takes_context=True)
def eventinfo(context, evt):
    event = None
    if isinstance(evt, basestring):
        if '_evt_cache' in context:
            event = context['_evt_cache'].get(evt)
        if event is None:
            event = utils.prepare_event(evt)
    else:
        event = evt
    return {
        'event': event
    }

class PreloadEventsNode(Node):
    def __init__(self, nodelist, event_names=None):
        self.nodelist = nodelist
        self.event_names = event_names or []

    def __repr__(self):
        return "<PreloadEventNode>"

    def render(self, context):
        cache = {}
        for evt in utils.load_event_models(self.event_names):
            if isinstance(evt, models.Session):
                evtkey = 'session:{0}'.format(evt.pk)
            else:
                evtkey = 'side:{0}'.format(evt.pk)
            cache[evtkey] = utils.prepare_event(evt)
        context.update({
            '_evt_cache': cache
            })
        output = self.nodelist.render(context)
        context.pop()
        return output

@register.tag('preloadevents')
def do_preloadevents(parser, token):
    bits = token.split_contents()
    event_names = bits[1:]
    if not event_names:
        raise TemplateSyntaxError("You have to specify at least one event")
    nodelist = parser.parse(('endpreloadevents',))
    parser.delete_first_token()
    return PreloadEventsNode(nodelist, event_names=event_names)<|endoftext|>


<commit_before>__version__ = "0.1.0"
<commit_msg>bump version<commit_after>__version__ = "0.2.0"
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import openerp
import openerp.addons.im.im as im
import json
import random
import jinja2
from openerp.osv import osv, fields
from openerp import tools
import openerp.addons.web.http as http
from openerp.addons.web.http import request

env = jinja2.Environment(
    loader=jinja2.PackageLoader('openerp.addons.im_livechat', "."),
    autoescape=False
)
env.filters["json"] = json.dumps

class LiveChatController(http.Controller):

    def _auth(self, db):
        reg = openerp.modules.registry.RegistryManager.get(db)
        uid = openerp.netsvc.dispatch_rpc('common', 'authenticate', [db, "anonymous", "anonymous", None])
        return reg, uid

    @http.route('/im_livechat/loader', auth="none")
    def loader(self, **kwargs):
        p = json.loads(kwargs["p"])
        db = p["db"]
        channel = p["channel"]
        user_name = p.get("user_name", None)

        reg, uid = self._auth(db)
        with reg.cursor() as cr:
            info = reg.get('im_livechat.channel').get_info_for_chat_src(cr, uid, channel)
            info["db"] = db
            info["channel"] = channel
            info["userName"] = user_name
            return request.make_response(env.get_template("loader.js").render(info),
                 headers=[('Content-Type', "text/javascript")])

    @http.route('/im_livechat/web_page', auth="none")
    def web_page(self, **kwargs):
        p = json.loads(kwargs["p"])
        db = p["db"]
        channel = p["channel"]
        reg, uid = self._auth(db)
        with reg.cursor() as cr:
            script = reg.get('im_livechat.channel').read(cr, uid, channel, ["script"])["script"]
            info = reg.get('im_livechat.channel').get_info_for_chat_src(cr, uid, channel)
            info["script"] = script
            return request.make_response(env.get_template("web_page.html").render(info),
                 headers=[('Content-Type', "text/html")])

    @http.route('/im_livechat/available', type='json', auth="none")
    def available(self, db, channel):
        reg, uid = self._auth(db)
        with reg.cursor() as cr:
            return len(reg.get('im_livechat.channel').get_available_users(cr, uid, channel)) > 0

class im_livechat_channel(osv.osv):
    _name = 'im_livechat.channel'

    def _get_default_image(self, cr, uid, context=None):
        image_path = openerp.modules.get_module_resource('im_livechat', 'static/src/img', 'default.png')
        return tools.image_resize_image_big(open(image_path, 'rb').read().encode('base64'))
    def _get_image(self, cr, uid, ids, name, args, context=None):
        result = dict.fromkeys(ids, False)
        for obj in self.browse(cr, uid, ids, context=context):
            result[obj.id] = tools.image_get_resized_images(obj.image)
        return result
    def _set_image(self, cr, uid, id, name, value, args, context=None):
        return self.write(cr, uid, [id], {'image': tools.image_resize_image_big(value)}, context=context)

    def _are_you_inside(self, cr, uid, ids, name, arg, context=None):
        res = {}
        for record in self.browse(cr, uid, ids, context=context):
            res[record.id] = False
            for user in record.user_ids:
                if user.id == uid:
                    res[record.id] = True
                    break
        return res

    def _script(self, cr, uid, ids, name, arg, context=None):
        res = {}
        for record in self.browse(cr, uid, ids, context=context):
            res[record.id] = env.get_template("include.html").render({
                "url": self.pool.get('ir.config_parameter').get_param(cr, uid, 'web.base.url'),
                "parameters": {"db":cr.dbname, "channel":record.id},
            })
        return res

    def _web_page(self, cr, uid, ids, name, arg, context=None):
        res = {}
        for record in self.browse(cr, uid, ids, context=context):
            res[record.id] = self.pool.get('ir.config_parameter').get_param(cr, uid, 'web.base.url') + \
                "/im_livechat/web_page?p=" + json.dumps({"db":cr.dbname, "channel":record.id})
        return res

    _columns = {
        'name': fields.char(string="Channel Name", size=200, required=True),
        'user_ids': fields.many2many('res.users', 'im_livechat_channel_im_user', 'channel_id', 'user_id', string="Users"),
        'are_you_inside': fields.function(_are_you_inside, type='boolean', string='Are you inside the matrix?', store=False),
        'script': fields.function(_script, type='text', string='Script', store=False),
        'web_page': fields.function(_web_page, type='url', string='Web Page', store=False, size="200"),
        'button_text': fields.char(string="Text of the Button", size=200),
        'input_placeholder': fields.char(string="Chat Input Placeholder", size=200),
        'default_message': fields.char(string="Welcome Message", size=200, help="This is an automated 'welcome' message that your visitor will see when they initiate a new chat session."),
        # image: all image fields are base64 encoded and PIL-supported
        'image': fields.binary("Photo",
            help="This field holds the image used as photo for the group, limited to 1024x1024px."),
        'image_medium': fields.function(_get_image, fnct_inv=_set_image,
            string="Medium-sized photo", type="binary", multi="_get_image",
            store={
                'im_livechat.channel': (lambda self, cr, uid, ids, c={}: ids, ['image'], 10),
            },
            help="Medium-sized photo of the group. It is automatically "\
                 "resized as a 128x128px image, with aspect ratio preserved. "\
                 "Use this field in form views or some kanban views."),
        'image_small': fields.function(_get_image, fnct_inv=_set_image,
            string="Small-sized photo", type="binary", multi="_get_image",
            store={
                'im_livechat.channel': (lambda self, cr, uid, ids, c={}: ids, ['image'], 10),
            },
            help="Small-sized photo of the group. It is automatically "\
                 "resized as a 64x64px image, with aspect ratio preserved. "\
                 "Use this field anywhere a small image is required."),
    }

    def _default_user_ids(self, cr, uid, context=None):
        return [(6, 0, [uid])]

    _defaults = {
        'button_text': "Have a Question? Chat with us.",
        'input_placeholder': "How may I help you?",
        'default_message': '',
        'user_ids': _default_user_ids,
        'image': _get_default_image,
    }

    def get_available_users(self, cr, uid, channel_id, context=None):
        channel = self.browse(cr, openerp.SUPERUSER_ID, channel_id, context=context)
        im_user_ids = self.pool.get("im.user").search(cr, uid, [["user_id", "in", [user.id for user in channel.user_ids]]], context=context)
        users = []
        for iuid in im_user_ids:
            imuser = self.pool.get("im.user").browse(cr, uid, iuid, context=context)
            if imuser.im_status:
                users.append(imuser)
        return users

    def get_session(self, cr, uid, channel_id, uuid, context=None):
        my_id = self.pool.get("im.user").get_my_id(cr, uid, uuid, context=context)
        users = self.get_available_users(cr, uid, channel_id, context=context)
        if len(users) == 0:
            return False
        user_id = random.choice(users).id
        session = self.pool.get("im.session").session_get(cr, uid, [user_id], uuid, context=context)
        self.pool.get("im.session").write(cr, openerp.SUPERUSER_ID, session.get("id"), {'channel_id': channel_id}, context=context)
        return session.get("id")

    def test_channel(self, cr, uid, channel, context=None):
        if not channel:
            return {}
        return {
            'url': self.browse(cr, uid, channel[0], context=context or {}).web_page,
            'type': 'ir.actions.act_url'
        }

    def get_info_for_chat_src(self, cr, uid, channel, context=None):
        url = self.pool.get('ir.config_parameter').get_param(cr, openerp.SUPERUSER_ID, 'web.base.url')
        chan = self.browse(cr, uid, channel, context=context)
        return {
            "url": url,
            'buttonText': chan.button_text,
            'inputPlaceholder': chan.input_placeholder,
            'defaultMessage': chan.default_message,
            "channelName": chan.name,
        }

    def join(self, cr, uid, ids, context=None):
        self.write(cr, uid, ids, {'user_ids': [(4, uid)]})
        return True

    def quit(self, cr, uid, ids, context=None):
        self.write(cr, uid, ids, {'user_ids': [(3, uid)]})
        return True

class im_session(osv.osv):
    _inherit = 'im.session'

    _columns = {
        'channel_id': fields.many2one("im.user", "Channel"),
    }
<commit_msg>[FIX] im_livechat: many2one channel_id of im_session to the right model.<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import openerp
import openerp.addons.im.im as im
import json
import random
import jinja2
from openerp.osv import osv, fields
from openerp import tools
import openerp.addons.web.http as http
from openerp.addons.web.http import request

env = jinja2.Environment(
    loader=jinja2.PackageLoader('openerp.addons.im_livechat', "."),
    autoescape=False
)
env.filters["json"] = json.dumps

class LiveChatController(http.Controller):

    def _auth(self, db):
        reg = openerp.modules.registry.RegistryManager.get(db)
        uid = openerp.netsvc.dispatch_rpc('common', 'authenticate', [db, "anonymous", "anonymous", None])
        return reg, uid

    @http.route('/im_livechat/loader', auth="none")
    def loader(self, **kwargs):
        p = json.loads(kwargs["p"])
        db = p["db"]
        channel = p["channel"]
        user_name = p.get("user_name", None)

        reg, uid = self._auth(db)
        with reg.cursor() as cr:
            info = reg.get('im_livechat.channel').get_info_for_chat_src(cr, uid, channel)
            info["db"] = db
            info["channel"] = channel
            info["userName"] = user_name
            return request.make_response(env.get_template("loader.js").render(info),
                 headers=[('Content-Type', "text/javascript")])

    @http.route('/im_livechat/web_page', auth="none")
    def web_page(self, **kwargs):
        p = json.loads(kwargs["p"])
        db = p["db"]
        channel = p["channel"]
        reg, uid = self._auth(db)
        with reg.cursor() as cr:
            script = reg.get('im_livechat.channel').read(cr, uid, channel, ["script"])["script"]
            info = reg.get('im_livechat.channel').get_info_for_chat_src(cr, uid, channel)
            info["script"] = script
            return request.make_response(env.get_template("web_page.html").render(info),
                 headers=[('Content-Type', "text/html")])

    @http.route('/im_livechat/available', type='json', auth="none")
    def available(self, db, channel):
        reg, uid = self._auth(db)
        with reg.cursor() as cr:
            return len(reg.get('im_livechat.channel').get_available_users(cr, uid, channel)) > 0

class im_livechat_channel(osv.osv):
    _name = 'im_livechat.channel'

    def _get_default_image(self, cr, uid, context=None):
        image_path = openerp.modules.get_module_resource('im_livechat', 'static/src/img', 'default.png')
        return tools.image_resize_image_big(open(image_path, 'rb').read().encode('base64'))
    def _get_image(self, cr, uid, ids, name, args, context=None):
        result = dict.fromkeys(ids, False)
        for obj in self.browse(cr, uid, ids, context=context):
            result[obj.id] = tools.image_get_resized_images(obj.image)
        return result
    def _set_image(self, cr, uid, id, name, value, args, context=None):
        return self.write(cr, uid, [id], {'image': tools.image_resize_image_big(value)}, context=context)

    def _are_you_inside(self, cr, uid, ids, name, arg, context=None):
        res = {}
        for record in self.browse(cr, uid, ids, context=context):
            res[record.id] = False
            for user in record.user_ids:
                if user.id == uid:
                    res[record.id] = True
                    break
        return res

    def _script(self, cr, uid, ids, name, arg, context=None):
        res = {}
        for record in self.browse(cr, uid, ids, context=context):
            res[record.id] = env.get_template("include.html").render({
                "url": self.pool.get('ir.config_parameter').get_param(cr, uid, 'web.base.url'),
                "parameters": {"db":cr.dbname, "channel":record.id},
            })
        return res

    def _web_page(self, cr, uid, ids, name, arg, context=None):
        res = {}
        for record in self.browse(cr, uid, ids, context=context):
            res[record.id] = self.pool.get('ir.config_parameter').get_param(cr, uid, 'web.base.url') + \
                "/im_livechat/web_page?p=" + json.dumps({"db":cr.dbname, "channel":record.id})
        return res

    _columns = {
        'name': fields.char(string="Channel Name", size=200, required=True),
        'user_ids': fields.many2many('res.users', 'im_livechat_channel_im_user', 'channel_id', 'user_id', string="Users"),
        'are_you_inside': fields.function(_are_you_inside, type='boolean', string='Are you inside the matrix?', store=False),
        'script': fields.function(_script, type='text', string='Script', store=False),
        'web_page': fields.function(_web_page, type='url', string='Web Page', store=False, size="200"),
        'button_text': fields.char(string="Text of the Button", size=200),
        'input_placeholder': fields.char(string="Chat Input Placeholder", size=200),
        'default_message': fields.char(string="Welcome Message", size=200, help="This is an automated 'welcome' message that your visitor will see when they initiate a new chat session."),
        # image: all image fields are base64 encoded and PIL-supported
        'image': fields.binary("Photo",
            help="This field holds the image used as photo for the group, limited to 1024x1024px."),
        'image_medium': fields.function(_get_image, fnct_inv=_set_image,
            string="Medium-sized photo", type="binary", multi="_get_image",
            store={
                'im_livechat.channel': (lambda self, cr, uid, ids, c={}: ids, ['image'], 10),
            },
            help="Medium-sized photo of the group. It is automatically "\
                 "resized as a 128x128px image, with aspect ratio preserved. "\
                 "Use this field in form views or some kanban views."),
        'image_small': fields.function(_get_image, fnct_inv=_set_image,
            string="Small-sized photo", type="binary", multi="_get_image",
            store={
                'im_livechat.channel': (lambda self, cr, uid, ids, c={}: ids, ['image'], 10),
            },
            help="Small-sized photo of the group. It is automatically "\
                 "resized as a 64x64px image, with aspect ratio preserved. "\
                 "Use this field anywhere a small image is required."),
    }

    def _default_user_ids(self, cr, uid, context=None):
        return [(6, 0, [uid])]

    _defaults = {
        'button_text': "Have a Question? Chat with us.",
        'input_placeholder': "How may I help you?",
        'default_message': '',
        'user_ids': _default_user_ids,
        'image': _get_default_image,
    }

    def get_available_users(self, cr, uid, channel_id, context=None):
        channel = self.browse(cr, openerp.SUPERUSER_ID, channel_id, context=context)
        im_user_ids = self.pool.get("im.user").search(cr, uid, [["user_id", "in", [user.id for user in channel.user_ids]]], context=context)
        users = []
        for iuid in im_user_ids:
            imuser = self.pool.get("im.user").browse(cr, uid, iuid, context=context)
            if imuser.im_status:
                users.append(imuser)
        return users

    def get_session(self, cr, uid, channel_id, uuid, context=None):
        my_id = self.pool.get("im.user").get_my_id(cr, uid, uuid, context=context)
        users = self.get_available_users(cr, uid, channel_id, context=context)
        if len(users) == 0:
            return False
        user_id = random.choice(users).id
        session = self.pool.get("im.session").session_get(cr, uid, [user_id], uuid, context=context)
        self.pool.get("im.session").write(cr, openerp.SUPERUSER_ID, session.get("id"), {'channel_id': channel_id}, context=context)
        return session.get("id")

    def test_channel(self, cr, uid, channel, context=None):
        if not channel:
            return {}
        return {
            'url': self.browse(cr, uid, channel[0], context=context or {}).web_page,
            'type': 'ir.actions.act_url'
        }

    def get_info_for_chat_src(self, cr, uid, channel, context=None):
        url = self.pool.get('ir.config_parameter').get_param(cr, openerp.SUPERUSER_ID, 'web.base.url')
        chan = self.browse(cr, uid, channel, context=context)
        return {
            "url": url,
            'buttonText': chan.button_text,
            'inputPlaceholder': chan.input_placeholder,
            'defaultMessage': chan.default_message,
            "channelName": chan.name,
        }

    def join(self, cr, uid, ids, context=None):
        self.write(cr, uid, ids, {'user_ids': [(4, uid)]})
        return True

    def quit(self, cr, uid, ids, context=None):
        self.write(cr, uid, ids, {'user_ids': [(3, uid)]})
        return True

class im_session(osv.osv):
    _inherit = 'im.session'

    _columns = {
        'channel_id': fields.many2one("im_livechat.channel", "Channel"),
    }
<|endoftext|>


<commit_before>import json
import logging

import requests
from django.conf import settings
from django.core.cache import cache
from django.core.mail import get_connection, send_mail
from django.utils import timezone
from django_q.tasks import async
from twilio.base.exceptions import TwilioRestException

from apostello.twilio import twilio_client
from apostello.utils import fetch_default_reply

logger = logging.getLogger('apostello')

# sending messages

def group_send_message_task(body, group_name, sent_by, eta):
    """Send message to all members of group."""
    from apostello.models import Recipient, RecipientGroup
    group = RecipientGroup.objects.filter(name=group_name, is_archived=False)

    for recipient in Recipient.objects.filter(groups__in=group):
        recipient.send_message(content=body, group=group_name, sent_by=sent_by, eta=eta)

def recipient_send_message_task(recipient_pk, body, group, sent_by):
    """Send a message asynchronously."""
    from apostello.models import Recipient
    recipient = Recipient.objects.get(pk=recipient_pk)
    if recipient.is_archived:
        # if recipient is not active, fail silently
        return

    from apostello.models import SmsOutbound, RecipientGroup
    # if %name% is present, replace:
    body = recipient.personalise(body)
    # send twilio message
    try:
        message = twilio_client.messages.create(body=body, to=str(recipient.number), from_=settings.TWILIO_FROM_NUM)
        # add to sms out table
        sms = SmsOutbound(sid=message.sid, content=body, time_sent=timezone.now(), recipient=recipient, sent_by=sent_by)
        if group is not None:
            sms.recipient_group = RecipientGroup.objects.filter(name=group)[0]
        sms.save()
    except TwilioRestException as e:
        if e.code == 21610:
            recipient.is_blocking = True
            recipient.save()
            async('apostello.tasks.blacklist_notify', recipient.pk, '', 'stop')
        else:
            raise e

def send_queued_sms():
    """Check for and send any queued messages."""
    from apostello.models import QueuedSms
    for sms in QueuedSms.objects.filter(sent=False, time_to_send__lte=timezone.now()):
        sms.send()

def ask_for_name(person_from_pk, sms_body, ask_for_name):
    """Asks a contact to provide their name."""
    if not ask_for_name:
        return
    from site_config.models import SiteConfiguration
    config = SiteConfiguration.get_solo()
    if not config.disable_all_replies:
        from apostello.models import Recipient
        contact = Recipient.objects.get(pk=person_from_pk)
        contact.send_message(content=fetch_default_reply('auto_name_request'), sent_by="auto name request")
        async(
            'apostello.tasks.notify_office_mail',
            '[Apostello] Unknown Contact!',
            'SMS: {0}\nFrom: {1}\n\n\nThis person is unknown and has been'
            ' asked for their name.'.format(sms_body, str(contact)),
        )

# SMS logging and consistency checks

def check_incoming_log():
    """Update incoming log."""
    from apostello.logs import check_incoming_log
    check_incoming_log()

def check_outgoing_log():
    """Update outgoing log."""
    from apostello.logs import check_outgoing_log
    check_outgoing_log()

def log_msg_in(p, t, from_pk):
    """Log incoming message."""
    from apostello.models import Keyword, SmsInbound, Recipient
    from_ = Recipient.objects.get(pk=from_pk)
    matched_keyword = Keyword.match(p['Body'].strip())
    SmsInbound.objects.create(
        sid=p['MessageSid'],
        content=p['Body'],
        time_received=t,
        sender_name=str(from_),
        sender_num=p['From'],
        matched_keyword=str(matched_keyword),
        matched_colour=Keyword.lookup_colour(p['Body'].strip())
    )
    # check log is consistent:
    async('apostello.tasks.check_incoming_log')

def update_msgs_name(person_pk):
    """Back date sender_name field on inbound sms."""
    from apostello.models import Recipient, SmsInbound
    person_ = Recipient.objects.get(pk=person_pk)
    name = str(person_)
    number = str(person_.number)
    for sms in SmsInbound.objects.filter(sender_num=number):
        sms.sender_name = name
        sms.save()

def cleanup_expired_sms():
    """Remove expired messages."""
    from apostello import logs
    logs.cleanup_expired_sms()

# notifications, email, slack etc

def send_async_mail(subject, body, to):
    """Send email."""
    # read email settings from DB, if they are empty, they will be read from
    # settings.py instead
    from site_config.models import SiteConfiguration
    s = SiteConfiguration.get_solo()
    from_ = s.email_from or settings.EMAIL_FROM
    send_mail(subject, body, from_, to)

def notify_office_mail(subject, body):
    """Send email to office."""
    from site_config.models import SiteConfiguration
    to_ = SiteConfiguration.get_solo().office_email
    send_async_mail(subject, body, [to_])

def blacklist_notify(recipient_pk, sms_body, keyword):
    """Send email to office when we discover we are blacklisted."""
    from apostello.models import Recipient
    recipient = Recipient.objects.get(pk=recipient_pk)
    if keyword == 'start':
        return
    if keyword == 'stop':
        async(
            'apostello.tasks.notify_office_mail',
            '[Apostello] Blacklist Update',
            "{0} ({1}) is now blocking us".format(
                str(recipient.number),
                str(recipient),
            ),
        )
        return
    if recipient.is_blocking:
        email_body = "{0} has blacklisted us in the past but has just sent " \
            "this message:\n\n\t{1}\n\n" \
            "You may need to email them as we cannot currently reply to them."
        email_body.format(str(recipient), sms_body)
        async(
            'apostello.tasks.notify_office_mail',
            '[Apostello] Blacklist Receipt Notice',
            email_body,
        )

def send_keyword_digest():
    """Send daily digest email."""
    from apostello import digest
    from apostello.models import Keyword
    for keyword in Keyword.objects.filter(is_archived=False):
        checked_time = timezone.now()
        new_responses = keyword.fetch_matches()
        if keyword.last_email_sent_time is not None:
            new_responses = new_responses.filter(time_received__gt=keyword.last_email_sent_time)
        # if any, loop over subscribers and send email
        if new_responses.count() > 0:
            subject = 'Daily update for "{0}" responses'.format(str(keyword)),
            body = digest.create_email_body(keyword, new_responses)
            for subscriber in keyword.subscribed_to_digest.all():
                async('apostello.tasks.send_async_mail', subject, body, [subscriber.email])

        keyword.last_email_sent_time = checked_time
        keyword.save()

def post_to_slack(attachments):
    """Post message to slack webhook."""
    from site_config.models import SiteConfiguration
    config = SiteConfiguration.get_solo()
    url = config.slack_url
    if url:
        data = {'username': 'apostello', 'icon_emoji': ':speech_balloon:', 'attachments': attachments}
        headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
        requests.post(url, data=json.dumps(data), headers=headers)

def sms_to_slack(sms_body, person_name, keyword_name):
    """Post message to slack webhook."""
    fallback = "{0}\nFrom: {1}\n(matched: {2})".format(sms_body, person_name, keyword_name)
    attachments = [
        {
            'fallback':
            fallback,
            'color':
            '#5b599c',
            'text':
            sms_body,
            'fields': [
                {
                    'title': 'From',
                    'value': person_name,
                    'short': True
                }, {
                    'title': 'Matched',
                    'value': keyword_name,
                    'short': True
                }
            ],
        },
    ]
    post_to_slack(attachments)

def send_cloud_messages():
    """Send messages to all registere cloud messaging IDs."""
    server_key = settings.CM_SERVER_KEY
    if not server_key:
        return
    from apostello.models import CloudMessageId
    cmids = CloudMessageId.objects.all()
    for cmid in cmids:
        if not (cmid.user.profile.is_staff or cmid.user.profile.can_see_incoming):
            # only send messages to people with permission
            continue
        headers={
            'Content-Length': '0',
            'TTL': '60',
        }
        if 'mozilla' not in cmid.url:
            headers['Authorization'] = f'key={server_key}'
        try:
            r = requests.post(
                cmid.url,
                headers = headers,
            )
            if r.status_code == 401:
                cmid.delete()
                logger.info('Deleting %s', str(cmid))
        except Exception as e:
            print(e)
            logger.info('Unable to push to %s', str(cmid))

# Elvanto import
def fetch_elvanto_groups(force=False):
    """Fetch all Elvanto groups."""
    from site_config.models import SiteConfiguration
    config = SiteConfiguration.get_solo()
    if force or config.sync_elvanto:
        from elvanto.models import ElvantoGroup
        ElvantoGroup.fetch_all_groups()

def pull_elvanto_groups(force=False):
    """Pull all the Elvanto groups that are set to sync."""
    from site_config.models import SiteConfiguration
    config = SiteConfiguration.get_solo()
    if force or config.sync_elvanto:
        from elvanto.models import ElvantoGroup
        ElvantoGroup.pull_all_groups()

# Onebody import
def pull_onebody_csv():
    from onebody.importer import import_onebody_csv
    import_onebody_csv()

#
def add_new_contact_to_groups(contact_pk):
    """Add contact to any groups that are in "auto populate with new contacts."""
    logger.info('Adding new person to designated groups')
    from apostello.models import Recipient
    from site_config.models import SiteConfiguration
    contact = Recipient.objects.get(pk=contact_pk)
    for grp in SiteConfiguration.get_solo().auto_add_new_groups.all():
        logger.info('Adding %s to %s', contact.full_name, grp.name)
        contact.groups.add(grp)
        contact.save()
        logger.info('Added %s to %s', contact.full_name, grp.name)

# Keyword number of matches cache:
def populate_keyword_response_count(pk=None):
    """
    Populate cache that counts matched keywords.

    If a pk is passed, only update cache for that pk.
    If no pk is passed, update cache for all keywords.
    """
    from apostello.models import Keyword
    if pk is None:
        keywords = Keyword.objects.all()
    else:
        keywords = [Keyword.objects.get(pk=pk)]
    for k in keywords:
        cache.set('keyword_{0}_num_resps'.format(pk), k.fetch_matches().count(), 600)
        cache.set('keyword_{0}_num_arch_resps'.format(pk), k.fetch_archived_matches().count(), 600)
<commit_msg>fix typo in send notifications task<commit_after>import json
import logging

import requests
from django.conf import settings
from django.core.cache import cache
from django.core.mail import get_connection, send_mail
from django.utils import timezone
from django_q.tasks import async
from twilio.base.exceptions import TwilioRestException

from apostello.twilio import twilio_client
from apostello.utils import fetch_default_reply

logger = logging.getLogger('apostello')

# sending messages

def group_send_message_task(body, group_name, sent_by, eta):
    """Send message to all members of group."""
    from apostello.models import Recipient, RecipientGroup
    group = RecipientGroup.objects.filter(name=group_name, is_archived=False)

    for recipient in Recipient.objects.filter(groups__in=group):
        recipient.send_message(content=body, group=group_name, sent_by=sent_by, eta=eta)

def recipient_send_message_task(recipient_pk, body, group, sent_by):
    """Send a message asynchronously."""
    from apostello.models import Recipient
    recipient = Recipient.objects.get(pk=recipient_pk)
    if recipient.is_archived:
        # if recipient is not active, fail silently
        return

    from apostello.models import SmsOutbound, RecipientGroup
    # if %name% is present, replace:
    body = recipient.personalise(body)
    # send twilio message
    try:
        message = twilio_client.messages.create(body=body, to=str(recipient.number), from_=settings.TWILIO_FROM_NUM)
        # add to sms out table
        sms = SmsOutbound(sid=message.sid, content=body, time_sent=timezone.now(), recipient=recipient, sent_by=sent_by)
        if group is not None:
            sms.recipient_group = RecipientGroup.objects.filter(name=group)[0]
        sms.save()
    except TwilioRestException as e:
        if e.code == 21610:
            recipient.is_blocking = True
            recipient.save()
            async('apostello.tasks.blacklist_notify', recipient.pk, '', 'stop')
        else:
            raise e

def send_queued_sms():
    """Check for and send any queued messages."""
    from apostello.models import QueuedSms
    for sms in QueuedSms.objects.filter(sent=False, time_to_send__lte=timezone.now()):
        sms.send()

def ask_for_name(person_from_pk, sms_body, ask_for_name):
    """Asks a contact to provide their name."""
    if not ask_for_name:
        return
    from site_config.models import SiteConfiguration
    config = SiteConfiguration.get_solo()
    if not config.disable_all_replies:
        from apostello.models import Recipient
        contact = Recipient.objects.get(pk=person_from_pk)
        contact.send_message(content=fetch_default_reply('auto_name_request'), sent_by="auto name request")
        async(
            'apostello.tasks.notify_office_mail',
            '[Apostello] Unknown Contact!',
            'SMS: {0}\nFrom: {1}\n\n\nThis person is unknown and has been'
            ' asked for their name.'.format(sms_body, str(contact)),
        )

# SMS logging and consistency checks

def check_incoming_log():
    """Update incoming log."""
    from apostello.logs import check_incoming_log
    check_incoming_log()

def check_outgoing_log():
    """Update outgoing log."""
    from apostello.logs import check_outgoing_log
    check_outgoing_log()

def log_msg_in(p, t, from_pk):
    """Log incoming message."""
    from apostello.models import Keyword, SmsInbound, Recipient
    from_ = Recipient.objects.get(pk=from_pk)
    matched_keyword = Keyword.match(p['Body'].strip())
    SmsInbound.objects.create(
        sid=p['MessageSid'],
        content=p['Body'],
        time_received=t,
        sender_name=str(from_),
        sender_num=p['From'],
        matched_keyword=str(matched_keyword),
        matched_colour=Keyword.lookup_colour(p['Body'].strip())
    )
    # check log is consistent:
    async('apostello.tasks.check_incoming_log')

def update_msgs_name(person_pk):
    """Back date sender_name field on inbound sms."""
    from apostello.models import Recipient, SmsInbound
    person_ = Recipient.objects.get(pk=person_pk)
    name = str(person_)
    number = str(person_.number)
    for sms in SmsInbound.objects.filter(sender_num=number):
        sms.sender_name = name
        sms.save()

def cleanup_expired_sms():
    """Remove expired messages."""
    from apostello import logs
    logs.cleanup_expired_sms()

# notifications, email, slack etc

def send_async_mail(subject, body, to):
    """Send email."""
    # read email settings from DB, if they are empty, they will be read from
    # settings.py instead
    from site_config.models import SiteConfiguration
    s = SiteConfiguration.get_solo()
    from_ = s.email_from or settings.EMAIL_FROM
    send_mail(subject, body, from_, to)

def notify_office_mail(subject, body):
    """Send email to office."""
    from site_config.models import SiteConfiguration
    to_ = SiteConfiguration.get_solo().office_email
    send_async_mail(subject, body, [to_])

def blacklist_notify(recipient_pk, sms_body, keyword):
    """Send email to office when we discover we are blacklisted."""
    from apostello.models import Recipient
    recipient = Recipient.objects.get(pk=recipient_pk)
    if keyword == 'start':
        return
    if keyword == 'stop':
        async(
            'apostello.tasks.notify_office_mail',
            '[Apostello] Blacklist Update',
            "{0} ({1}) is now blocking us".format(
                str(recipient.number),
                str(recipient),
            ),
        )
        return
    if recipient.is_blocking:
        email_body = "{0} has blacklisted us in the past but has just sent " \
            "this message:\n\n\t{1}\n\n" \
            "You may need to email them as we cannot currently reply to them."
        email_body.format(str(recipient), sms_body)
        async(
            'apostello.tasks.notify_office_mail',
            '[Apostello] Blacklist Receipt Notice',
            email_body,
        )

def send_keyword_digest():
    """Send daily digest email."""
    from apostello import digest
    from apostello.models import Keyword
    for keyword in Keyword.objects.filter(is_archived=False):
        checked_time = timezone.now()
        new_responses = keyword.fetch_matches()
        if keyword.last_email_sent_time is not None:
            new_responses = new_responses.filter(time_received__gt=keyword.last_email_sent_time)
        # if any, loop over subscribers and send email
        if new_responses.count() > 0:
            subject = 'Daily update for "{0}" responses'.format(str(keyword)),
            body = digest.create_email_body(keyword, new_responses)
            for subscriber in keyword.subscribed_to_digest.all():
                async('apostello.tasks.send_async_mail', subject, body, [subscriber.email])

        keyword.last_email_sent_time = checked_time
        keyword.save()

def post_to_slack(attachments):
    """Post message to slack webhook."""
    from site_config.models import SiteConfiguration
    config = SiteConfiguration.get_solo()
    url = config.slack_url
    if url:
        data = {'username': 'apostello', 'icon_emoji': ':speech_balloon:', 'attachments': attachments}
        headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
        requests.post(url, data=json.dumps(data), headers=headers)

def sms_to_slack(sms_body, person_name, keyword_name):
    """Post message to slack webhook."""
    fallback = "{0}\nFrom: {1}\n(matched: {2})".format(sms_body, person_name, keyword_name)
    attachments = [
        {
            'fallback':
            fallback,
            'color':
            '#5b599c',
            'text':
            sms_body,
            'fields': [
                {
                    'title': 'From',
                    'value': person_name,
                    'short': True
                }, {
                    'title': 'Matched',
                    'value': keyword_name,
                    'short': True
                }
            ],
        },
    ]
    post_to_slack(attachments)

def send_cloud_messages():
    """Send messages to all registere cloud messaging IDs."""
    server_key = settings.CM_SERVER_KEY
    if not server_key:
        return
    from apostello.models import CloudMessageId
    cmids = CloudMessageId.objects.all()
    for cmid in cmids:
        if not (cmid.user.is_staff or cmid.user.profile.can_see_incoming):
            # only send messages to people with permission
            continue
        headers = {
            'Content-Length': '0',
            'TTL': '60',
        }
        if 'mozilla' not in cmid.url:
            headers['Authorization'] = f'key={server_key}'
        try:
            r = requests.post(
                cmid.url,
                headers=headers,
            )
            if r.status_code == 401:
                cmid.delete()
                logger.info('Deleting %s', str(cmid))
        except Exception as e:
            print(e)
            logger.info('Unable to push to %s', str(cmid))

# Elvanto import
def fetch_elvanto_groups(force=False):
    """Fetch all Elvanto groups."""
    from site_config.models import SiteConfiguration
    config = SiteConfiguration.get_solo()
    if force or config.sync_elvanto:
        from elvanto.models import ElvantoGroup
        ElvantoGroup.fetch_all_groups()

def pull_elvanto_groups(force=False):
    """Pull all the Elvanto groups that are set to sync."""
    from site_config.models import SiteConfiguration
    config = SiteConfiguration.get_solo()
    if force or config.sync_elvanto:
        from elvanto.models import ElvantoGroup
        ElvantoGroup.pull_all_groups()

# Onebody import
def pull_onebody_csv():
    from onebody.importer import import_onebody_csv
    import_onebody_csv()

#
def add_new_contact_to_groups(contact_pk):
    """Add contact to any groups that are in "auto populate with new contacts."""
    logger.info('Adding new person to designated groups')
    from apostello.models import Recipient
    from site_config.models import SiteConfiguration
    contact = Recipient.objects.get(pk=contact_pk)
    for grp in SiteConfiguration.get_solo().auto_add_new_groups.all():
        logger.info('Adding %s to %s', contact.full_name, grp.name)
        contact.groups.add(grp)
        contact.save()
        logger.info('Added %s to %s', contact.full_name, grp.name)

# Keyword number of matches cache:
def populate_keyword_response_count(pk=None):
    """
    Populate cache that counts matched keywords.

    If a pk is passed, only update cache for that pk.
    If no pk is passed, update cache for all keywords.
    """
    from apostello.models import Keyword
    if pk is None:
        keywords = Keyword.objects.all()
    else:
        keywords = [Keyword.objects.get(pk=pk)]
    for k in keywords:
        cache.set('keyword_{0}_num_resps'.format(pk), k.fetch_matches().count(), 600)
        cache.set('keyword_{0}_num_arch_resps'.format(pk), k.fetch_archived_matches().count(), 600)
<|endoftext|>


<commit_before># coding: utf8

from core.ParsingEntities import Entity

class ModificationEntity(Entity):
    def __add__(self, other):
        if isinstance(self, ModificationEntity) and isinstance(other, ModificationEntity):
            modification_operator = ModificationOperator()
            return modification_operator
        else:
            raise TypeError("Operands have to be ModificationEntity's subclasses")

class ModificationCondition(ModificationEntity):
    pass

class ModificationOperator(ModificationEntity):
    pass<commit_msg>Implement ModificationOperator and ModificationCondition classes<commit_after># coding: utf8

from core.ParsingEntities import Entity

class ModificationEntity(Entity):
    def __add__(self, other):
        if isinstance(self, ModificationEntity) and isinstance(other, ModificationEntity):
            modification_operator = ModificationOperator()
            return modification_operator
        else:
            raise TypeError("Operands have to be ModificationEntity's subclasses")

class ModificationOperator(ModificationEntity):
    def __init__(self, operand_a, operand_b):
        self.operandA = operand_a
        self.operandB = operand_b

    def check(self, element, ref_position):
        pass

class ModificationCondition(ModificationEntity):
    def __init__(self, character, operation_type, rel_position=0):
        self.rel_position = rel_position
        self.operationType = operation_type
        self.character = character

    def check(self, element, ref_position=0):
        pass
<|endoftext|>


<commit_before>#!/home/arthurzey/deltawerx.com/env/bin/python

from os.path import expanduser, join
from os import environ

from stormpath.client import Client

# This is the absolute path to the file ~/.stormpath/apiKey.properties
# downloaded in the previous section.  This will work on any OS.
API_KEY_FILE = join(expanduser('~'), '.stormpath', 'apiKey.properties')

client = Client(api_key_file=API_KEY_FILE)

application = client.applications.search({'name': 'My Application'})[0]

url = environ['SCRIPT_URI'] + "?" + environ['QUERY_STRING']

result = application.handle_stormpath_callback(url)

# print "Location: http://www.deltawerx.com/toyproblems/interviews/stormpath/"
print "Content-type: text/html\n\n"

print dir(result.account.full_name)
# print "You are being redirected to http://www.deltawerx.com/toyproblems/interviews/stormpath/"<commit_msg>stormpath: providing login congratulations message<commit_after>#!/home/arthurzey/deltawerx.com/env/bin/python

from os.path import expanduser, join
from os import environ

from stormpath.client import Client

# This is the absolute path to the file ~/.stormpath/apiKey.properties
# downloaded in the previous section.  This will work on any OS.
API_KEY_FILE = join(expanduser('~'), '.stormpath', 'apiKey.properties')

client = Client(api_key_file=API_KEY_FILE)

application = client.applications.search({'name': 'My Application'})[0]

url = environ['SCRIPT_URI'] + "?" + environ['QUERY_STRING']

result = application.handle_stormpath_callback(url)

# print "Location: http://www.deltawerx.com/toyproblems/interviews/stormpath/"
print "Content-type: text/html\n\n"

print "Congratulations! " + result.account.full_name + " has been logged in successfully!"
# print "You are being redirected to http://www.deltawerx.com/toyproblems/interviews/stormpath/"<|endoftext|>


<commit_before>'''
Declarations used to elucidate MARC model
'''

#Just set up some flags
#BOUND_TO_WORK = object()
#BOUND_TO_INSTANCE = object()

#Full MARC field list: http://www.loc.gov/marc/bibliographic/ecbdlist.html

MATERIALIZE = {
'100': ('creator', {'marcrType': 'Person'}),
'110': ('creator', {'marcrType': 'Organization'}),
'111': ('creator', {'marcrType': 'Meeting'}),

'130': ('uniformTitle', {'marcrType': 'Title'}),
'240': ('uniformTitle', {'marcrType': 'Title'}),
'243': ('uniformTitle', {'marcrType': 'Title'}),
'730': ('uniformTitle', {'marcrType': 'Title'}),
'830': ('uniformTitle', {'marcrType': 'Title'}),

'260a': ('place', {'marcrType': 'Place'}),
'260b': ('provider', {'marcrType': 'Organization'}),
'260e': ('place', {'marcrType': 'Place'}),
'260f': ('provider', {'marcrType': 'Organization'}),

'300': ('physicalDescription', {'marcrType': 'Measurement'}),

'600': ('subject', {'marcrType': 'Person'}),
'610': ('subject', {'marcrType': 'Organization'}),
'611': ('subject', {'marcrType': 'Meeting'}),

'630': ('uniformTitle', {'marcrType': 'Title'}),
'650': ('subject', {'marcrType': 'Topic'}),
'651': ('subject', {'marcrType': 'Geographic'}),
#'655': ('genre', {'marcrType': 'Genre'}),

'700': ('creator', {'marcrType': 'Person'}),
'710': ('creator', {'marcrType': 'Organization'}),
'711': ('creator', {'marcrType': 'Meeting'}),
}

MATERIALIZE_VIA_ANNOTATION = {
#'852': ('institution', {'marcrType': 'Organization'}, {'marcrType': 'Holdings'}),
'852': ('institution', 'Holdings', {'holderType': 'Organization'},),
}

FIELD_RENAMINGS = {
'010a': 'lccn',
#Don't do a simple field renaming of ISBN because
#'020a': 'isbn',
'022a': 'issn',
'034a': 'cartographicMathematicalDataScaleStatement', #Rebecca & Sally suggested this should effectively be a merge with 034a
'034b': 'cartographicMathematicalDataProjectionStatement',
'034c': 'cartographicMathematicalDataCoordinateStatement',
'050a': 'lcCallNumber',
'0503': 'material',
'082a': 'deweyNumber',

'100a': 'label',
'100b': 'numeration',
'100c': 'titles',
'100d': 'date',  #Note: there has been discussion about removing this, but we're no sure we get reliable ID.LOC lookups without it.  If it is removed, update augment.py
'110a': 'label',
'110d': 'date',
'111a': 'label',
'111d': 'date',

'130a': 'label',
'240a': 'label',
'730a': 'label',
'830a': 'label',

'130l': 'language',
'041a': 'language',

'245a': 'title',
'245b': 'titleRemainder',
'245c': 'titleStatement',
'245f': 'titleInclusiveDates',
'245n': 'titleNumberParts',
'245p': 'titleNameParts',
'247a': 'formerTitle',
'250a': 'edition',
'250b': 'edition',
'254a': 'musicalPresentation',
'255a': 'cartographicMathematicalDataScaleStatement',
'255b': 'cartographicMathematicalDataProjectionStatement',
'255c': 'cartographicMathematicalDataCoordinateStatement',
'256a': 'computerFilecharacteristics',
'260a': 'label',
'260b': 'label',
'260c': 'date',
'260e': 'place',
'260f': 'label',
'260g': 'date',

'300a': 'extent',
'300b': 'physicalDesc',
'300c': 'dimensions',
'300e': 'accompanyingMaterial',
'300f': 'typeOfunit',
'300g': 'size',
'3003': 'materials',

'490a': 'seriesStatement',

'500a': 'note',
'501a': 'note',
'502a': 'note',
'502b': 'note',
'502c': 'note',
'502d': 'note',
'502g': 'note',
'502o': 'note',
'504a': 'note',
'505a': 'formatedContentsNote',
'506a': 'note',
'506b': 'note',
'506c': 'note',
'506u': 'note',
'507a': 'note',
'507b': 'note',
'508a': 'note',
'510a': 'note',
'510b': 'note',
'510c': 'note',
'510u': 'note',
'511a': 'note',
'513a': 'note',
'513b': 'note',
'515a': 'note',
'516a': 'note',
'518a': 'note',
'518d': 'note',
'518o': 'note',
'518p': 'note',
'520a': 'summary',
'520b': 'summary',
'521a': 'targetAudienceNote',
'521b': 'note',
'522a': 'coverage',
'525a': 'note',

'600a': 'label',
'600d': 'date',
'610a': 'label',
'610d': 'date',  #Note: there has been discussion about removing this, but we're no sure we get reliable ID.LOC lookups without it.  If it is removed, update augment.py
'650a': 'label',
'650d': 'date',
'651a': 'label',
'651d': 'date',
'630a': 'uniformTitle',
'630l': 'language',

'630a': 'label',
'630h': 'medium',
'630v': 'formSubdivision',
'630x': 'generalSubdivision',
'630y': 'chronologicalSubdivision',
'630z': 'geographicSubdivision',

'650a': 'label',
'650c': 'locationOfEvent',
'650v': 'formSubdivision',
'650x': 'generalSubdivision',
'650y': 'chronologicalSubdivision',
'650z': 'geographicSubdivision',

'651a': 'label',
'651v': 'formSubdivision',
'651x': 'generalSubdivision',
'651y': 'chronologicalSubdivision',
'651z': 'geographicSubdivision',

'700a': 'label',
'700b': 'numeration',
'700c': 'titles',
'700d': 'date',  #Note: there has been discussion about removing this, but we're no sure we get reliable ID.LOC lookups without it.  If it is removed, update augment.py
'710a': 'label',
'710d': 'date',
'711a': 'label',
'711d': 'date',

#'852a': 'institution',
'852a': 'label',
'852h': 'callNumber', #Need to verify this one, since it seems to contradict the rest of the 852 pattern
'852n': 'code',
'852u': 'link',
'852e': 'streetAddress',

'856u': 'link',
}

WORK_FIELDS = set([
'010',
'028',
'035',
'040',
'041',
'050a', #Note: should be able to link directly to authority @ id.loc.gov authority/classification/####
'082',
'100',
'110',
'111',
'111',
'130',
'210',
'222',
'240',
'243',
'245',
'245',
'246',
'247',
'310',
'310',
'321',
'321',
'362',
'490',
'500',
'502',
'504',
'505',
'507',
'508',
'511',
'513',
'515',
'516',
'518',
'520',
'521',
'522',
'525',
'583',
'600',
'610',
'611',
'630',
'650',
'651',
'700',
'710',
'711',
'730',
])

INSTANCE_FIELDS = set([
'020',
'022',
'055',
'060',
'070',
'086',
'250',
'254',
'255',
'256',
'257',
'260',
'263',
'300',
'306',
'340',
'351',
'501',
'510',
'506',
'850',
'852',
'856',
])

ANNOTATIONS_FIELDS = set([
'852h',
])

#HOLDINGS_FIELDS = set([
#'852',
#])

<commit_msg>updating contributor and contentNotes<commit_after>'''
Declarations used to elucidate MARC model
'''

#Just set up some flags
#BOUND_TO_WORK = object()
#BOUND_TO_INSTANCE = object()

#Full MARC field list: http://www.loc.gov/marc/bibliographic/ecbdlist.html

MATERIALIZE = {
'100': ('creator', {'marcrType': 'Person'}),
'110': ('creator', {'marcrType': 'Organization'}),
'111': ('creator', {'marcrType': 'Meeting'}),

'130': ('uniformTitle', {'marcrType': 'Title'}),
'240': ('uniformTitle', {'marcrType': 'Title'}),
'243': ('uniformTitle', {'marcrType': 'Title'}),
'730': ('uniformTitle', {'marcrType': 'Title'}),
'830': ('uniformTitle', {'marcrType': 'Title'}),

'260a': ('place', {'marcrType': 'Place'}),
'260b': ('provider', {'marcrType': 'Organization'}),
'260e': ('place', {'marcrType': 'Place'}),
'260f': ('provider', {'marcrType': 'Organization'}),

'300': ('physicalDescription', {'marcrType': 'Measurement'}),

'600': ('subject', {'marcrType': 'Person'}),
'610': ('subject', {'marcrType': 'Organization'}),
'611': ('subject', {'marcrType': 'Meeting'}),

'630': ('uniformTitle', {'marcrType': 'Title'}),
'650': ('subject', {'marcrType': 'Topic'}),
'651': ('subject', {'marcrType': 'Geographic'}),
#'655': ('genre', {'marcrType': 'Genre'}),

'700': ('contributor', {'marcrType': 'Person'}),
'710': ('contributor', {'marcrType': 'Organization'}),
'711': ('contributor', {'marcrType': 'Meeting'}),
}

MATERIALIZE_VIA_ANNOTATION = {
#'852': ('institution', {'marcrType': 'Organization'}, {'marcrType': 'Holdings'}),
'852': ('institution', 'Holdings', {'holderType': 'Organization'},),
}

FIELD_RENAMINGS = {
'010a': 'lccn',
#Don't do a simple field renaming of ISBN because
#'020a': 'isbn',
'022a': 'issn',
'034a': 'cartographicMathematicalDataScaleStatement', #Rebecca & Sally suggested this should effectively be a merge with 034a
'034b': 'cartographicMathematicalDataProjectionStatement',
'034c': 'cartographicMathematicalDataCoordinateStatement',
'050a': 'lcCallNumber',
'0503': 'material',
'082a': 'deweyNumber',

'100a': 'label',
'100b': 'numeration',
'100c': 'titles',
'100d': 'date',  #Note: there has been discussion about removing this, but we're no sure we get reliable ID.LOC lookups without it.  If it is removed, update augment.py
'110a': 'label',
'110d': 'date',
'111a': 'label',
'111d': 'date',

'130a': 'label',
'240a': 'label',
'730a': 'label',
'830a': 'label',

'130l': 'language',
'041a': 'language',

'245a': 'title',
'245b': 'titleRemainder',
'245c': 'titleStatement',
'245f': 'titleInclusiveDates',
'245n': 'titleNumberParts',
'245p': 'titleNameParts',
'247a': 'formerTitle',
'250a': 'edition',
'250b': 'edition',
'254a': 'musicalPresentation',
'255a': 'cartographicMathematicalDataScaleStatement',
'255b': 'cartographicMathematicalDataProjectionStatement',
'255c': 'cartographicMathematicalDataCoordinateStatement',
'256a': 'computerFilecharacteristics',
'260a': 'label',
'260b': 'label',
'260c': 'date',
'260e': 'place',
'260f': 'label',
'260g': 'date',

'300a': 'extent',
'300b': 'physicalDesc',
'300c': 'dimensions',
'300e': 'accompanyingMaterial',
'300f': 'typeOfunit',
'300g': 'size',
'3003': 'materials',

'490a': 'seriesStatement',

'500a': 'note',
'501a': 'note',
'502a': 'note',
'502b': 'note',
'502c': 'note',
'502d': 'note',
'502g': 'note',
'502o': 'note',
'504a': 'note',
'505a': 'contentsNote',
'506a': 'note',
'506b': 'note',
'506c': 'note',
'506u': 'note',
'507a': 'note',
'507b': 'note',
'508a': 'note',
'510a': 'note',
'510b': 'note',
'510c': 'note',
'510u': 'note',
'511a': 'note',
'513a': 'note',
'513b': 'note',
'515a': 'note',
'516a': 'note',
'518a': 'note',
'518d': 'note',
'518o': 'note',
'518p': 'note',
'520a': 'summary',
'520b': 'summary',
'521a': 'targetAudienceNote',
'521b': 'note',
'522a': 'coverage',
'525a': 'note',

'600a': 'label',
'600d': 'date',
'610a': 'label',
'610d': 'date',  #Note: there has been discussion about removing this, but we're no sure we get reliable ID.LOC lookups without it.  If it is removed, update augment.py
'650a': 'label',
'650d': 'date',
'651a': 'label',
'651d': 'date',
'630a': 'uniformTitle',
'630l': 'language',

'630a': 'label',
'630h': 'medium',
'630v': 'formSubdivision',
'630x': 'generalSubdivision',
'630y': 'chronologicalSubdivision',
'630z': 'geographicSubdivision',

'650a': 'label',
'650c': 'locationOfEvent',
'650v': 'formSubdivision',
'650x': 'generalSubdivision',
'650y': 'chronologicalSubdivision',
'650z': 'geographicSubdivision',

'651a': 'label',
'651v': 'formSubdivision',
'651x': 'generalSubdivision',
'651y': 'chronologicalSubdivision',
'651z': 'geographicSubdivision',

'700a': 'label',
'700b': 'numeration',
'700c': 'titles',
'700d': 'date',  #Note: there has been discussion about removing this, but we're no sure we get reliable ID.LOC lookups without it.  If it is removed, update augment.py
'710a': 'label',
'710d': 'date',
'711a': 'label',
'711d': 'date',

#'852a': 'institution',
'852a': 'label',
'852h': 'callNumber', #Need to verify this one, since it seems to contradict the rest of the 852 pattern
'852n': 'code',
'852u': 'link',
'852e': 'streetAddress',

'856u': 'link',
}

WORK_FIELDS = set([
'010',
'028',
'035',
'040',
'041',
'050a', #Note: should be able to link directly to authority @ id.loc.gov authority/classification/####
'082',
'100',
'110',
'111',
'111',
'130',
'210',
'222',
'240',
'243',
'245',
'245',
'246',
'247',
'310',
'310',
'321',
'321',
'362',
'490',
'500',
'502',
'504',
'507',
'508',
'511',
'513',
'515',
'516',
'518',
'520',
'521',
'522',
'525',
'583',
'600',
'610',
'611',
'630',
'650',
'651',
'700',
'710',
'711',
'730',
])

INSTANCE_FIELDS = set([
'020',
'022',
'055',
'060',
'070',
'086',
'250',
'254',
'255',
'256',
'257',
'260',
'263',
'300',
'306',
'340',
'351',
'501',
'505',
'510',
'506',
'850',
'852',
'856',
])

ANNOTATIONS_FIELDS = set([
'852h',
])

#HOLDINGS_FIELDS = set([
#'852',
#])

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2015-2019 CERN.
#
# Invenio is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.

"""REST API serializers."""

import json
import warnings
from time import sleep

from flask import current_app, request, url_for
from invenio_rest.serializer import BaseSchema as InvenioRestBaseSchema
from marshmallow import fields, missing, post_dump

from .errors import FilesException
from .models import Bucket, MultipartObject, ObjectVersion, Part

class BaseSchema(InvenioRestBaseSchema):
    """Base schema for all serializations."""

    created = fields.DateTime(dump_only=True)
    updated = fields.DateTime(dump_only=True)
    links = fields.Method('dump_links', dump_only=True)

    def dump_links(self, o):
        """Get base links."""
        return missing

class BucketSchema(BaseSchema):
    """Schema for bucket."""

    id = fields.UUID()
    size = fields.Integer()
    quota_size = fields.Integer()
    max_file_size = fields.Integer()
    locked = fields.Boolean()

    def dump_links(self, o):
        """Dump links."""
        return {
            'self': url_for('.bucket_api', bucket_id=o.id, _external=True),
            'versions': url_for(
                '.bucket_api', bucket_id=o.id, _external=True) + '?versions',
            'uploads': url_for(
                '.bucket_api', bucket_id=o.id, _external=True) + '?uploads',
        }

class ObjectVersionSchema(BaseSchema):
    """Schema for ObjectVersions."""

    key = fields.Str()
    version_id = fields.UUID()
    is_head = fields.Boolean()
    mimetype = fields.Str()
    size = fields.Integer(attribute='file.size')
    checksum = fields.String(attribute='file.checksum')
    delete_marker = fields.Boolean(attribute='deleted')
    tags = fields.Method('dump_tags', dump_only=True)

    def dump_tags(self, o):
        """Dump tags."""
        return o.get_tags()

    def dump_links(self, o):
        """Dump links."""
        params = {'versionId': o.version_id}
        data = {
            'self': url_for(
                '.object_api',
                bucket_id=o.bucket_id,
                key=o.key,
                _external=True,
                **(params if not o.is_head or o.deleted else {})
            ),
            'version': url_for(
                '.object_api',
                bucket_id=o.bucket_id,
                key=o.key,
                _external=True,
                **params
            )
        }

        if o.is_head and not o.deleted:
            data.update({'uploads': url_for(
                '.object_api',
                bucket_id=o.bucket_id,
                key=o.key,
                _external=True
            ) + '?uploads', })

        return data

    @post_dump(pass_many=True)
    def wrap(self, data, many):
        """Wrap response in envelope."""
        if not many:
            return data
        else:
            data = {'contents': data}
            bucket = self.context.get('bucket')
            if bucket:
                data.update(BucketSchema().dump(bucket).data)
            return data

class MultipartObjectSchema(BaseSchema):
    """Schema for ObjectVersions."""

    _endpoint = '.object_api'

    id = fields.UUID(attribute='upload_id', dump_only=True)
    bucket = fields.UUID(attribute='bucket_id', dump_only=True)
    key = fields.Str(dump_only=True)
    completed = fields.Boolean(dump_only=True)
    size = fields.Integer()
    part_size = fields.Integer(attribute='chunk_size')
    last_part_number = fields.Integer(dump_only=True)
    last_part_size = fields.Integer(dump_only=True)

    def dump_links(self, o):
        """Dump links."""
        links = {
            'self': url_for(
                '.object_api',
                bucket_id=o.bucket_id,
                key=o.key,
                uploadId=o.upload_id,
                _external=True,
            ),
            'object': url_for(
                '.object_api',
                bucket_id=o.bucket_id,
                key=o.key,
                _external=True,
            ),
        }

        version_id = self.context.get('object_version_id')
        if version_id:
            links.update({
                'object_version': url_for(
                    '.object_api',
                    bucket_id=o.bucket_id,
                    key=o.key,
                    versionId=version_id,
                    _external=True,
                )
            })

        bucket = self.context.get('bucket')
        if bucket:
            links.update({
                'bucket': url_for(
                    '.bucket_api',
                    bucket_id=o.bucket_id,
                    _external=True,
                )
            })

        return links

class PartSchema(BaseSchema):
    """Schema for parts."""

    part_number = fields.Integer()
    start_byte = fields.Integer()
    end_byte = fields.Integer()
    checksum = fields.Str()

    links = None

    @post_dump(pass_many=True)
    def wrap(self, data, many):
        """Wrap response in envelope."""
        if not many:
            return data
        else:
            data = {'parts': data}
            multipart = self.context.get('multipart')
            if multipart:
                data.update(MultipartObjectSchema(context={
                    'bucket': multipart.bucket}).dump(multipart).data)
            return data

serializer_mapping = {
    Bucket: BucketSchema,
    ObjectVersion: ObjectVersionSchema,
    MultipartObject: MultipartObjectSchema,
    Part: PartSchema,
}

def schema_from_context(context):
    """Determine which schema to use."""
    item_class = context.get('class')
    return (
        serializer_mapping[item_class] if item_class else BaseSchema,
        context.get('many', False)
    )

def _format_args():
    if request and request.args.get('prettyprint'):
        return dict(
            indent=2,
            separators=(', ', ': '),
        )
    else:
        return dict(
            indent=None,
            separators=(',', ':'),
        )

def wait_for_taskresult(task_result, content, interval, max_rounds):
    """Get helper to wait for async task result to finish.

    The task will periodically send whitespace to prevent the connection from
    being closed.

    :param task_result: The async task to wait for.
    :param content: The content to return when the task is ready.
    :param interval: The duration of a sleep period before check again if the
        task is ready.
    :param max_rounds: The maximum number of intervals the function check
        before returning an Exception.
    :returns: An iterator on the content or a
        :class:`invenio_files_rest.errors.FilesException` exception if the
        timeout happened or the job failed.
    """
    assert max_rounds > 0

    def _whitespace_waiting():
        current = 0
        while current < max_rounds and current != -1:
            if task_result.ready():
                # Task is done and we return
                current = -1
                if task_result.successful():
                    yield content
                else:
                    yield FilesException(
                        description='Job failed.'
                    ).get_body()
            else:
                # Send whitespace to prevent connection from closing.
                current += 1
                sleep(interval)
                yield b' '

        # Timed-out reached
        if current == max_rounds:
            yield FilesException(
                description='Job timed out.'
            ).get_body()

    return _whitespace_waiting()

def json_serializer(data=None, code=200, headers=None, context=None,
                    etag=None, task_result=None):
    """Build a json flask response using the given data.

    :param data: The data to serialize. (Default: ``None``)
    :param code: The HTTP status code. (Default: ``200``)
    :param headers: The HTTP headers to include. (Default: ``None``)
    :param context: The schema class context. (Default: ``None``)
    :param etag: The ETag header. (Default: ``None``)
    :param task_result: Optionally you can pass async task to wait for.
        (Default: ``None``)
    :returns: A Flask response with json data.
    :rtype: :py:class:`flask.Response`
    """
    schema_class, many = schema_from_context(context or {})

    if data is not None:
        # Generate JSON response
        data = json.dumps(
            schema_class(context=context).dump(data, many=many).data,
            **_format_args()
        )

        interval = current_app.config['FILES_REST_TASK_WAIT_INTERVAL']
        max_rounds = int(
            current_app.config['FILES_REST_TASK_WAIT_MAX_SECONDS'] // interval
        )

        response = current_app.response_class(
            # Stream response if waiting for task result.
            data if task_result is None else wait_for_taskresult(
                task_result, data, interval, max_rounds, ),
            mimetype='application/json'
        )
    else:
        response = current_app.response_class(mimetype='application/json')

    response.status_code = code
    if headers is not None:
        response.headers.extend(headers)

    if etag:
        response.set_etag(etag)

    return response
<commit_msg>serializer: make serializer methods more versatile<commit_after># -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2015-2019 CERN.
#
# Invenio is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.

"""REST API serializers."""

import json
import warnings
from time import sleep

from flask import current_app, request, url_for
from invenio_rest.serializer import BaseSchema as InvenioRestBaseSchema
from marshmallow import fields, missing, post_dump

from .errors import FilesException
from .models import Bucket, MultipartObject, ObjectVersion, Part

class BaseSchema(InvenioRestBaseSchema):
    """Base schema for all serializations."""

    created = fields.DateTime(dump_only=True)
    updated = fields.DateTime(dump_only=True)
    links = fields.Method('dump_links', dump_only=True)

    def dump_links(self, o):
        """Get base links."""
        return missing

class BucketSchema(BaseSchema):
    """Schema for bucket."""

    id = fields.UUID()
    size = fields.Integer()
    quota_size = fields.Integer()
    max_file_size = fields.Integer()
    locked = fields.Boolean()

    def dump_links(self, o):
        """Dump links."""
        return {
            'self': url_for('.bucket_api', bucket_id=o.id, _external=True),
            'versions': url_for(
                '.bucket_api', bucket_id=o.id, _external=True) + '?versions',
            'uploads': url_for(
                '.bucket_api', bucket_id=o.id, _external=True) + '?uploads',
        }

class ObjectVersionSchema(BaseSchema):
    """Schema for ObjectVersions."""

    key = fields.Str()
    version_id = fields.UUID()
    is_head = fields.Boolean()
    mimetype = fields.Str()
    size = fields.Integer(attribute='file.size')
    checksum = fields.String(attribute='file.checksum')
    delete_marker = fields.Boolean(attribute='deleted')
    tags = fields.Method('dump_tags', dump_only=True)

    def dump_tags(self, o):
        """Dump tags."""
        return o.get_tags()

    def dump_links(self, o):
        """Dump links."""
        params = {'versionId': o.version_id}
        data = {
            'self': url_for(
                '.object_api',
                bucket_id=o.bucket_id,
                key=o.key,
                _external=True,
                **(params if not o.is_head or o.deleted else {})
            ),
            'version': url_for(
                '.object_api',
                bucket_id=o.bucket_id,
                key=o.key,
                _external=True,
                **params
            )
        }

        if o.is_head and not o.deleted:
            data.update({'uploads': url_for(
                '.object_api',
                bucket_id=o.bucket_id,
                key=o.key,
                _external=True
            ) + '?uploads', })

        return data

    @post_dump(pass_many=True)
    def wrap(self, data, many):
        """Wrap response in envelope."""
        if not many:
            return data
        else:
            data = {'contents': data}
            bucket = self.context.get('bucket')
            if bucket:
                data.update(BucketSchema().dump(bucket).data)
            return data

class MultipartObjectSchema(BaseSchema):
    """Schema for ObjectVersions."""

    _endpoint = '.object_api'

    id = fields.UUID(attribute='upload_id', dump_only=True)
    bucket = fields.UUID(attribute='bucket_id', dump_only=True)
    key = fields.Str(dump_only=True)
    completed = fields.Boolean(dump_only=True)
    size = fields.Integer()
    part_size = fields.Integer(attribute='chunk_size')
    last_part_number = fields.Integer(dump_only=True)
    last_part_size = fields.Integer(dump_only=True)

    def dump_links(self, o):
        """Dump links."""
        links = {
            'self': url_for(
                '.object_api',
                bucket_id=o.bucket_id,
                key=o.key,
                uploadId=o.upload_id,
                _external=True,
            ),
            'object': url_for(
                '.object_api',
                bucket_id=o.bucket_id,
                key=o.key,
                _external=True,
            ),
        }

        version_id = self.context.get('object_version_id')
        if version_id:
            links.update({
                'object_version': url_for(
                    '.object_api',
                    bucket_id=o.bucket_id,
                    key=o.key,
                    versionId=version_id,
                    _external=True,
                )
            })

        bucket = self.context.get('bucket')
        if bucket:
            links.update({
                'bucket': url_for(
                    '.bucket_api',
                    bucket_id=o.bucket_id,
                    _external=True,
                )
            })

        return links

class PartSchema(BaseSchema):
    """Schema for parts."""

    part_number = fields.Integer()
    start_byte = fields.Integer()
    end_byte = fields.Integer()
    checksum = fields.Str()

    links = None

    @post_dump(pass_many=True)
    def wrap(self, data, many):
        """Wrap response in envelope."""
        if not many:
            return data
        else:
            data = {'parts': data}
            multipart = self.context.get('multipart')
            if multipart:
                data.update(MultipartObjectSchema(context={
                    'bucket': multipart.bucket}).dump(multipart).data)
            return data

serializer_mapping = {
    Bucket: BucketSchema,
    ObjectVersion: ObjectVersionSchema,
    MultipartObject: MultipartObjectSchema,
    Part: PartSchema,
}

def schema_from_context(context, serializer_mapping):
    """Determine which schema to use."""
    item_class = context.get('class')
    return (
        serializer_mapping[item_class] if item_class else BaseSchema,
        context.get('many', False)
    )

def _format_args():
    if request and request.args.get('prettyprint'):
        return dict(
            indent=2,
            separators=(', ', ': '),
        )
    else:
        return dict(
            indent=None,
            separators=(',', ':'),
        )

def wait_for_taskresult(task_result, content, interval, max_rounds):
    """Get helper to wait for async task result to finish.

    The task will periodically send whitespace to prevent the connection from
    being closed.

    :param task_result: The async task to wait for.
    :param content: The content to return when the task is ready.
    :param interval: The duration of a sleep period before check again if the
        task is ready.
    :param max_rounds: The maximum number of intervals the function check
        before returning an Exception.
    :returns: An iterator on the content or a
        :class:`invenio_files_rest.errors.FilesException` exception if the
        timeout happened or the job failed.
    """
    assert max_rounds > 0

    def _whitespace_waiting():
        current = 0
        while current < max_rounds and current != -1:
            if task_result.ready():
                # Task is done and we return
                current = -1
                if task_result.successful():
                    yield content
                else:
                    yield FilesException(
                        description='Job failed.'
                    ).get_body()
            else:
                # Send whitespace to prevent connection from closing.
                current += 1
                sleep(interval)
                yield b' '

        # Timed-out reached
        if current == max_rounds:
            yield FilesException(
                description='Job timed out.'
            ).get_body()

    return _whitespace_waiting()

def json_serializer(data=None, code=200, headers=None, context=None,
                    etag=None, task_result=None,
                    serializer_mapping=serializer_mapping, view_name=None):
    """Build a json flask response using the given data.

    :param data: The data to serialize. (Default: ``None``)
    :param code: The HTTP status code. (Default: ``200``)
    :param headers: The HTTP headers to include. (Default: ``None``)
    :param context: The schema class context. (Default: ``None``)
    :param etag: The ETag header. (Default: ``None``)
    :param task_result: Optionally you can pass async task to wait for.
        (Default: ``None``)
    :param serializer_mapping: Optionally provide the serializer with a
        different mapping.
    :param view_name: Optionally push to the marshmallow context the view
        name prefix,usefull in case of multiple routes pointing to the same
        blueprint.
    :returns: A Flask response with json data.
    :rtype: :py:class:`flask.Response`
    """
    context = context or {}
    if view_name:
        context.update({'view_name': view_name})
    schema_class, many = schema_from_context(context, serializer_mapping)

    if data is not None:
        # Generate JSON response
        data = json.dumps(
            schema_class(context=context).dump(data, many=many).data,
            **_format_args()
        )

        interval = current_app.config['FILES_REST_TASK_WAIT_INTERVAL']
        max_rounds = int(
            current_app.config['FILES_REST_TASK_WAIT_MAX_SECONDS'] // interval
        )

        response = current_app.response_class(
            # Stream response if waiting for task result.
            data if task_result is None else wait_for_taskresult(
                task_result, data, interval, max_rounds, ),
            mimetype='application/json'
        )
    else:
        response = current_app.response_class(mimetype='application/json')

    response.status_code = code
    if headers is not None:
        response.headers.extend(headers)

    if etag:
        response.set_etag(etag)

    return response
<|endoftext|>


<commit_before>__version__ = '0.30.0
<commit_msg>Fix typo in version code<commit_after>__version__ = '0.30.0'
<|endoftext|>