<commit_before>#!/usr/bin/env python

import time
import random

import pickle
import json

import logging
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)

from envparse import env, ConfigurationError # pip install envparse
env.read_envfile()

from flask import Flask, request, g, current_app
#from flask_apscheduler import APScheduler # pip install Flask-APScheduler
from werkzeug.local import LocalProxy

import fbmq
from fbmq import Attachment, Template, QuickReply

from funkyquizbot import data
from funkyquizbot.data import Datastore

APIVERSION = '0.1'
SECRET_CHALLENGE = env('SECRET_CHALLENGE')
SECRET_URI = '/{}'.format(env('SECRET_URI'))
PAGE_ACCESS_TOKEN = env('PAGE_ACCESS_TOKEN')

app = Flask(__name__)
from logging.handlers import RotatingFileHandler
file_handler = RotatingFileHandler('python.log', maxBytes=1024 * 1024 * 100, backupCount=20)
file_handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
file_handler.setFormatter(formatter)
app.logger.addHandler(file_handler)

page = fbmq.Page(PAGE_ACCESS_TOKEN)

def setup_quizes():
    quizes = getattr(g, 'quizes', None)
    if quizes is None:
        quizes = g.quizes = getquizdata()
    return quizes

quizes = LocalProxy(setup_quizes)

def setup_quizprizes():
    p = getattr(g, 'quizprizes', None)
    if p is None:
        p = g.quizprizes = getquizprizes()
    return p

quizprizes = LocalProxy(setup_quizprizes)

def setup_giphys():
    giphys = getattr(g, 'giphys', None)
    if giphys is None:
        giphys = g.giphys = getgiphys()
    return giphys

giphys = LocalProxy(setup_giphys)

@app.route(SECRET_URI, methods=['GET'])
def handle_verification():
    'Get a GET request and try to verify it'
    app.logger.debug('About to read a challenge')
    token = request.args.get('hub.verify_token')
    if request.args.get('hub.mode', '') == 'subscribe' and \
        token is not None and token == SECRET_CHALLENGE:
        return request.args.get('hub.challenge', 'Oops')
    else:
        return 'You dont belong here'

@app.route(SECRET_URI, methods=['POST'])
def handle_message():
    'Get a POST request and treat it like an incoming message'
    app.logger.debug('Incoming payload')
    postdata = request.get_data(as_text=True)

    page.handle_webhook(postdata) # fbmq distributes according to @decorators
    return 'OK' # return quickly

@page.after_send
def after_send(payload, response):
    """:type event: fbmq.Payload"""
    app.logger.debug("complete")

def receipt(payload, response):
    "a callback that receives a message receipt"
    app.logger.debug('response : ' + response.text)

def encode_payload(prefix, data):
    """Return a <data> as a string, prefixed with <prefix>, for use as callback payload.
    Raises ValueError if content is invalid as a payload. E.g. length>1000."""
    r = """{}___{}""".format(prefix, json.dumps(data))
    # do some formal checks, as per
    # https://developers.facebook.com/docs/messenger-platform/send-api-reference/postback-button
    if len(r) > 1000:
        # postback content has max length of 1000
        raise ValueError
    return r

def decode_payload(s):
    "Decode a payload encoded with encode_payload(). Returns (prefix, data)"
    prefix, data = s.split('___', 2)
    return (prefix, json.loads(data))

@page.handle_message
def message_handler(event):
    """:type event: fbmq.Event"""
    sender_id = event.sender_id
    message = event.message_text
    app.logger.debug('New msg from %s: %r', sender_id, message)
    page.typing_on(sender_id)
    if message is None:
        app.logger.debug("message is none, is this a thumbs up?")
    elif message.lower() in ['quiz',]:
        quiz(event)
    elif event.is_postback:
        app.logger.debug("this is postback, someone else must handle it")
    elif event.is_quick_reply:
        app.logger.debug("this is quickreply, someone else must handle it")
    else:
        page.send(sender_id, "thank you, '%s' yourself! type 'quiz' to start it :)" % message, callback=receipt)

def quiz(event, previous=None):
    "start or continue a quiz"
    sender_id = event.sender_id
    message = event.message_text
    # Send a gif
    #page.send(sender_id, Attachment.Image('https://media.giphy.com/media/3o7bu57lYhUEFiYDSM/giphy.gif'))

    # the first question is special
    if previous is None:
        # a brand new quiz
        page.send(sender_id, "Welcome to a brand new quiz! If you get seven in a row, you get a prize")
        previous = [ ]  # a list to keep previous quiz id's
    else:
        if len(previous) >= 7:
            # we have made 7 in a row
            send_prize(event, previous)
            return
    # ask a question
    try:
        quiz = random.choice(quizes) # get a random quiz
        while quiz.qid in previous:
            quiz = random.choice(quizes) # we've had this ques before, get a new onone
    except IndexError:
        # no quizes in list, yikes
        page.send(sender_id, "We have no available quizes for you, pls try again later 8)")
        return
    previous.append(quiz.qid) # remember what we've seen|
    buttons = []
    for text in quiz.incorrectanswers:
        buttons.append(
            QuickReply(title=text, payload=encode_payload('ANSWER', {'previous':previous, 'correct':False}))
        )
    # TODO: quick_replies is limited to 11, prune incorrect answers if too many
    buttons.append(
        QuickReply(title=quiz.correct, payload=encode_payload('ANSWER', {'previous':previous, 'correct':True})),
    )
    random.shuffle(buttons) # hide  correct answer
    app.logger.debug("sending quiz: %s", quiz)
    page.send(sender_id, quiz.question, quick_replies=buttons)
    page.typing_off(sender_id)

def send_prize(event, previous=None):
    "send a prize"
    sender_id = event.sender_id
    message = event.message_text
    page.typing_on(sender_id)
    page.send(sender_id, "wow, you're on a nice streak. Here's a prize!")
    # Send a gif prize
    prize = random.choice(quizprizes)
    while not prize.is_embargoed: # make sure we can publish this
        prize = random.choice(quizprizes)
    if prize.media_type == 'image':
        att = Attachment.Image(prize.url)
    elif prize.media_type == 'video':
        att = Attachment.Video(prize.url)
    page.send(sender_id, att)

def get_giphy(context):
    "Get a random giphy that fits the context 'CORRECT'/'WRONG'"

@page.callback(['ANSWER_.+'])
def callback_answer(payload, event):
    "A callback for any ANSWER payload we get. "
    sender_id = event.sender_id
    page.typing_on(sender_id)
    prefix, metadata = decode_payload(payload)
    app.logger.debug('Got ANSWER: {} (correct? {})'.format(metadata, 'YES' if metadata['correct'] else 'NON'))
    page.send(sender_id, "Your reply was {}".format('CORRECT' if metadata['correct'] else 'INCORRECT :('))
    if random.random() > 0.9: # ten percent of the time, send a gif
        giph = get_giphy('CORRECT' if metadata['correct'] else 'WRONG')
        page.send(sender_id, Attachment.Image(giph.url))

    # TODO check how many we have correct
    if metadata['correct']:
        # answer is correct, you may continue
        _prev = metadata['previous']
        page.send(sender_id, "you have {} correct questions, only {} to go!".format(len(_prev),
                                                                                7-len(_prev)))
        quiz(event, _prev)

@page.handle_delivery
def delivery_handler(event):
    """:type event: fbmq.Event
    This callback will occur when a message a page has sent has been delivered."""
    sender_id = event.sender_id
    watermark = event.delivery.get('watermark', None)
    messages = event.delivery.get('mids', [])
    #logger.debug('Message from me ({}) delivered: {}'.format(sender_id, messages or watermark))

@page.handle_read
def read_handler(event):
    """:type event: fbmq.Event
    This callback will occur when a message a page has sent has been read by the user.
    """
    sender_id = event.sender_id
    watermark = event.read.get('watermark', None)
    #logger.debug('Message from me ({}) has been read: {}'.format(sender_id, watermark))

optin_handler = message_handler

def getpickles(env_key):
    "Helper to unpickle from file at env_key, returns empty list on errors"
    try:
        return pickle.load(open(env(env_key), 'rb'))
    except FileNotFoundError:
        app.logger.warning('Could not load cached values from "{}"->{!r}'.format(env_key, env(env_key)))
        return []

def getquizdata():
    "Background task to periodically update quizes"
    quizes = getpickles('CACHEFILE_QUIZQUESTIONS')
    app.logger.debug("Read {} questions".format(len(quizes)))
    return quizes

def getquizprizes():
    "Background task to periodically update quizesprizes"
    quizprizes = getpickles('CACHEFILE_QUIZPRIZES')
    app.logger.debug("Read {} questions".format(len(quizprizes)))
    return quizprizes

def getgiphys():
    "Background task to periodically update giphys"
    giphys = getpickles('CACHEFILE_GIPHYS')
    app.logger.debug("Read {} questions".format(len(giphys)))
    return giphys

#with app.app_context():
    ## within this block, current_app points to app.
    #print(current_app.name)
    #app.before_first_request(getquizdata)
    #app.before_first_request(getquizprizes)
    #app.before_first_request(getgiphys)

class Config(object):
    JOBS = [
        {
            'id': 'getquizdata',
            'func': 'funkyquizbot.app:getquizdata',
            'args': (),
            'trigger': 'interval',
            'minutes': 60
        },
        {
            'id': 'getquizprizes',
            'func': 'funkyquizbot.app:getquizprizes',
            'args': (),
            'trigger': 'interval',
            'hours': 6
        },
        {
            'id': 'getgiphys',
            'func': 'funkyquizbot.app:getgiphys',
            'args': (),
            'trigger': 'interval',
            'hours': 6
        },
    ]
    SCHEDULER_API_ENABLED = False # REST api to jobs
    SCHEDULER_TIMEZONE = 'Europe/Oslo'

#app.config.from_object(Config())
# get quizes
#scheduler = APScheduler()
#scheduler.init_app(app)
#scheduler.start()

if __name__ == '__main__':
    # start server
    #getquizdata()
    #getquizprizes()
    #getgiphys()
    app.run(host='0.0.0.0', port=8000, debug=False, threaded=True)
<commit_msg>dont bomb on errors<commit_after>#!/usr/bin/env python

import time
import random

import pickle
import json

import logging
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)

from envparse import env, ConfigurationError # pip install envparse
env.read_envfile()

from flask import Flask, request, g, current_app
#from flask_apscheduler import APScheduler # pip install Flask-APScheduler
from werkzeug.local import LocalProxy

import fbmq
from fbmq import Attachment, Template, QuickReply

from funkyquizbot import data
from funkyquizbot.data import Datastore

APIVERSION = '0.1'
SECRET_CHALLENGE = env('SECRET_CHALLENGE')
SECRET_URI = '/{}'.format(env('SECRET_URI'))
PAGE_ACCESS_TOKEN = env('PAGE_ACCESS_TOKEN')

app = Flask(__name__)
from logging.handlers import RotatingFileHandler
file_handler = RotatingFileHandler('python.log', maxBytes=1024 * 1024 * 100, backupCount=20)
file_handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
file_handler.setFormatter(formatter)
app.logger.addHandler(file_handler)

page = fbmq.Page(PAGE_ACCESS_TOKEN)

def setup_quizes():
    quizes = getattr(g, 'quizes', None)
    if quizes is None:
        quizes = g.quizes = getquizdata()
    return quizes

quizes = LocalProxy(setup_quizes)

def setup_quizprizes():
    p = getattr(g, 'quizprizes', None)
    if p is None:
        p = g.quizprizes = getquizprizes()
    return p

quizprizes = LocalProxy(setup_quizprizes)

def setup_giphys():
    giphys = getattr(g, 'giphys', None)
    if giphys is None:
        giphys = g.giphys = getgiphys()
    return giphys

giphys = LocalProxy(setup_giphys)

@app.route(SECRET_URI, methods=['GET'])
def handle_verification():
    'Get a GET request and try to verify it'
    app.logger.debug('About to read a challenge')
    token = request.args.get('hub.verify_token')
    if request.args.get('hub.mode', '') == 'subscribe' and \
        token is not None and token == SECRET_CHALLENGE:
        return request.args.get('hub.challenge', 'Oops')
    else:
        return 'You dont belong here'

@app.route(SECRET_URI, methods=['POST'])
def handle_message():
    'Get a POST request and treat it like an incoming message'
    app.logger.debug('Incoming payload')
    postdata = request.get_data(as_text=True)

    page.handle_webhook(postdata) # fbmq distributes according to @decorators
    return 'OK' # return quickly

@page.after_send
def after_send(payload, response):
    """:type event: fbmq.Payload"""
    app.logger.debug("complete")

def receipt(payload, response):
    "a callback that receives a message receipt"
    app.logger.debug('response : ' + response.text)

def encode_payload(prefix, data):
    """Return a <data> as a string, prefixed with <prefix>, for use as callback payload.
    Raises ValueError if content is invalid as a payload. E.g. length>1000."""
    r = """{}___{}""".format(prefix, json.dumps(data))
    # do some formal checks, as per
    # https://developers.facebook.com/docs/messenger-platform/send-api-reference/postback-button
    if len(r) > 1000:
        # postback content has max length of 1000
        raise ValueError
    return r

def decode_payload(s):
    "Decode a payload encoded with encode_payload(). Returns (prefix, data)"
    prefix, data = s.split('___', 2)
    return (prefix, json.loads(data))

@page.handle_message
def message_handler(event):
    """:type event: fbmq.Event"""
    sender_id = event.sender_id
    message = event.message_text
    app.logger.debug('New msg from %s: %r', sender_id, message)
    page.typing_on(sender_id)
    if message is None:
        app.logger.debug("message is none, is this a thumbs up?")
    elif message.lower() in ['quiz',]:
        quiz(event)
    elif event.is_postback:
        app.logger.debug("this is postback, someone else must handle it")
    elif event.is_quick_reply:
        app.logger.debug("this is quickreply, someone else must handle it")
    else:
        page.send(sender_id, "thank you, '%s' yourself! type 'quiz' to start it :)" % message, callback=receipt)

def quiz(event, previous=None):
    "start or continue a quiz"
    sender_id = event.sender_id
    message = event.message_text
    # Send a gif
    #page.send(sender_id, Attachment.Image('https://media.giphy.com/media/3o7bu57lYhUEFiYDSM/giphy.gif'))

    # the first question is special
    if previous is None:
        # a brand new quiz
        page.send(sender_id, "Welcome to a brand new quiz! If you get seven in a row, you get a prize")
        previous = [ ]  # a list to keep previous quiz id's
    else:
        if len(previous) >= 7:
            # we have made 7 in a row
            send_prize(event, previous)
            return
    # ask a question
    try:
        quiz = random.choice(quizes) # get a random quiz
        while quiz.qid in previous:
            quiz = random.choice(quizes) # we've had this ques before, get a new onone
    except IndexError:
        # no quizes in list, yikes
        page.send(sender_id, "We have no available quizes for you, pls try again later 8)")
        return
    previous.append(quiz.qid) # remember what we've seen|
    buttons = []
    for text in quiz.incorrectanswers:
        buttons.append(
            QuickReply(title=text, payload=encode_payload('ANSWER', {'previous':previous, 'correct':False}))
        )
    # TODO: quick_replies is limited to 11, prune incorrect answers if too many
    buttons.append(
        QuickReply(title=quiz.correct, payload=encode_payload('ANSWER', {'previous':previous, 'correct':True})),
    )
    random.shuffle(buttons) # hide  correct answer
    app.logger.debug("sending quiz: %s", quiz)
    page.send(sender_id, quiz.question, quick_replies=buttons)
    page.typing_off(sender_id)

def send_prize(event, previous=None):
    "send a prize"
    sender_id = event.sender_id
    message = event.message_text
    page.typing_on(sender_id)
    page.send(sender_id, "wow, you're on a nice streak. Here's a prize!")
    # Send a gif prize
    prize = random.choice(quizprizes)
    while not prize.is_embargoed: # make sure we can publish this
        prize = random.choice(quizprizes)
    if prize.media_type == 'image':
        att = Attachment.Image(prize.url)
    elif prize.media_type == 'video':
        att = Attachment.Video(prize.url)
    page.send(sender_id, att)

def get_giphy(context):
    "Get a random giphy that fits the context 'CORRECT'/'WRONG'"
    try:
        return random.choice([x for x in giphys if x.context == context])
    except IndexError: # no giphs available
        return None

@page.callback(['ANSWER_.+'])
def callback_answer(payload, event):
    "A callback for any ANSWER payload we get. "
    sender_id = event.sender_id
    page.typing_on(sender_id)
    prefix, metadata = decode_payload(payload)
    app.logger.debug('Got ANSWER: {} (correct? {})'.format(metadata, 'YES' if metadata['correct'] else 'NON'))
    page.send(sender_id, "Your reply was {}".format('CORRECT' if metadata['correct'] else 'INCORRECT :('))
    if random.random() > 0.9: # ten percent of the time, send a gif
        giph = get_giphy('CORRECT' if metadata['correct'] else 'WRONG')
        page.send(sender_id, Attachment.Image(giph.url))

    # TODO check how many we have correct
    if metadata['correct']:
        # answer is correct, you may continue
        _prev = metadata['previous']
        page.send(sender_id, "you have {} correct questions, only {} to go!".format(len(_prev),
                                                                                7-len(_prev)))
        quiz(event, _prev)

@page.handle_delivery
def delivery_handler(event):
    """:type event: fbmq.Event
    This callback will occur when a message a page has sent has been delivered."""
    sender_id = event.sender_id
    watermark = event.delivery.get('watermark', None)
    messages = event.delivery.get('mids', [])
    #logger.debug('Message from me ({}) delivered: {}'.format(sender_id, messages or watermark))

@page.handle_read
def read_handler(event):
    """:type event: fbmq.Event
    This callback will occur when a message a page has sent has been read by the user.
    """
    sender_id = event.sender_id
    watermark = event.read.get('watermark', None)
    #logger.debug('Message from me ({}) has been read: {}'.format(sender_id, watermark))

optin_handler = message_handler

def getpickles(env_key):
    "Helper to unpickle from file at env_key, returns empty list on errors"
    try:
        return pickle.load(open(env(env_key), 'rb'))
    except FileNotFoundError:
        app.logger.warning('Could not load cached values from "{}"->{!r}'.format(env_key, env(env_key)))
        return []

def getquizdata():
    "Background task to periodically update quizes"
    quizes = getpickles('CACHEFILE_QUIZQUESTIONS')
    app.logger.debug("Read {} questions".format(len(quizes)))
    return quizes

def getquizprizes():
    "Background task to periodically update quizesprizes"
    quizprizes = getpickles('CACHEFILE_QUIZPRIZES')
    app.logger.debug("Read {} questions".format(len(quizprizes)))
    return quizprizes

def getgiphys():
    "Background task to periodically update giphys"
    giphys = getpickles('CACHEFILE_GIPHYS')
    app.logger.debug("Read {} questions".format(len(giphys)))
    return giphys

#with app.app_context():
    ## within this block, current_app points to app.
    #print(current_app.name)
    #app.before_first_request(getquizdata)
    #app.before_first_request(getquizprizes)
    #app.before_first_request(getgiphys)

class Config(object):
    JOBS = [
        {
            'id': 'getquizdata',
            'func': 'funkyquizbot.app:getquizdata',
            'args': (),
            'trigger': 'interval',
            'minutes': 60
        },
        {
            'id': 'getquizprizes',
            'func': 'funkyquizbot.app:getquizprizes',
            'args': (),
            'trigger': 'interval',
            'hours': 6
        },
        {
            'id': 'getgiphys',
            'func': 'funkyquizbot.app:getgiphys',
            'args': (),
            'trigger': 'interval',
            'hours': 6
        },
    ]
    SCHEDULER_API_ENABLED = False # REST api to jobs
    SCHEDULER_TIMEZONE = 'Europe/Oslo'

#app.config.from_object(Config())
# get quizes
#scheduler = APScheduler()
#scheduler.init_app(app)
#scheduler.start()

if __name__ == '__main__':
    # start server
    #getquizdata()
    #getquizprizes()
    #getgiphys()
    app.run(host='0.0.0.0', port=8000, debug=False, threaded=True)
<|endoftext|>


<commit_before>import datetime
import pytz
from collections import defaultdict

from django.conf import settings
from django.contrib.auth.models import User
from django.core.mail import EmailMultiAlternatives
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from django.template.loader import get_template

from opencivicdata.models import Bill
from ...models import EmailRecord

class BillAccumulator:
    def __init__(self, days):
        self.days = days
        bills = self.get_bills_with_actions_since(days)
        print('Accumulated {} bills for {} days...'.format(len(bills), days))
        self.make_buckets(bills)

    def get_bills_with_actions_since(self, days):
        """ get list of all bills w/ actions in last N days """
        today = datetime.date.today()
        yesterday = (today - datetime.timedelta(days=days)).strftime('%Y-%m-%d')
        today = today.strftime('%Y-%m-%d')
        bills = Bill.objects.filter(actions__date__range=(yesterday, today))
        # check created/updated date on root bill?

        # annotate bills w/ actions
        for bill in bills:
            bill.latest_action = bill.actions.all().order_by('-order')[0]
        return bills

    def make_buckets(self, bills):
        """
            given list of modified bills, return mappings by
            legislator, location, and subject
        """
        # map bills
        self.by_legislator = defaultdict(list)
        self.by_location = defaultdict(list)
        self.by_subject = defaultdict(list)

        for bill in bills:
            for sponsor_id in bill.sponsorships.values_list('person_id', flat=True):
                self.by_legislator[sponsor_id].append(bill)
            for location in bill.extras['places']:
                self.by_location[location].append(bill)
            for subject in bill.subject:
                self.by_subject[subject].append(bill)

    def bills_for_user(self, user):
        """ for a user's interests, get all relevant bills """
        people = user.person_follows.values_list('person_id', flat=True)
        locations = user.location_follows.values_list('location', flat=True)
        topics = user.topic_follows.values_list('topic', flat=True)

        bills = set()
        for person_id in people:
            bills.update(self.by_legislator[person_id])
        for location in locations:
            bills.update(self.by_location[location])
        for topic in topics:
            bills.update(self.by_subject[topic])

        return bills

class Command(BaseCommand):
    help = 'Send emails for all updates in last N days'

    def add_arguments(self, parser):
        parser.add_argument('--week', action='store_true',
                            default=False)
        parser.add_argument('--force', action='store_true',
                            default=False)

    def handle(self, *args, **options):
        week = options['week']
        force = options['force']

        if week:
            days = 7
            email_freq = 'W'
            subject = 'Tabs on Tallahasse: Week of {}'.format(
                datetime.date.today().strftime('%B %d, %Y')
            )
        else:
            days = 1
            email_freq = 'D'
            subject = 'Tabs on Tallahasse: {}'.format(
                datetime.date.today().strftime('%B %d, %Y')
            )

        # get all modified bills
        bill_accumulator = BillAccumulator(days)

        # email users that get emails on this interval
        users = User.objects.filter(preferences__email_frequency=email_freq)
        print('{} users to process...'.format(len(users)))

        sent_recently = 0
        no_bills = 0
        sent = 0

        for user in users:
            # skip user if last email was sent in the last [1|7] days
            last_email = user.email_tasks.latest('created_at')
            now = pytz.utc.localize(datetime.datetime.utcnow())

            if not force and now - last_email.created_at < datetime.timedelta(days=days):
                sent_recently += 1
                continue

            bills = bill_accumulator.bills_for_user(user)
            if not bills:
                no_bills += 1
                continue

            # send & record sending together
            with transaction.atomic():
                er = EmailRecord.objects.create(
                    user=user,
                    bills=len(bills),
                )

                # build email message
                text_template = get_template('email/updates.txt')
                context = {
                    'bills': bills,
                    'subject': subject,
                    'unsubscribe_guid': er.unsubscribe_guid,
                }
                text_content = text_template.render(context)
                msg = EmailMultiAlternatives(subject,
                                                text_content,
                                                settings.DEFAULT_FROM_EMAIL,
                                                [user.email]
                                                )
                # add html if user prefers it
                if user.preferences.email_type == 'H':
                    html_template = get_template('email/updates.html')
                    html_content = html_template.render(context)
                    msg.attach_alternative(html_content, "text/html")

                # if this fails it'll break the transaction & roll it back
                msg.send()
                sent += 1
        print('''{} users sent too recently...
{} users w/ no bills...
{} emails sent...'''.format(sent_recently, no_bills, sent)
              )
<commit_msg>fix first email sending<commit_after>import datetime
import pytz
from collections import defaultdict

from django.conf import settings
from django.contrib.auth.models import User
from django.core.mail import EmailMultiAlternatives
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from django.template.loader import get_template

from opencivicdata.models import Bill
from ...models import EmailRecord

class BillAccumulator:
    def __init__(self, days):
        self.days = days
        bills = self.get_bills_with_actions_since(days)
        print('Accumulated {} bills for {} days...'.format(len(bills), days))
        self.make_buckets(bills)

    def get_bills_with_actions_since(self, days):
        """ get list of all bills w/ actions in last N days """
        today = datetime.date.today()
        yesterday = (today - datetime.timedelta(days=days)).strftime('%Y-%m-%d')
        today = today.strftime('%Y-%m-%d')
        bills = Bill.objects.filter(actions__date__range=(yesterday, today))
        # check created/updated date on root bill?

        # annotate bills w/ actions
        for bill in bills:
            bill.latest_action = bill.actions.all().order_by('-order')[0]
        return bills

    def make_buckets(self, bills):
        """
            given list of modified bills, return mappings by
            legislator, location, and subject
        """
        # map bills
        self.by_legislator = defaultdict(list)
        self.by_location = defaultdict(list)
        self.by_subject = defaultdict(list)

        for bill in bills:
            for sponsor_id in bill.sponsorships.values_list('person_id', flat=True):
                self.by_legislator[sponsor_id].append(bill)
            for location in bill.extras['places']:
                self.by_location[location].append(bill)
            for subject in bill.subject:
                self.by_subject[subject].append(bill)

    def bills_for_user(self, user):
        """ for a user's interests, get all relevant bills """
        people = user.person_follows.values_list('person_id', flat=True)
        locations = user.location_follows.values_list('location', flat=True)
        topics = user.topic_follows.values_list('topic', flat=True)

        bills = set()
        for person_id in people:
            bills.update(self.by_legislator[person_id])
        for location in locations:
            bills.update(self.by_location[location])
        for topic in topics:
            bills.update(self.by_subject[topic])

        return bills

class Command(BaseCommand):
    help = 'Send emails for all updates in last N days'

    def add_arguments(self, parser):
        parser.add_argument('--week', action='store_true',
                            default=False)
        parser.add_argument('--force', action='store_true',
                            default=False)

    def handle(self, *args, **options):
        week = options['week']
        force = options['force']

        if week:
            days = 7
            email_freq = 'W'
            subject = 'Tabs on Tallahasse: Week of {}'.format(
                datetime.date.today().strftime('%B %d, %Y')
            )
        else:
            days = 1
            email_freq = 'D'
            subject = 'Tabs on Tallahasse: {}'.format(
                datetime.date.today().strftime('%B %d, %Y')
            )

        # get all modified bills
        bill_accumulator = BillAccumulator(days)

        # email users that get emails on this interval
        users = User.objects.filter(preferences__email_frequency=email_freq)
        print('{} users to process...'.format(len(users)))

        sent_recently = 0
        no_bills = 0
        sent = 0

        for user in users:
            # skip user if last email was sent in the last [1|7] days
            try:
                last_email = user.email_tasks.latest('created_at')
                now = pytz.utc.localize(datetime.datetime.utcnow())

                if not force and now - last_email.created_at < datetime.timedelta(days=days):
                    sent_recently += 1
                    continue
            except EmailRecord.DoesNotExist:
                pass

            bills = bill_accumulator.bills_for_user(user)
            if not bills:
                no_bills += 1
                continue

            # send & record sending together
            with transaction.atomic():
                er = EmailRecord.objects.create(
                    user=user,
                    bills=len(bills),
                )

                # build email message
                text_template = get_template('email/updates.txt')
                context = {
                    'bills': bills,
                    'subject': subject,
                    'unsubscribe_guid': er.unsubscribe_guid,
                }
                text_content = text_template.render(context)
                msg = EmailMultiAlternatives(subject,
                                                text_content,
                                                settings.DEFAULT_FROM_EMAIL,
                                                [user.email]
                                                )
                # add html if user prefers it
                if user.preferences.email_type == 'H':
                    html_template = get_template('email/updates.html')
                    html_content = html_template.render(context)
                    msg.attach_alternative(html_content, "text/html")

                # if this fails it'll break the transaction & roll it back
                msg.send()
                sent += 1
        print('''{} users sent too recently...
{} users w/ no bills...
{} emails sent...'''.format(sent_recently, no_bills, sent)
              )
<|endoftext|>


<commit_before>import os,subprocess
from shlex import quote
from igf_data.utils.fileutils import check_file_path,get_temp_dir,remove_dir,copy_local_file

class GATK_tools:
  '''
  A python class for running gatk tools

  :param gatk_exe: Gatk exe path
  :param java_param: Java parameter, default '-XX:ParallelGCThreads=1 -Xmx4g'
  :param ref_fasta: Input reference fasta filepath
  :param use_ephemeral_space: A toggle for temp dir settings, default False
  '''
  def __init__(self,gatk_exe,ref_fasta,use_ephemeral_space=False,
               java_param='-XX:ParallelGCThreads=1 -Xmx4g'):
    self.gatk_exe = gatk_exe
    self.ref_fasta = ref_fasta
    self.use_ephemeral_space = use_ephemeral_space
    temp_dir = \
      get_temp_dir(use_ephemeral_space=self.use_ephemeral_space)
    self.java_param = java_param + ' -Djava.io.tmpdir={0}'.format(temp_dir)

  def _run_gatk_checks(self):
    '''
    An internal method for running checks before GATK run
    '''
    try:
      check_file_path(self.gatk_exe)
      check_file_path(self.ref_fasta)
    except Exception as e:
      raise ValueError(
              "Failed to run GATK checks, error: {0}".\
                format(e))

  def run_BaseRecalibrator(self,input_bam,output_table,known_snp_sites=None,
                           known_indel_sites=None,force=False,dry_run=False,
                           gatk_param_list=None):
    '''
    A method for running GATK BaseRecalibrator

    :param input_bam: An input bam file
    :param output_table: An output table filepath for recalibration results
    :param known_snp_sites: Known snp sites (e.g. dbSNP vcf file), default None
    :param known_indel_sites: Known indel sites (e.g.Mills_and_1000G_gold_standard indels vcf), default None
    :param force: Overwrite output file, if force is True
    :param dry_run: Return GATK command, if its true, default False
    :param gatk_param_list: List of additional params for BQSR, default None
    :returns: GATK commandline
    '''
    try:
      self._run_gatk_checks()                                                   # run initial checks
      check_file_path(input_bam)
      temp_dir = \
        get_temp_dir(use_ephemeral_space=self.use_ephemeral_space)              # get temp dir
      temp_output = \
        os.path.join(
          temp_dir,
          os.path.basename(output_table))
      gatk_cmd = [
        quote(self.gatk_exe),
        "BaseRecalibrator",
        "-I",quote(input_bam),
        "-O",quote(temp_output),
        "--reference",quote(self.ref_fasta),
        "--java-options",quote(self.java_param)]
      if known_snp_sites is not None:
        check_file_path(known_snp_sites)
        gatk_cmd.\
          extend([
            "--known-sites",
            quote(known_snp_sites)])

      if known_indel_sites:
        check_file_path(known_indel_sites)
        gatk_cmd.\
          extend([
            "--known-sites",
            quote(known_indel_sites)])
      if gatk_param_list is not None and \
         isinstance(gatk_param_list,list) and \
         len(gatk_param_list) > 0:
        gatk_cmd.extend(gatk_param_list)                                        # additional params
      gatk_cmd = ' '.join(gatk_cmd)
      if dry_run:
        return gatk_cmd

      subprocess.check_call(gatk_cmd,shell=True)                                # run gatk cmd
      copy_local_file(
        source_path=temp_output,
        destinationa_path=output_table,
        force=force)                                                            # copy output file
      remove_dir(temp_dir)                                                      # remove temp dir
      return gatk_cmd
    except Exception as e:
      raise ValueError(
              "Failed to run GATK BaseRecalibrator, error: {0}".\
                format(e))

  def run_ApplyBQSR(self,bqsr_recal_file,input_bam,output_bam_path,force=False,
                    dry_run=False,gatk_param_list=None):
    '''
    A method for running GATK ApplyBQSR

    :param input_bam: An input bam file
    :param bqsr_recal_file: An bqsr table filepath
    :param output_bam_path: A bam output file
    :param force: Overwrite output file, if force is True
    :param dry_run: Return GATK command, if its true, default False
    :param gatk_param_list: List of additional params for BQSR, default None
    :returns: GATK commandline
    '''
    try:
      self._run_gatk_checks()                                                   # run initial checks
      check_file_path(input_bam)
      check_file_path(bqsr_recal_file)
      temp_dir = \
        get_temp_dir(use_ephemeral_space=self.use_ephemeral_space)              # get temp dir
      temp_output = \
        os.path.join(
          temp_dir,
          os.path.basename(output_bam_path))
      gatk_cmd = [
        quote(self.gatk_exe),
        "ApplyBQSR",
        "--bqsr-recal-file",quote(bqsr_recal_file),
        "--create-output-bam-index",
        "--emit-original-quals",
        "-I",quote(input_bam),
        "-O",quote(temp_output),
        "--reference",quote(self.ref_fasta),
        "--java-options",quote(self.java_param)]
      if gatk_param_list is not None and \
         isinstance(gatk_param_list,list) and \
         len(gatk_param_list) > 0:
        gatk_cmd.extend(gatk_param_list)                                        # additional params

      gatk_cmd = ' '.join(gatk_cmd)
      if dry_run:
        return gatk_cmd

      subprocess.check_call(gatk_cmd,shell=True)
      copy_local_file(
        source_path=temp_output,
        destinationa_path=output_bam_path,
        force=force)
      remove_dir(temp_dir)
      return gatk_cmd
    except Exception as e:
      raise ValueError(
              "Failed to run GATK ApplyBQSR, error: {0}".\
                format(e))

  def run_AnalyzeCovariates(self,before_report_file,after_report_file,output_pdf_path,
                            force=False,dry_run=False,gatk_param_list=None):
    '''
    A method for running GATK AnalyzeCovariates tool

    :param before_report_file: A file containing bqsr output before recalibration
    :param after_report_file: A file containing bqsr output after recalibration
    :param output_pdf_path: An output pdf filepath
    :param force: Overwrite output file, if force is True
    :param dry_run: Return GATK command, if its true, default False
    :param gatk_param_list: List of additional params for BQSR, default None
    :returns: GATK commandline
    '''
    try:
      self._run_gatk_checks()                                                   # run initial checks
      check_file_path(before_report_file)
      check_file_path(after_report_file)
      temp_dir = \
        get_temp_dir(use_ephemeral_space=self.use_ephemeral_space)              # get temp dir
      temp_output = \
        os.path.join(
          temp_dir,
          os.path.basename(output_pdf_path))
      gatk_cmd = [
        quote(self.gatk_exe),
        "AnalyzeCovariates",
        "--before-report-file",quote(before_report_file),
        "--after-report-file",quote(after_report_file),
        "--plots-report-file",quote(temp_output),
        "--java-options",quote(self.java_param)]
      if gatk_param_list is not None and \
         isinstance(gatk_param_list,list) and \
         len(gatk_param_list) > 0:
        gatk_cmd.extend(gatk_param_list)                                        # additional params
      gatk_cmd = ' '.join(gatk_cmd)
      if dry_run:
        return gatk_cmd

      subprocess.check_call(gatk_cmd,shell=True)
      copy_local_file(
        source_path=temp_output,
        destinationa_path=output_pdf_path,
        force=force)
      remove_dir(temp_dir)
      return gatk_cmd
    except Exception as e:
      raise ValueError(
              "Failed to run GATK AnalyzeCovariates, error: {0}".\
                format(e))

  def run_HaplotypeCaller(self,input_bam,output_vcf_path,dbsnp_vcf,emit_gvcf=True,
                          force=False,dry_run=False,gatk_param_list=None):
    '''
    A method for running GATK HaplotypeCaller

    :param input_bam: A input bam file
    :param output_vcf_path: A output vcf filepath
    :param dbsnp_vcf: A dbsnp vcf file
    :param emit_gvcf: A toggle for GVCF generation, default True
    :param force: Overwrite output file, if force is True
    :param dry_run: Return GATK command, if its true, default False
    :param gatk_param_list: List of additional params for BQSR, default None
    :returns: GATK commandline
    '''
    try:
      self._run_gatk_checks()                                                   # run initial checks
      check_file_path(input_bam)
      check_file_path(dbsnp_vcf)
      temp_dir = \
        get_temp_dir(use_ephemeral_space=self.use_ephemeral_space)              # get temp dir
      temp_output = \
        os.path.join(
          temp_dir,
          os.path.basename(output_vcf_path))
      gatk_cmd = [
        quote(self.gatk_exe),
        "HaplotypeCaller",
        "-I",quote(input_bam),
        "-O",quote(temp_output),
        "--reference",quote(self.ref_fasta),
        "--dbsnp",quote(dbsnp_vcf),
        "--java-options",quote(self.java_param)]
      if emit_gvcf:
        gatk_cmd.extend(["--emit-ref-confidence","GVCF"])
      if gatk_param_list is not None and \
         isinstance(gatk_param_list,list) and \
         len(gatk_param_list) > 0:
        gatk_cmd.extend(gatk_param_list)                                        # additional params
      gatk_cmd = ' '.join(gatk_cmd)
      if dry_run:
        return gatk_cmd

      subprocess.check_call(gatk_cmd,shell=True)
      copy_local_file(
        source_path=temp_output,
        destinationa_path=output_vcf_path,
        force=force)
      remove_dir(temp_dir)
      return gatk_cmd
    except Exception as e:
      raise ValueError(
              "Failed to run GATK HaplotypeCaller, error: {0}".\
                format(e))

<commit_msg>copy bai index to work dir<commit_after>import os,subprocess
from shlex import quote
from igf_data.utils.fileutils import check_file_path,get_temp_dir,remove_dir,copy_local_file

class GATK_tools:
  '''
  A python class for running gatk tools

  :param gatk_exe: Gatk exe path
  :param java_param: Java parameter, default '-XX:ParallelGCThreads=1 -Xmx4g'
  :param ref_fasta: Input reference fasta filepath
  :param use_ephemeral_space: A toggle for temp dir settings, default False
  '''
  def __init__(self,gatk_exe,ref_fasta,use_ephemeral_space=False,
               java_param='-XX:ParallelGCThreads=1 -Xmx4g'):
    self.gatk_exe = gatk_exe
    self.ref_fasta = ref_fasta
    self.use_ephemeral_space = use_ephemeral_space
    temp_dir = \
      get_temp_dir(use_ephemeral_space=self.use_ephemeral_space)
    self.java_param = java_param + ' -Djava.io.tmpdir={0}'.format(temp_dir)

  def _run_gatk_checks(self):
    '''
    An internal method for running checks before GATK run
    '''
    try:
      check_file_path(self.gatk_exe)
      check_file_path(self.ref_fasta)
    except Exception as e:
      raise ValueError(
              "Failed to run GATK checks, error: {0}".\
                format(e))

  def run_BaseRecalibrator(self,input_bam,output_table,known_snp_sites=None,
                           known_indel_sites=None,force=False,dry_run=False,
                           gatk_param_list=None):
    '''
    A method for running GATK BaseRecalibrator

    :param input_bam: An input bam file
    :param output_table: An output table filepath for recalibration results
    :param known_snp_sites: Known snp sites (e.g. dbSNP vcf file), default None
    :param known_indel_sites: Known indel sites (e.g.Mills_and_1000G_gold_standard indels vcf), default None
    :param force: Overwrite output file, if force is True
    :param dry_run: Return GATK command, if its true, default False
    :param gatk_param_list: List of additional params for BQSR, default None
    :returns: GATK commandline
    '''
    try:
      self._run_gatk_checks()                                                   # run initial checks
      check_file_path(input_bam)
      temp_dir = \
        get_temp_dir(use_ephemeral_space=self.use_ephemeral_space)              # get temp dir
      temp_output = \
        os.path.join(
          temp_dir,
          os.path.basename(output_table))
      gatk_cmd = [
        quote(self.gatk_exe),
        "BaseRecalibrator",
        "-I",quote(input_bam),
        "-O",quote(temp_output),
        "--reference",quote(self.ref_fasta),
        "--java-options",quote(self.java_param)]
      if known_snp_sites is not None:
        check_file_path(known_snp_sites)
        gatk_cmd.\
          extend([
            "--known-sites",
            quote(known_snp_sites)])

      if known_indel_sites:
        check_file_path(known_indel_sites)
        gatk_cmd.\
          extend([
            "--known-sites",
            quote(known_indel_sites)])
      if gatk_param_list is not None and \
         isinstance(gatk_param_list,list) and \
         len(gatk_param_list) > 0:
        gatk_cmd.extend(gatk_param_list)                                        # additional params
      gatk_cmd = ' '.join(gatk_cmd)
      if dry_run:
        return gatk_cmd

      subprocess.check_call(gatk_cmd,shell=True)                                # run gatk cmd
      copy_local_file(
        source_path=temp_output,
        destinationa_path=output_table,
        force=force)                                                            # copy output file
      remove_dir(temp_dir)                                                      # remove temp dir
      return gatk_cmd
    except Exception as e:
      raise ValueError(
              "Failed to run GATK BaseRecalibrator, error: {0}".\
                format(e))

  def run_ApplyBQSR(self,bqsr_recal_file,input_bam,output_bam_path,force=False,
                    dry_run=False,gatk_param_list=None):
    '''
    A method for running GATK ApplyBQSR

    :param input_bam: An input bam file
    :param bqsr_recal_file: An bqsr table filepath
    :param output_bam_path: A bam output file
    :param force: Overwrite output file, if force is True
    :param dry_run: Return GATK command, if its true, default False
    :param gatk_param_list: List of additional params for BQSR, default None
    :returns: GATK commandline
    '''
    try:
      self._run_gatk_checks()                                                   # run initial checks
      check_file_path(input_bam)
      check_file_path(bqsr_recal_file)
      temp_dir = \
        get_temp_dir(use_ephemeral_space=self.use_ephemeral_space)              # get temp dir
      temp_output = \
        os.path.join(
          temp_dir,
          os.path.basename(output_bam_path))
      gatk_cmd = [
        quote(self.gatk_exe),
        "ApplyBQSR",
        "--bqsr-recal-file",quote(bqsr_recal_file),
        "--create-output-bam-index",
        "--emit-original-quals",
        "-I",quote(input_bam),
        "-O",quote(temp_output),
        "--reference",quote(self.ref_fasta),
        "--java-options",quote(self.java_param)]
      if gatk_param_list is not None and \
         isinstance(gatk_param_list,list) and \
         len(gatk_param_list) > 0:
        gatk_cmd.extend(gatk_param_list)                                        # additional params

      gatk_cmd = ' '.join(gatk_cmd)
      if dry_run:
        return gatk_cmd

      subprocess.check_call(gatk_cmd,shell=True)
      copy_local_file(
        source_path=temp_output,
        destinationa_path=output_bam_path,
        force=force)
      copy_local_file(
        source_path='{0}.bai'.format(temp_output),
        destinationa_path='{0}.bai'.format(output_bam_path),
        force=force)
      remove_dir(temp_dir)
      return gatk_cmd
    except Exception as e:
      raise ValueError(
              "Failed to run GATK ApplyBQSR, error: {0}".\
                format(e))

  def run_AnalyzeCovariates(self,before_report_file,after_report_file,output_pdf_path,
                            force=False,dry_run=False,gatk_param_list=None):
    '''
    A method for running GATK AnalyzeCovariates tool

    :param before_report_file: A file containing bqsr output before recalibration
    :param after_report_file: A file containing bqsr output after recalibration
    :param output_pdf_path: An output pdf filepath
    :param force: Overwrite output file, if force is True
    :param dry_run: Return GATK command, if its true, default False
    :param gatk_param_list: List of additional params for BQSR, default None
    :returns: GATK commandline
    '''
    try:
      self._run_gatk_checks()                                                   # run initial checks
      check_file_path(before_report_file)
      check_file_path(after_report_file)
      temp_dir = \
        get_temp_dir(use_ephemeral_space=self.use_ephemeral_space)              # get temp dir
      temp_output = \
        os.path.join(
          temp_dir,
          os.path.basename(output_pdf_path))
      gatk_cmd = [
        quote(self.gatk_exe),
        "AnalyzeCovariates",
        "--before-report-file",quote(before_report_file),
        "--after-report-file",quote(after_report_file),
        "--plots-report-file",quote(temp_output),
        "--java-options",quote(self.java_param)]
      if gatk_param_list is not None and \
         isinstance(gatk_param_list,list) and \
         len(gatk_param_list) > 0:
        gatk_cmd.extend(gatk_param_list)                                        # additional params
      gatk_cmd = ' '.join(gatk_cmd)
      if dry_run:
        return gatk_cmd

      subprocess.check_call(gatk_cmd,shell=True)
      copy_local_file(
        source_path=temp_output,
        destinationa_path=output_pdf_path,
        force=force)
      remove_dir(temp_dir)
      return gatk_cmd
    except Exception as e:
      raise ValueError(
              "Failed to run GATK AnalyzeCovariates, error: {0}".\
                format(e))

  def run_HaplotypeCaller(self,input_bam,output_vcf_path,dbsnp_vcf,emit_gvcf=True,
                          force=False,dry_run=False,gatk_param_list=None):
    '''
    A method for running GATK HaplotypeCaller

    :param input_bam: A input bam file
    :param output_vcf_path: A output vcf filepath
    :param dbsnp_vcf: A dbsnp vcf file
    :param emit_gvcf: A toggle for GVCF generation, default True
    :param force: Overwrite output file, if force is True
    :param dry_run: Return GATK command, if its true, default False
    :param gatk_param_list: List of additional params for BQSR, default None
    :returns: GATK commandline
    '''
    try:
      self._run_gatk_checks()                                                   # run initial checks
      check_file_path(input_bam)
      check_file_path(dbsnp_vcf)
      temp_dir = \
        get_temp_dir(use_ephemeral_space=self.use_ephemeral_space)              # get temp dir
      temp_output = \
        os.path.join(
          temp_dir,
          os.path.basename(output_vcf_path))
      gatk_cmd = [
        quote(self.gatk_exe),
        "HaplotypeCaller",
        "-I",quote(input_bam),
        "-O",quote(temp_output),
        "--reference",quote(self.ref_fasta),
        "--dbsnp",quote(dbsnp_vcf),
        "--java-options",quote(self.java_param)]
      if emit_gvcf:
        gatk_cmd.extend(["--emit-ref-confidence","GVCF"])
      if gatk_param_list is not None and \
         isinstance(gatk_param_list,list) and \
         len(gatk_param_list) > 0:
        gatk_cmd.extend(gatk_param_list)                                        # additional params
      gatk_cmd = ' '.join(gatk_cmd)
      if dry_run:
        return gatk_cmd

      subprocess.check_call(gatk_cmd,shell=True)
      copy_local_file(
        source_path=temp_output,
        destinationa_path=output_vcf_path,
        force=force)
      remove_dir(temp_dir)
      return gatk_cmd
    except Exception as e:
      raise ValueError(
              "Failed to run GATK HaplotypeCaller, error: {0}".\
                format(e))

<|endoftext|>


<commit_before>'''
This script calculates the gross emissions in tonnes CO2e/ha for every loss pixel.
The properties of each pixel determine the appropriate emissions equation, the constants for the equation, and the
carbon pool values that go into the equation.
Unlike all other flux model components, this one uses C++ to quickly iterate through every pixel in each tile.
Before running the model, the C++ script must be compiled.
From carbon-budget/emissions/, do c++ ./cpp_util/calc_gross_emissions_biomass_soil.cpp -o ./cpp_util/calc_gross_emissions_biomass_soil.exe -lgdal
calc_emissions_v3_biomass_soil.exe should appear in the directory.
Run mp_calculate_gross_emissions.py by typing python mp_calculate_gross_emissions.py -p [POOL_OPTION]. The Python script will call the
compiled C++ code as needed.
The other C++ scripts (equations.cpp and flu_val.cpp) do not need to be compiled.
The --pools-to-use argument specifies whether to calculate gross emissions from biomass+soil or just from soil.
Emissions from each driver (including loss that had no driver assigned) gets its own tile, as does all emissions combined.
Emissions from all drivers is also output as emissions due to CO2 only and emissions due to other GHG (CH4 and N2O).
The other output shows which branch of the decision tree that determines the emissions equation applies to each pixel.
These codes are summarized in carbon-budget/emissions/node_codes.txt
'''

import multiprocessing
import argparse
import os
import calculate_gross_emissions
from functools import partial
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

def main ():

    tile_list = uu.tile_list(cn.AGC_emis_year_dir)
    # tile_list = ['00N_110E', '30N_080W', '40N_050E', '50N_100E', '80N_020E'] # test tiles
    # tile_list = ['20N_010E'] # test tiles
    # tile_list = ['00N_110E', '80N_020E', '30N_080W', '00N_020E'] # test tiles: no mangrove or planted forest, mangrove only, planted forest only, mangrove and planted forest
    print tile_list
    print "There are {} tiles to process".format(str(len(tile_list)))

    # One argument for the script: whether only emissions from biomass (soil_only) is being calculated or emissions from biomass and soil (biomass_soil)
    parser = argparse.ArgumentParser(description='Calculate gross emissions')
    parser.add_argument('--pools-to-use', '-p', required=True,
                        help='Options are soil_only or biomass_soil. Former only considers emissions from soil. Latter considers emissions from biomass and soil.')
    args = parser.parse_args()

    pools = args.pools_to_use

    # Checks the validity of the argument.
    if (pools not in ['soil_only', 'biomass_soil']):
        raise Exception('Invalid pool input. Please choose soil_only or biomass_soil.')

    # For downloading all tiles in the folders.
    # This takes about 40 minutes.
    download_list = [
                     cn.AGC_emis_year_dir, cn.BGC_emis_year_dir, cn.deadwood_emis_year_2000_dir, cn.litter_emis_year_2000_dir, cn.soil_C_emis_year_2000_dir,
                     cn.peat_mask_dir, cn.ifl_primary_processed_dir, cn.planted_forest_type_unmasked_dir, cn.drivers_processed_dir, cn.climate_zone_processed_dir,
                     cn.bor_tem_trop_processed_dir, cn.burn_year_dir,
                     cn.plant_pre_2000_processed_dir,
                     cn.loss_dir
                    ]

    for input in download_list:
        uu.s3_folder_download(input, './cpp_util/')

    # # For copying individual tiles to s3 for testing
    # for tile in tile_list:
    #
    #     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.AGC_emis_year_dir, tile, cn.pattern_AGC_emis_year), './cpp_util/')
    #     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.BGC_emis_year_dir, tile, cn.pattern_BGC_emis_year), './cpp_util/')
    #     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.deadwood_emis_year_2000_dir, tile, cn.pattern_deadwood_emis_year_2000), './cpp_util/')
    #     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.litter_emis_year_2000_dir, tile, cn.pattern_litter_emis_year_2000), './cpp_util/')
    #     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.soil_C_emis_year_2000_dir, tile, cn.pattern_soil_C_emis_year_2000), './cpp_util/')
    #     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.drivers_processed_dir, tile, cn.pattern_drivers), './cpp_util/')
    #     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.climate_zone_processed_dir, tile, cn.pattern_climate_zone), './cpp_util/')
    #     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.bor_tem_trop_processed_dir, tile, cn.pattern_bor_tem_trop_processed), './cpp_util/')
    #     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.burn_year_dir, tile, cn.pattern_burn_year), './cpp_util/')
    #     uu.s3_file_download('{0}{1}.tif'.format(cn.loss_dir, tile), './cpp_util/')
    #     try:
    #         uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.plant_pre_2000_processed_dir, tile, cn.pattern_plant_pre_2000), './cpp_util/')
    #     except:
    #         print "No pre-2000 plantations in", tile
    #     try:
    #         uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.peat_mask_dir, tile, cn.pattern_peat_mask), './cpp_util/')
    #     except:
    #         print "No peat mask for", tile
    #     try:
    #         uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.planted_forest_type_unmasked_dir, tile, cn.pattern_planted_forest_type_unmasked), './cpp_util/')
    #     except:
    #         print "No plantations in", tile
    #     try:
    #         uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.ifl_primary_processed_dir, tile, cn.pattern_ifl_primary), './cpp_util/')
    #     except:
    #         print "No IFL/primary forest in", tile

    print "Removing loss pixels from plantations that existed in Indonesia and Malaysia before 2000..."
    # Pixels that were in plantations that existed before 2000 should not be included in gross emissions.
    # Pre-2000 plantations have not previously been masked, so that is done here.
    # There are only 8 tiles to process, so count/2 will cover all of them in one go.
    count = multiprocessing.cpu_count()
    pool = multiprocessing.Pool(count/2)
    pool.map(calculate_gross_emissions.mask_pre_2000_plant, tile_list)

    # # For single processor use
    # for tile in tile_list:
    #
    #       calculate_gross_emissions.mask_pre_2000_plant(tile)

    # The C++ code expects a plantations tile for every input 10x10.
    # However, not all Hansen tiles have plantations.
    # This function creates "dummy" plantation tiles for all Hansen tiles that do not have plantations.
    # That way, the C++ script gets all the necessary input files
    folder = 'cpp_util/'

    # All of the inputs that need to have dummy tiles made in order to match the tile list of the carbon pools
    pattern_list = [cn.pattern_planted_forest_type_unmasked, cn.pattern_peat_mask, cn.pattern_ifl_primary,
                    cn.pattern_drivers, cn.pattern_bor_tem_trop_processed]

    for pattern in pattern_list:
        count = multiprocessing.cpu_count()
        pool = multiprocessing.Pool(count-10)
        pool.map(partial(uu.make_blank_tile, pattern=pattern, folder=folder), tile_list)
        pool.close()
        pool.join()

    # # For single processor use
    # for pattern in pattern_list:
    #     for tile in tile_list:
    #
    #         uu.make_blank_tile(tile, pattern, folder)

    # Checks if the correct c++ script has been compiled for the pool option selected
    if pools == 'biomass_soil':
        if os.path.exists('./cpp_util/calc_gross_emissions_biomass_soil.exe'):
            print "C++ for biomass+soil already compiled."
        else:
            raise Exception('Must compile biomass+soil C++...')

    elif pools == 'soil_only':
        if os.path.exists('./cpp_util/calc_gross_emissions_soil_only.exe'):
            print "C++ for soil_only already compiled."
        else:
            raise Exception('Must compile soil_only C++...')

    else:
        raise Exception('Pool option not valid')

    # Calculates gross emissions for each tile
    # count/4 uses about 390 GB on a r4.16xlarge spot machine.
    # processes=18 uses about 440 GB on an r4.16xlarge spot machine.
    count = multiprocessing.cpu_count()
    pool = multiprocessing.Pool(processes=18)
    pool.map(partial(calculate_gross_emissions.calc_emissions, pools=pools), tile_list)

    # # For single processor use
    # for tile in tile_list:
    #
    #       calculate_gross_emissions.calc_emissions(tile, pools)

    # Uploads emissions to default directory for biomass+soil
    if pools == 'biomass_soil':

        uu.upload_final_set(cn.gross_emis_commod_biomass_soil_dir, cn.pattern_gross_emis_commod_biomass_soil)
        uu.upload_final_set(cn.gross_emis_shifting_ag_biomass_soil_dir, cn.pattern_gross_emis_shifting_ag_biomass_soil)
        uu.upload_final_set(cn.gross_emis_forestry_biomass_soil_dir, cn.pattern_gross_emis_forestry_biomass_soil)
        uu.upload_final_set(cn.gross_emis_wildfire_biomass_soil_dir, cn.pattern_gross_emis_wildfire_biomass_soil)
        uu.upload_final_set(cn.gross_emis_urban_biomass_soil_dir, cn.pattern_gross_emis_urban_biomass_soil)
        uu.upload_final_set(cn.gross_emis_no_driverv_biomass_soil_dir, cn.pattern_gross_emis_no_driver_biomass_soil)
        uu.upload_final_set(cn.gross_emis_all_gases_all_drivers_biomass_soil_dir, cn.pattern_gross_emis_all_gases_all_drivers_biomass_soil)
        uu.upload_final_set(cn.gross_emis_co2_only_all_drivers_biomass_soil_dir, cn.pattern_gross_emis_co2_only_all_drivers_biomass_soil)
        uu.upload_final_set(cn.gross_emis_non_co2_all_drivers_biomass_soil_dir, cn.pattern_gross_emis_non_co2_all_drivers_biomass_soil)
        uu.upload_final_set(cn.gross_emis_nodes_biomass_soil_dir, cn.pattern_gross_emis_nodes_biomass_soil)

    # If soil_only was chosen, the output directories need to be changed.
    elif pools == 'soil_only':

        uu.upload_final_set(cn.gross_emis_commod_soil_only_dir, cn.pattern_gross_emis_commod_soil_only)
        uu.upload_final_set(cn.gross_emis_shifting_ag_soil_only_dir, cn.pattern_gross_emis_shifting_ag_soil_only)
        uu.upload_final_set(cn.gross_emis_forestry_soil_only_dir, cn.pattern_gross_emis_forestry_soil_only)
        uu.upload_final_set(cn.gross_emis_wildfire_soil_only_dir, cn.pattern_gross_emis_wildfire_soil_only)
        uu.upload_final_set(cn.gross_emis_urban_soil_only_dir, cn.pattern_gross_emis_urban_soil_only)
        uu.upload_final_set(cn.gross_emis_no_driver_soil_only_dir, cn.pattern_gross_emis_no_driver_soil_only)
        uu.upload_final_set(cn.gross_emis_all_gases_all_drivers_soil_only_dir, cn.pattern_gross_emis_all_gases_all_drivers_soil_only)
        uu.upload_final_set(cn.gross_emis_co2_only_all_drivers_soil_only_dir, cn.pattern_gross_emis_co2_only_all_drivers_soil_only)
        uu.upload_final_set(cn.gross_emis_non_co2_all_drivers_soil_only_dir, cn.pattern_gross_emis_non_co2_all_drivers_soil_only)
        uu.upload_final_set(cn.gross_emis_nodes_soil_only_dir, cn.pattern_gross_emis_nodes_soil_only)

    else:
        raise Exception('Pool option not valid')

if __name__ == '__main__':
    main()<commit_msg>Creating biomass+soil emissions for model v1.1.1.<commit_after>'''
This script calculates the gross emissions in tonnes CO2e/ha for every loss pixel.
The properties of each pixel determine the appropriate emissions equation, the constants for the equation, and the
carbon pool values that go into the equation.
Unlike all other flux model components, this one uses C++ to quickly iterate through every pixel in each tile.
Before running the model, the C++ script must be compiled.
From carbon-budget/emissions/, do c++ ./cpp_util/calc_gross_emissions_biomass_soil.cpp -o ./cpp_util/calc_gross_emissions_biomass_soil.exe -lgdal
calc_emissions_v3_biomass_soil.exe should appear in the directory.
Run mp_calculate_gross_emissions.py by typing python mp_calculate_gross_emissions.py -p [POOL_OPTION]. The Python script will call the
compiled C++ code as needed.
The other C++ scripts (equations.cpp and flu_val.cpp) do not need to be compiled.
The --pools-to-use argument specifies whether to calculate gross emissions from biomass+soil or just from soil.
Emissions from each driver (including loss that had no driver assigned) gets its own tile, as does all emissions combined.
Emissions from all drivers is also output as emissions due to CO2 only and emissions due to other GHG (CH4 and N2O).
The other output shows which branch of the decision tree that determines the emissions equation applies to each pixel.
These codes are summarized in carbon-budget/emissions/node_codes.txt
'''

import multiprocessing
import argparse
import os
import calculate_gross_emissions
from functools import partial
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

def main ():

    tile_list = uu.tile_list(cn.AGC_emis_year_dir)
    # tile_list = ['00N_110E', '30N_080W', '40N_050E', '50N_100E', '80N_020E'] # test tiles
    # tile_list = ['20N_010E'] # test tiles
    # tile_list = ['00N_110E', '80N_020E', '30N_080W', '00N_020E'] # test tiles: no mangrove or planted forest, mangrove only, planted forest only, mangrove and planted forest
    print tile_list
    print "There are {} tiles to process".format(str(len(tile_list)))

    # One argument for the script: whether only emissions from biomass (soil_only) is being calculated or emissions from biomass and soil (biomass_soil)
    parser = argparse.ArgumentParser(description='Calculate gross emissions')
    parser.add_argument('--pools-to-use', '-p', required=True,
                        help='Options are soil_only or biomass_soil. Former only considers emissions from soil. Latter considers emissions from biomass and soil.')
    args = parser.parse_args()

    pools = args.pools_to_use

    # Checks the validity of the argument.
    if (pools not in ['soil_only', 'biomass_soil']):
        raise Exception('Invalid pool input. Please choose soil_only or biomass_soil.')

    # For downloading all tiles in the folders.
    # This takes about 40 minutes.
    download_list = [
                     cn.AGC_emis_year_dir, cn.BGC_emis_year_dir, cn.deadwood_emis_year_2000_dir, cn.litter_emis_year_2000_dir, cn.soil_C_emis_year_2000_dir,
                     cn.peat_mask_dir, cn.ifl_primary_processed_dir, cn.planted_forest_type_unmasked_dir,
                     cn.drivers_processed_dir, cn.climate_zone_processed_dir,
                     cn.bor_tem_trop_processed_dir, cn.burn_year_dir,
                     cn.plant_pre_2000_processed_dir,
                     cn.loss_dir
                    ]

    for input in download_list:
        uu.s3_folder_download(input, './cpp_util/')

    # # For copying individual tiles to s3 for testing
    # for tile in tile_list:
    #
    #     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.AGC_emis_year_dir, tile, cn.pattern_AGC_emis_year), './cpp_util/')
    #     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.BGC_emis_year_dir, tile, cn.pattern_BGC_emis_year), './cpp_util/')
    #     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.deadwood_emis_year_2000_dir, tile, cn.pattern_deadwood_emis_year_2000), './cpp_util/')
    #     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.litter_emis_year_2000_dir, tile, cn.pattern_litter_emis_year_2000), './cpp_util/')
    #     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.soil_C_emis_year_2000_dir, tile, cn.pattern_soil_C_emis_year_2000), './cpp_util/')
    #     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.drivers_processed_dir, tile, cn.pattern_drivers), './cpp_util/')
    #     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.climate_zone_processed_dir, tile, cn.pattern_climate_zone), './cpp_util/')
    #     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.bor_tem_trop_processed_dir, tile, cn.pattern_bor_tem_trop_processed), './cpp_util/')
    #     uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.burn_year_dir, tile, cn.pattern_burn_year), './cpp_util/')
    #     uu.s3_file_download('{0}{1}.tif'.format(cn.loss_dir, tile), './cpp_util/')
    #     try:
    #         uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.plant_pre_2000_processed_dir, tile, cn.pattern_plant_pre_2000), './cpp_util/')
    #     except:
    #         print "No pre-2000 plantations in", tile
    #     try:
    #         uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.peat_mask_dir, tile, cn.pattern_peat_mask), './cpp_util/')
    #     except:
    #         print "No peat mask for", tile
    #     try:
    #         uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.planted_forest_type_unmasked_dir, tile, cn.pattern_planted_forest_type_unmasked), './cpp_util/')
    #     except:
    #         print "No plantations in", tile
    #     try:
    #         uu.s3_file_download('{0}{1}_{2}.tif'.format(cn.ifl_primary_processed_dir, tile, cn.pattern_ifl_primary), './cpp_util/')
    #     except:
    #         print "No IFL/primary forest in", tile

    print "Removing loss pixels from plantations that existed in Indonesia and Malaysia before 2000..."
    # Pixels that were in plantations that existed before 2000 should not be included in gross emissions.
    # Pre-2000 plantations have not previously been masked, so that is done here.
    # There are only 8 tiles to process, so count/2 will cover all of them in one go.
    count = multiprocessing.cpu_count()
    pool = multiprocessing.Pool(count/2)
    pool.map(calculate_gross_emissions.mask_pre_2000_plant, tile_list)

    # # For single processor use
    # for tile in tile_list:
    #
    #       calculate_gross_emissions.mask_pre_2000_plant(tile)

    # The C++ code expects a plantations tile for every input 10x10.
    # However, not all Hansen tiles have plantations.
    # This function creates "dummy" plantation tiles for all Hansen tiles that do not have plantations.
    # That way, the C++ script gets all the necessary input files
    folder = 'cpp_util/'

    # All of the inputs that need to have dummy tiles made in order to match the tile list of the carbon pools
    pattern_list = [cn.pattern_planted_forest_type_unmasked, cn.pattern_peat_mask, cn.pattern_ifl_primary,
                    cn.pattern_drivers, cn.pattern_bor_tem_trop_processed]

    for pattern in pattern_list:
        count = multiprocessing.cpu_count()
        pool = multiprocessing.Pool(count-10)
        pool.map(partial(uu.make_blank_tile, pattern=pattern, folder=folder), tile_list)
        pool.close()
        pool.join()

    # # For single processor use
    # for pattern in pattern_list:
    #     for tile in tile_list:
    #
    #         uu.make_blank_tile(tile, pattern, folder)

    # Checks if the correct c++ script has been compiled for the pool option selected
    if pools == 'biomass_soil':
        if os.path.exists('./cpp_util/calc_gross_emissions_biomass_soil.exe'):
            print "C++ for biomass+soil already compiled."
        else:
            raise Exception('Must compile biomass+soil C++...')

    elif pools == 'soil_only':
        if os.path.exists('./cpp_util/calc_gross_emissions_soil_only.exe'):
            print "C++ for soil_only already compiled."
        else:
            raise Exception('Must compile soil_only C++...')

    else:
        raise Exception('Pool option not valid')

    # Calculates gross emissions for each tile
    # count/4 uses about 390 GB on a r4.16xlarge spot machine.
    # processes=18 uses about 440 GB on an r4.16xlarge spot machine.
    count = multiprocessing.cpu_count()
    pool = multiprocessing.Pool(processes=18)
    pool.map(partial(calculate_gross_emissions.calc_emissions, pools=pools), tile_list)

    # # For single processor use
    # for tile in tile_list:
    #
    #       calculate_gross_emissions.calc_emissions(tile, pools)

    # Uploads emissions to default directory for biomass+soil
    if pools == 'biomass_soil':

        uu.upload_final_set(cn.gross_emis_commod_biomass_soil_dir, cn.pattern_gross_emis_commod_biomass_soil)
        uu.upload_final_set(cn.gross_emis_shifting_ag_biomass_soil_dir, cn.pattern_gross_emis_shifting_ag_biomass_soil)
        uu.upload_final_set(cn.gross_emis_forestry_biomass_soil_dir, cn.pattern_gross_emis_forestry_biomass_soil)
        uu.upload_final_set(cn.gross_emis_wildfire_biomass_soil_dir, cn.pattern_gross_emis_wildfire_biomass_soil)
        uu.upload_final_set(cn.gross_emis_urban_biomass_soil_dir, cn.pattern_gross_emis_urban_biomass_soil)
        uu.upload_final_set(cn.gross_emis_no_driverv_biomass_soil_dir, cn.pattern_gross_emis_no_driver_biomass_soil)
        uu.upload_final_set(cn.gross_emis_all_gases_all_drivers_biomass_soil_dir, cn.pattern_gross_emis_all_gases_all_drivers_biomass_soil)
        uu.upload_final_set(cn.gross_emis_co2_only_all_drivers_biomass_soil_dir, cn.pattern_gross_emis_co2_only_all_drivers_biomass_soil)
        uu.upload_final_set(cn.gross_emis_non_co2_all_drivers_biomass_soil_dir, cn.pattern_gross_emis_non_co2_all_drivers_biomass_soil)
        uu.upload_final_set(cn.gross_emis_nodes_biomass_soil_dir, cn.pattern_gross_emis_nodes_biomass_soil)

    # If soil_only was chosen, the output directories need to be changed.
    elif pools == 'soil_only':

        uu.upload_final_set(cn.gross_emis_commod_soil_only_dir, cn.pattern_gross_emis_commod_soil_only)
        uu.upload_final_set(cn.gross_emis_shifting_ag_soil_only_dir, cn.pattern_gross_emis_shifting_ag_soil_only)
        uu.upload_final_set(cn.gross_emis_forestry_soil_only_dir, cn.pattern_gross_emis_forestry_soil_only)
        uu.upload_final_set(cn.gross_emis_wildfire_soil_only_dir, cn.pattern_gross_emis_wildfire_soil_only)
        uu.upload_final_set(cn.gross_emis_urban_soil_only_dir, cn.pattern_gross_emis_urban_soil_only)
        uu.upload_final_set(cn.gross_emis_no_driver_soil_only_dir, cn.pattern_gross_emis_no_driver_soil_only)
        uu.upload_final_set(cn.gross_emis_all_gases_all_drivers_soil_only_dir, cn.pattern_gross_emis_all_gases_all_drivers_soil_only)
        uu.upload_final_set(cn.gross_emis_co2_only_all_drivers_soil_only_dir, cn.pattern_gross_emis_co2_only_all_drivers_soil_only)
        uu.upload_final_set(cn.gross_emis_non_co2_all_drivers_soil_only_dir, cn.pattern_gross_emis_non_co2_all_drivers_soil_only)
        uu.upload_final_set(cn.gross_emis_nodes_soil_only_dir, cn.pattern_gross_emis_nodes_soil_only)

    else:
        raise Exception('Pool option not valid')

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>from django.db import models
from django.contrib.auth.models import User
from django.utils.encoding import python_2_unicode_compatible

PUBLISHED_CHOICES = (
    ('private', 'private'),
    ('shared', 'shared'),
    ('public', 'public')
)

@python_2_unicode_compatible
class Photo(models.Model):
    image = models.ImageField(upload_to='photo_files/%Y-%m-%d')
    user = models.ForeignKey(
        User,
        null=False
    )
    title = models.CharField(max_length=128)
    description = models.TextField(help_text="Describe your photo.")
    date_created = models.DateField(auto_now_add=True)
    date_modified = models.DateField(auto_now=True)
    date_published = models.DateField(auto_now=True)
    published = models.CharField(max_length=256,
                                 choices=PUBLISHED_CHOICES,
                                 default='private')
    def __str__(self):
        return self.title

@python_2_unicode_compatible
class Album(models.Model):
    user = models.ForeignKey(User, null=False)
    photos = models.ManyToManyField(
        Photo,
        related_name='albums',
        through='PhotoInAlbum')
    title = models.CharField(max_length=128)
    description = models.TextField()
    date_created = models.DateField(auto_now_add=True)
    date_modified = models.DateField(auto_now=True)
    date_published = models.DateField(null=True, blank=True)
    published = models.CharField(max_length=256,
                                 choices=PUBLISHED_CHOICES,
                                 default='private')
    cover = models.ForeignKey(Photo, related_name='cover_for')

    def __str__(self):
        return self.title

    @property
    def cover(self):
        qs = self.photos
        if qs.filter(is_cover=True).exists():
            qs = qs.filter(is_cover=True)
        return qs.order_by('?').first()

@python_2_unicode_compatible
class PhotoInAlbum(models.Model):
    photo = models.ForeignKey(Photo)
    album = models.ForeignKey(Album)
    is_cover = models.BooleanField(default=False)

    def __str__(self):
        return "{}: in album {}".format(self.photo, self.album)<commit_msg>fixed models<commit_after>from django.db import models
from django.contrib.auth.models import User
from django.utils.encoding import python_2_unicode_compatible

PUBLISHED_CHOICES = (
    ('private', 'private'),
    ('shared', 'shared'),
    ('public', 'public')
)

@python_2_unicode_compatible
class Photo(models.Model):
    image = models.ImageField(upload_to='photo_files/%Y-%m-%d')
    user = models.ForeignKey(
        User,
        null=False
    )
    title = models.CharField(max_length=128)
    description = models.TextField(help_text="Describe your photo.")
    date_created = models.DateField(auto_now_add=True)
    date_modified = models.DateField(auto_now=True)
    date_published = models.DateField(auto_now=True)
    published = models.CharField(max_length=256,
                                 choices=PUBLISHED_CHOICES,
                                 default='private')
    def __str__(self):
        return self.title

@python_2_unicode_compatible
class Album(models.Model):
    user = models.ForeignKey(User, null=False)
    photos = models.ManyToManyField(
        Photo,
        related_name='albums',
        blank=True)
    title = models.CharField(max_length=128)
    description = models.TextField()
    date_created = models.DateField(auto_now_add=True)
    date_modified = models.DateField(auto_now=True)
    date_published = models.DateField(null=True, blank=True)
    published = models.CharField(max_length=256,
                                 choices=PUBLISHED_CHOICES,
                                 default='private')
    cover = models.ForeignKey(Photo, related_name='cover_for')

    def __str__(self):
        return self.title

    @property
    def cover(self):
        qs = self.photos
        if qs.filter(is_cover=True).exists():
            qs = qs.filter(is_cover=True)
        return qs.order_by('?').first()

@python_2_unicode_compatible
class PhotoInAlbum(models.Model):
    photo = models.ForeignKey(Photo)
    album = models.ForeignKey(Album)
    is_cover = models.BooleanField(default=False)

    def __str__(self):
        return "{}: in album {}".format(self.photo, self.album)<|endoftext|>


<commit_before>"""Define image models."""

from django.db import models
from imager_profile.models import UserProfile
from django.utils.encoding import python_2_unicode_compatible
from taggit.managers import TaggableManager

# Create yo  models here.

SHARING = (
    ('PRIVATE', "Private"),
    ('SHARED', "Shared"),
    ('PUBLIC', "Public"),
)

@python_2_unicode_compatible
class Photo(models.Model):
    """One individaul photo."""

    image_file = models.ImageField(upload_to="")
    title = models.CharField(max_length=255, blank=True, null=True)
    description = models.TextField(max_length=255, blank=True, null=True)
    date_uploaded = models.DateField(auto_now_add=True)
    date_modified = models.DateField(auto_now=True)
    date_published = models.DateField(auto_now_add=True)

    photographer = models.ForeignKey(UserProfile,
                                     related_name='photo',
                                     blank=True,
                                     null=True
                                     )

    published = models.CharField(max_length=255,
                                 choices=SHARING,
                                 default='PUBLIC')
    tags = TaggableManager()

    def __str__(self):
        """String representation of UserProfile."""
        return self.title

@python_2_unicode_compatible
class Album(models.Model):
    """Album class, one user, many photos."""

    owner = models.ForeignKey(UserProfile,
                              related_name="albums",
                              blank=True,
                              null=True
                              )

    contents = models.ManyToManyField(Photo,
                                      related_name='in_album',
                                      blank=True)

    title = models.CharField(max_length=255, blank=True, null=True)
    description = models.TextField(max_length=255, blank=True, null=True)
    date_uploaded = models.DateField(auto_now_add=True)
    date_modified = models.DateField(auto_now=True)
    date_published = models.DateField(auto_now_add=True)

    published = models.CharField(max_length=255,
                                 choices=SHARING,
                                 default='PUBLIC')

    cover_photo = models.ForeignKey(Photo,
                                    related_name='cover',
                                    blank=True,
                                    null=True)
    tags = TaggableManager()
<commit_msg>removed tags in model for album<commit_after>"""Define image models."""

from django.db import models
from imager_profile.models import UserProfile
from django.utils.encoding import python_2_unicode_compatible
from taggit.managers import TaggableManager

# Create yo  models here.

SHARING = (
    ('PRIVATE', "Private"),
    ('SHARED', "Shared"),
    ('PUBLIC', "Public"),
)

@python_2_unicode_compatible
class Photo(models.Model):
    """One individaul photo."""

    image_file = models.ImageField(upload_to="")
    title = models.CharField(max_length=255, blank=True, null=True)
    description = models.TextField(max_length=255, blank=True, null=True)
    date_uploaded = models.DateField(auto_now_add=True)
    date_modified = models.DateField(auto_now=True)
    date_published = models.DateField(auto_now_add=True)

    photographer = models.ForeignKey(UserProfile,
                                     related_name='photo',
                                     blank=True,
                                     null=True
                                     )

    published = models.CharField(max_length=255,
                                 choices=SHARING,
                                 default='PUBLIC')
    tags = TaggableManager()

    def __str__(self):
        """String representation of UserProfile."""
        return self.title

@python_2_unicode_compatible
class Album(models.Model):
    """Album class, one user, many photos."""

    owner = models.ForeignKey(UserProfile,
                              related_name="albums",
                              blank=True,
                              null=True
                              )

    contents = models.ManyToManyField(Photo,
                                      related_name='in_album',
                                      blank=True)

    title = models.CharField(max_length=255, blank=True, null=True)
    description = models.TextField(max_length=255, blank=True, null=True)
    date_uploaded = models.DateField(auto_now_add=True)
    date_modified = models.DateField(auto_now=True)
    date_published = models.DateField(auto_now_add=True)

    published = models.CharField(max_length=255,
                                 choices=SHARING,
                                 default='PUBLIC')

    cover_photo = models.ForeignKey(Photo,
                                    related_name='cover',
                                    blank=True,
                                    null=True)
<|endoftext|>


<commit_before>import cv2
import numpy

from filter import Filter

class OverExposed(Filter):

    """Filter for detecting overexposure"""

    def __init__(self):
        self.name = "over_exposure"
        self.parameters = {}

    def required(self):
        return {'image'}

    def run(self):
        """ Checks if image is over-exposed
        """
        image = self.parameters['image']
        histogram = cv2.calcHist([image], [0], None, [256], [0, 256])
        clip = clipping_percentage(histogram, 250) * 50
        return min(1, clip)

def clipping_percentage(histogram, threshold):
    total = numpy.sum(histogram)
    if total < 0.0005:  # avoid division by zero
        return 0

    return float(numpy.sum(histogram[threshold:])) / total
<commit_msg>bugfix<commit_after>import cv2
import numpy

from filter import Filter

class OverExposed(Filter):

    """Filter for detecting overexposure"""

    def __init__(self):
        self.name = "over_exposure"
        self.parameters = {}

    def required(self):
        return {'image'}

    def run(self):
        """ Checks if image is over-exposed
        """
        image = self.parameters['image']
        histogram = cv2.calcHist([image], [0], None, [256], [0, 256])
        # Normalize:
        clip = clipping_percentage(histogram, 250) * 50
        return min(1, clip)

def clipping_percentage(histogram, threshold):
    total = numpy.sum(histogram)
    if total < 0.0005:  # avoid division by zero
        return 0
    return float(numpy.sum(histogram[threshold:])) / total
<|endoftext|>


<commit_before># File: badgeeventreminder.py
# Version: 1
# Description: Easter egg
# License: MIT
# Authors: Renze Nicolai <renze@rnplus.nl>

import virtualtimers, time, appglue, badge

# Tue Aug  8 13:30:00 2017 (CEST)
whenToTrigger = 1502191800 - 600

def ber_task():
    global whenToTrigger
    now = time.time()
    if now >= whenToTrigger and now < whenToTrigger + 600 + 3600:
        badge.nvs_set_u8('badge','evrt',1)
        print("BADGE EVENT REMINDER ACTIVATED")
        appglue.start_app("badge_event_reminder")
    idleFor = whenToTrigger - now
    if idleFor < 0:
        idleFor = 0
    return idleFor * 1000

def enable():
    if badge.nvs_get_u8('badge','evrt',0)==0:
        virtualtimers.new(1, ber_task)

def disable():
    virtualtimers.delete(ber_task)
<commit_msg>work around time.time() brokenness<commit_after># File: badgeeventreminder.py
# Version: 1
# Description: Easter egg
# License: MIT
# Authors: Renze Nicolai <renze@rnplus.nl>

import virtualtimers, time, appglue, badge

# Tue Aug  8 13:30:00 2017 (CEST)
# (warn 10 minutes before the event)
whenToTrigger = 1502191800 - 600

def ber_task():
    global whenToTrigger
    # braindead time.time() implementation. it's 2 hours in the
    # future.
    now = time.time() - 7200
    if now >= whenToTrigger and now < whenToTrigger + 600 + 3600:
        badge.nvs_set_u8('badge','evrt',1)
        print("BADGE EVENT REMINDER ACTIVATED")
        appglue.start_app("badge_event_reminder")
    idleFor = whenToTrigger - now
    if idleFor < 0:
        idleFor = 0
    return idleFor * 1000

def enable():
    if badge.nvs_get_u8('badge','evrt',0)==0:
        virtualtimers.new(1, ber_task)

def disable():
    virtualtimers.delete(ber_task)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.utils.six.moves.urllib.parse import urljoin
from django.utils.encoding import python_2_unicode_compatible
from djangocms_text_ckeditor.fields import HTMLField
import reversion
from shop.money.fields import MoneyField
from shop.models.product import BaseProduct, BaseProductManager
from myshop.models.properties import Manufacturer, ProductPage, ProductImage

class ProductManager(BaseProductManager):
    def select_lookup(self, term):
        query = models.Q(name__icontains=term) | models.Q(slug__icontains=term)
        return self.get_queryset().filter(query)

@python_2_unicode_compatible
class SmartCard(BaseProduct):
    # common product fields
    name = models.CharField(max_length=255, verbose_name=_("Name"))
    slug = models.SlugField(verbose_name=_("Slug"))
    unit_price = MoneyField(_("Unit price"), decimal_places=3,
        help_text=_("Net price for this product"))
    description = HTMLField(verbose_name=_("Description"),
                            help_text=_("Description for the list view of products."))

    # product properties
    manufacturer = models.ForeignKey(Manufacturer, verbose_name=_("Manufacturer"))
    CARD_TYPE = (2 * ('{}{}'.format(s, t),)
                 for t in ('SD', 'SDXC', 'SDHC', 'SDHC II') for s in ('', 'micro '))
    card_type = models.CharField(_("Card Type"), choices=CARD_TYPE, max_length=15)
    SPEED = ((str(s), "{} MB/s".format(s)) for s in (4, 20, 30, 40, 48, 80, 95, 280))
    speed = models.CharField(_("Transfer Speed"), choices=SPEED, max_length=8)
    product_code = models.CharField(_("Product code"), max_length=255, unique=True)
    storage = models.PositiveIntegerField(_("Storage Capacity"),
        help_text=_("Storage capacity in GB"))

    # controlling the catalog
    order = models.PositiveIntegerField(verbose_name=_("Sort by"), db_index=True)
    cms_pages = models.ManyToManyField('cms.Page', through=ProductPage, null=True,
        help_text=_("Choose list view this product shall appear on."))
    images = models.ManyToManyField('filer.Image', through=ProductImage, null=True)

    objects = ProductManager()

    class Meta:
        verbose_name = _("Smart Card")
        verbose_name_plural = _("Smart Cards")
        ordering = ('order',)

    def __str__(self):
        return self.name

    @property
    def product_name(self):
        return self.name

    @property
    def product_code(self):
        return self.slug

    @property
    def sample_image(self):
        return self.images.first()

    def get_price(self, request):
        return self.unit_price

    def get_absolute_url(self):
        # sorting by highest level, so that the canonical URL associates with the
        # most generic category
        cms_page = self.cms_pages.order_by('depth').last()
        if cms_page is None:
            return urljoin('category-not-assigned', self.slug)
        return urljoin(cms_page.get_absolute_url(), self.slug)

reversion.register(SmartCard)
<commit_msg>removed property for product_code<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.utils.six.moves.urllib.parse import urljoin
from django.utils.encoding import python_2_unicode_compatible
from djangocms_text_ckeditor.fields import HTMLField
import reversion
from shop.money.fields import MoneyField
from shop.models.product import BaseProduct, BaseProductManager
from myshop.models.properties import Manufacturer, ProductPage, ProductImage

class ProductManager(BaseProductManager):
    def select_lookup(self, term):
        query = models.Q(name__icontains=term) | models.Q(slug__icontains=term)
        return self.get_queryset().filter(query)

@python_2_unicode_compatible
class SmartCard(BaseProduct):
    # common product fields
    name = models.CharField(max_length=255, verbose_name=_("Name"))
    slug = models.SlugField(verbose_name=_("Slug"))
    unit_price = MoneyField(_("Unit price"), decimal_places=3,
        help_text=_("Net price for this product"))
    description = HTMLField(verbose_name=_("Description"),
                            help_text=_("Description for the list view of products."))

    # product properties
    manufacturer = models.ForeignKey(Manufacturer, verbose_name=_("Manufacturer"))
    CARD_TYPE = (2 * ('{}{}'.format(s, t),)
                 for t in ('SD', 'SDXC', 'SDHC', 'SDHC II') for s in ('', 'micro '))
    card_type = models.CharField(_("Card Type"), choices=CARD_TYPE, max_length=15)
    SPEED = ((str(s), "{} MB/s".format(s)) for s in (4, 20, 30, 40, 48, 80, 95, 280))
    speed = models.CharField(_("Transfer Speed"), choices=SPEED, max_length=8)
    product_code = models.CharField(_("Product code"), max_length=255, unique=True)
    storage = models.PositiveIntegerField(_("Storage Capacity"),
        help_text=_("Storage capacity in GB"))

    # controlling the catalog
    order = models.PositiveIntegerField(verbose_name=_("Sort by"), db_index=True)
    cms_pages = models.ManyToManyField('cms.Page', through=ProductPage, null=True,
        help_text=_("Choose list view this product shall appear on."))
    images = models.ManyToManyField('filer.Image', through=ProductImage, null=True)

    objects = ProductManager()

    class Meta:
        verbose_name = _("Smart Card")
        verbose_name_plural = _("Smart Cards")
        ordering = ('order',)

    def __str__(self):
        return self.name

    @property
    def product_name(self):
        return self.name

    @property
    def sample_image(self):
        return self.images.first()

    def get_price(self, request):
        return self.unit_price

    def get_absolute_url(self):
        # sorting by highest level, so that the canonical URL associates with the
        # most generic category
        cms_page = self.cms_pages.order_by('depth').last()
        if cms_page is None:
            return urljoin('category-not-assigned', self.slug)
        return urljoin(cms_page.get_absolute_url(), self.slug)

reversion.register(SmartCard)
<|endoftext|>