<commit_before>import random
from queue import Queue
import threading
import json
import logging

import requests
import pymysql

from .connection import CustomServerConnection

log = logging.getLogger('tyggbot')

class Whisper:
    def __init__(self, target, message):
        self.target = target
        self.message = message

class WhisperConnection:
    def __init__(self, conn, name, oauth):
        self.conn = conn
        self.num_msgs_sent = 0
        self.name = name
        self.oauth = oauth

    def reduce_msgs_sent(self):
        self.num_msgs_sent -= 1

class WhisperConnectionManager:
    def __init__(self, reactor, tyggbot, target, message_limit, time_interval, num_of_conns=30):
        self.reactor = reactor
        self.tyggbot = tyggbot
        self.message_limit = message_limit
        self.time_interval = time_interval
        self.num_of_conns = num_of_conns
        self.whisper_thread = None

        self.connlist = []
        self.whispers = Queue()

        self.maintenance_lock = False

    def __contains__(self, connection):
        return connection in [c.conn for c in self.connlist]

    def start(self, accounts=[]):
        log.debug("Starting connection manager")
        try:
            # Update available group servers.
            # This will also be run at an interval to make sure it's up to date
            self.update_servers_list()
            self.reactor.execute_every(3600, self.update_servers_list)

            # Run the maintenance function every 4 seconds.
            # The maintenance function is responsible for reconnecting lost connections.
            self.reactor.execute_every(4, self.run_maintenance)

            # Fetch additional whisper accounts from the database
            self.tyggbot.sqlconn.ping()
            cursor = self.tyggbot.sqlconn.cursor(pymysql.cursors.DictCursor)
            cursor.execute("SELECT `username`, `oauth` FROM `tb_whisper_account` WHERE `enabled`=1 ORDER BY RAND() LIMIT %s", self.num_of_conns)
            for row in cursor:
                accounts.append(row)

            # Start the connections.
            t = threading.Thread(target=self.start_connections, args=[accounts], name='WhisperConnectionStarterThread')
            t.daemon = True
            t.start()

            return True
        except:
            log.exception("WhisperConnectionManager: Unhandled exception")
            return False

    def start_connections(self, accounts):
        for account in accounts:
            newconn = self.make_new_connection(account['username'], account['oauth'])
            self.connlist.append(newconn)

        self.whisper_thread = threading.Thread(target=self.whisper_sender, name='WhisperThread')  # start a loop sending whispers in a thread
        self.whisper_thread.daemon = True
        self.whisper_thread.start()

    def quit(self):
        for connection in self.connlist:
            connection.conn.quit('bye')

    def update_servers_list(self):
            log.debug("Refreshing list of whisper servers")
            servers_list = json.loads(requests.get("http://tmi.twitch.tv/servers?cluster=group").text)
            self.servers_list = servers_list['servers']

    def whisper_sender(self):
        while True:
            whisp = self.whispers.get()
            username = whisp.target
            message = whisp.message

            i = 0
            while((not self.connlist[i].conn.is_connected()) or self.connlist[i].num_msgs_sent >= self.message_limit):
                i += 1  # find a usable connection
                if i >= len(self.connlist):
                    i = 0

            log.debug('Sending whisper: {0} {1}'.format(username, message))
            self.connlist[i].conn.privmsg('#jtv', '/w {0} {1}'.format(username, message))
            self.connlist[i].num_msgs_sent += 1
            self.connlist[i].conn.execute_delayed(self.time_interval, self.connlist[i].reduce_msgs_sent)

    def run_maintenance(self):
        if self.maintenance_lock:
            return

        self.maintenance_lock = True
        for connection in self.connlist:
            if not connection.conn.is_connected():
                connection.conn.close()
                self.connlist.remove(connection)
                newconn = self.make_new_connection(connection.name, connection.oauth)
                self.connlist.append(newconn)

        self.maintenance_lock = False

    def get_main_conn(self):
        for connection in self.connlist:
            if connection.conn.is_connected():
                return connection.conn

        log.error("No connection with is_connected() found in WhisperConnectionManager")

    def make_new_connection(self, name, oauth):
        server = random.choice(self.servers_list)
        ip, port = server.split(':')
        port = int(port)
        log.debug("Whispers: Connection to server {0}".format(server))

        newconn = CustomServerConnection(self.reactor)
        with self.reactor.mutex:
            self.reactor.connections.append(newconn)
        newconn.connect(ip, port, name, oauth, name)
        newconn.cap('REQ', 'twitch.tv/commands')
        return WhisperConnection(newconn, name, oauth)

    def on_disconnect(self, conn):
        conn.reconnect()
        conn.cap('REQ', 'twitch.tv/commands')
        return

    def whisper(self, target, message):
        if not target:
            target = self.target
        self.whispers.put(Whisper(target, message))
<commit_msg>whisper_sender now catches all exceptions.<commit_after>import random
from queue import Queue
import threading
import json
import logging

import requests
import pymysql

from .connection import CustomServerConnection

log = logging.getLogger('tyggbot')

class Whisper:
    def __init__(self, target, message):
        self.target = target
        self.message = message

class WhisperConnection:
    def __init__(self, conn, name, oauth):
        self.conn = conn
        self.num_msgs_sent = 0
        self.name = name
        self.oauth = oauth

    def reduce_msgs_sent(self):
        self.num_msgs_sent -= 1

class WhisperConnectionManager:
    def __init__(self, reactor, tyggbot, target, message_limit, time_interval, num_of_conns=30):
        self.reactor = reactor
        self.tyggbot = tyggbot
        self.message_limit = message_limit
        self.time_interval = time_interval
        self.num_of_conns = num_of_conns
        self.whisper_thread = None

        self.connlist = []
        self.whispers = Queue()

        self.maintenance_lock = False

    def __contains__(self, connection):
        return connection in [c.conn for c in self.connlist]

    def start(self, accounts=[]):
        log.debug("Starting connection manager")
        try:
            # Update available group servers.
            # This will also be run at an interval to make sure it's up to date
            self.update_servers_list()
            self.reactor.execute_every(3600, self.update_servers_list)

            # Run the maintenance function every 4 seconds.
            # The maintenance function is responsible for reconnecting lost connections.
            self.reactor.execute_every(4, self.run_maintenance)

            # Fetch additional whisper accounts from the database
            self.tyggbot.sqlconn.ping()
            cursor = self.tyggbot.sqlconn.cursor(pymysql.cursors.DictCursor)
            cursor.execute("SELECT `username`, `oauth` FROM `tb_whisper_account` WHERE `enabled`=1 ORDER BY RAND() LIMIT %s", self.num_of_conns)
            for row in cursor:
                accounts.append(row)

            # Start the connections.
            t = threading.Thread(target=self.start_connections, args=[accounts], name='WhisperConnectionStarterThread')
            t.daemon = True
            t.start()

            return True
        except:
            log.exception("WhisperConnectionManager: Unhandled exception")
            return False

    def start_connections(self, accounts):
        for account in accounts:
            newconn = self.make_new_connection(account['username'], account['oauth'])
            self.connlist.append(newconn)

        self.whisper_thread = threading.Thread(target=self.whisper_sender, name='WhisperThread')  # start a loop sending whispers in a thread
        self.whisper_thread.daemon = True
        self.whisper_thread.start()

    def quit(self):
        for connection in self.connlist:
            connection.conn.quit('bye')

    def update_servers_list(self):
            log.debug("Refreshing list of whisper servers")
            servers_list = json.loads(requests.get("http://tmi.twitch.tv/servers?cluster=group").text)
            self.servers_list = servers_list['servers']

    def whisper_sender(self):
        while True:
            try:
                whisp = self.whispers.get()
                username = whisp.target
                message = whisp.message

                i = 0
                while((not self.connlist[i].conn.is_connected()) or self.connlist[i].num_msgs_sent >= self.message_limit):
                    i += 1  # find a usable connection
                    if i >= len(self.connlist):
                        i = 0

                log.debug('Sending whisper: {0} {1}'.format(username, message))
                self.connlist[i].conn.privmsg('#jtv', '/w {0} {1}'.format(username, message))
                self.connlist[i].num_msgs_sent += 1
                self.connlist[i].conn.execute_delayed(self.time_interval, self.connlist[i].reduce_msgs_sent)
            except:
                log.exception('Caught an exception in the whisper_sender function')

    def run_maintenance(self):
        if self.maintenance_lock:
            return

        self.maintenance_lock = True
        for connection in self.connlist:
            if not connection.conn.is_connected():
                connection.conn.close()
                self.connlist.remove(connection)
                newconn = self.make_new_connection(connection.name, connection.oauth)
                self.connlist.append(newconn)

        self.maintenance_lock = False

    def get_main_conn(self):
        for connection in self.connlist:
            if connection.conn.is_connected():
                return connection.conn

        log.error("No connection with is_connected() found in WhisperConnectionManager")

    def make_new_connection(self, name, oauth):
        server = random.choice(self.servers_list)
        ip, port = server.split(':')
        port = int(port)
        log.debug("Whispers: Connection to server {0}".format(server))

        newconn = CustomServerConnection(self.reactor)
        with self.reactor.mutex:
            self.reactor.connections.append(newconn)
        newconn.connect(ip, port, name, oauth, name)
        newconn.cap('REQ', 'twitch.tv/commands')
        return WhisperConnection(newconn, name, oauth)

    def on_disconnect(self, conn):
        conn.reconnect()
        conn.cap('REQ', 'twitch.tv/commands')
        return

    def whisper(self, target, message):
        if not target:
            target = self.target
        self.whispers.put(Whisper(target, message))
<|endoftext|>


<commit_before># coding: utf-8

""" SCOPE tests. """

from __future__ import division, print_function

__author__ = "Andy Casey <acasey@mso.anu.edu.au>"

# Standard library
import logging
import os
import re

from glob import glob

# Third-party
import numpy as np
import pyfits
import yaml

# Module imports
import config

folder_path = os.path.abspath(os.path.dirname(__file__))

def test_assert():
    assert True

def test_scipy():
    from scipy.interpolate import griddata

def test_default_configuration():

    configuration = config.load(os.path.join(folder_path, '../config.yml'))

    # Verify everything except models

    # Check the normalisation
    normalisation_priors = config.verify_normalisation(configuration)

    # Check the smoothing
    smoothing_priors = config.verify_smoothing(configuration)

    # Check the doppler corrections
    doppler_priors = config.verify_doppler(configuration)

    # Establish all of the priors
    priors_to_expect = doppler_priors + smoothing_priors + normalisation_priors

    # Verify that we have priors established for all the priors
    # we expect, and the stellar parameters we plan to solve for
    config.verify_priors(configuration, priors_to_expect)
<commit_msg>Run additional model tests on localhost<commit_after># coding: utf-8

""" SCOPE tests. """

from __future__ import division, print_function

__author__ = "Andy Casey <acasey@mso.anu.edu.au>"

# Standard library
import logging
import os
import re

from glob import glob

# Third-party
import numpy as np
import pyfits
import scipy
import yaml

# Module imports
import config

folder_path = os.path.abspath(os.path.dirname(__file__))
environment = ['remote', 'localhost'][folder_path.startswith('/Users/andycasey/')]

def test_assert():
    assert True

def test_default_configuration():

    configuration = config.load(os.path.join(folder_path, '../config.yml'))

    # Verify everything except models
    if environment == 'localhost':
        config.verify_models(configuration)

    # Check the normalisation
    normalisation_priors = config.verify_normalisation(configuration)

    # Check the smoothing
    smoothing_priors = config.verify_smoothing(configuration)

    # Check the doppler corrections
    doppler_priors = config.verify_doppler(configuration)

    # Establish all of the priors
    priors_to_expect = doppler_priors + smoothing_priors + normalisation_priors

    # Verify that we have priors established for all the priors
    # we expect, and the stellar parameters we plan to solve for
    config.verify_priors(configuration, priors_to_expect)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

# rss is the interface to rss parsing,
# processing and storing.
#
# @author Jin Yuan
# @contact jinyuan@baidu.com
# @created Jul. 20, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

from administration import database as db_feeds
import calendar
import database as db_news
from datetime import datetime, timedelta
from data_processor import summarizer
from data_processor import transcoder
from data_processor import tts_provider
from image_processor import image_helper
from image_processor import thumbnail
import memory
import random
import rss_parser
import time

from config import CATEGORY_IMAGE_SIZE
from config import DATABASE_REMOVAL_DAYS
from config import HOT_IMAGE_SIZE
from config import LANGUAGES
from config import MEMORY_EXPIRATION_DAYS
from config import MIN_IMAGE_SIZE
from config import THUMBNAIL_LANDSCAPE_SIZE
from config import THUMBNAIL_PORTRAIT_SIZE
from config import THUMBNAIL_STYLE

def _value_added_process(entries=None, language=None, transcoder_type='chengdujin'):
    """
    add more value to an entry
    tts, transcode, images, redis_entry_expiration, database_entry_expiration
    """
    if not entries:
        return None
    if not language or language not in LANGUAGES:
        raise Exception("ERROR: language not found or not supported!")

    entries_new = []
    for entry in entries:
        # get a random int from 100 million possibilities
        try:
            # [MUST-HAVE] transcoding
            rand = random.randint(0, 100000000)
            transcoded_relative_path = '%s_%s_%s_%i' % (
                entry['language'], entry['feed_id'], entry['updated'], rand)
            # high chances transcoder cannot work properly
            entry['transcoded'], entry['transcoded_local'], raw_transcoded_content, images_from_transcoded = transcoder.convert(
                entry['language'], entry['title'], entry['link'], transcoder_type, transcoded_relative_path)

            # [MUST-HAVE] summary
            entry['summary'] = summarizer.extract(
                entry['summary'], raw_transcoded_content, entry['language'])

            # process images found in the transcoded data
            if images_from_transcoded:
                # images from transcoded are already normalized
                entry['images'].extend(images_from_transcoded)
                # remove duplicated images
                entry['images'] = image_helper.dedupe_images(
                    entry['images']) if entry.has_key('images') and entry['images'] else None
            # make images none if nothing's there, instead of a []
            entry['images'] = entry['images'] if entry.has_key(
                'images') and entry['images'] else None

            # [OPTIONAL] generate 3 types of images: thumbnail, category image and hot news image
            if entry.has_key('images') and entry['images']:
                biggest = image_helper.find_biggest_image(entry['images'])
                if biggest:
                    try:
                        rand = random.randint(0, 100000000)
                        image_relative_path = '%s_%s_%s_%i' % (
                            entry['language'], entry['feed_id'], entry['updated'], rand)

                        # hot news image
                        hot_web, hot_local = image_helper.scale_image(
                            image=biggest, size_expected=HOT_IMAGE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_hotnews' % image_relative_path)
                        entry['hot_news_image'] = hot_web if hot_web else None
                        entry[
                            'hot_news_image_local'] = hot_local if hot_local else None

                        # category image
                        category_web, category_local = image_helper.scale_image(
                            image=biggest, size_expected=CATEGORY_IMAGE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_category' % image_relative_path)
                        entry[
                            'category_image'] = category_web if category_web else None
                        entry[
                            'category_image_local'] = category_local if category_local else None

                        # thumbnail image
                        # thumbnail_landscape_size
                        if float(biggest['width']) / float(biggest['height']) >= THUMBNAIL_STYLE:
                            thumbnail_web, thumbnail_local = image_helper.scale_image(
                                image=biggest, size_expected=THUMBNAIL_LANDSCAPE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_thumbnail' % image_relative_path)
                        else:  # else thumbnail_portrait_size
                            thumbnail_web, thumbnail_local = image_helper.scale_image(
                                image=biggest, size_expected=THUMBNAIL_PORTRAIT_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_thumbnail' % image_relative_path)
                        entry[
                            'thumbnail_image'] = thumbnail_web if thumbnail_web else None
                        entry[
                            'thumbnail_image_local'] = thumbnail_local if thumbnail_local else None

                        # for older version users
                        entry['image'] = entry['thumbnail_image'][
                            'url'] if thumbnail_web else None
                    except IOError as k:
                        entry['error'].append(str(k) + '\n')

            # [OPTIONAL] google tts not for indonesian
            """
            if entry['language'] != 'ind':
                try:
                    tts_relative_path = '%s_%s_%s_%i.mp3' % (
                        entry['language'], entry['feed_id'], entry['updated'], rand)
                    entry['mp3'], entry['mp3_local'] = tts_provider.google(
                        entry['language'], entry['title'], tts_relative_path)
                except Exception as k:
                    print k, '... cannot generate TTS for %s' % entry['link']
                    entry['error'].append(k + '\n')
                    entry['mp3'] = None
                    entry['mp3_local'] = None
            """

            # [MUST-HAVE] add expiration data
            def _expired(updated, days_to_deadline):
                """
                compute expiration information
                return time string and unix time
                """
                deadline = datetime.utcfromtimestamp(
                    updated) + timedelta(days=days_to_deadline)
                return time.asctime(time.gmtime(calendar.timegm(deadline.timetuple())))

            entry['memory_expired'] = _expired(
                entry['updated'], MEMORY_EXPIRATION_DAYS)
            entry['database_expired'] = _expired(
                entry['updated'], DATABASE_REMOVAL_DAYS)

            entry['error'] = entry['error'] if entry['error'] else None
            entries_new.append(entry)
            print
            print
        except Exception as k:
            print k
    return entries_new

# TODO: code to remove added items if things suck at database/memory
def update(feed_link=None, feed_id=None, language=None, categories=None, transcoder_type='chengdujin'):
    """
    update could be called
    1. from task procedure: feed_id
    2. after an rss is added: feed_id
    3. manually for testing purpose: feed_link, language
    Note. categories are ids of category item
    Notel categories are kept for manual testing
    """
    if not feed_id and not (feed_link and language):
        raise Exception(
            "ERROR: Method signature not well formed!")

    if feed_id:
        feed = db_feeds.get(feed_id=feed_id)
    else:
        feed = db_feeds.get(feed_link=feed_link, language=language)

    feed_id = str(feed['_id']) if feed else feed_id
    feed_link = feed['feed_link'] if feed else feed_link
    language = feed['language'] if feed else language
    categories = feed['categories'] if feed else categories
    feed_title = feed['feed_title'] if feed and 'feed_title' in feed else None
    transcoder_type = feed['transcoder'] if feed else transcoder_type
    etag = feed['etag'] if feed and 'etag' in feed else None
    modified = feed['modified'] if feed and 'modified' in feed else None

    # parse rss reading from remote rss servers
    entries, status_new, feed_title_new, etag_new, modified_new = rss_parser.parse(
        feed_link, feed_id, feed_title, language, categories, etag, modified)

    # filter out existing entries in db_news
    # there are some possible exceptions -- yet let it be
    entries = db_news.dedup(entries, language)
    # and do tts, big_images, image as well as transcode.
    entries = _value_added_process(entries, language, transcoder_type)

    # update new entries to db_news
    # each entry is added with _id
    entries = db_news.update(entries, language)
    # and some data, like feed_title, etag and modified to db_feeds
    # only feed_id is necessary, others are optional **kwargs
    db_feeds.update(feed_id=feed_id, status=status_new,
                    feed_title=feed_title_new, etag=etag_new, modified=modified_new)

    # store in memory
    memory.update(entries, language, categories)
    return entries
<commit_msg>renamed import administration to feed_manager<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

# rss is the interface to rss parsing,
# processing and storing.
#
# @author Jin Yuan
# @contact jinyuan@baidu.com
# @created Jul. 20, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

from feed_manager import database as db_feeds
import calendar
import database as db_news
from datetime import datetime, timedelta
from data_processor import summarizer
from data_processor import transcoder
from data_processor import tts_provider
from image_processor import image_helper
from image_processor import thumbnail
import memory
import random
import rss_parser
import time

from config import CATEGORY_IMAGE_SIZE
from config import DATABASE_REMOVAL_DAYS
from config import HOT_IMAGE_SIZE
from config import LANGUAGES
from config import MEMORY_EXPIRATION_DAYS
from config import MIN_IMAGE_SIZE
from config import THUMBNAIL_LANDSCAPE_SIZE
from config import THUMBNAIL_PORTRAIT_SIZE
from config import THUMBNAIL_STYLE

def _value_added_process(entries=None, language=None, transcoder_type='chengdujin'):
    """
    add more value to an entry
    tts, transcode, images, redis_entry_expiration, database_entry_expiration
    """
    if not entries:
        return None
    if not language or language not in LANGUAGES:
        raise Exception("ERROR: language not found or not supported!")

    entries_new = []
    for entry in entries:
        # get a random int from 100 million possibilities
        try:
            # [MUST-HAVE] transcoding
            rand = random.randint(0, 100000000)
            transcoded_relative_path = '%s_%s_%s_%i' % (
                entry['language'], entry['feed_id'], entry['updated'], rand)
            # high chances transcoder cannot work properly
            entry['transcoded'], entry['transcoded_local'], raw_transcoded_content, images_from_transcoded = transcoder.convert(
                entry['language'], entry['title'], entry['link'], transcoder_type, transcoded_relative_path)

            # [MUST-HAVE] summary
            entry['summary'] = summarizer.extract(
                entry['summary'], raw_transcoded_content, entry['language'])

            # process images found in the transcoded data
            if images_from_transcoded:
                # images from transcoded are already normalized
                entry['images'].extend(images_from_transcoded)
                # remove duplicated images
                entry['images'] = image_helper.dedupe_images(
                    entry['images']) if entry.has_key('images') and entry['images'] else None
            # make images none if nothing's there, instead of a []
            entry['images'] = entry['images'] if entry.has_key(
                'images') and entry['images'] else None

            # [OPTIONAL] generate 3 types of images: thumbnail, category image and hot news image
            if entry.has_key('images') and entry['images']:
                biggest = image_helper.find_biggest_image(entry['images'])
                if biggest:
                    try:
                        rand = random.randint(0, 100000000)
                        image_relative_path = '%s_%s_%s_%i' % (
                            entry['language'], entry['feed_id'], entry['updated'], rand)

                        # hot news image
                        hot_web, hot_local = image_helper.scale_image(
                            image=biggest, size_expected=HOT_IMAGE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_hotnews' % image_relative_path)
                        entry['hot_news_image'] = hot_web if hot_web else None
                        entry[
                            'hot_news_image_local'] = hot_local if hot_local else None

                        # category image
                        category_web, category_local = image_helper.scale_image(
                            image=biggest, size_expected=CATEGORY_IMAGE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_category' % image_relative_path)
                        entry[
                            'category_image'] = category_web if category_web else None
                        entry[
                            'category_image_local'] = category_local if category_local else None

                        # thumbnail image
                        # thumbnail_landscape_size
                        if float(biggest['width']) / float(biggest['height']) >= THUMBNAIL_STYLE:
                            thumbnail_web, thumbnail_local = image_helper.scale_image(
                                image=biggest, size_expected=THUMBNAIL_LANDSCAPE_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_thumbnail' % image_relative_path)
                        else:  # else thumbnail_portrait_size
                            thumbnail_web, thumbnail_local = image_helper.scale_image(
                                image=biggest, size_expected=THUMBNAIL_PORTRAIT_SIZE, resize_by_width=True, crop_by_center=False, relative_path='%s_thumbnail' % image_relative_path)
                        entry[
                            'thumbnail_image'] = thumbnail_web if thumbnail_web else None
                        entry[
                            'thumbnail_image_local'] = thumbnail_local if thumbnail_local else None

                        # for older version users
                        entry['image'] = entry['thumbnail_image'][
                            'url'] if thumbnail_web else None
                    except IOError as k:
                        entry['error'].append(str(k) + '\n')

            # [OPTIONAL] google tts not for indonesian
            """
            if entry['language'] != 'ind':
                try:
                    tts_relative_path = '%s_%s_%s_%i.mp3' % (
                        entry['language'], entry['feed_id'], entry['updated'], rand)
                    entry['mp3'], entry['mp3_local'] = tts_provider.google(
                        entry['language'], entry['title'], tts_relative_path)
                except Exception as k:
                    print k, '... cannot generate TTS for %s' % entry['link']
                    entry['error'].append(k + '\n')
                    entry['mp3'] = None
                    entry['mp3_local'] = None
            """

            # [MUST-HAVE] add expiration data
            def _expired(updated, days_to_deadline):
                """
                compute expiration information
                return time string and unix time
                """
                deadline = datetime.utcfromtimestamp(
                    updated) + timedelta(days=days_to_deadline)
                return time.asctime(time.gmtime(calendar.timegm(deadline.timetuple())))

            entry['memory_expired'] = _expired(
                entry['updated'], MEMORY_EXPIRATION_DAYS)
            entry['database_expired'] = _expired(
                entry['updated'], DATABASE_REMOVAL_DAYS)

            entry['error'] = entry['error'] if entry['error'] else None
            entries_new.append(entry)
            print
            print
        except Exception as k:
            print k
    return entries_new

# TODO: code to remove added items if things suck at database/memory
def update(feed_link=None, feed_id=None, language=None, categories=None, transcoder_type='chengdujin'):
    """
    update could be called
    1. from task procedure: feed_id
    2. after an rss is added: feed_id
    3. manually for testing purpose: feed_link, language
    Note. categories are ids of category item
    Notel categories are kept for manual testing
    """
    if not feed_id and not (feed_link and language):
        raise Exception(
            "ERROR: Method signature not well formed!")

    if feed_id:
        feed = db_feeds.get(feed_id=feed_id)
    else:
        feed = db_feeds.get(feed_link=feed_link, language=language)

    feed_id = str(feed['_id']) if feed else feed_id
    feed_link = feed['feed_link'] if feed else feed_link
    language = feed['language'] if feed else language
    categories = feed['categories'] if feed else categories
    feed_title = feed['feed_title'] if feed and 'feed_title' in feed else None
    transcoder_type = feed['transcoder'] if feed else transcoder_type
    etag = feed['etag'] if feed and 'etag' in feed else None
    modified = feed['modified'] if feed and 'modified' in feed else None

    # parse rss reading from remote rss servers
    entries, status_new, feed_title_new, etag_new, modified_new = rss_parser.parse(
        feed_link, feed_id, feed_title, language, categories, etag, modified)

    # filter out existing entries in db_news
    # there are some possible exceptions -- yet let it be
    entries = db_news.dedup(entries, language)
    # and do tts, big_images, image as well as transcode.
    entries = _value_added_process(entries, language, transcoder_type)

    # update new entries to db_news
    # each entry is added with _id
    entries = db_news.update(entries, language)
    # and some data, like feed_title, etag and modified to db_feeds
    # only feed_id is necessary, others are optional **kwargs
    db_feeds.update(feed_id=feed_id, status=status_new,
                    feed_title=feed_title_new, etag=etag_new, modified=modified_new)

    # store in memory
    memory.update(entries, language, categories)
    return entries
<|endoftext|>


<commit_before>from datetime import datetime

import six
import json
import pytz
import logging

logger = logging.getLogger()

def timestamp():
    return pytz.utc.localize(datetime.utcnow()).isoformat()

def copy_to_unicode(element):
    """ used to transform the lxml version of unicode to a
    standard version of unicode that can be pickalable -
    necessary for linting """

    if isinstance(element, dict):
        for key, val in element.items():
            element[key] = copy_to_unicode(val)
    elif isinstance(element, list):
        for idx, item in enumerate(element):
            element[idx] = copy_to_unicode(item)
    else:
        try:
            # A dirty way to convert to unicode in python 2 + 3.3+
            element = u''.join(element)
        except TypeError:
            pass
    return element

def stamp_from_raw(raw_doc, **kwargs):
    kwargs['normalizeFinished'] = timestamp()
    stamps = raw_doc['timestamps']
    stamps.update(kwargs)
    return stamps

def format_date_with_slashes(date):
    return date.strftime('%m/%d/%Y')

def json_without_bytes(jobj):
    """
        An ugly hack.

        Before we treat a structure as JSON, ensure that bytes are decoded to str.
    """
    # Create a JSON-compatible copy of the attributes for validation
    jobj = jobj.copy()
    for k, v in jobj.items():
        if isinstance(v, six.binary_type):
            jobj[k] = v.decode('utf8')
    return jobj
<commit_msg>Remove unused json<commit_after>from datetime import datetime

import six
import pytz
import logging

logger = logging.getLogger()

def timestamp():
    return pytz.utc.localize(datetime.utcnow()).isoformat()

def copy_to_unicode(element):
    """ used to transform the lxml version of unicode to a
    standard version of unicode that can be pickalable -
    necessary for linting """

    if isinstance(element, dict):
        for key, val in element.items():
            element[key] = copy_to_unicode(val)
    elif isinstance(element, list):
        for idx, item in enumerate(element):
            element[idx] = copy_to_unicode(item)
    else:
        try:
            # A dirty way to convert to unicode in python 2 + 3.3+
            element = u''.join(element)
        except TypeError:
            pass
    return element

def stamp_from_raw(raw_doc, **kwargs):
    kwargs['normalizeFinished'] = timestamp()
    stamps = raw_doc['timestamps']
    stamps.update(kwargs)
    return stamps

def format_date_with_slashes(date):
    return date.strftime('%m/%d/%Y')

def json_without_bytes(jobj):
    """
        An ugly hack.

        Before we treat a structure as JSON, ensure that bytes are decoded to str.
    """
    # Create a JSON-compatible copy of the attributes for validation
    jobj = jobj.copy()
    for k, v in jobj.items():
        if isinstance(v, six.binary_type):
            jobj[k] = v.decode('utf8')
    return jobj
<|endoftext|>


<commit_before>import copy
import sys
import subprocess
from subprocess import PIPE

from pprint import pprint

class FrameworkTestType:
  '''Interface between a test type (json, query, plaintext, etc) and
  the rest of TFB. A test type defines a number of keys it expects
  to find in the benchmark_config, and this base class handles extracting
  those keys and injecting them into the test. For example, if
  benchmark_config contains a line `"spam" : "foobar"` and a subclasses X
  passes an argument list of ['spam'], then after parsing there will
  exist a member `X.spam = 'foobar'`.
  '''

  accept_json = "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"
  accept_html = "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
  accept_plaintext = "Accept: text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"

  def __init__(self, name, requires_db = False, accept_header = None, args = []):
    self.name = name
    self.requires_db = requires_db
    self.args = args
    self.out = sys.stdout
    self.err = sys.stderr
    self.accept_header = accept_header
    if accept_header is None:
      self.accept_header = self.accept_plaintext

    self.passed = None
    self.failed = None
    self.warned = None

  def setup_out_err(self, out, err):
    '''Sets up file-like objects for logging. Used in
    cases where it is hard just return the output. Any
    output sent to these file objects is also printed to
    the console

    NOTE: I detest this. It would be much better to use
    logging like it's intended
    '''
    self.out = out
    self.err = err

  def parse(self, test_keys):
    '''Takes the dict of key/value pairs describing a FrameworkTest
    and collects all variables needed by this FrameworkTestType

    Raises AttributeError if required keys are missing
    '''
    if all (arg in test_keys for arg in self.args):
      self.__dict__.update({ arg:test_keys[arg] for arg in self.args})
      return self
    else: # This is quite common - most tests don't support all types
      raise AttributeError("A %s requires the benchmark_config to contain %s"%(self.name,self.args))

  def _curl(self, url):
    '''Downloads a URL and returns the HTTP body'''
    # Use -m 15 to make curl stop trying after 15sec.
    # Use -i to output response with headers
    # Don't use -f so that the HTTP response code is ignored.
    # Use -sS to hide progress bar, but show errors.
    print "Accessing URL %s" % url
    self.err.write("Accessing URL %s \n" % url)
    self.out.write("Accessing URL %s \n" % url)
    p = subprocess.Popen(["curl", "-m", "15", "-i", "-sS", url], stderr=PIPE, stdout=PIPE)
    (out, err) = p.communicate()
    self.err.write(err+'\n')
    self.out.write(out+'\n')
    if p.returncode != 0:
      return None
    # Get response body
    p = subprocess.Popen(["curl", "-m", "15", "-s", url], stdout=PIPE, stderr=PIPE)
    (out, err) = p.communicate()
    return out

  def verify(self, base_url):
    '''Accesses URL used by this test type and checks the return
    values for correctness. Most test types run multiple checks,
    so this returns a list of results. Each result is a 3-tuple
    of (String result, String reason, String urlTested).

    - result : 'pass','warn','fail'
    - reason : Short human-readable reason if result was
        warn or fail. Please do not print the response as part of this,
        other parts of TFB will do that based upon the current logging
        settings if this method indicates a failure happened
    - urlTested: The exact URL that was queried

    Subclasses should make a best-effort attempt to report as many
    failures and warnings as they can to help users avoid needing
    to run TFB repeatedly while debugging
    '''
    # TODO make String result into an enum to enforce
    raise NotImplementedError("Subclasses must provide verify")

  def get_url(self):
    '''Returns the URL for this test, like '/json'''
    # This is a method because each test type uses a different key
    # for their URL so the base class can't know which arg is the URL
    raise NotImplementedError("Subclasses must provide verify")

  def copy(self):
    '''Returns a copy that can be safely modified. Use before calling
    parse'''
    return copy.copy(self)

<commit_msg>Print summary of response<commit_after>import copy
import sys
import subprocess
from subprocess import PIPE

from pprint import pprint

class FrameworkTestType:
  '''Interface between a test type (json, query, plaintext, etc) and
  the rest of TFB. A test type defines a number of keys it expects
  to find in the benchmark_config, and this base class handles extracting
  those keys and injecting them into the test. For example, if
  benchmark_config contains a line `"spam" : "foobar"` and a subclasses X
  passes an argument list of ['spam'], then after parsing there will
  exist a member `X.spam = 'foobar'`.
  '''

  accept_json = "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"
  accept_html = "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
  accept_plaintext = "Accept: text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"

  def __init__(self, name, requires_db = False, accept_header = None, args = []):
    self.name = name
    self.requires_db = requires_db
    self.args = args
    self.out = sys.stdout
    self.err = sys.stderr
    self.accept_header = accept_header
    if accept_header is None:
      self.accept_header = self.accept_plaintext

    self.passed = None
    self.failed = None
    self.warned = None

  def setup_out_err(self, out, err):
    '''Sets up file-like objects for logging. Used in
    cases where it is hard just return the output. Any
    output sent to these file objects is also printed to
    the console

    NOTE: I detest this. It would be much better to use
    logging like it's intended
    '''
    self.out = out
    self.err = err

  def parse(self, test_keys):
    '''Takes the dict of key/value pairs describing a FrameworkTest
    and collects all variables needed by this FrameworkTestType

    Raises AttributeError if required keys are missing
    '''
    if all (arg in test_keys for arg in self.args):
      self.__dict__.update({ arg:test_keys[arg] for arg in self.args})
      return self
    else: # This is quite common - most tests don't support all types
      raise AttributeError("A %s requires the benchmark_config to contain %s"%(self.name,self.args))

  def _curl(self, url):
    '''Downloads a URL and returns the HTTP body'''
    # Use -m 15 to make curl stop trying after 15sec.
    # Use -i to output response with headers
    # Don't use -f so that the HTTP response code is ignored.
    # Use -sS to hide progress bar, but show errors.
    print "Accessing URL %s" % url
    self.err.write("Accessing URL %s \n" % url)
    self.out.write("Accessing URL %s \n" % url)
    p = subprocess.Popen(["curl", "-m", "15", "-i", "-sS", url], stderr=PIPE, stdout=PIPE)
    (out, err) = p.communicate()
    self.err.write(err+'\n')
    self.out.write("Response: \n\"" + out+ "\"\n")
    if p.returncode != 0:
      return None
    # Get response body
    p = subprocess.Popen(["curl", "-m", "15", "-s", url], stdout=PIPE, stderr=PIPE)
    (out, err) = p.communicate()
    print "  Response (trimmed to 40 bytes): \"%s\"" % out[:40]
    return out

  def verify(self, base_url):
    '''Accesses URL used by this test type and checks the return
    values for correctness. Most test types run multiple checks,
    so this returns a list of results. Each result is a 3-tuple
    of (String result, String reason, String urlTested).

    - result : 'pass','warn','fail'
    - reason : Short human-readable reason if result was
        warn or fail. Please do not print the response as part of this,
        other parts of TFB will do that based upon the current logging
        settings if this method indicates a failure happened
    - urlTested: The exact URL that was queried

    Subclasses should make a best-effort attempt to report as many
    failures and warnings as they can to help users avoid needing
    to run TFB repeatedly while debugging
    '''
    # TODO make String result into an enum to enforce
    raise NotImplementedError("Subclasses must provide verify")

  def get_url(self):
    '''Returns the URL for this test, like '/json'''
    # This is a method because each test type uses a different key
    # for their URL so the base class can't know which arg is the URL
    raise NotImplementedError("Subclasses must provide verify")

  def copy(self):
    '''Returns a copy that can be safely modified. Use before calling
    parse'''
    return copy.copy(self)

<|endoftext|>


<commit_before>import functools
import inspect

import numpy as np
from astropy import units as u
from astropy.units import UnitsWarning
from plasmapy.constants import c
import warnings
from plasmapy.utils.exceptions import RelativityWarning, RelativityError
from textwrap import dedent

def check_quantity(validations):
    """
    Raise an exception if an annotated argument in a decorated function
    is not a `~astropy.units.Quantity` with correct units and valid
    numerical values.

    Parameters
    ----------
    validations : `dict`
        Validation dictionary.

    Raises
    ------
    `TypeError`
        If the argument is not a `~astropy.units.Quantity`, units is not
        entirely units or `argname` does not have a type annotation.

    `~astropy.units.UnitConversionError`
        If the argument is not in acceptable units.

    `ValueError`
        If the argument contains `~numpy.nan` or other invalid values as
        determined by the keywords.

    Returns
    -------
    function
        Decorated function.

    Examples
    --------
    >>> from astropy import units as u
    >>> @check_quantity({
    ... "x": {"units": u.m},
    ... "y": {"units": u.s,
    ...       "can_be_negative": False,
    ...       "can_be_complex": True,
    ...       "can_be_inf": False}
    ... })
    ... def func(x: u.m, y: u.s=1 * u.s):
    ...     return x
    ...
    >>> func(1 * u.m)
    <Quantity 1. m>
    >>> func(1 * u.m, 2 * u.m)
    Traceback (most recent call last):
      ...
    astropy.units.core.UnitConversionError: The argument y to func should be a Quantity with the following units: s

    """
    def decorator(f):
        wrapped_sign = inspect.signature(f)
        fname = f.__name__

        @functools.wraps(f)
        def wrapper(*args, **kwargs):
            # combine args and kwargs into dictionary
            bound_args = wrapped_sign.bind(*args, **kwargs)
            bound_args.apply_defaults()
            given_params_values = bound_args.arguments
            given_params = set(given_params_values.keys())

            # names of params to check
            validated_params = set(validations.keys())

            missing_params = [
                param for param in (validated_params - given_params)
            ]

            if len(missing_params) > 0:
                params_str = ", ".join(missing_params)
                raise TypeError(
                    f"Call to {fname} is missing "
                    f"validated params {params_str}")

            for param_to_check, validation_settings in validations.items():
                value_to_check = given_params_values[param_to_check]

                can_be_negative = validation_settings.get('can_be_negative', True)
                can_be_complex = validation_settings.get('can_be_complex', False)
                can_be_inf = validation_settings.get('can_be_inf', True)
                can_be_nan = validation_settings.get('can_be_nan', False)

                validated_value = _check_quantity(value_to_check,
                                                  param_to_check,
                                                  fname,
                                                  validation_settings['units'],
                                                  can_be_negative=can_be_negative,
                                                  can_be_complex=can_be_complex,
                                                  can_be_inf=can_be_inf,
                                                  can_be_nan=can_be_nan)
                given_params_values[param_to_check] = validated_value

            return f(**given_params_values)
        return wrapper
    return decorator

def _check_quantity(arg, argname, funcname, units, can_be_negative=True,
                    can_be_complex=False, can_be_inf=True, can_be_nan=False):
    """
    Raise an exception if an object is not a `~astropy.units.Quantity`
    with correct units and valid numerical values.

    Parameters
    ----------
    arg : ~astropy.units.Quantity
        The object to be tested.

    argname : str
        The name of the argument to be printed in error messages.

    funcname : str
        The name of the original function to be printed in error messages.

    units : `~astropy.units.Unit` or list of `~astropy.unit.Unit`
        Acceptable units for `arg`.

    can_be_negative : bool, optional
        `True` if the `~astropy.units.Quantity` can be negative,
        `False` otherwise.  Defaults to `True`.

    can_be_complex : bool, optional
        `True` if the `~astropy.units.Quantity` can be a complex number,
        `False` otherwise.  Defaults to `False`.

    can_be_inf : bool, optional
        `True` if the `~astropy.units.Quantity` can contain infinite
        values, `False` otherwise.  Defaults to `True`.

    can_be_inf : bool, optional
        `True` if the `~astropy.units.Quantity` can contain NaN
        values, `False` otherwise.  Defaults to `True`.

    Raises
    ------
    TypeError
        If the argument is not a `~astropy.units.Quantity` or units is
        not entirely units.

    ~astropy.units.UnitConversionError
        If the argument is not in acceptable units.

    ~astropy.units.UnitsError
        If after the assumption checks, the argument is still not in acceptable
        units.

    ValueError
        If the argument contains any `~numpy.nan` or other invalid
        values as determined by the keywords.

    Warns
    -----
    ~astropy.units.UnitsWarning
        If a `~astropy.units.Quantity` is not provided and unique units
        are provided, a `UnitsWarning` will be raised and the inputted
        units will be assumed.

    Examples
    --------
    >>> from astropy import units as u
    >>> _check_quantity(4*u.T, 'B', 'f', u.T)
    <Quantity 4. T>
    >>> _check_quantity(4, 'B', 'f', u.T)
    <Quantity 4. T>

    """

    # TODO: Replace `funcname` with func.__name__?

    if not isinstance(units, list):
        units = [units]

    for unit in units:
        if not isinstance(unit, (u.Unit, u.CompositeUnit, u.IrreducibleUnit)):
            raise TypeError(
                "The keyword 'units' to check_quantity must be "
                "a unit or a list/tuple containing only units.")

    # Create a generic error message

    typeerror_message = (
        f"The argument {argname} to {funcname} should be a Quantity with "
    )

    if len(units) == 1:
        typeerror_message += f"the following units: {str(units[0])}"
    else:
        typeerror_message += "one of the following units: "
        for unit in units:
            typeerror_message += str(unit)
            if unit != units[-1]:
                typeerror_message += ", "

    if isinstance(arg, (u.Unit, u.CompositeUnit, u.IrreducibleUnit)):
        raise TypeError(typeerror_message)

    # Make sure arg is a quantity with correct units

    unit_casting_warning = dedent(
            f"""No units are specified for {argname} = {arg} in {funcname}. Assuming units of {str(units[0])}.
                To silence this warning, explicitly pass in an Astropy Quantity (from astropy.units)
                (see http://docs.astropy.org/en/stable/units/)""")

    # TODO include explicit note on how to pass in Astropy Quantity

    if not isinstance(arg, (u.Quantity)):
        if len(units) != 1:
            raise TypeError(typeerror_message)
        else:
            try:
                arg = arg*units[0]
            except Exception:
                raise TypeError(typeerror_message)
            else:
                warnings.warn(UnitsWarning(unit_casting_warning))
    if not isinstance(arg, u.Quantity):
        raise u.UnitsError("{} is still not a Quantity after checks!".format(arg))

    in_acceptable_units = []

    for unit in units:
        try:
            arg.unit.to(unit, equivalencies=u.temperature_energy())
        except Exception:
            in_acceptable_units.append(False)
        else:
            in_acceptable_units.append(True)

    if not np.any(in_acceptable_units):
        raise u.UnitConversionError(typeerror_message)

    # Make sure that the quantity has valid numerical values

    valueerror_message = (
        f"The argument {argname} to function {funcname} cannot contain"
    )

    if np.any(np.isnan(arg.value)) and not can_be_nan:
        raise ValueError(f"{valueerror_message} NaNs.")
    elif np.any(np.iscomplex(arg.value)) and not can_be_complex:
        raise ValueError(f"{valueerror_message} complex numbers.")
    elif not can_be_negative and np.any(arg.value < 0):
        raise ValueError(f"{valueerror_message} negative numbers.")
    elif not can_be_inf and np.any(np.isinf(arg.value)):
        raise ValueError(f"{valueerror_message} infs.")

    return arg

def check_relativistic(func=None, betafrac=0.1):
    r"""
    Warns or raises an exception when the output of the decorated
    function is greater than `betafrac` times the speed of light.

    Parameters
    ----------
    func : `function`, optional
        The function to decorate.

    betafrac : float, optional
        The minimum fraction of the speed of light that will raise a
        `~plasmapy.utils.RelativityWarning`. Defaults to 0.1.

    Returns
    -------
    function
        Decorated function.

    Raises
    ------
    TypeError
        If `V` is not a `~astropy.units.Quantity`.

    ~astropy.units.UnitConversionError
        If `V` is not in units of velocity.

    ValueError
        If `V` contains any `~numpy.nan` values.

    ~plasmapy.utils.RelativityError
        If `V` is greater than or equal to the speed of light.

    Warns
    -----
    ~plasmapy.utils.RelativityWarning
        If `V` is greater than or equal to `betafrac` times the speed of light,
        but less than the speed of light.

    Examples
    --------
    >>> from astropy import units as u
    >>> @check_relativistic
    ... def speed():
    ...     return 1 * u.m / u.s

    Passing in a custom `betafrac`:

    >>> @check_relativistic(betafrac=0.01)
    ... def speed():
    ...     return 1 * u.m / u.s

    """
    def decorator(f):
        @functools.wraps(f)
        def wrapper(*args, **kwargs):
            return_ = f(*args, **kwargs)
            _check_relativistic(return_, f.__name__, betafrac=betafrac)
            return return_
        return wrapper
    if func:
        return decorator(func)
    return decorator

def _check_relativistic(V, funcname, betafrac=0.1):
    r"""
    Warn or raise error for relativistic or superrelativistic
    velocities.

    Parameters
    ----------
    V : ~astropy.units.Quantity
        A velocity.

    funcname : str
        The name of the original function to be printed in the error
        messages.

    betafrac : float
        The minimum fraction of the speed of light that will generate
        a warning.

    Raises
    ------
    TypeError
        If `V` is not a `~astropy.units.Quantity`.

    ~astropy.units.UnitConversionError
        If `V` is not in units of velocity.

    ValueError
        If `V` contains any `~numpy.nan` values.

    RelativityError
        If `V` is greater than or equal to the speed of light.

    Warns
    -----
    ~plasmapy.utils.RelativityWarning
        If `V` is greater than or equal to the specified fraction of the
        speed of light.

    Examples
    --------
    >>> from astropy import units as u
    >>> _check_relativistic(1*u.m/u.s, 'function_calling_this')

    """

    # TODO: Replace `funcname` with func.__name__?

    errmsg = ("V must be a Quantity with units of velocity in"
              "_check_relativistic")

    if not isinstance(V, u.Quantity):
        raise TypeError(errmsg)

    if V.si.unit != u.m / u.s:
        raise u.UnitConversionError(errmsg)

    if np.any(np.isnan(V.value)):
        raise ValueError(f"V includes NaNs in {funcname}")

    beta = np.max(np.abs((V.si / c).value))

    if beta == np.inf:
        raise RelativityError(f"{funcname} is yielding an infinite velocity.")
    elif beta >= 1:
        raise RelativityError(
            f"{funcname} is yielding a velocity that is {str(round(beta, 3))} "
            f"times the speed of light.")
    elif beta >= betafrac:
        warnings.warn(
            f"{funcname} is yielding a velocity that is "
            f"{str(round(beta * 100, 3))}% of the speed of "
            f"light. Relativistic effects may be important.",
            RelativityWarning)
<commit_msg>Change default fraction of c to warn at to 5%<commit_after>import functools
import inspect

import numpy as np
from astropy import units as u
from astropy.units import UnitsWarning
from plasmapy.constants import c
import warnings
from plasmapy.utils.exceptions import RelativityWarning, RelativityError
from textwrap import dedent

def check_quantity(validations):
    """
    Raise an exception if an annotated argument in a decorated function
    is not a `~astropy.units.Quantity` with correct units and valid
    numerical values.

    Parameters
    ----------
    validations : `dict`
        Validation dictionary.

    Raises
    ------
    `TypeError`
        If the argument is not a `~astropy.units.Quantity`, units is not
        entirely units or `argname` does not have a type annotation.

    `~astropy.units.UnitConversionError`
        If the argument is not in acceptable units.

    `ValueError`
        If the argument contains `~numpy.nan` or other invalid values as
        determined by the keywords.

    Returns
    -------
    function
        Decorated function.

    Examples
    --------
    >>> from astropy import units as u
    >>> @check_quantity({
    ... "x": {"units": u.m},
    ... "y": {"units": u.s,
    ...       "can_be_negative": False,
    ...       "can_be_complex": True,
    ...       "can_be_inf": False}
    ... })
    ... def func(x: u.m, y: u.s=1 * u.s):
    ...     return x
    ...
    >>> func(1 * u.m)
    <Quantity 1. m>
    >>> func(1 * u.m, 2 * u.m)
    Traceback (most recent call last):
      ...
    astropy.units.core.UnitConversionError: The argument y to func should be a Quantity with the following units: s

    """
    def decorator(f):
        wrapped_sign = inspect.signature(f)
        fname = f.__name__

        @functools.wraps(f)
        def wrapper(*args, **kwargs):
            # combine args and kwargs into dictionary
            bound_args = wrapped_sign.bind(*args, **kwargs)
            bound_args.apply_defaults()
            given_params_values = bound_args.arguments
            given_params = set(given_params_values.keys())

            # names of params to check
            validated_params = set(validations.keys())

            missing_params = [
                param for param in (validated_params - given_params)
            ]

            if len(missing_params) > 0:
                params_str = ", ".join(missing_params)
                raise TypeError(
                    f"Call to {fname} is missing "
                    f"validated params {params_str}")

            for param_to_check, validation_settings in validations.items():
                value_to_check = given_params_values[param_to_check]

                can_be_negative = validation_settings.get('can_be_negative', True)
                can_be_complex = validation_settings.get('can_be_complex', False)
                can_be_inf = validation_settings.get('can_be_inf', True)
                can_be_nan = validation_settings.get('can_be_nan', False)

                validated_value = _check_quantity(value_to_check,
                                                  param_to_check,
                                                  fname,
                                                  validation_settings['units'],
                                                  can_be_negative=can_be_negative,
                                                  can_be_complex=can_be_complex,
                                                  can_be_inf=can_be_inf,
                                                  can_be_nan=can_be_nan)
                given_params_values[param_to_check] = validated_value

            return f(**given_params_values)
        return wrapper
    return decorator

def _check_quantity(arg, argname, funcname, units, can_be_negative=True,
                    can_be_complex=False, can_be_inf=True, can_be_nan=False):
    """
    Raise an exception if an object is not a `~astropy.units.Quantity`
    with correct units and valid numerical values.

    Parameters
    ----------
    arg : ~astropy.units.Quantity
        The object to be tested.

    argname : str
        The name of the argument to be printed in error messages.

    funcname : str
        The name of the original function to be printed in error messages.

    units : `~astropy.units.Unit` or list of `~astropy.unit.Unit`
        Acceptable units for `arg`.

    can_be_negative : bool, optional
        `True` if the `~astropy.units.Quantity` can be negative,
        `False` otherwise.  Defaults to `True`.

    can_be_complex : bool, optional
        `True` if the `~astropy.units.Quantity` can be a complex number,
        `False` otherwise.  Defaults to `False`.

    can_be_inf : bool, optional
        `True` if the `~astropy.units.Quantity` can contain infinite
        values, `False` otherwise.  Defaults to `True`.

    can_be_inf : bool, optional
        `True` if the `~astropy.units.Quantity` can contain NaN
        values, `False` otherwise.  Defaults to `True`.

    Raises
    ------
    TypeError
        If the argument is not a `~astropy.units.Quantity` or units is
        not entirely units.

    ~astropy.units.UnitConversionError
        If the argument is not in acceptable units.

    ~astropy.units.UnitsError
        If after the assumption checks, the argument is still not in acceptable
        units.

    ValueError
        If the argument contains any `~numpy.nan` or other invalid
        values as determined by the keywords.

    Warns
    -----
    ~astropy.units.UnitsWarning
        If a `~astropy.units.Quantity` is not provided and unique units
        are provided, a `UnitsWarning` will be raised and the inputted
        units will be assumed.

    Examples
    --------
    >>> from astropy import units as u
    >>> _check_quantity(4*u.T, 'B', 'f', u.T)
    <Quantity 4. T>
    >>> _check_quantity(4, 'B', 'f', u.T)
    <Quantity 4. T>

    """

    # TODO: Replace `funcname` with func.__name__?

    if not isinstance(units, list):
        units = [units]

    for unit in units:
        if not isinstance(unit, (u.Unit, u.CompositeUnit, u.IrreducibleUnit)):
            raise TypeError(
                "The keyword 'units' to check_quantity must be "
                "a unit or a list/tuple containing only units.")

    # Create a generic error message

    typeerror_message = (
        f"The argument {argname} to {funcname} should be a Quantity with "
    )

    if len(units) == 1:
        typeerror_message += f"the following units: {str(units[0])}"
    else:
        typeerror_message += "one of the following units: "
        for unit in units:
            typeerror_message += str(unit)
            if unit != units[-1]:
                typeerror_message += ", "

    if isinstance(arg, (u.Unit, u.CompositeUnit, u.IrreducibleUnit)):
        raise TypeError(typeerror_message)

    # Make sure arg is a quantity with correct units

    unit_casting_warning = dedent(
            f"""No units are specified for {argname} = {arg} in {funcname}. Assuming units of {str(units[0])}.
                To silence this warning, explicitly pass in an Astropy Quantity (from astropy.units)
                (see http://docs.astropy.org/en/stable/units/)""")

    # TODO include explicit note on how to pass in Astropy Quantity

    if not isinstance(arg, (u.Quantity)):
        if len(units) != 1:
            raise TypeError(typeerror_message)
        else:
            try:
                arg = arg*units[0]
            except Exception:
                raise TypeError(typeerror_message)
            else:
                warnings.warn(UnitsWarning(unit_casting_warning))
    if not isinstance(arg, u.Quantity):
        raise u.UnitsError("{} is still not a Quantity after checks!".format(arg))

    in_acceptable_units = []

    for unit in units:
        try:
            arg.unit.to(unit, equivalencies=u.temperature_energy())
        except Exception:
            in_acceptable_units.append(False)
        else:
            in_acceptable_units.append(True)

    if not np.any(in_acceptable_units):
        raise u.UnitConversionError(typeerror_message)

    # Make sure that the quantity has valid numerical values

    valueerror_message = (
        f"The argument {argname} to function {funcname} cannot contain"
    )

    if np.any(np.isnan(arg.value)) and not can_be_nan:
        raise ValueError(f"{valueerror_message} NaNs.")
    elif np.any(np.iscomplex(arg.value)) and not can_be_complex:
        raise ValueError(f"{valueerror_message} complex numbers.")
    elif not can_be_negative and np.any(arg.value < 0):
        raise ValueError(f"{valueerror_message} negative numbers.")
    elif not can_be_inf and np.any(np.isinf(arg.value)):
        raise ValueError(f"{valueerror_message} infs.")

    return arg

def check_relativistic(func=None, betafrac=0.05):
    r"""
    Warns or raises an exception when the output of the decorated
    function is greater than `betafrac` times the speed of light.

    Parameters
    ----------
    func : `function`, optional
        The function to decorate.

    betafrac : float, optional
        The minimum fraction of the speed of light that will raise a
        `~plasmapy.utils.RelativityWarning`. Defaults to 5%.

    Returns
    -------
    function
        Decorated function.

    Raises
    ------
    TypeError
        If `V` is not a `~astropy.units.Quantity`.

    ~astropy.units.UnitConversionError
        If `V` is not in units of velocity.

    ValueError
        If `V` contains any `~numpy.nan` values.

    ~plasmapy.utils.RelativityError
        If `V` is greater than or equal to the speed of light.

    Warns
    -----
    ~plasmapy.utils.RelativityWarning
        If `V` is greater than or equal to `betafrac` times the speed of light,
        but less than the speed of light.

    Examples
    --------
    >>> from astropy import units as u
    >>> @check_relativistic
    ... def speed():
    ...     return 1 * u.m / u.s

    Passing in a custom `betafrac`:

    >>> @check_relativistic(betafrac=0.01)
    ... def speed():
    ...     return 1 * u.m / u.s

    """
    def decorator(f):
        @functools.wraps(f)
        def wrapper(*args, **kwargs):
            return_ = f(*args, **kwargs)
            _check_relativistic(return_, f.__name__, betafrac=betafrac)
            return return_
        return wrapper
    if func:
        return decorator(func)
    return decorator

def _check_relativistic(V, funcname, betafrac=0.05):
    r"""
    Warn or raise error for relativistic or superrelativistic
    velocities.

    Parameters
    ----------
    V : ~astropy.units.Quantity
        A velocity.

    funcname : str
        The name of the original function to be printed in the error
        messages.

    betafrac : float, optional
        The minimum fraction of the speed of light that will generate
        a warning. Defaults to 5%.

    Raises
    ------
    TypeError
        If `V` is not a `~astropy.units.Quantity`.

    ~astropy.units.UnitConversionError
        If `V` is not in units of velocity.

    ValueError
        If `V` contains any `~numpy.nan` values.

    RelativityError
        If `V` is greater than or equal to the speed of light.

    Warns
    -----
    ~plasmapy.utils.RelativityWarning
        If `V` is greater than or equal to the specified fraction of the
        speed of light.

    Examples
    --------
    >>> from astropy import units as u
    >>> _check_relativistic(1*u.m/u.s, 'function_calling_this')

    """

    # TODO: Replace `funcname` with func.__name__?

    errmsg = ("V must be a Quantity with units of velocity in"
              "_check_relativistic")

    if not isinstance(V, u.Quantity):
        raise TypeError(errmsg)

    if V.si.unit != u.m / u.s:
        raise u.UnitConversionError(errmsg)

    if np.any(np.isnan(V.value)):
        raise ValueError(f"V includes NaNs in {funcname}")

    beta = np.max(np.abs((V.si / c).value))

    if beta == np.inf:
        raise RelativityError(f"{funcname} is yielding an infinite velocity.")
    elif beta >= 1:
        raise RelativityError(
            f"{funcname} is yielding a velocity that is {str(round(beta, 3))} "
            f"times the speed of light.")
    elif beta >= betafrac:
        warnings.warn(
            f"{funcname} is yielding a velocity that is "
            f"{str(round(beta * 100, 3))}% of the speed of "
            f"light. Relativistic effects may be important.",
            RelativityWarning)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from datetime import datetime
from itertools import chain

from blinker import Signal
from flask import url_for
from mongoengine.signals import pre_save, post_save
from werkzeug import cached_property

from udata.core.storages import avatars, default_image_basename
from udata.frontend.markdown import mdstrip
from udata.models import db, BadgeMixin, WithMetrics
from udata.i18n import lazy_gettext as _

__all__ = (
    'Organization', 'Team', 'Member', 'MembershipRequest',
    'ORG_ROLES', 'MEMBERSHIP_STATUS', 'PUBLIC_SERVICE', 'CERTIFIED'
)

ORG_ROLES = {
    'admin': _('Administrateur'),
    'editor': _('Editor'),
}
DEFAULT_ROLE = 'editor'

MEMBERSHIP_STATUS = {
    'pending': _('Pending'),
    'accepted': _('Accepted'),
    'refused': _('Refused'),
}

LOGO_SIZES = [100, 60, 25]

PUBLIC_SERVICE = 'public-service'
CERTIFIED = 'certified'

class Team(db.EmbeddedDocument):
    name = db.StringField(required=True)
    slug = db.SlugField(
        max_length=255, required=True, populate_from='name', update=True,
        unique=False)
    description = db.StringField()

    members = db.ListField(db.ReferenceField('User'))

class Member(db.EmbeddedDocument):
    user = db.ReferenceField('User')
    role = db.StringField(choices=ORG_ROLES.keys(), default=DEFAULT_ROLE)
    since = db.DateTimeField(default=datetime.now, required=True)

    @property
    def label(self):
        return ORG_ROLES[self.role]

class MembershipRequest(db.EmbeddedDocument):
    '''
    Pending organization membership requests
    '''
    id = db.AutoUUIDField()
    user = db.ReferenceField('User')
    status = db.StringField(
        choices=MEMBERSHIP_STATUS.keys(), default='pending')

    created = db.DateTimeField(default=datetime.now, required=True)

    handled_on = db.DateTimeField()
    handled_by = db.ReferenceField('User')

    comment = db.StringField()
    refusal_comment = db.StringField()

    @property
    def status_label(self):
        return MEMBERSHIP_STATUS[self.status]

class OrganizationQuerySet(db.BaseQuerySet):
    def visible(self):
        return self(deleted=None)

    def hidden(self):
        return self(deleted__ne=None)

    def get_by_id_or_slug(self, id_or_slug):
        return self(slug=id_or_slug).first() or self(id=id_or_slug).first()

class Organization(WithMetrics, BadgeMixin, db.Datetimed, db.Document):
    name = db.StringField(required=True)
    acronym = db.StringField(max_length=128)
    slug = db.SlugField(
        max_length=255, required=True, populate_from='name', update=True)
    description = db.StringField(required=True)
    url = db.StringField()
    image_url = db.StringField()
    logo = db.ImageField(
        fs=avatars, basename=default_image_basename, thumbnails=LOGO_SIZES)

    members = db.ListField(db.EmbeddedDocumentField(Member))
    teams = db.ListField(db.EmbeddedDocumentField(Team))
    requests = db.ListField(db.EmbeddedDocumentField(MembershipRequest))

    ext = db.MapField(db.GenericEmbeddedDocumentField())
    zone = db.StringField()
    extras = db.ExtrasField()

    deleted = db.DateTimeField()

    meta = {
        'indexes': ['-created_at', 'slug'],
        'ordering': ['-created_at'],
        'queryset_class': OrganizationQuerySet,
    }

    def __str__(self):
        return self.name or ''

    __unicode__ = __str__

    __badges__ = {
        PUBLIC_SERVICE: _('Public Service'),
        CERTIFIED: _('Certified'),
    }

    before_save = Signal()
    after_save = Signal()
    on_create = Signal()
    on_update = Signal()
    before_delete = Signal()
    after_delete = Signal()

    @classmethod
    def pre_save(cls, sender, document, **kwargs):
        cls.before_save.send(document)

    @classmethod
    def post_save(cls, sender, document, **kwargs):
        cls.after_save.send(document)
        if kwargs.get('created'):
            cls.on_create.send(document)
        else:
            cls.on_update.send(document)

    def url_for(self, *args, **kwargs):
        return url_for('organizations.show', org=self, *args, **kwargs)

    display_url = property(url_for)

    @property
    def external_url(self):
        return self.url_for(_external=True)

    @property
    def pending_requests(self):
        return [r for r in self.requests if r.status == 'pending']

    @property
    def refused_requests(self):
        return [r for r in self.requests if r.status == 'refused']

    @property
    def accepted_requests(self):
        return [r for r in self.requests if r.status == 'accepted']

    @property
    def certified(self):
        return any(b.kind == CERTIFIED for b in self.badges)

    @property
    def public_service(self):
        is_public_service = any(b.kind == PUBLIC_SERVICE for b in self.badges)
        return self.certified and is_public_service

    def member(self, user):
        for member in self.members:
            if member.user == user:
                return member
        return None

    def is_member(self, user):
        return self.member(user) is not None

    def is_admin(self, user):
        member = self.member(user)
        return member is not None and member.role == 'admin'

    def pending_request(self, user):
        for request in self.requests:
            if request.user == user and request.status == 'pending':
                return request
        return None

    @classmethod
    def get(cls, id_or_slug):
        obj = cls.objects(slug=id_or_slug).first()
        return obj or cls.objects.get_or_404(id=id_or_slug)

    def by_role(self, role):
        return filter(lambda m: m.role == role, self.members)

    def check_availability(self):
        from udata.models import Dataset  # Circular imports.
        # Performances: only check the first 20 datasets for now.
        return chain(
            *[dataset.check_availability()
              for dataset in Dataset.objects(organization=self).visible()[:20]]
        )

    @cached_property
    def json_ld(self):
        type_ = 'GovernmentOrganization' if self.public_service \
                else 'Organization'

        result = {
            '@context': 'http://schema.org',
            '@type': type_,
            'alternateName': self.slug,
            'url': url_for('organizations.show', org=self, _external=True),
            'name': self.name,
        }

        if self.description:
            result['description'] = mdstrip(self.description)

        logo = self.logo(external=True)
        if logo:
            result['logo'] = logo

        return result

pre_save.connect(Organization.pre_save, sender=Organization)
post_save.connect(Organization.post_save, sender=Organization)
<commit_msg>Define a max size for organization logos<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from datetime import datetime
from itertools import chain

from blinker import Signal
from flask import url_for
from mongoengine.signals import pre_save, post_save
from werkzeug import cached_property

from udata.core.storages import avatars, default_image_basename
from udata.frontend.markdown import mdstrip
from udata.models import db, BadgeMixin, WithMetrics
from udata.i18n import lazy_gettext as _

__all__ = (
    'Organization', 'Team', 'Member', 'MembershipRequest',
    'ORG_ROLES', 'MEMBERSHIP_STATUS', 'PUBLIC_SERVICE', 'CERTIFIED'
)

ORG_ROLES = {
    'admin': _('Administrateur'),
    'editor': _('Editor'),
}
DEFAULT_ROLE = 'editor'

MEMBERSHIP_STATUS = {
    'pending': _('Pending'),
    'accepted': _('Accepted'),
    'refused': _('Refused'),
}

LOGO_MAX_SIZE = 500
LOGO_SIZES = [100, 60, 25]

PUBLIC_SERVICE = 'public-service'
CERTIFIED = 'certified'

class Team(db.EmbeddedDocument):
    name = db.StringField(required=True)
    slug = db.SlugField(
        max_length=255, required=True, populate_from='name', update=True,
        unique=False)
    description = db.StringField()

    members = db.ListField(db.ReferenceField('User'))

class Member(db.EmbeddedDocument):
    user = db.ReferenceField('User')
    role = db.StringField(choices=ORG_ROLES.keys(), default=DEFAULT_ROLE)
    since = db.DateTimeField(default=datetime.now, required=True)

    @property
    def label(self):
        return ORG_ROLES[self.role]

class MembershipRequest(db.EmbeddedDocument):
    '''
    Pending organization membership requests
    '''
    id = db.AutoUUIDField()
    user = db.ReferenceField('User')
    status = db.StringField(
        choices=MEMBERSHIP_STATUS.keys(), default='pending')

    created = db.DateTimeField(default=datetime.now, required=True)

    handled_on = db.DateTimeField()
    handled_by = db.ReferenceField('User')

    comment = db.StringField()
    refusal_comment = db.StringField()

    @property
    def status_label(self):
        return MEMBERSHIP_STATUS[self.status]

class OrganizationQuerySet(db.BaseQuerySet):
    def visible(self):
        return self(deleted=None)

    def hidden(self):
        return self(deleted__ne=None)

    def get_by_id_or_slug(self, id_or_slug):
        return self(slug=id_or_slug).first() or self(id=id_or_slug).first()

class Organization(WithMetrics, BadgeMixin, db.Datetimed, db.Document):
    name = db.StringField(required=True)
    acronym = db.StringField(max_length=128)
    slug = db.SlugField(
        max_length=255, required=True, populate_from='name', update=True)
    description = db.StringField(required=True)
    url = db.StringField()
    image_url = db.StringField()
    logo = db.ImageField(fs=avatars, basename=default_image_basename,
                         max_size=LOGO_MAX_SIZE, thumbnails=LOGO_SIZES)

    members = db.ListField(db.EmbeddedDocumentField(Member))
    teams = db.ListField(db.EmbeddedDocumentField(Team))
    requests = db.ListField(db.EmbeddedDocumentField(MembershipRequest))

    ext = db.MapField(db.GenericEmbeddedDocumentField())
    zone = db.StringField()
    extras = db.ExtrasField()

    deleted = db.DateTimeField()

    meta = {
        'indexes': ['-created_at', 'slug'],
        'ordering': ['-created_at'],
        'queryset_class': OrganizationQuerySet,
    }

    def __str__(self):
        return self.name or ''

    __unicode__ = __str__

    __badges__ = {
        PUBLIC_SERVICE: _('Public Service'),
        CERTIFIED: _('Certified'),
    }

    before_save = Signal()
    after_save = Signal()
    on_create = Signal()
    on_update = Signal()
    before_delete = Signal()
    after_delete = Signal()

    @classmethod
    def pre_save(cls, sender, document, **kwargs):
        cls.before_save.send(document)

    @classmethod
    def post_save(cls, sender, document, **kwargs):
        cls.after_save.send(document)
        if kwargs.get('created'):
            cls.on_create.send(document)
        else:
            cls.on_update.send(document)

    def url_for(self, *args, **kwargs):
        return url_for('organizations.show', org=self, *args, **kwargs)

    display_url = property(url_for)

    @property
    def external_url(self):
        return self.url_for(_external=True)

    @property
    def pending_requests(self):
        return [r for r in self.requests if r.status == 'pending']

    @property
    def refused_requests(self):
        return [r for r in self.requests if r.status == 'refused']

    @property
    def accepted_requests(self):
        return [r for r in self.requests if r.status == 'accepted']

    @property
    def certified(self):
        return any(b.kind == CERTIFIED for b in self.badges)

    @property
    def public_service(self):
        is_public_service = any(b.kind == PUBLIC_SERVICE for b in self.badges)
        return self.certified and is_public_service

    def member(self, user):
        for member in self.members:
            if member.user == user:
                return member
        return None

    def is_member(self, user):
        return self.member(user) is not None

    def is_admin(self, user):
        member = self.member(user)
        return member is not None and member.role == 'admin'

    def pending_request(self, user):
        for request in self.requests:
            if request.user == user and request.status == 'pending':
                return request
        return None

    @classmethod
    def get(cls, id_or_slug):
        obj = cls.objects(slug=id_or_slug).first()
        return obj or cls.objects.get_or_404(id=id_or_slug)

    def by_role(self, role):
        return filter(lambda m: m.role == role, self.members)

    def check_availability(self):
        from udata.models import Dataset  # Circular imports.
        # Performances: only check the first 20 datasets for now.
        return chain(
            *[dataset.check_availability()
              for dataset in Dataset.objects(organization=self).visible()[:20]]
        )

    @cached_property
    def json_ld(self):
        type_ = 'GovernmentOrganization' if self.public_service \
                else 'Organization'

        result = {
            '@context': 'http://schema.org',
            '@type': type_,
            'alternateName': self.slug,
            'url': url_for('organizations.show', org=self, _external=True),
            'name': self.name,
        }

        if self.description:
            result['description'] = mdstrip(self.description)

        logo = self.logo(external=True)
        if logo:
            result['logo'] = logo

        return result

pre_save.connect(Organization.pre_save, sender=Organization)
post_save.connect(Organization.post_save, sender=Organization)
<|endoftext|>


<commit_before><commit_msg>remove obsolete print statement<commit_after><|endoftext|>


<commit_before>from datetime import date, datetime

from django import forms
from django.core.exceptions import ValidationError
from django.core.urlresolvers import get_callable
from django.db import models
from django.db.models import Q
from django.utils.translation import ugettext_lazy as _

import plata
from plata.fields import CurrencyField
from plata.product.models import Category, Product, ProductVariation
from plata.shop.processors import ProcessorBase
from plata.utils import JSONFieldDescriptor

class DiscountBase(models.Model):
    AMOUNT_EXCL_TAX = 10
    AMOUNT_INCL_TAX = 20
    PERCENTAGE = 30

    TYPE_CHOICES = (
        (AMOUNT_EXCL_TAX, _('amount excl. tax')),
        (AMOUNT_INCL_TAX, _('amount incl. tax')),
        (PERCENTAGE, _('percentage')),
        )

    CONFIG_OPTIONS = [
        ('all', {
            'title': _('All products'),
            }),
        ('exclude_sale', {
            'title': _('Exclude sale prices'),
            'orderitem_query': lambda **values: Q(is_sale=False),
            }),
        ('products', {
            'title': _('Explicitly define discountable products'),
            'form_fields': [
                ('products', forms.ModelMultipleChoiceField(
                    Product.objects.all(),
                    label=_('products'),
                    required=True)),
                ],
            'variation_query': lambda products: Q(product__in=products),
            }),
        ('only_categories', {
            'title': _('Only products from selected categories'),
            'form_fields': [
                ('categories', forms.ModelMultipleChoiceField(
                    Category.objects.all(),
                    label=_('categories'),
                    required=True)),
                ],
            'variation_query': lambda categories: Q(product__categories__in=categories),
            }),
        ]

    name = models.CharField(_('name'), max_length=100)

    # TODO currency handling. Maybe split type/value into amount, tax, currency, percentage?
    type = models.PositiveIntegerField(_('type'), choices=TYPE_CHOICES)
    value = models.DecimalField(_('value'), max_digits=10, decimal_places=2)

    config_json = models.TextField(_('configuration'), blank=True)
    config = JSONFieldDescriptor('config_json')

    class Meta:
        abstract = True

    def __unicode__(self):
        return self.name

    def eligible_products(self, order, items, products=None):

        if not products:
            shop = plata.shop_instance()
            products = shop.product_model._default_manager.all()

        variations = ProductVariation.objects.filter(
            id__in=[item.variation_id for item in items])
        orderitems = shop.orderitem_model.objects.filter(
            id__in=[item.id for item in items])

        for key, parameters in self.config.items():
            parameters = dict((str(k), v) for k, v in parameters.items())

            cfg = dict(self.CONFIG_OPTIONS)[key]

            if 'variation_query' in cfg:
                variations = variations.filter(cfg['variation_query'](**parameters))
            if 'orderitem_query' in cfg:
                orderitems = orderitems.filter(cfg['orderitem_query'](**parameters))

        return products.filter(id__in=variations.values('product_id')).filter(id__in=orderitems.values('id'))

    def apply(self, order, items, **kwargs):
        if not items:
            return

        if self.type == self.AMOUNT_EXCL_TAX:
            self.apply_amount_discount(order, items, tax_included=False)
        elif self.type == self.AMOUNT_INCL_TAX:
            self.apply_amount_discount(order, items, tax_included=True)
        elif self.type == self.PERCENTAGE:
            self.apply_percentage_discount(order, items)
        else:
            raise NotImplementedError, 'Unknown discount type %s' % self.type

    def apply_amount_discount(self, order, items, tax_included):
        eligible_products = self.eligible_products(order, items).values_list('id', flat=True)

        eligible_items = [item for item in items if item.variation.product_id in eligible_products]

        if tax_included:
            # TODO how should this value be calculated in the presence of multiple tax rates?
            tax_rate = items[0].tax_class.rate
            discount = self.value / (1 + tax_rate/100)
        else:
            discount = self.value

        items_subtotal = sum([item.discounted_subtotal_excl_tax for item in eligible_items], 0)

        if items_subtotal < discount:
            remaining = discount - items_subtotal
            discount = items_subtotal

        for item in eligible_items:
            item._line_item_discount += item.discounted_subtotal_excl_tax / items_subtotal * discount

    def apply_percentage_discount(self, order, items):
        eligible_products = self.eligible_products(order, items).values_list('id', flat=True)

        factor = self.value / 100

        for item in items:
            if item.variation.product_id not in eligible_products:
                continue

            item._line_item_discount += item.discounted_subtotal_excl_tax * factor

class Discount(DiscountBase):
    code = models.CharField(_('code'), max_length=30, unique=True)

    is_active = models.BooleanField(_('is active'), default=True)
    valid_from = models.DateField(_('valid from'), default=date.today)
    valid_until = models.DateField(_('valid until'), blank=True, null=True)

    allowed_uses = models.IntegerField(_('number of allowed uses'),
        blank=True, null=True,
        help_text=_('Leave empty if there is no limit on the number of uses of this discount.'))
    used = models.IntegerField(_('number of times already used'), default=0)

    class Meta:
        verbose_name = _('discount')
        verbose_name_plural = _('discounts')

    def validate(self, order):
        messages = []
        if not self.is_active:
            messages.append(_('Discount is inactive.'))

        today = date.today()
        if today < self.valid_from:
            messages.append(_('Discount is not active yet.'))
        if self.valid_until and today > self.valid_until:
            messages.append(_('Discount is expired.'))

        if messages:
            raise ValidationError(messages)

        return True

class DiscountProcessor(ProcessorBase):
    def process(self, instance, items):
        for applied in instance.applied_discounts.all():
            applied.apply(instance, items)
<commit_msg>Discounting: Fix determination of eligible products (again)<commit_after>from datetime import date, datetime

from django import forms
from django.core.exceptions import ValidationError
from django.core.urlresolvers import get_callable
from django.db import models
from django.db.models import Q
from django.utils.translation import ugettext_lazy as _

import plata
from plata.fields import CurrencyField
from plata.product.models import Category, Product, ProductVariation
from plata.shop.processors import ProcessorBase
from plata.utils import JSONFieldDescriptor

class DiscountBase(models.Model):
    AMOUNT_EXCL_TAX = 10
    AMOUNT_INCL_TAX = 20
    PERCENTAGE = 30

    TYPE_CHOICES = (
        (AMOUNT_EXCL_TAX, _('amount excl. tax')),
        (AMOUNT_INCL_TAX, _('amount incl. tax')),
        (PERCENTAGE, _('percentage')),
        )

    CONFIG_OPTIONS = [
        ('all', {
            'title': _('All products'),
            }),
        ('exclude_sale', {
            'title': _('Exclude sale prices'),
            'orderitem_query': lambda **values: Q(is_sale=False),
            }),
        ('products', {
            'title': _('Explicitly define discountable products'),
            'form_fields': [
                ('products', forms.ModelMultipleChoiceField(
                    Product.objects.all(),
                    label=_('products'),
                    required=True)),
                ],
            'variation_query': lambda products: Q(product__in=products),
            }),
        ('only_categories', {
            'title': _('Only products from selected categories'),
            'form_fields': [
                ('categories', forms.ModelMultipleChoiceField(
                    Category.objects.all(),
                    label=_('categories'),
                    required=True)),
                ],
            'variation_query': lambda categories: Q(product__categories__in=categories),
            }),
        ]

    name = models.CharField(_('name'), max_length=100)

    # TODO currency handling. Maybe split type/value into amount, tax, currency, percentage?
    type = models.PositiveIntegerField(_('type'), choices=TYPE_CHOICES)
    value = models.DecimalField(_('value'), max_digits=10, decimal_places=2)

    config_json = models.TextField(_('configuration'), blank=True)
    config = JSONFieldDescriptor('config_json')

    class Meta:
        abstract = True

    def __unicode__(self):
        return self.name

    def eligible_products(self, order, items, products=None):
        if not products:
            shop = plata.shop_instance()
            products = shop.product_model._default_manager.all()

        variations = ProductVariation.objects.filter(
            id__in=[item.variation_id for item in items])
        orderitems = shop.orderitem_model.objects.filter(
            id__in=[item.id for item in items])

        for key, parameters in self.config.items():
            parameters = dict((str(k), v) for k, v in parameters.items())

            cfg = dict(self.CONFIG_OPTIONS)[key]

            if 'variation_query' in cfg:
                variations = variations.filter(cfg['variation_query'](**parameters))
            if 'orderitem_query' in cfg:
                orderitems = orderitems.filter(cfg['orderitem_query'](**parameters))

        return products.filter(id__in=variations.values('product_id')).filter(id__in=orderitems.values('variation__product__id'))

    def apply(self, order, items, **kwargs):
        if not items:
            return

        if self.type == self.AMOUNT_EXCL_TAX:
            self.apply_amount_discount(order, items, tax_included=False)
        elif self.type == self.AMOUNT_INCL_TAX:
            self.apply_amount_discount(order, items, tax_included=True)
        elif self.type == self.PERCENTAGE:
            self.apply_percentage_discount(order, items)
        else:
            raise NotImplementedError, 'Unknown discount type %s' % self.type

    def apply_amount_discount(self, order, items, tax_included):
        eligible_products = self.eligible_products(order, items).values_list('id', flat=True)

        eligible_items = [item for item in items if item.variation.product_id in eligible_products]

        if tax_included:
            # TODO how should this value be calculated in the presence of multiple tax rates?
            tax_rate = items[0].tax_class.rate
            discount = self.value / (1 + tax_rate/100)
        else:
            discount = self.value

        items_subtotal = sum([item.discounted_subtotal_excl_tax for item in eligible_items], 0)

        if items_subtotal < discount:
            remaining = discount - items_subtotal
            discount = items_subtotal

        for item in eligible_items:
            item._line_item_discount += item.discounted_subtotal_excl_tax / items_subtotal * discount

    def apply_percentage_discount(self, order, items):
        eligible_products = self.eligible_products(order, items).values_list('id', flat=True)

        factor = self.value / 100

        for item in items:
            if item.variation.product_id not in eligible_products:
                continue

            item._line_item_discount += item.discounted_subtotal_excl_tax * factor

class Discount(DiscountBase):
    code = models.CharField(_('code'), max_length=30, unique=True)

    is_active = models.BooleanField(_('is active'), default=True)
    valid_from = models.DateField(_('valid from'), default=date.today)
    valid_until = models.DateField(_('valid until'), blank=True, null=True)

    allowed_uses = models.IntegerField(_('number of allowed uses'),
        blank=True, null=True,
        help_text=_('Leave empty if there is no limit on the number of uses of this discount.'))
    used = models.IntegerField(_('number of times already used'), default=0)

    class Meta:
        verbose_name = _('discount')
        verbose_name_plural = _('discounts')

    def validate(self, order):
        messages = []
        if not self.is_active:
            messages.append(_('Discount is inactive.'))

        today = date.today()
        if today < self.valid_from:
            messages.append(_('Discount is not active yet.'))
        if self.valid_until and today > self.valid_until:
            messages.append(_('Discount is expired.'))

        if messages:
            raise ValidationError(messages)

        return True

class DiscountProcessor(ProcessorBase):
    def process(self, instance, items):
        for applied in instance.applied_discounts.all():
            applied.apply(instance, items)
<|endoftext|>