<commit_before>from django.utils.translation import ugettext as _

DJANGO_COUNTRIES_INDEX = 0
DJANGO_COUNTRIES_VALUE_INDEX = 1

MARKETPLACE_PARAM_QUERY = 'query'
MARKETPLACE_PARAM_COUNTRY = 'country'
MARKETPLACE_PARAM_CITY = 'city'

ORG_PROBLEM_TYPE_DELIVERY_SERVICES = 'delivery-services'
ORG_PROBLEM_TYPE_FUNDING = 'funding'
ORG_PROBLEM_TYPE_HEALTH_CARE_SUPPLIES = 'healthcare-supplies'
ORG_PROBLEM_TYPE_HUMAN_RESOURCES = 'human-resources'
ORG_PROBLEM_TYPE_MENTORSHIP = 'mentorship'
ORG_PROBLEM_TYPE_ONLINE_TRAINING = 'online-training'
ORG_PROBLEM_TYPE_OTHER = 'other-problem'
ORG_PROBLEM_TYPE_REMOTE_WORKING_TOOLS = 'remote-working-tools'

ORG_PROBLEM_CHOICES = [
    (ORG_PROBLEM_TYPE_HEALTH_CARE_SUPPLIES, _('Healthcare Supplies')),
    (ORG_PROBLEM_TYPE_REMOTE_WORKING_TOOLS, _('Remote Working Tools')),
    (ORG_PROBLEM_TYPE_ONLINE_TRAINING, _('Online Training')),
    (ORG_PROBLEM_TYPE_DELIVERY_SERVICES, _('Delivery Services')),
    (ORG_PROBLEM_TYPE_MENTORSHIP, _('Mentorship')),
    (ORG_PROBLEM_TYPE_FUNDING, _('Funding')),
    (ORG_PROBLEM_TYPE_HUMAN_RESOURCES, _('Human Resources')),
]

ORG_PROBLEM_TEMPLATE_CHOICES = [
    (ORG_PROBLEM_TYPE_HEALTH_CARE_SUPPLIES, _('Healthcare Supplies')),
    (ORG_PROBLEM_TYPE_REMOTE_WORKING_TOOLS, _('Remote Working Tools')),
    (ORG_PROBLEM_TYPE_ONLINE_TRAINING, _('Online Training')),
    (ORG_PROBLEM_TYPE_DELIVERY_SERVICES, _('Delivery Services')),
    (ORG_PROBLEM_TYPE_MENTORSHIP, _('Mentorship')),
    (ORG_PROBLEM_TYPE_FUNDING, _('Funding')),
    (ORG_PROBLEM_TYPE_HUMAN_RESOURCES, _('Human Resources')),
    (ORG_PROBLEM_TYPE_OTHER, _('Other'))
]

ORG_SECTOR_HEALTH_AND_WELL_BEING = 'health-and-well-being'
ORG_SECTOR_EDUCATION = 'education'
ORG_SECTOR_GENDER_EQUALITY = 'gender-equality'
ORG_SECTOR_SANITATION = 'sanitation'
ORG_SECTOR_CLIMATE_CHANGES = 'climate-changes'
ORG_SECTOR_CLEAN_ENERGY = 'clean-energy'
ORG_SECTOR_ENVIRONMENTAL_CONSERVATION = 'environmental-conservation'
ORG_SECTOR_WORK_AND_ECONOMIC_GROWTH = 'work-and-economic-growth'
ORG_SECTOR_HUMAN_RIGHTS = 'human-rights'
ORG_SECTOR_SOCIAL_JUSTICE = 'social-justice'
ORG_SECTOR_ART_AND_CULTURE = 'art-and-culture'
ORG_SECTOR_OTHER = 'other-sector'

ORG_SECTOR_CHOICES = [
    (ORG_SECTOR_HEALTH_AND_WELL_BEING, _('Health & Well-being')),
    (ORG_SECTOR_EDUCATION, _('Education')),
    (ORG_SECTOR_GENDER_EQUALITY, _('Gender Equality')),
    (ORG_SECTOR_SANITATION, _('Sanitation')),
    (ORG_SECTOR_CLIMATE_CHANGES, _('Climate Change')),
    (ORG_SECTOR_CLEAN_ENERGY, _('Clean Energy')),
    (ORG_SECTOR_ENVIRONMENTAL_CONSERVATION, _('Environmental Conservation')),
    (ORG_SECTOR_WORK_AND_ECONOMIC_GROWTH, _('Work & Economic Growth')),
    (ORG_SECTOR_HUMAN_RIGHTS, _('Human Rights')),
    (ORG_SECTOR_SOCIAL_JUSTICE, _('Social Justice')),
    (ORG_SECTOR_ART_AND_CULTURE, _('Art & Culture'))
]

ORG_SECTOR_TEMPLATE_CHOICES = [
    (ORG_SECTOR_HEALTH_AND_WELL_BEING, _('Health & Well-being')),
    (ORG_SECTOR_EDUCATION, _('Education')),
    (ORG_SECTOR_GENDER_EQUALITY, _('Gender Equality')),
    (ORG_SECTOR_SANITATION, _('Sanitation')),
    (ORG_SECTOR_CLIMATE_CHANGES, _('Climate Change')),
    (ORG_SECTOR_CLEAN_ENERGY, _('Clean Energy')),
    (ORG_SECTOR_ENVIRONMENTAL_CONSERVATION, _('Environmental Conservation')),
    (ORG_SECTOR_WORK_AND_ECONOMIC_GROWTH, _('Work & Economic Growth')),
    (ORG_SECTOR_HUMAN_RIGHTS, _('Human Rights')),
    (ORG_SECTOR_SOCIAL_JUSTICE, _('Social Justice')),
    (ORG_SECTOR_ART_AND_CULTURE, _('Art & Culture')),
    (ORG_SECTOR_OTHER, _('Other'))
]

USER_SERVICES_DELIVERY_SERVICES = 'delivery-services'
USER_SERVICES_FUNDING = 'funding'
USER_SERVICES_HEALTH_CARE_SUPPLIES = 'healthcare-supplies'
USER_SERVICES_MENTORSHIP = 'mentorship'
USER_SERVICES_ONLINE_TRAINING = 'online-training'
USER_SERVICES_PEOPLE_POWER = 'people-power'

USER_SERVICES_CHOICES = (
    (USER_SERVICES_HEALTH_CARE_SUPPLIES, _('Healthcare Supplies')),
    (USER_SERVICES_PEOPLE_POWER, _('People Power')),
    (USER_SERVICES_ONLINE_TRAINING, _('Online Training')),
    (USER_SERVICES_DELIVERY_SERVICES, _('Delivery Services')),
    (USER_SERVICES_MENTORSHIP, _('Mentorship')),
    (USER_SERVICES_FUNDING, _('Funding')),
)

PUBLISHED_DATE_FORMAT = '%B %d, %Y'
<commit_msg>refractored the constants file for LP-1966<commit_after>from django.utils.translation import ugettext as _

DJANGO_COUNTRIES_INDEX = 0
DJANGO_COUNTRIES_VALUE_INDEX = 1

MARKETPLACE_PARAM_QUERY = 'query'
MARKETPLACE_PARAM_COUNTRY = 'country'
MARKETPLACE_PARAM_CITY = 'city'

ORG_PROBLEM_TYPE_DELIVERY_SERVICES = 'delivery-services'
ORG_PROBLEM_TYPE_FUNDING = 'funding'
ORG_PROBLEM_TYPE_HEALTH_CARE_SUPPLIES = 'healthcare-supplies'
ORG_PROBLEM_TYPE_HUMAN_RESOURCES = 'human-resources'
ORG_PROBLEM_TYPE_MENTORSHIP = 'mentorship'
ORG_PROBLEM_TYPE_ONLINE_TRAINING = 'online-training'
ORG_PROBLEM_TYPE_OTHER = 'other-problem'
ORG_PROBLEM_TYPE_REMOTE_WORKING_TOOLS = 'remote-working-tools'

ORG_PROBLEM_CHOICES = [
    (ORG_PROBLEM_TYPE_HEALTH_CARE_SUPPLIES, _('Healthcare Supplies')),
    (ORG_PROBLEM_TYPE_REMOTE_WORKING_TOOLS, _('Remote Working Tools')),
    (ORG_PROBLEM_TYPE_ONLINE_TRAINING, _('Online Training')),
    (ORG_PROBLEM_TYPE_DELIVERY_SERVICES, _('Delivery Services')),
    (ORG_PROBLEM_TYPE_MENTORSHIP, _('Mentorship')),
    (ORG_PROBLEM_TYPE_FUNDING, _('Funding')),
    (ORG_PROBLEM_TYPE_HUMAN_RESOURCES, _('Human Resources')),
]

ORG_PROBLEM_TEMPLATE_CHOICES = []
ORG_PROBLEM_TEMPLATE_CHOICES.extend(ORG_PROBLEM_CHOICES)
ORG_PROBLEM_TEMPLATE_CHOICES.append(ORG_PROBLEM_TYPE_OTHER, _('Other'))

ORG_SECTOR_HEALTH_AND_WELL_BEING = 'health-and-well-being'
ORG_SECTOR_EDUCATION = 'education'
ORG_SECTOR_GENDER_EQUALITY = 'gender-equality'
ORG_SECTOR_SANITATION = 'sanitation'
ORG_SECTOR_CLIMATE_CHANGES = 'climate-changes'
ORG_SECTOR_CLEAN_ENERGY = 'clean-energy'
ORG_SECTOR_ENVIRONMENTAL_CONSERVATION = 'environmental-conservation'
ORG_SECTOR_WORK_AND_ECONOMIC_GROWTH = 'work-and-economic-growth'
ORG_SECTOR_HUMAN_RIGHTS = 'human-rights'
ORG_SECTOR_SOCIAL_JUSTICE = 'social-justice'
ORG_SECTOR_ART_AND_CULTURE = 'art-and-culture'
ORG_SECTOR_OTHER = 'other-sector'

ORG_SECTOR_CHOICES = [
    (ORG_SECTOR_HEALTH_AND_WELL_BEING, _('Health & Well-being')),
    (ORG_SECTOR_EDUCATION, _('Education')),
    (ORG_SECTOR_GENDER_EQUALITY, _('Gender Equality')),
    (ORG_SECTOR_SANITATION, _('Sanitation')),
    (ORG_SECTOR_CLIMATE_CHANGES, _('Climate Change')),
    (ORG_SECTOR_CLEAN_ENERGY, _('Clean Energy')),
    (ORG_SECTOR_ENVIRONMENTAL_CONSERVATION, _('Environmental Conservation')),
    (ORG_SECTOR_WORK_AND_ECONOMIC_GROWTH, _('Work & Economic Growth')),
    (ORG_SECTOR_HUMAN_RIGHTS, _('Human Rights')),
    (ORG_SECTOR_SOCIAL_JUSTICE, _('Social Justice')),
    (ORG_SECTOR_ART_AND_CULTURE, _('Art & Culture'))
]

ORG_SECTOR_TEMPLATE_CHOICES = []
ORG_SECTOR_TEMPLATE_CHOICES.extend(ORG_SECTOR_CHOICES)
ORG_SECTOR_TEMPLATE_CHOICES.append(ORG_SECTOR_OTHER, _('Other'))

USER_SERVICES_DELIVERY_SERVICES = 'delivery-services'
USER_SERVICES_FUNDING = 'funding'
USER_SERVICES_HEALTH_CARE_SUPPLIES = 'healthcare-supplies'
USER_SERVICES_MENTORSHIP = 'mentorship'
USER_SERVICES_ONLINE_TRAINING = 'online-training'
USER_SERVICES_PEOPLE_POWER = 'people-power'

USER_SERVICES_CHOICES = (
    (USER_SERVICES_HEALTH_CARE_SUPPLIES, _('Healthcare Supplies')),
    (USER_SERVICES_PEOPLE_POWER, _('People Power')),
    (USER_SERVICES_ONLINE_TRAINING, _('Online Training')),
    (USER_SERVICES_DELIVERY_SERVICES, _('Delivery Services')),
    (USER_SERVICES_MENTORSHIP, _('Mentorship')),
    (USER_SERVICES_FUNDING, _('Funding')),
)

PUBLISHED_DATE_FORMAT = '%B %d, %Y'
<|endoftext|>


<commit_before>#! /usr/bin/python

import argparse
import glob
import ConfigParser
import time
import os
import os.path
import smtplib
from email.mime.text import MIMEText
from email import charset
import hashlib
import json
import urllib2

import flickr_api
import jinja2

def write_state(state):
    with open('state.ini', 'w') as state_file:
        state.write(state_file)

def main():
    parser = argparse.ArgumentParser(description='Send email when new pictures are posted to Flickr.')

    parser.add_argument('--user_add_interactive', '-i', action='store_true',
                        help='add a user interactively based on OAuth')
    parser.add_argument('--user_add', '-a', action='store_true', help='prepare adding a user based on OAuth')
    parser.add_argument('--user_auth_verifier', '-v', action='store', help='finish adding the user based on OAuth')
    parser.add_argument('--user_old_auth', '-o', action='store', help='add a user based on a pre-OAuth token')
    parser.add_argument('--user_delete', '-d', action='store', help='remove a user')
    parser.add_argument('--since_days', '-s', action='store', type=int, help='remove a user')

    args = parser.parse_args()

    state = ConfigParser.RawConfigParser({
        'last_date': int(time.time() - 24 * 60 * 60),
        'smtp_from': 'admin@example.com',
        'smtp_to': 'clients@example.com',
        'smtp_subject': 'New photos',
        'smtp_port': 25,
        'smtp_server': 'localhost',
        'smtp_tls': 'false',
    })
    state.add_section('main')
    state.read("state.ini")

    if args.since_days:
        state.set('main', 'last_date', int(time.time() - args.since_days * 24 * 60 * 60))

    # set up flickr_api app credentials
    try:
        flickr_api.set_keys(api_key=state.get('main', 'api_key'), api_secret=state.get('main', 'api_secret'))
    except ConfigParser.NoOptionError:
        state.set('main', 'api_key', 'API_KEY')
        state.set('main', 'api_secret', 'API_SECRET')
        write_state(state)
        exit("Please fill in the api_key and api_secret in state.ini")

    if args.user_add_interactive:
        a = flickr_api.auth.AuthHandler()
        print "Open this URL in a web browser and authorize", a.get_authorization_url("read")

        oauth_token = raw_input("Paste the oauth_verifier parameter here: ")

        a.set_verifier(oauth_token)
        flickr_api.set_auth_handler(a)

        user = flickr_api.test.login()
        print "Authorized user:", user

        if not os.path.exists('users'):
            os.mkdir('users')
        a.save("users/" + user.username)
        exit()

    if args.user_add:
        a = flickr_api.auth.AuthHandler()
        print "Open this URL in a web browser and authorize", a.get_authorization_url("read")
        print "Then run the command again with --user_auth_verifier <oauth_verifier> to finish authorizing"

        if not os.path.exists('tmp_users'):
            os.mkdir('tmp_users')

        # can't use a.save() because that method requires an access_token and all we have is a request_token
        with open("tmp_users/" + a.request_token.key, "w") as f:
            f.write("\n".join([a.request_token.key,
                               a.request_token.secret]))
        exit()

    if args.user_auth_verifier:
        # we don't know which of the temp users matches the verifier, try each of them in turn
        for f in glob.glob("tmp_users/*"):
            try:
                with open(f, "r") as f1:
                    request_token = f1.read().split("\n")

                a = flickr_api.auth.AuthHandler(
                    request_token_key=request_token[0],
                    request_token_secret=request_token[1]
                )

                # this will throw an exception if the verifier doesn't match the request_token
                a.set_verifier(args.user_auth_verifier)

                flickr_api.set_auth_handler(a)

                user = flickr_api.test.login()
                print "Authorized user:", user

                if not os.path.exists('users'):
                    os.mkdir('users')
                a.save("users/" + user.username)

                os.remove(f)

                exit()
            except SystemExit:
                exit()
            except:
                # continue trying
                pass

        exit("No matching pre-authorization found")

    if args.user_delete:
        try:
            os.remove('users/' + args.user_delete)
        except:
            print "Could not remove the user"
            raise
        exit()

    if args.user_old_auth:
        # handle old-style auth directly, since flickr_api only handled OAuth
        m = hashlib.md5()
        m.update(state.get('main', 'api_secret'))
        m.update('api_key')
        m.update(state.get('main', 'api_key'))
        m.update('auth_token')
        m.update(args.user_old_auth)
        m.update('format')
        m.update('json')
        m.update('method')
        m.update('flickr.auth.oauth.getAccessToken')
        m.update('nojsoncallback')
        m.update('1')

        url = "https://api.flickr.com/services/rest/" \
              "?method=flickr.auth.oauth.getAccessToken" \
              "&api_key={0}" \
              "&auth_token={1}" \
              "&format=json" \
              "&nojsoncallback=1" \
              "&api_sig={2}".format(
            state.get('main', 'api_key'),
            args.user_old_auth,
            m.hexdigest())

        resp = urllib2.urlopen(url)
        response = json.load(resp)

        if response['stat'] != 'ok':
            exit("Request failed, Flickr responded: " + response['message'])

        access_token = response['auth']['access_token']

        # exchange old-style auth token for OAuth credentials; Flickr disables the old token immediately
        a = flickr_api.auth.AuthHandler(
            access_token_key=str(access_token['oauth_token']),
            access_token_secret=str(access_token['oauth_token_secret']))

        flickr_api.set_auth_handler(a)

        user = flickr_api.test.login()
        print "Authorized user:", user

        if not os.path.exists('users'):
            os.mkdir('users')
        a.save("users/" + user.username)
        exit()

    # get photos and send email
    user_photos = {}
    user_photos_by_taken = {}
    users = {}

    for f in glob.glob("users/*"):
        flickr_api.set_auth_handler(flickr_api.auth.AuthHandler.load(f))

        username = os.path.basename(f)

        last_date = state.getint('main', 'last_date')
        photos = flickr_api.Photo.recentlyUpdated(min_date=last_date, extras=['url_m', 'url_o', 'description', 'date_taken'])

        if photos.data:
            info = flickr_api.test.login().getInfo()

            info['buddyicon'] = "http://farm{iconfarm}.staticflickr.com/{iconserver}/buddyicons/{nsid}.jpg"\
                .format(**info)

            for photo in photos.data:
                photo._set_properties(
                    flickr_url="https://www.flickr.com/photos/{0}/{1}/".format(username, photo.id)
                )

            users[username] = info
            user_photos[username] = photos.data
            user_photos_by_taken[username] = sorted(photos.data, key=lambda x: x.datetaken)

    state.set('main', 'last_date', int(time.time()))
    write_state(state)

    if len(users) == 0:
        print "No new content"
        exit(0)

    env = jinja2.Environment(loader=jinja2.FileSystemLoader('.'))
    template = env.get_template('email.jinja2')
    text = template.render(user_photos=user_photos, users=users, user_photos_by_taken=user_photos_by_taken)

    try:
        charset.add_charset('utf-8', charset.QP, charset.QP)
        msg = MIMEText(text, 'html', 'utf8')
        msg['From'] = state.get('main', 'smtp_from')
        msg['To'] = state.get('main', 'smtp_to')
        msg['Subject'] = state.get('main', 'smtp_subject')

        s = smtplib.SMTP()
        s.connect(state.get('main', 'smtp_server'), state.getint('main', 'smtp_port'))

        s.ehlo()

        if state.getboolean('main', 'smtp_tls'):
            s.starttls()

        if state.has_option('main', 'smtp_user') and state.has_option('main', 'smtp_password'):
            s.login(state.get('main', 'smtp_user'), state.get('main', 'smtp_password'))

        s.sendmail(state.get('main', 'smtp_from'), state.get('main', 'smtp_to'), msg.as_string())
        s.quit()
    except:
        print "Can't send email, you can set SMTP options in state.ini: " \
              "set smtp_tls=true, smtp_user, smtp_pass for encrypted and authenticated SMTP"
        raise

    print "Sent email containing {0} photos from {1} users".format(
        len([item for sublist in user_photos.values() for item in sublist]),
        len(users)
    )

if __name__ == '__main__':
    main()<commit_msg>Add support for sending emails via bcc<commit_after>#! /usr/bin/python

import argparse
import glob
import ConfigParser
import time
import os
import os.path
import smtplib
from email.mime.text import MIMEText
from email import charset
import hashlib
import json
import urllib2

import flickr_api
import jinja2

def write_state(state):
    with open('state.ini', 'w') as state_file:
        state.write(state_file)

def main():
    parser = argparse.ArgumentParser(description='Send email when new pictures are posted to Flickr.')

    parser.add_argument('--user_add_interactive', '-i', action='store_true',
                        help='add a user interactively based on OAuth')
    parser.add_argument('--user_add', '-a', action='store_true', help='prepare adding a user based on OAuth')
    parser.add_argument('--user_auth_verifier', '-v', action='store', help='finish adding the user based on OAuth')
    parser.add_argument('--user_old_auth', '-o', action='store', help='add a user based on a pre-OAuth token')
    parser.add_argument('--user_delete', '-d', action='store', help='remove a user')
    parser.add_argument('--since_days', '-s', action='store', type=int, help='remove a user')

    args = parser.parse_args()

    state = ConfigParser.RawConfigParser({
        'last_date': int(time.time() - 24 * 60 * 60),
        'smtp_from': 'admin@example.com',
        'smtp_to': 'clients@example.com',
        'smtp_cc': '',
        'smtp_bcc': '',
        'smtp_subject': 'New photos',
        'smtp_port': 25,
        'smtp_server': 'localhost',
        'smtp_tls': 'false',
    })
    state.add_section('main')
    state.read("state.ini")

    if args.since_days:
        state.set('main', 'last_date', int(time.time() - args.since_days * 24 * 60 * 60))

    # set up flickr_api app credentials
    try:
        flickr_api.set_keys(api_key=state.get('main', 'api_key'), api_secret=state.get('main', 'api_secret'))
    except ConfigParser.NoOptionError:
        state.set('main', 'api_key', 'API_KEY')
        state.set('main', 'api_secret', 'API_SECRET')
        write_state(state)
        exit("Please fill in the api_key and api_secret in state.ini")

    if args.user_add_interactive:
        a = flickr_api.auth.AuthHandler()
        print "Open this URL in a web browser and authorize", a.get_authorization_url("read")

        oauth_token = raw_input("Paste the oauth_verifier parameter here: ")

        a.set_verifier(oauth_token)
        flickr_api.set_auth_handler(a)

        user = flickr_api.test.login()
        print "Authorized user:", user

        if not os.path.exists('users'):
            os.mkdir('users')
        a.save("users/" + user.username)
        exit()

    if args.user_add:
        a = flickr_api.auth.AuthHandler()
        print "Open this URL in a web browser and authorize", a.get_authorization_url("read")
        print "Then run the command again with --user_auth_verifier <oauth_verifier> to finish authorizing"

        if not os.path.exists('tmp_users'):
            os.mkdir('tmp_users')

        # can't use a.save() because that method requires an access_token and all we have is a request_token
        with open("tmp_users/" + a.request_token.key, "w") as f:
            f.write("\n".join([a.request_token.key,
                               a.request_token.secret]))
        exit()

    if args.user_auth_verifier:
        # we don't know which of the temp users matches the verifier, try each of them in turn
        for f in glob.glob("tmp_users/*"):
            try:
                with open(f, "r") as f1:
                    request_token = f1.read().split("\n")

                a = flickr_api.auth.AuthHandler(
                    request_token_key=request_token[0],
                    request_token_secret=request_token[1]
                )

                # this will throw an exception if the verifier doesn't match the request_token
                a.set_verifier(args.user_auth_verifier)

                flickr_api.set_auth_handler(a)

                user = flickr_api.test.login()
                print "Authorized user:", user

                if not os.path.exists('users'):
                    os.mkdir('users')
                a.save("users/" + user.username)

                os.remove(f)

                exit()
            except SystemExit:
                exit()
            except:
                # continue trying
                pass

        exit("No matching pre-authorization found")

    if args.user_delete:
        try:
            os.remove('users/' + args.user_delete)
        except:
            print "Could not remove the user"
            raise
        exit()

    if args.user_old_auth:
        # handle old-style auth directly, since flickr_api only handled OAuth
        m = hashlib.md5()
        m.update(state.get('main', 'api_secret'))
        m.update('api_key')
        m.update(state.get('main', 'api_key'))
        m.update('auth_token')
        m.update(args.user_old_auth)
        m.update('format')
        m.update('json')
        m.update('method')
        m.update('flickr.auth.oauth.getAccessToken')
        m.update('nojsoncallback')
        m.update('1')

        url = "https://api.flickr.com/services/rest/" \
              "?method=flickr.auth.oauth.getAccessToken" \
              "&api_key={0}" \
              "&auth_token={1}" \
              "&format=json" \
              "&nojsoncallback=1" \
              "&api_sig={2}".format(
            state.get('main', 'api_key'),
            args.user_old_auth,
            m.hexdigest())

        resp = urllib2.urlopen(url)
        response = json.load(resp)

        if response['stat'] != 'ok':
            exit("Request failed, Flickr responded: " + response['message'])

        access_token = response['auth']['access_token']

        # exchange old-style auth token for OAuth credentials; Flickr disables the old token immediately
        a = flickr_api.auth.AuthHandler(
            access_token_key=str(access_token['oauth_token']),
            access_token_secret=str(access_token['oauth_token_secret']))

        flickr_api.set_auth_handler(a)

        user = flickr_api.test.login()
        print "Authorized user:", user

        if not os.path.exists('users'):
            os.mkdir('users')
        a.save("users/" + user.username)
        exit()

    # get photos and send email
    user_photos = {}
    user_photos_by_taken = {}
    users = {}

    for f in glob.glob("users/*"):
        flickr_api.set_auth_handler(flickr_api.auth.AuthHandler.load(f))

        username = os.path.basename(f)

        last_date = state.getint('main', 'last_date')
        photos = flickr_api.Photo.recentlyUpdated(min_date=last_date, extras=['url_m', 'url_o', 'description', 'date_taken'])

        if photos.data:
            info = flickr_api.test.login().getInfo()

            info['buddyicon'] = "http://farm{iconfarm}.staticflickr.com/{iconserver}/buddyicons/{nsid}.jpg"\
                .format(**info)

            for photo in photos.data:
                photo._set_properties(
                    flickr_url="https://www.flickr.com/photos/{0}/{1}/".format(username, photo.id)
                )

            users[username] = info
            user_photos[username] = photos.data
            user_photos_by_taken[username] = sorted(photos.data, key=lambda x: x.datetaken)

    state.set('main', 'last_date', int(time.time()))
    write_state(state)

    if len(users) == 0:
        print "No new content"
        exit(0)

    env = jinja2.Environment(loader=jinja2.FileSystemLoader('.'))
    template = env.get_template('email.jinja2')
    text = template.render(user_photos=user_photos, users=users, user_photos_by_taken=user_photos_by_taken)

    try:
        charset.add_charset('utf-8', charset.QP, charset.QP)
        msg = MIMEText(text, 'html', 'utf8')
        msg['From'] = state.get('main', 'smtp_from')
        msg['To'] = state.get('main', 'smtp_to')
        msg['Cc'] = state.get('main', 'smtp_cc')
        msg['Bcc'] = state.get('main', 'smtp_bcc')
        msg['Subject'] = state.get('main', 'smtp_subject')

        s = smtplib.SMTP()
        s.connect(state.get('main', 'smtp_server'), state.getint('main', 'smtp_port'))

        s.ehlo()

        if state.getboolean('main', 'smtp_tls'):
            s.starttls()

        if state.has_option('main', 'smtp_user') and state.has_option('main', 'smtp_password'):
            s.login(state.get('main', 'smtp_user'), state.get('main', 'smtp_password'))

        s.sendmail(state.get('main', 'smtp_from'), state.get('main', 'smtp_to'), msg.as_string())
        s.quit()
    except:
        print "Can't send email, you can set SMTP options in state.ini: " \
              "set smtp_tls=true, smtp_user, smtp_pass for encrypted and authenticated SMTP"
        raise

    print "Sent email containing {0} photos from {1} users".format(
        len([item for sublist in user_photos.values() for item in sublist]),
        len(users)
    )

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>from __future__ import unicode_literals
import os
import sys

try:
    import urlparse
except ImportError:
    import urllib.parse as urlparse

import csv
import cssutils

import requests
from lxml import etree
from inlinestyler.cssselect import CSSSelector, ExpressionError

class Conversion(object):
    def __init__(self):
        self.CSSErrors = []
        self.CSSUnsupportErrors = dict()
        self.supportPercentage = 100
        self.convertedHTML = ""

    def perform(self, document, sourceHTML, sourceURL, encoding='unicode'):
        aggregate_css = ""

        # Retrieve CSS rel links from html pasted and aggregate into one string
        CSSRelSelector = CSSSelector("link[rel=stylesheet],link[rel=StyleSheet],link[rel=STYLESHEET]")
        matching = CSSRelSelector.evaluate(document)
        for element in matching:
            try:
                csspath = element.get("href")
                if len(sourceURL):
                    if element.get("href").lower().find("http://", 0) < 0:
                        parsed_url = urlparse.urlparse(sourceURL)
                        csspath = urlparse.urljoin(parsed_url.scheme + "://" + parsed_url.hostname, csspath)

                css_content = requests.get(csspath).text
                aggregate_css += ''.join(css_content)

                element.getparent().remove(element)
            except:
                raise IOError('The stylesheet ' + element.get("href") + ' could not be found')

        # Include inline style elements
        CSSStyleSelector = CSSSelector("style,Style")
        matching = CSSStyleSelector.evaluate(document)
        for element in matching:
            aggregate_css += element.text
            element.getparent().remove(element)

        # Convert document to a style dictionary compatible with etree
        styledict = self.get_view(document, aggregate_css)

        # Set inline style attribute if not one of the elements not worth styling
        ignore_list = ['html', 'head', 'title', 'meta', 'link', 'script']
        for element, style in styledict.items():
            if element.tag not in ignore_list:
                v = style.getCssText(separator='')
                element.set('style', v)

        self.convertedHTML = etree.tostring(document, method="html", pretty_print=True, encoding=encoding)
        return self

    def styleattribute(self, element):
        """
          returns css.CSSStyleDeclaration of inline styles, for html: @style
          """
        css_text = element.get('style')
        if css_text:
            return cssutils.css.CSSStyleDeclaration(cssText=css_text)
        else:
            return None

    def get_view(self, document, css):

        view = {}
        specificities = {}
        supportratios = {}
        support_failrate = 0
        support_totalrate = 0
        compliance = dict()

        mycsv = csv.DictReader(open(os.path.join(os.path.dirname(__file__), "css_compliance.csv")), delimiter=str(','))

        for row in mycsv:
            # Count clients so we can calculate an overall support percentage later
            client_count = len(row)
            compliance[row['property'].strip()] = dict(row)

        # Decrement client count to account for first col which is property name
        client_count -= 1

        sheet = cssutils.parseString(css)

        rules = (rule for rule in sheet if rule.type == rule.STYLE_RULE)
        for rule in rules:

            for selector in rule.selectorList:
                try:
                    cssselector = CSSSelector(selector.selectorText)
                    matching = cssselector.evaluate(document)

                    for element in matching:
                        # add styles for all matching DOM elements
                        if element not in view:
                            # add initial
                            view[element] = cssutils.css.CSSStyleDeclaration()
                            specificities[element] = {}

                            # add inline style if present
                            inlinestyletext = element.get('style')
                            if inlinestyletext:
                                inlinestyle = cssutils.css.CSSStyleDeclaration(cssText=inlinestyletext)
                            else:
                                inlinestyle = None
                            if inlinestyle:
                                for p in inlinestyle:
                                    # set inline style specificity
                                    view[element].setProperty(p)
                                    specificities[element][p.name] = (1, 0, 0, 0)

                        for p in rule.style:
                            if p.name not in supportratios:
                                supportratios[p.name] = {'usage': 0, 'failedClients': 0}

                            supportratios[p.name]['usage'] += 1

                            try:
                                if p.name not in self.CSSUnsupportErrors:
                                    for client, support in compliance[p.name].items():
                                        if support == "N" or support == "P":
                                            # Increment client failure count for this property
                                            supportratios[p.name]['failedClients'] += 1
                                            if p.name not in self.CSSUnsupportErrors:
                                                if support == "P":
                                                    self.CSSUnsupportErrors[p.name] = [client + ' (partial support)']
                                                else:
                                                    self.CSSUnsupportErrors[p.name] = [client]
                                            else:
                                                if support == "P":
                                                    self.CSSUnsupportErrors[p.name].append(client + ' (partial support)')
                                                else:
                                                    self.CSSUnsupportErrors[p.name].append(client)

                            except KeyError:
                                pass

                            # update styles
                            if p not in view[element]:
                                view[element].setProperty(p.name, p.value, p.priority)
                                specificities[element][p.name] = selector.specificity
                            else:
                                sameprio = (p.priority == view[element].getPropertyPriority(p.name))
                                if not sameprio and bool(p.priority) or (sameprio and selector.specificity >= specificities[element][p.name]):
                                    # later, more specific or higher prio
                                    view[element].setProperty(p.name, p.value, p.priority)

                except ExpressionError:
                    if str(sys.exc_info()[1]) not in self.CSSErrors:
                        self.CSSErrors.append(str(sys.exc_info()[1]))
                    pass

        for props, propvals in supportratios.items():
            support_failrate += (propvals['usage']) * int(propvals['failedClients'])
            support_totalrate += int(propvals['usage']) * client_count

        if support_failrate and support_totalrate:
            self.supportPercentage = 100 - ((float(support_failrate) / float(support_totalrate)) * 100)
        return view
<commit_msg>Ensure file gets closed after opening<commit_after>from __future__ import unicode_literals
import os
import sys

try:
    import urlparse
except ImportError:
    import urllib.parse as urlparse

import csv
import cssutils

import requests
from lxml import etree
from inlinestyler.cssselect import CSSSelector, ExpressionError

class Conversion(object):
    def __init__(self):
        self.CSSErrors = []
        self.CSSUnsupportErrors = dict()
        self.supportPercentage = 100
        self.convertedHTML = ""

    def perform(self, document, sourceHTML, sourceURL, encoding='unicode'):
        aggregate_css = ""

        # Retrieve CSS rel links from html pasted and aggregate into one string
        CSSRelSelector = CSSSelector("link[rel=stylesheet],link[rel=StyleSheet],link[rel=STYLESHEET]")
        matching = CSSRelSelector.evaluate(document)
        for element in matching:
            try:
                csspath = element.get("href")
                if len(sourceURL):
                    if element.get("href").lower().find("http://", 0) < 0:
                        parsed_url = urlparse.urlparse(sourceURL)
                        csspath = urlparse.urljoin(parsed_url.scheme + "://" + parsed_url.hostname, csspath)

                css_content = requests.get(csspath).text
                aggregate_css += ''.join(css_content)

                element.getparent().remove(element)
            except:
                raise IOError('The stylesheet ' + element.get("href") + ' could not be found')

        # Include inline style elements
        CSSStyleSelector = CSSSelector("style,Style")
        matching = CSSStyleSelector.evaluate(document)
        for element in matching:
            aggregate_css += element.text
            element.getparent().remove(element)

        # Convert document to a style dictionary compatible with etree
        styledict = self.get_view(document, aggregate_css)

        # Set inline style attribute if not one of the elements not worth styling
        ignore_list = ['html', 'head', 'title', 'meta', 'link', 'script']
        for element, style in styledict.items():
            if element.tag not in ignore_list:
                v = style.getCssText(separator='')
                element.set('style', v)

        self.convertedHTML = etree.tostring(document, method="html", pretty_print=True, encoding=encoding)
        return self

    def styleattribute(self, element):
        """
          returns css.CSSStyleDeclaration of inline styles, for html: @style
          """
        css_text = element.get('style')
        if css_text:
            return cssutils.css.CSSStyleDeclaration(cssText=css_text)
        else:
            return None

    def get_view(self, document, css):

        view = {}
        specificities = {}
        supportratios = {}
        support_failrate = 0
        support_totalrate = 0
        compliance = dict()

        with open(os.path.join(os.path.dirname(__file__), "css_compliance.csv")) as csv_file:
            mycsv = csv.DictReader(csv_file, delimiter=str(','))

        for row in mycsv:
            # Count clients so we can calculate an overall support percentage later
            client_count = len(row)
            compliance[row['property'].strip()] = dict(row)

        # Decrement client count to account for first col which is property name
        client_count -= 1

        sheet = cssutils.parseString(css)

        rules = (rule for rule in sheet if rule.type == rule.STYLE_RULE)
        for rule in rules:

            for selector in rule.selectorList:
                try:
                    cssselector = CSSSelector(selector.selectorText)
                    matching = cssselector.evaluate(document)

                    for element in matching:
                        # add styles for all matching DOM elements
                        if element not in view:
                            # add initial
                            view[element] = cssutils.css.CSSStyleDeclaration()
                            specificities[element] = {}

                            # add inline style if present
                            inlinestyletext = element.get('style')
                            if inlinestyletext:
                                inlinestyle = cssutils.css.CSSStyleDeclaration(cssText=inlinestyletext)
                            else:
                                inlinestyle = None
                            if inlinestyle:
                                for p in inlinestyle:
                                    # set inline style specificity
                                    view[element].setProperty(p)
                                    specificities[element][p.name] = (1, 0, 0, 0)

                        for p in rule.style:
                            if p.name not in supportratios:
                                supportratios[p.name] = {'usage': 0, 'failedClients': 0}

                            supportratios[p.name]['usage'] += 1

                            try:
                                if p.name not in self.CSSUnsupportErrors:
                                    for client, support in compliance[p.name].items():
                                        if support == "N" or support == "P":
                                            # Increment client failure count for this property
                                            supportratios[p.name]['failedClients'] += 1
                                            if p.name not in self.CSSUnsupportErrors:
                                                if support == "P":
                                                    self.CSSUnsupportErrors[p.name] = [client + ' (partial support)']
                                                else:
                                                    self.CSSUnsupportErrors[p.name] = [client]
                                            else:
                                                if support == "P":
                                                    self.CSSUnsupportErrors[p.name].append(client + ' (partial support)')
                                                else:
                                                    self.CSSUnsupportErrors[p.name].append(client)

                            except KeyError:
                                pass

                            # update styles
                            if p not in view[element]:
                                view[element].setProperty(p.name, p.value, p.priority)
                                specificities[element][p.name] = selector.specificity
                            else:
                                sameprio = (p.priority == view[element].getPropertyPriority(p.name))
                                if not sameprio and bool(p.priority) or (sameprio and selector.specificity >= specificities[element][p.name]):
                                    # later, more specific or higher prio
                                    view[element].setProperty(p.name, p.value, p.priority)

                except ExpressionError:
                    if str(sys.exc_info()[1]) not in self.CSSErrors:
                        self.CSSErrors.append(str(sys.exc_info()[1]))
                    pass

        for props, propvals in supportratios.items():
            support_failrate += (propvals['usage']) * int(propvals['failedClients'])
            support_totalrate += int(propvals['usage']) * client_count

        if support_failrate and support_totalrate:
            self.supportPercentage = 100 - ((float(support_failrate) / float(support_totalrate)) * 100)
        return view
<|endoftext|>


<commit_before>PLUGIN_VERSION = '2.6.2'
# The line above is auto-generated by tag_release.py. Do not change it manually.

try:
    from .common import shared as G
    assert G
except ImportError:
    from common import shared as G

G.__VERSION__ = '0.11'
G.__PLUGIN_VERSION__ = PLUGIN_VERSION
<commit_msg>Tag new release: 2.6.3<commit_after>PLUGIN_VERSION = '2.6.3'
# The line above is auto-generated by tag_release.py. Do not change it manually.

try:
    from .common import shared as G
    assert G
except ImportError:
    from common import shared as G

G.__VERSION__ = '0.11'
G.__PLUGIN_VERSION__ = PLUGIN_VERSION
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Plot flux graphs."""
import numpy as np
from astropy import log
from astropy.time import Time
import copy

import config
import graph

############
# CONSTANTS
############
DEFAULT_MIN_METEORS = 100  # meteors
DEFAULT_MIN_ECA = 50000    # km^2 h
DEFAULT_GAMMA = 1.5
DEFAULT_POPINDEX = 2.2     # population index

##########
# CLASSES
#########

class BaseProfile(object):
    """Abstract base class."""

    def __init__(self, fluxdb):
        self.fluxdb = fluxdb

    def field(self, key):
        """Returns a data field"""
        return np.array([row[key] for row in self.fluxes])

    def get_response(self):
        """Returns the flux profile in JSON format.

        Returns
        -------
        dict
        """
        result = {}
        if self.fluxes is None or len(self.fluxes) == 0:
            log.error('No suitable data found.')
            result['status'] = 'WARNING'
            result['msg'] = 'No suitable data found.'
        else:
            result['status'] = 'OK'
            result['graph'] = self.save_graph()
            result['flux'] = []
            for row in self.fluxes:
                newrow = []
                # Averaged profiles do not have a time field
                if row['time']:
                    newrow.append(str(row['time'])[0:16])
                newrow.extend(('{:.3f}'.format(row['solarlon']),
                              '{:.1f}'.format(row['teff']),
                              '{:.1f}'.format(row['eca']),
                              '{0}'.format(row['met']),
                              '{:.1f} ± {:.1f}'.format(row['flux'], row['e_flux']),
                              '{:.0f}'.format(row['zhr'])))
                result['flux'].append(newrow)
        return result

    def save_graph(self):
        """Creates a graph and returns the filename.
        """
        mygraph = self.graph()
        return mygraph.save()

class VideoProfile(BaseProfile):

    def __init__(self, fluxdb,
                 shower, start, stop,
                 min_interval=1, max_interval=24,
                 min_meteors=DEFAULT_MIN_METEORS,
                 min_eca=DEFAULT_MIN_ECA,
                 min_alt=10, min_eca_station=0.5,
                 gamma=DEFAULT_GAMMA,
                 popindex=DEFAULT_POPINDEX):
        """

        Parameters
        ----------
        shower : string
            IMO shower code

        start : string
            ISO timestamp

        stop : string
            ISO timestamp

        min_meteors : int
            Minimum number of meteors in each bin.

        min_eca : float [10^3 km^2 h]
            Minimum ECA in each bin.

        min_interval : float [hours]

        max_interval : float [hours]

        min_alt : float [degrees]
            Minimum radiant altitude for a flux record to be included.

        min_eca_station : float [degrees]
            Minimum ECA for a flux record to be included.

        gamma : float
            Zenith correction exponent.

        popindex : float
            Population index.

        Returns
        -------
        Result of the query.
        """
        BaseProfile.__init__(self, fluxdb)

        if isinstance(start, Time):
            self.start = start
        else:
            self.start = Time(start, scale='utc')

        if isinstance(stop, Time):
            self.stop = stop
        else:
            self.stop = Time(stop, scale='utc')

        self.popindex = popindex
        self.gamma = gamma
        self.fluxes = self.fluxdb.query("""SELECT * FROM
                                    VideoProfile(%s,
                                                 %s::timestamp,
                                                 %s::timestamp,
                                                 %s, %s,
                                                 '%s hours'::interval,
                                                 '%s hours'::interval,
                                                 %s, %s, %s, %s)
                                """, (shower,
                                      self.start.isot,
                                      self.stop.isot,
                                      min_meteors, min_eca,
                                      min_interval, max_interval,
                                      min_alt, min_eca_station,
                                      gamma, popindex, ))

    def graph(self):
        mygraph = graph.VideoGraph(self)
        mygraph.plot()
        return mygraph

class SolVideoProfile(BaseProfile):

    def __init__(self, fluxdb, shower,
                 year, start, stop,
                 min_interval=1, max_interval=24,
                 min_meteors=DEFAULT_MIN_METEORS,
                 min_eca=DEFAULT_MIN_ECA,
                 min_alt=10, min_eca_station=0.5,
                 gamma=DEFAULT_GAMMA,
                 popindex=DEFAULT_POPINDEX,
                 label=None,
                 marker='s'):
        """

        Parameters
        ----------
        shower : string
            IMO shower code

        years : int
            e.g. 2012

        start : float [degrees]
            Solar longitude.

        stop : float [degrees]
            Solar longitude.

        min_meteors : int
            Minimum number of meteors in each bin.

        min_eca : float [10^3 km^2 h]
            Minimum ECA in each bin.

        min_interval : float [hours]

        max_interval : float [hours]

        min_alt : float [degrees]
            Minimum radiant altitude for a flux record to be included.

        min_eca_station : float [degrees]
            Minimum ECA for a flux record to be included.

        gamma : float
            Zenith correction exponent.

        popindex : float
            Population index.

        Returns
        -------
        Result of the query.
        """
        BaseProfile.__init__(self, fluxdb)
        self.shower = shower
        self.year = year
        self.start = start
        self.stop = stop
        self.popindex = popindex
        self.gamma = gamma
        if label != None:
            self.label = label
        else:
            #self.label = '{0} {1}'.format(shower, year)
            self.label = str(year)
        self.marker = marker
        self.fluxes = self.fluxdb.query("""SELECT * FROM
                                    SolVideoProfile(%s,
                                                 %s, %s, %s,
                                                 %s, %s,
                                                 %s, %s,
                                                 %s, %s, %s, %s)
                                """, (shower,
                                      year, start, stop,
                                      min_meteors, min_eca,
                                      min_interval, max_interval,
                                      min_alt, min_eca_station,
                                      gamma, popindex, ))

    def graph(self):
        mygraph = graph.SolVideoGraph(self)
        mygraph.plot()
        return mygraph

class AvgVideoProfile(BaseProfile):

    def __init__(self, fluxdb, shower,
                 years, start, stop,
                 min_interval=1, max_interval=24,
                 min_meteors=DEFAULT_MIN_METEORS,
                 min_eca=DEFAULT_MIN_ECA,
                 min_alt=10, min_eca_station=0.5,
                 gamma=DEFAULT_GAMMA,
                 popindex=DEFAULT_POPINDEX,
                 label=None,
                 marker='s'):
        """

        Parameters
        ----------
        shower : string
            IMO shower code

        years : list
            e.g. [2011,2012]

        start : float [degrees]
            Solar longitude.

        stop : float [degrees]
            Solar longitude.

        min_meteors : int
            Minimum number of meteors in each bin.

        min_eca : float [10^3 km^2 h]
            Minimum ECA in each bin.

        min_interval : float [hours]

        max_interval : float [hours]

        min_alt : float [degrees]
            Minimum radiant altitude for a flux record to be included.

        min_eca_station : float [degrees]
            Minimum ECA for a flux record to be included.

        gamma : float
            Zenith correction exponent.

        popindex : float
            Population index.

        Returns
        -------
        Result of the query.
        """
        BaseProfile.__init__(self, fluxdb)
        self.shower = shower
        self.years = years
        self.start = start
        self.stop = stop
        self.popindex = popindex
        self.gamma = gamma
        if label != None:
            self.label = label
        elif len(years) == 1:
            self.label = years[0]
        else:
            #self.label = '{0} {1}'.format(shower, year)
            self.label = shower
        self.marker = marker
        self.fluxes = self.fluxdb.query("""SELECT * FROM
                                    AvgVideoProfile(%s,
                                                 %s::int[], %s, %s,
                                                 %s, %s,
                                                 %s, %s,
                                                 %s, %s, %s, %s)
                                """, (shower,
                                      years, start, stop,
                                      min_meteors, min_eca,
                                      min_interval, max_interval,
                                      min_alt, min_eca_station,
                                      gamma, popindex, ))

    def graph(self):
        mygraph = graph.SolVideoGraph(self)
        mygraph.plot()
        return mygraph
<commit_msg>ymax wasnt passed through as a parameter<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Plot flux graphs."""
import numpy as np
from astropy import log
from astropy.time import Time
import copy

import config
import graph

############
# CONSTANTS
############
DEFAULT_MIN_METEORS = 100  # meteors
DEFAULT_MIN_ECA = 50000    # km^2 h
DEFAULT_GAMMA = 1.5
DEFAULT_POPINDEX = 2.2     # population index

##########
# CLASSES
#########

class BaseProfile(object):
    """Abstract base class."""

    def __init__(self, fluxdb, ymax=None):
        self.fluxdb = fluxdb
        self.ymax = ymax

    def field(self, key):
        """Returns a data field"""
        return np.array([row[key] for row in self.fluxes])

    def get_response(self):
        """Returns the flux profile in JSON format.

        Returns
        -------
        dict
        """
        result = {}
        if self.fluxes is None or len(self.fluxes) == 0:
            log.error('No suitable data found.')
            result['status'] = 'WARNING'
            result['msg'] = 'No suitable data found.'
        else:
            result['status'] = 'OK'
            result['graph'] = self.save_graph()
            result['flux'] = []
            for row in self.fluxes:
                newrow = []
                # Averaged profiles do not have a time field
                if row['time']:
                    newrow.append(str(row['time'])[0:16])
                newrow.extend(('{:.3f}'.format(row['solarlon']),
                              '{:.1f}'.format(row['teff']),
                              '{:.1f}'.format(row['eca']),
                              '{0}'.format(row['met']),
                              '{:.1f} ± {:.1f}'.format(row['flux'], row['e_flux']),
                              '{:.0f}'.format(row['zhr'])))
                result['flux'].append(newrow)
        return result

    def save_graph(self):
        """Creates a graph and returns the filename.
        """
        mygraph = self.graph()
        return mygraph.save()

class VideoProfile(BaseProfile):

    def __init__(self, fluxdb,
                 shower, start, stop,
                 min_interval=1, max_interval=24,
                 min_meteors=DEFAULT_MIN_METEORS,
                 min_eca=DEFAULT_MIN_ECA,
                 min_alt=10, min_eca_station=0.5,
                 gamma=DEFAULT_GAMMA,
                 popindex=DEFAULT_POPINDEX,
                 ymax=None):
        """

        Parameters
        ----------
        shower : string
            IMO shower code

        start : string
            ISO timestamp

        stop : string
            ISO timestamp

        min_meteors : int
            Minimum number of meteors in each bin.

        min_eca : float [10^3 km^2 h]
            Minimum ECA in each bin.

        min_interval : float [hours]

        max_interval : float [hours]

        min_alt : float [degrees]
            Minimum radiant altitude for a flux record to be included.

        min_eca_station : float [degrees]
            Minimum ECA for a flux record to be included.

        gamma : float
            Zenith correction exponent.

        popindex : float
            Population index.

        Returns
        -------
        Result of the query.
        """
        BaseProfile.__init__(self, fluxdb, ymax=ymax)

        if isinstance(start, Time):
            self.start = start
        else:
            self.start = Time(start, scale='utc')

        if isinstance(stop, Time):
            self.stop = stop
        else:
            self.stop = Time(stop, scale='utc')

        self.popindex = popindex
        self.gamma = gamma
        self.fluxes = self.fluxdb.query("""SELECT * FROM
                                    VideoProfile(%s,
                                                 %s::timestamp,
                                                 %s::timestamp,
                                                 %s, %s,
                                                 '%s hours'::interval,
                                                 '%s hours'::interval,
                                                 %s, %s, %s, %s)
                                """, (shower,
                                      self.start.isot,
                                      self.stop.isot,
                                      min_meteors, min_eca,
                                      min_interval, max_interval,
                                      min_alt, min_eca_station,
                                      gamma, popindex, ))

    def graph(self):
        mygraph = graph.VideoGraph(self, ymax=self.ymax)
        mygraph.plot()
        return mygraph

class SolVideoProfile(BaseProfile):

    def __init__(self, fluxdb, shower,
                 year, start, stop,
                 min_interval=1, max_interval=24,
                 min_meteors=DEFAULT_MIN_METEORS,
                 min_eca=DEFAULT_MIN_ECA,
                 min_alt=10, min_eca_station=0.5,
                 gamma=DEFAULT_GAMMA,
                 popindex=DEFAULT_POPINDEX,
                 ymax=None,
                 label=None,
                 marker='s'):
        """

        Parameters
        ----------
        shower : string
            IMO shower code

        years : int
            e.g. 2012

        start : float [degrees]
            Solar longitude.

        stop : float [degrees]
            Solar longitude.

        min_meteors : int
            Minimum number of meteors in each bin.

        min_eca : float [10^3 km^2 h]
            Minimum ECA in each bin.

        min_interval : float [hours]

        max_interval : float [hours]

        min_alt : float [degrees]
            Minimum radiant altitude for a flux record to be included.

        min_eca_station : float [degrees]
            Minimum ECA for a flux record to be included.

        gamma : float
            Zenith correction exponent.

        popindex : float
            Population index.

        Returns
        -------
        Result of the query.
        """
        BaseProfile.__init__(self, fluxdb, ymax=ymax)
        self.shower = shower
        self.year = year
        self.start = start
        self.stop = stop
        self.popindex = popindex
        self.gamma = gamma
        if label != None:
            self.label = label
        else:
            #self.label = '{0} {1}'.format(shower, year)
            self.label = str(year)
        self.marker = marker
        self.fluxes = self.fluxdb.query("""SELECT * FROM
                                    SolVideoProfile(%s,
                                                 %s, %s, %s,
                                                 %s, %s,
                                                 %s, %s,
                                                 %s, %s, %s, %s)
                                """, (shower,
                                      year, start, stop,
                                      min_meteors, min_eca,
                                      min_interval, max_interval,
                                      min_alt, min_eca_station,
                                      gamma, popindex, ))

    def graph(self):
        mygraph = graph.SolVideoGraph(self ymax=self.ymax)
        mygraph.plot()
        return mygraph

class AvgVideoProfile(BaseProfile):

    def __init__(self, fluxdb, shower,
                 years, start, stop,
                 min_interval=1, max_interval=24,
                 min_meteors=DEFAULT_MIN_METEORS,
                 min_eca=DEFAULT_MIN_ECA,
                 min_alt=10, min_eca_station=0.5,
                 gamma=DEFAULT_GAMMA,
                 popindex=DEFAULT_POPINDEX,
                 ymax=None,
                 label=None,
                 marker='s'):
        """

        Parameters
        ----------
        shower : string
            IMO shower code

        years : list
            e.g. [2011,2012]

        start : float [degrees]
            Solar longitude.

        stop : float [degrees]
            Solar longitude.

        min_meteors : int
            Minimum number of meteors in each bin.

        min_eca : float [10^3 km^2 h]
            Minimum ECA in each bin.

        min_interval : float [hours]

        max_interval : float [hours]

        min_alt : float [degrees]
            Minimum radiant altitude for a flux record to be included.

        min_eca_station : float [degrees]
            Minimum ECA for a flux record to be included.

        gamma : float
            Zenith correction exponent.

        popindex : float
            Population index.

        Returns
        -------
        Result of the query.
        """
        BaseProfile.__init__(self, fluxdb, ymax=ymax)
        self.shower = shower
        self.years = years
        self.start = start
        self.stop = stop
        self.popindex = popindex
        self.gamma = gamma
        if label != None:
            self.label = label
        elif len(years) == 1:
            self.label = years[0]
        else:
            #self.label = '{0} {1}'.format(shower, year)
            self.label = shower
        self.marker = marker
        self.fluxes = self.fluxdb.query("""SELECT * FROM
                                    AvgVideoProfile(%s,
                                                 %s::int[], %s, %s,
                                                 %s, %s,
                                                 %s, %s,
                                                 %s, %s, %s, %s)
                                """, (shower,
                                      years, start, stop,
                                      min_meteors, min_eca,
                                      min_interval, max_interval,
                                      min_alt, min_eca_station,
                                      gamma, popindex, ))

    def graph(self):
        mygraph = graph.SolVideoGraph(self, ymax=self.ymax)
        mygraph.plot()
        return mygraph
<|endoftext|>


<commit_before>import numpy as np
import matplotlib.pyplot as pl

from sklearn.cluster import KMeans
from scipy.linalg import inv

class FoamEM(object):
    """

    Run mixture of factor analyzers, under a noise model with
    optional jitter.

    internal variables:

    `K`:           Number of components
    `M`:           Latent dimensionality
    `D`:           Data dimensionality
    `N`:           Number of data points
    `data`:        (N,D) array of observations
    `latents`:     (K,M,N) array of latent variables
    `latent_covs`: (K,M,M,N) array of latent covariances
    `lambdas`:     (K,M,D) array of loadings
    `psis`:        (K,D) array of diagonal variance values
    `rs`:          (K,N) array of responsibilities
    `amps`:        (K) array of component amplitudes

    """
    def __init__(self,data,K,M,
                 PPCA=False,lock_psis=False,
                 rs_clip = 0.0,
                 max_condition_number=1.e6,
                 init=True,init_ppca=True):

        # required
        self.K     = K
        self.M     = M
        self.data  = np.atleast_2d(data)
        self.dataT = self.data.T # INSANE DATA DUPLICATION
        self.N     = self.data.shape[0]
        self.D     = self.data.shape[1]

        # options
        self.PPCA                 = PPCA
        self.lock_psis            = lock_psis
        self.rs_clip              = rs_clip
        self.max_condition_number = float(max_condition_number)
        assert rs_clip >= 0.0

        # empty arrays to be filled
        self.betas       = np.zeros((self.K,self.M,self.D))
        self.latents     = np.zeros((self.K,self.M,self.N))
        self.latent_covs = np.zeros((self.K,self.M,self.M,self.N))
        self.kmeans_rs   = np.zeros(self.N, dtype=int)
        self.rs          = np.zeros((self.K,self.N))

        # initialize
        if init:
            self._initialize(init_ppca)

    def _initialize(self,init_ppca,maxiter=200, tol=1e-4):

        # Run K-means
        self.run_kmeans()

        # Randomly assign factor loadings
        self.lambdas = np.random.randn(self.K,self.D,self.M) / \
            np.sqrt(self.max_condition_number)

        # Set (high rank) variance to variance of all data, along a dimension
        self.psis = np.tile(np.var(self.data,axis=0)[None,:],(self.K,1))

        # Set initial covs
        self.covs = np.zeros((self.K,self.D,self.D))
        self.inv_covs = 0. * self.covs
        self._update_covs()

        # Randomly assign the amplitudes.
        self.amps = np.random.rand(self.K)
        self.amps /= np.sum(self.amps)

        if init_ppca:

            # for each cluster, run a PPCA
            for k in range(self.K):

                ind = self.kmeans_rs==k
                self.rs[k,ind] = 1

                sumrs = np.sum(self.rs[k])

                # run em
                L = None
                for i in xrange(maxiter):
                    self._one_component_E_step(k)
                    newL = self._log_sum(
                        self._log_multi_gauss(k,self.data[ind]))
                    newL = np.sum(newL)
                    self._one_component_M_step(k,sumrs,True)
                    self._update_covs()
                    if L!=None:
                        dL = np.abs((newL - L) / L)
                        if i > 5 and dL < tol:
                            break
                    L = newL

    def run_kmeans(self, Ninit=10):
        """
        Run the K-means algorithm using the scikit-learn's
        implementation.
        """
        km = KMeans(init='k-means++', n_clusters=self.K, n_init=Ninit)
        km.fit(self.data)
        self.means = km.cluster_centroids
        print self.means
        print self.means.shape
        assert False

    def run_em(self, maxiter=400, tol=1e-4, verbose=True):
        """
        Run the EM algorithm.

        :param maxiter:
            The maximum number of iterations to try.

        :param tol:
            The tolerance on the relative change in the loss function that
            controls convergence.

        :param verbose:
            Print all the messages?

        """
        L = None
        for i in xrange(maxiter):
            self._E_step()
            newL = self.logLs.sum()
            if i == 0 and verbose:
                print("Initial NLL =", -newL)

            self._M_step()
            if L!=None:
                dL = np.abs((newL - L) / L)
                if i > 5 and dL < tol:
                    break
            L = newL

        if i < maxiter - 1:
            if verbose:
                print("EM converged after {0} iterations".format(i))
                print("Final NLL = {0}".format(-newL))
        else:
            print("Warning: EM didn't converge after {0} iterations"
                    .format(i))

    def take_EM_step(self):
        """
        Do one E step and then do one M step.  Duh!
        """
        self._E_step()
        self._M_step()

    def _E_step(self):
        """
        Expectation step.  See docs for details.
        """
        # resposibilities and likelihoods
        self.logLs, self.rs = self._calc_probs()

        for k in range(self.K):
            self._one_component_E_step(k)

    def _M_step(self):
        """
        Maximization step.  See docs for details.

        This assumes that `_E_step()` has been run.
        """
        sumrs = np.sum(self.rs,axis=1)

        # maximize for each component
        for k in range(self.K):
            self._one_component_M_step(k,sumrs[k],self.PPCA)
            self.amps[k] = sumrs[k] / self.N

        if self.lock_psis:
            psi = np.dot(sumrs, self.psis) / np.sum(sumrs)
            for k in range(self.K):
                self.psis[k] = psi

        self._update_covs()

    def _one_component_E_step(self,k):
        """
        Calculate the E step for one component.
        """
        # beta
        self.betas[k] = np.dot(self.lambdas[k].T,self.inv_covs[k])

        # latent values
        zeroed = self.dataT - self.means[k, :, None]
        self.latents[k] = np.dot(self.betas[k], zeroed)

        # latent empirical covariance
        step1   = self.latents[k, :, None, :] * self.latents[k, None, :, :]
        step2   = np.dot(self.betas[k], self.lambdas[k])
        self.latent_covs[k] = np.eye(self.M)[:,:,None] - step2[:,:,None] + step1

    def _one_component_M_step(self,k,sumrs,PPCA):
        """
        Calculate the M step for one component.
        """
        # means
        lambdalatents = np.dot(self.lambdas[k], self.latents[k])
        self.means[k] = np.sum(self.rs[k] * (self.dataT - lambdalatents),
                        axis=1) / sumrs

        # lambdas
        zeroed = self.dataT - self.means[k,:, None]
        self.lambdas[k] = np.dot(np.dot(zeroed[:,None,:] *
                                        self.latents[k,None,:,:],self.rs[k]),
                                 inv(np.dot(self.latent_covs[k],self.rs[k])))
        # psis
        # hacking a floor for psis
        psis = np.dot((zeroed - lambdalatents) * zeroed,self.rs[k]) / sumrs
        maxpsi = np.max(psis)
        maxlam = np.max(np.sum(self.lambdas[k] * self.lambdas[k], axis=0))
        minpsi = np.max([maxpsi, maxlam]) / self.max_condition_number
        psis   = np.clip(psis, minpsi, np.Inf)
        if PPCA:
            psis = np.mean(psis) * np.ones(self.D)
        self.psis[k] = psis

    def _update_covs(self):
        """
        Update self.cov for responsibility, logL calc
        """
        for k in range(self.K):
            self.covs[k] = np.dot(self.lambdas[k],self.lambdas[k].T) + \
                np.diag(self.psis[k])
            self.inv_covs[k] = self._invert_cov(k)

    def _calc_probs(self):
        """
        Calculate log likelihoods, responsibilites for each datum
        under each component.
        """
        logrs = np.zeros((self.K, self.N))
        for k in range(self.K):
            logrs[k] = np.log(self.amps[k]) + self._log_multi_gauss(k, self.data)

        # here lies some ghetto log-sum-exp...
        # nothing like a little bit of overflow to make your day better!
        L = self._log_sum(logrs)
        logrs -= L[None, :]
        if self.rs_clip > 0.0:
            logrs = np.clip(logrs,np.log(self.rs_clip),np.Inf)

        return L, np.exp(logrs)

    def _log_multi_gauss(self, k, D):
        """
        Gaussian log likelihood of the data for component k.
        """
        sgn, logdet = np.linalg.slogdet(self.covs[k])
        assert sgn > 0
        X1 = (D - self.means[k]).T
        X2 = np.dot(self.inv_covs[k], X1)
        p = -0.5 * np.sum(X1 * X2, axis=0)
        return -0.5 * np.log(2 * np.pi) * self.D - 0.5 * logdet + p

    def _log_sum(self,loglikes):
        """
        Calculate sum of log likelihoods
        """
        loglikes = np.atleast_2d(loglikes)
        a = np.max(loglikes, axis=0)
        return a + np.log(np.sum(np.exp(loglikes - a[None, :]), axis=0))

    def _invert_cov(self,k):
        """
        Calculate inverse covariance of mofa or ppca model,
        using inversion lemma
        """
        psiI = inv(np.diag(self.psis[k]))
        lam  = self.lambdas[k]
        lamT = lam.T
        step = inv(np.eye(self.M) + np.dot(lamT,np.dot(psiI,lam)))
        step = np.dot(step,np.dot(lamT,psiI))
        step = np.dot(psiI,np.dot(lam,step))
        return psiI - step

<commit_msg>still just mofa<commit_after>import numpy as np
import matplotlib.pyplot as pl

from sklearn.cluster import KMeans
from scipy.linalg import inv

class FoamEM(object):
    """

    Run mixture of factor analyzers, under a noise model with
    optional jitter.

    internal variables:

    `K`:           Number of components
    `M`:           Latent dimensionality
    `D`:           Data dimensionality
    `N`:           Number of data points
    `data`:        (N,D) array of observations
    `latents`:     (K,M,N) array of latent variables
    `latent_covs`: (K,M,M,N) array of latent covariances
    `lambdas`:     (K,M,D) array of loadings
    `psis`:        (K,D) array of diagonal variance values
    `rs`:          (K,N) array of responsibilities
    `amps`:        (K) array of component amplitudes

    """
    def __init__(self,data,K,M,
                 PPCA=False,lock_psis=False,
                 rs_clip = 0.0,
                 max_condition_number=1.e6,
                 init=True,init_ppca=True):

        # required
        self.K     = K
        self.M     = M
        self.data  = np.atleast_2d(data)
        self.dataT = self.data.T # INSANE DATA DUPLICATION
        self.N     = self.data.shape[0]
        self.D     = self.data.shape[1]

        # options
        self.PPCA                 = PPCA
        self.lock_psis            = lock_psis
        self.rs_clip              = rs_clip
        self.max_condition_number = float(max_condition_number)
        assert rs_clip >= 0.0

        # empty arrays to be filled
        self.betas       = np.zeros((self.K,self.M,self.D))
        self.latents     = np.zeros((self.K,self.M,self.N))
        self.latent_covs = np.zeros((self.K,self.M,self.M,self.N))
        self.kmeans_rs   = np.zeros(self.N, dtype=int)
        self.rs          = np.zeros((self.K,self.N))

        # initialize
        if init:
            self._initialize(init_ppca)

    def _initialize(self,init_ppca,maxiter=200, tol=1e-4):

        # Run K-means
        self.run_kmeans()

        # Randomly assign factor loadings
        self.lambdas = np.random.randn(self.K,self.D,self.M) / \
            np.sqrt(self.max_condition_number)

        # Set (high rank) variance to variance of all data, along a dimension
        self.psis = np.tile(np.var(self.data,axis=0)[None,:],(self.K,1))

        # Set initial covs
        self.covs = np.zeros((self.K,self.D,self.D))
        self.inv_covs = 0. * self.covs
        self._update_covs()

        # Randomly assign the amplitudes.
        self.amps = np.random.rand(self.K)
        self.amps /= np.sum(self.amps)

        if init_ppca:

            # for each cluster, run a PPCA
            for k in range(self.K):

                ind = self.kmeans_rs==k
                self.rs[k,ind] = 1

                sumrs = np.sum(self.rs[k])

                # run em
                L = None
                for i in xrange(maxiter):
                    self._one_component_E_step(k)
                    newL = self._log_sum(
                        self._log_multi_gauss(k,self.data[ind]))
                    newL = np.sum(newL)
                    self._one_component_M_step(k,sumrs,True)
                    self._update_covs()
                    if L!=None:
                        dL = np.abs((newL - L) / L)
                        if i > 5 and dL < tol:
                            break
                    L = newL

    def run_kmeans(self, Ninit=10):
        """
        Run the K-means algorithm using the scikit-learn's
        implementation.
        """
        km = KMeans(init='k-means++', n_clusters=self.K, n_init=Ninit)
        km.fit(self.data)
        self.means = km.cluster_centers_

    def run_em(self, maxiter=400, tol=1e-4, verbose=True):
        """
        Run the EM algorithm.

        :param maxiter:
            The maximum number of iterations to try.

        :param tol:
            The tolerance on the relative change in the loss function that
            controls convergence.

        :param verbose:
            Print all the messages?

        """
        L = None
        for i in xrange(maxiter):
            self._E_step()
            newL = self.logLs.sum()
            if i == 0 and verbose:
                print("Initial NLL =", -newL)

            self._M_step()
            if L!=None:
                dL = np.abs((newL - L) / L)
                if i > 5 and dL < tol:
                    break
            L = newL

        if i < maxiter - 1:
            if verbose:
                print("EM converged after {0} iterations".format(i))
                print("Final NLL = {0}".format(-newL))
        else:
            print("Warning: EM didn't converge after {0} iterations"
                    .format(i))

    def take_EM_step(self):
        """
        Do one E step and then do one M step.  Duh!
        """
        self._E_step()
        self._M_step()

    def _E_step(self):
        """
        Expectation step.  See docs for details.
        """
        # resposibilities and likelihoods
        self.logLs, self.rs = self._calc_probs()

        for k in range(self.K):
            self._one_component_E_step(k)

    def _M_step(self):
        """
        Maximization step.  See docs for details.

        This assumes that `_E_step()` has been run.
        """
        sumrs = np.sum(self.rs,axis=1)

        # maximize for each component
        for k in range(self.K):
            self._one_component_M_step(k,sumrs[k],self.PPCA)
            self.amps[k] = sumrs[k] / self.N

        if self.lock_psis:
            psi = np.dot(sumrs, self.psis) / np.sum(sumrs)
            for k in range(self.K):
                self.psis[k] = psi

        self._update_covs()

    def _one_component_E_step(self,k):
        """
        Calculate the E step for one component.
        """
        # beta
        self.betas[k] = np.dot(self.lambdas[k].T,self.inv_covs[k])

        # latent values
        zeroed = self.dataT - self.means[k, :, None]
        self.latents[k] = np.dot(self.betas[k], zeroed)

        # latent empirical covariance
        step1   = self.latents[k, :, None, :] * self.latents[k, None, :, :]
        step2   = np.dot(self.betas[k], self.lambdas[k])
        self.latent_covs[k] = np.eye(self.M)[:,:,None] - step2[:,:,None] + step1

    def _one_component_M_step(self,k,sumrs,PPCA):
        """
        Calculate the M step for one component.
        """
        # means
        lambdalatents = np.dot(self.lambdas[k], self.latents[k])
        self.means[k] = np.sum(self.rs[k] * (self.dataT - lambdalatents),
                        axis=1) / sumrs

        # lambdas
        zeroed = self.dataT - self.means[k,:, None]
        self.lambdas[k] = np.dot(np.dot(zeroed[:,None,:] *
                                        self.latents[k,None,:,:],self.rs[k]),
                                 inv(np.dot(self.latent_covs[k],self.rs[k])))
        # psis
        # hacking a floor for psis
        psis = np.dot((zeroed - lambdalatents) * zeroed,self.rs[k]) / sumrs
        maxpsi = np.max(psis)
        maxlam = np.max(np.sum(self.lambdas[k] * self.lambdas[k], axis=0))
        minpsi = np.max([maxpsi, maxlam]) / self.max_condition_number
        psis   = np.clip(psis, minpsi, np.Inf)
        if PPCA:
            psis = np.mean(psis) * np.ones(self.D)
        self.psis[k] = psis

    def _update_covs(self):
        """
        Update self.cov for responsibility, logL calc
        """
        for k in range(self.K):
            self.covs[k] = np.dot(self.lambdas[k],self.lambdas[k].T) + \
                np.diag(self.psis[k])
            self.inv_covs[k] = self._invert_cov(k)

    def _calc_probs(self):
        """
        Calculate log likelihoods, responsibilites for each datum
        under each component.
        """
        logrs = np.zeros((self.K, self.N))
        for k in range(self.K):
            logrs[k] = np.log(self.amps[k]) + self._log_multi_gauss(k, self.data)

        # here lies some ghetto log-sum-exp...
        # nothing like a little bit of overflow to make your day better!
        L = self._log_sum(logrs)
        logrs -= L[None, :]
        if self.rs_clip > 0.0:
            logrs = np.clip(logrs,np.log(self.rs_clip),np.Inf)

        return L, np.exp(logrs)

    def _log_multi_gauss(self, k, D):
        """
        Gaussian log likelihood of the data for component k.
        """
        sgn, logdet = np.linalg.slogdet(self.covs[k])
        assert sgn > 0
        X1 = (D - self.means[k]).T
        X2 = np.dot(self.inv_covs[k], X1)
        p = -0.5 * np.sum(X1 * X2, axis=0)
        return -0.5 * np.log(2 * np.pi) * self.D - 0.5 * logdet + p

    def _log_sum(self,loglikes):
        """
        Calculate sum of log likelihoods
        """
        loglikes = np.atleast_2d(loglikes)
        a = np.max(loglikes, axis=0)
        return a + np.log(np.sum(np.exp(loglikes - a[None, :]), axis=0))

    def _invert_cov(self,k):
        """
        Calculate inverse covariance of mofa or ppca model,
        using inversion lemma
        """
        psiI = inv(np.diag(self.psis[k]))
        lam  = self.lambdas[k]
        lamT = lam.T
        step = inv(np.eye(self.M) + np.dot(lamT,np.dot(psiI,lam)))
        step = np.dot(step,np.dot(lamT,psiI))
        step = np.dot(psiI,np.dot(lam,step))
        return psiI - step

<|endoftext|>


<commit_before>"""
Base implementation of Frappy framework.
"""

try:
    import urllib.request as urllib_request
    import urllib.error as urllib_error
except ImportError:
    import urllib2 as urllib_request
    import urllib2 as urllib_error

try:
    import json
except ImportError:
    import simplejson as json

class APIError(Exception):
    """
    Base Exception thrown by the APICall object when there is a
    general error interacting with the API.
    """

    pass

class APIHTTPError(APIError):
    """
    Base Exception thrown by the APICall object when there is a
    general error interacting with the API.
    """

    def __init__(self, e, uri, req_format, uriparts):
        """Initalize error object"""

        self.e = e
        self.uri = uri
        self.req_format = req_format
        self.uriparts = uriparts

    def __str__(self):
        """Stringify error"""

        return (
            "API sent status %i for URL: %s.%s using parameters: "
            "(%s)\ndetails: %s" % (
                self.e.code, self.uri, self.req_format, self.uriparts,
                self.e.fp.read()))

class APICall(object):
    """
    Base implementation of API call.

    This class is very generic and should provide most of the send/retrieve
    functionality for API access.  Thus, you should be able to subclass it,
    and provide a basic __init__ method.
    """

    def __init__(self, auth, req_format, domain, uriparts=None, secure=True,
                 post_actions=None, debug=False):

        """Initialize call API object"""

        self.auth = auth
        self.req_format = req_format
        self.domain = domain
        self.uri = ""
        self.secure = secure
        self.post_actions = post_actions

        if self.post_actions is None:
            self.post_actions = []

        self.response = None
        self.request_headers = {}
        self.response_headers = None

        self.method = "GET"
        self.body = None
        self.arg_data = ""

        self.uriparts = uriparts
        if self.uriparts is None:
            self.uriparts = ()

        # Set to True to test requests without Internet access (useful for unit
        # testing and just verifying that calls generate correct request uris)
        self.debug = debug

    def __getattr__(self, k):
        """
        Look for attribute k in base object, other wise append to uri

        This is allows for a very powerful and expressive syntax for creating
        API calls that map closely to the uri they query.  For example a
        Twitter call: <object>.statuses.public_timeline() will map to
        <domain>/statuses/public_timeline.
        """

        self.uriparts += (k,)
        return self

    def service_build_uri(self, **kwargs):
        """
        Service specific build uri

        This method is meant to be overriden by child classes to have the last
        opportunity to verify self.uri and add additional elements to it, etc.
        """
        return

    def _build_uri(self, *args, **kwargs):
        """Build up the final uri for the request"""

        uriparts = []
        for uripart in self.uriparts:
            # If this part matches a keyword argument, use the
            # supplied value otherwise, just use the part.
            uriparts.append(str(kwargs.pop(uripart, uripart)))

        self.uri += '/'.join(uriparts)

        # Don't use join here b/c not all arguments are required to be strings
        for arg in args:
            self.uri += '/%s' % (arg)

        secure_str = ''
        if self.secure:
            secure_str = 's'

        self.uri = "http%s://%s/%s" % (secure_str, self.domain, self.uri)

        # Wrapper for child classes to customize creation of the uri
        self.service_build_uri(**kwargs)

        # method is GET by default
        for action in self.post_actions:
            if self.uri.find(action) > -1:
                self.method = "POST"
                break

       # Append any authentication specified to request
        self._handle_auth(**kwargs)

    def _handle_auth(self, **kwargs):
        """
        Attach requested authentication to request and encoded body and
        parameters
        """

        if self.auth:
            self.request_headers.update(self.auth.generate_headers())
            self.arg_data = self.auth.encode_params(self.uri, self.method,
                                                    kwargs)

            if self.method == 'GET':
                self.uri += '?' + self.arg_data
            else:
                self.body = self.arg_data.encode('utf8')

    def __call__(self, *args, **kwargs):
        """
        Finish building uri with leftover arguments, append authentication, and
        send off request
        """

        self._build_uri(*args, **kwargs)

        req = urllib_request.Request(self.uri, self.body, self.request_headers)

        return self._handle_response(req, self.arg_data)

    def _handle_response(self, req, arg_data):
        """Verify response code and format data accordingly"""

        try:
            handle = urllib_request.urlopen(req)
            self.response_headers = handle.headers

            if "json" == self.req_format:
                self.response = json.loads(handle.read().decode('utf8'))
            else:
                self.response = handle.read().decode('utf8')

            return self
        except urllib_error.HTTPError as e:
            if (e.code == 304):
                return []
            # Allows for testing without Internet access
            elif self.debug:
                return self
            else:
                raise APIHTTPError(e, self.uri, self.req_format, arg_data)
        except urllib_error.URLError:
            # Allows for testing without Internet access
            if self.debug:
                return self

        finally:
            # Roll over request to prepare for new one
            self._reset_uri()

    def _reset_uri(self):
        """Clear active request uri to make way for another request"""

        # Save off the current uri request just for testing and inspection
        self.requested_uri = self.uri
        self.uri = ""
        self.uriparts = ()
<commit_msg>Cleanup pylint convention complains like short variable names and calling parent object<commit_after>"""
Base implementation of Frappy framework.
"""

try:
    import urllib.request as urllib_request
    import urllib.error as urllib_error
except ImportError:
    import urllib2 as urllib_request
    import urllib2 as urllib_error

try:
    import json
except ImportError:
    import simplejson as json

class APIError(Exception):
    """
    Base Exception thrown by the APICall object when there is a
    general error interacting with the API.
    """

    pass

class APIHTTPError(APIError):
    """
    Base Exception thrown by the APICall object when there is a
    general error interacting with the API.
    """

    def __init__(self, err, uri, req_format, uriparts):
        """Initalize error object"""

        self.err = err
        self.uri = uri
        self.req_format = req_format
        self.uriparts = uriparts

        APIError.__init__(self)

    def __str__(self):
        """Stringify error"""

        return (
            "API sent status %i for URL: %s.%s using parameters: "
            "(%s)\ndetails: %s" % (
                self.eerr.code, self.uri, self.req_format, self.uriparts,
                self.err.fp.read()))

class APICall(object):
    """
    Base implementation of API call.

    This class is very generic and should provide most of the send/retrieve
    functionality for API access.  Thus, you should be able to subclass it,
    and provide a basic __init__ method.
    """

    def __init__(self, auth, req_format, domain, uriparts=None, secure=True,
                 post_actions=None, debug=False):

        """Initialize call API object"""

        self.auth = auth
        self.req_format = req_format
        self.domain = domain
        self.uri = ""
        self.requested_uri = ""
        self.secure = secure
        self.post_actions = post_actions

        if self.post_actions is None:
            self.post_actions = []

        self.response = None
        self.request_headers = {}
        self.response_headers = None

        self.method = "GET"
        self.body = None
        self.arg_data = ""

        self.uriparts = uriparts
        if self.uriparts is None:
            self.uriparts = ()

        # Set to True to test requests without Internet access (useful for unit
        # testing and just verifying that calls generate correct request uris)
        self.debug = debug

    def __getattr__(self, k):
        """
        Look for attribute k in base object, other wise append to uri

        This is allows for a very powerful and expressive syntax for creating
        API calls that map closely to the uri they query.  For example a
        Twitter call: <object>.statuses.public_timeline() will map to
        <domain>/statuses/public_timeline.
        """

        self.uriparts += (k,)
        return self

    def service_build_uri(self, **kwargs):
        """
        Service specific build uri

        This method is meant to be overriden by child classes to have the last
        opportunity to verify self.uri and add additional elements to it, etc.
        """
        return

    def _build_uri(self, *args, **kwargs):
        """Build up the final uri for the request"""

        uriparts = []
        for uripart in self.uriparts:
            # If this part matches a keyword argument, use the
            # supplied value otherwise, just use the part.
            uriparts.append(str(kwargs.pop(uripart, uripart)))

        self.uri += '/'.join(uriparts)

        # Don't use join here b/c not all arguments are required to be strings
        for arg in args:
            self.uri += '/%s' % (arg)

        secure_str = ''
        if self.secure:
            secure_str = 's'

        self.uri = "http%s://%s/%s" % (secure_str, self.domain, self.uri)

        # Wrapper for child classes to customize creation of the uri
        self.service_build_uri(**kwargs)

        # method is GET by default
        for action in self.post_actions:
            if self.uri.find(action) > -1:
                self.method = "POST"
                break

       # Append any authentication specified to request
        self._handle_auth(**kwargs)

    def _handle_auth(self, **kwargs):
        """
        Attach requested authentication to request and encoded body and
        parameters
        """

        if self.auth:
            self.request_headers.update(self.auth.generate_headers())
            self.arg_data = self.auth.encode_params(self.uri, self.method,
                                                    kwargs)

            if self.method == 'GET':
                self.uri += '?' + self.arg_data
            else:
                self.body = self.arg_data.encode('utf8')

    def __call__(self, *args, **kwargs):
        """
        Finish building uri with leftover arguments, append authentication, and
        send off request
        """

        self._build_uri(*args, **kwargs)

        req = urllib_request.Request(self.uri, self.body, self.request_headers)

        return self._handle_response(req, self.arg_data)

    def _handle_response(self, req, arg_data):
        """Verify response code and format data accordingly"""

        try:
            handle = urllib_request.urlopen(req)
            self.response_headers = handle.headers

            if "json" == self.req_format:
                self.response = json.loads(handle.read().decode('utf8'))
            else:
                self.response = handle.read().decode('utf8')

            return self
        except urllib_error.HTTPError as err:
            if (err.code == 304):
                return []
            # Allows for testing without Internet access
            elif self.debug:
                return self
            else:
                raise APIHTTPError(err, self.uri, self.req_format, arg_data)
        except urllib_error.URLError:
            # Allows for testing without Internet access
            if self.debug:
                return self

        finally:
            # Roll over request to prepare for new one
            self._reset_uri()

    def _reset_uri(self):
        """Clear active request uri to make way for another request"""

        # Save off the current uri request just for testing and inspection
        self.requested_uri = self.uri
        self.uri = ""
        self.uriparts = ()
<|endoftext|>